diff --git a/Makefile.arm-wince-mingw32ce-gcc b/Makefile.arm-wince-mingw32ce-gcc index d1ca389..323b7ea 100644 --- a/Makefile.arm-wince-mingw32ce-gcc +++ b/Makefile.arm-wince-mingw32ce-gcc @@ -103,15 +103,15 @@ SHPREFIX = # #TCL_FLAGS = #TCL_FLAGS = -DSTATIC_BUILD=1 -TCL_FLAGS = -I/home/drh/tcltk/8.4linux -#TCL_FLAGS = -I/home/drh/tcltk/8.4win -DSTATIC_BUILD=1 +TCL_FLAGS = -I/home/drh/tcltk/8.5linux +#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1 #TCL_FLAGS = -I/home/drh/tcltk/8.3hpux #### Linker options needed to link against the TCL library. # #LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcltk/8.4linux/libtcl8.4g.a -lm -ldl -#LIBTCL = /home/drh/tcltk/8.4win/libtcl84s.a -lmsvcrt +LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl +#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt #LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc #### Additional objects for SQLite library when TCL support is enabled. diff --git a/Makefile.in b/Makefile.in index 4974194..5d2c1d9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -748,7 +748,7 @@ tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DTCL_USE_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c + $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la $(LTLINK) -o $@ tclsqlite-shell.lo \ diff --git a/Makefile.linux-gcc b/Makefile.linux-gcc index 2bb6184..1c9f24f 100644 --- a/Makefile.linux-gcc +++ b/Makefile.linux-gcc @@ -93,15 +93,15 @@ SHPREFIX = lib # #TCL_FLAGS = #TCL_FLAGS = -DSTATIC_BUILD=1 -TCL_FLAGS = -I/home/drh/tcltk/8.4linux -#TCL_FLAGS = -I/home/drh/tcltk/8.4win -DSTATIC_BUILD=1 +TCL_FLAGS = -I/home/drh/tcltk/8.5linux +#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1 #TCL_FLAGS = -I/home/drh/tcltk/8.3hpux #### Linker options needed to link against the TCL library. # #LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcltk/8.4linux/libtcl8.4g.a -lm -ldl -#LIBTCL = /home/drh/tcltk/8.4win/libtcl84s.a -lmsvcrt +LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl +#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt #LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc #### Additional objects for SQLite library when TCL support is enabled. diff --git a/Makefile.vxworks b/Makefile.vxworks index b3f648e..993e557 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -110,15 +110,15 @@ SHPREFIX = lib # #TCL_FLAGS = #TCL_FLAGS = -DSTATIC_BUILD=1 -TCL_FLAGS = -I/home/drh/tcltk/8.4linux -#TCL_FLAGS = -I/home/drh/tcltk/8.4win -DSTATIC_BUILD=1 +TCL_FLAGS = -I/home/drh/tcltk/8.5linux +#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1 #TCL_FLAGS = -I/home/drh/tcltk/8.3hpux #### Linker options needed to link against the TCL library. # #LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcltk/8.4linux/libtcl8.4g.a -lm -ldl -#LIBTCL = /home/drh/tcltk/8.4win/libtcl84s.a -lmsvcrt +LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl +#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt #LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc #### Additional objects for SQLite library when TCL support is enabled. diff --git a/VERSION b/VERSION index 1680c69..0b2eb36 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.0.1 +3.7.2 diff --git a/configure b/configure index c8c0c1f..3ac3077 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.0.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.2. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.0.1' -PACKAGE_STRING='sqlite 3.7.0.1' +PACKAGE_VERSION='3.7.2' +PACKAGE_STRING='sqlite 3.7.2' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1487,7 +1487,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.0.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1552,7 +1552,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.0.1:";; + short | recursive ) echo "Configuration of sqlite 3.7.2:";; esac cat <<\_ACEOF @@ -1670,7 +1670,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.0.1 +sqlite configure 3.7.2 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1684,7 +1684,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.0.1, which was +It was created by sqlite $as_me 3.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -12259,7 +12259,7 @@ USE_AMALGAMATION=1 # if not, then we fall back to plain tclsh. # TODO: try other versions before falling back? # -for ac_prog in tclsh8.4 tclsh +for ac_prog in tclsh8.5 tclsh do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -13972,7 +13972,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.0.1, which was +This file was extended by sqlite $as_me 3.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14025,7 +14025,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.0.1 +sqlite config.status 3.7.2 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/configure.ac b/configure.ac index 441cf78..ea8c23f 100644 --- a/configure.ac +++ b/configure.ac @@ -139,7 +139,7 @@ USE_AMALGAMATION=1 # if not, then we fall back to plain tclsh. # TODO: try other versions before falling back? # -AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.4 tclsh], none) +AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.5 tclsh], none) if test "$TCLSH_CMD" = "none"; then # If we can't find a local tclsh, then building the amalgamation will fail. # We act as though --disable-amalgamation has been used. diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index fe347ba..16c95ed 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2653,7 +2653,7 @@ static void hashDestroy(void *p){ ** used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point a the "simple" tokenizer implementation. +** to by the argument to point to the "simple" tokenizer implementation. ** Function ...PorterTokenizerModule() sets *pModule to point to the ** porter tokenizer/stemmer implementation. */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 7542c28..008ba81 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -78,7 +78,6 @@ int sqlite3_fts3_enable_parentheses = 0; #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 #include "fts3Int.h" -#include #include #include @@ -104,7 +103,7 @@ struct ParseContext { ** negative values). */ static int fts3isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } /* diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 4e30778..5963abc 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -30,7 +30,6 @@ #include #include #include -#include #include "fts3_tokenizer.h" diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index de739ac..d67f7ac 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -16,7 +16,6 @@ #include "fts3Int.h" #include #include -#include /* diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index ab90be8..54a91bf 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -32,7 +32,6 @@ #include "fts3Int.h" #include -#include #include /* diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index 36c5a2f..432c35d 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -30,7 +30,6 @@ #include #include #include -#include #include "fts3_tokenizer.h" @@ -53,6 +52,9 @@ typedef struct simple_tokenizer_cursor { static int simpleDelim(simple_tokenizer *t, unsigned char c){ return c<0x80 && t->delim[c]; } +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} /* ** Create a new tokenizer instance. @@ -87,7 +89,7 @@ static int simpleCreate( /* Mark non-alphanumeric ASCII characters as delimiters */ int i; for(i=1; i<0x80; i++){ - t->delim[i] = !isalnum(i) ? -1 : 0; + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; } } @@ -193,7 +195,7 @@ static int simpleNext( ** case-insensitivity. */ unsigned char ch = p[iStartOffset+i]; - c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch); + c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); } *ppToken = c->pToken; *pnBytes = n; diff --git a/ext/icu/README.txt b/ext/icu/README.txt index 5c995cc..c5cadb5 100644 --- a/ext/icu/README.txt +++ b/ext/icu/README.txt @@ -139,7 +139,7 @@ SQLite. Documentation follows. 3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro Passing very long patterns to the built-in SQLite LIKE operator can - cause a stack overflow. To curb this problem, SQLite defines the + cause excessive CPU usage. To curb this problem, SQLite defines the SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a pattern in bytes (irrespective of encoding). The default value is defined in internal header file "limits.h". @@ -167,4 +167,3 @@ SQLite. Documentation follows. malicious users may execute arbitrary SQL (i.e. gears), they should be prevented from invoking the icu_load_collation() function, possibly using the authorisation callback. - diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 6576db1..0bc4dd7 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1072,11 +1072,10 @@ static int rtreeFilter( ** idxNum idxStr Strategy ** ------------------------------------------------ ** 1 Unused Direct lookup by rowid. -** 2 See below R-tree query. -** 3 Unused Full table scan. +** 2 See below R-tree query or full-table scan. ** ------------------------------------------------ ** -** If strategy 1 or 3 is used, then idxStr is not meaningful. If strategy +** If strategy 1 is used, then idxStr is not meaningful. If strategy ** 2 is used, idxStr is formatted to contain 2 bytes for each ** constraint used. The first two bytes of idxStr correspond to ** the constraint in sqlite3_index_info.aConstraintUsage[] with diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index d23a74e..bab596e 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -88,8 +88,8 @@ do_test rtree6.2.2 { do_test rtree6.2.3 { query_plan {SELECT * FROM t1,t2 WHERE k=ii} } [list \ - {TABLE t2} \ - {TABLE t1 VIRTUAL TABLE INDEX 1:} \ + {TABLE t1 VIRTUAL TABLE INDEX 2:} \ + {TABLE t2 USING PRIMARY KEY} \ ] do_test rtree6.2.4 { @@ -102,8 +102,8 @@ do_test rtree6.2.4 { do_test rtree6.2.5 { query_plan {SELECT * FROM t1,t2 WHERE k=ii AND x1pNewTable = pNew; pNew->nRef = 1; - pNew->dbMem = pTab->dbMem; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; diff --git a/src/analyze.c b/src/analyze.c index 94cf3a3..5984945 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -490,21 +490,20 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** If the Index.aSample variable is not NULL, delete the aSample[] array ** and its contents. */ -void sqlite3DeleteIndexSamples(Index *pIdx){ +void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #ifdef SQLITE_ENABLE_STAT2 if( pIdx->aSample ){ int j; - sqlite3 *dbMem = pIdx->pTable->dbMem; for(j=0; jaSample[j]; if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(pIdx->pTable->dbMem, p->u.z); + sqlite3DbFree(db, p->u.z); } } - sqlite3DbFree(dbMem, pIdx->aSample); - pIdx->aSample = 0; + sqlite3DbFree(db, pIdx->aSample); } #else + UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); #endif } @@ -543,7 +542,8 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); - sqlite3DeleteIndexSamples(pIdx); + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; } /* Check to make sure the sqlite_stat1 table exists */ @@ -587,18 +587,17 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase); if( pIdx ){ int iSample = sqlite3_column_int(pStmt, 1); - sqlite3 *dbMem = pIdx->pTable->dbMem; - assert( dbMem==db || dbMem==0 ); if( iSample=0 ){ int eType = sqlite3_column_type(pStmt, 2); if( pIdx->aSample==0 ){ static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocZero(dbMem, sz); + pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); if( pIdx->aSample==0 ){ db->mallocFailed = 1; break; } + memset(pIdx->aSample, 0, sz); } assert( pIdx->aSample ); @@ -621,10 +620,8 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ if( n < 1){ pSample->u.z = 0; }else{ - pSample->u.z = sqlite3DbMallocRaw(dbMem, n); - if( pSample->u.z ){ - memcpy(pSample->u.z, z, n); - }else{ + pSample->u.z = sqlite3DbStrNDup(0, z, n); + if( pSample->u.z==0 ){ db->mallocFailed = 1; break; } diff --git a/src/backup.c b/src/backup.c index 41fa2c3..1fc01e1 100644 --- a/src/backup.c +++ b/src/backup.c @@ -221,6 +221,15 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ rc = SQLITE_READONLY; } +#ifdef SQLITE_HAS_CODEC + /* Backup is not possible if the page size of the destination is changing + ** a a codec is in use. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + rc = SQLITE_READONLY; + } +#endif + /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. diff --git a/src/bitvec.c b/src/bitvec.c index c0e25a4..47d33ea 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -37,7 +37,7 @@ #include "sqliteInt.h" /* Size of the Bitvec structure in bytes. */ -#define BITVEC_SZ (sizeof(void*)*128) /* 512 on 32bit. 1024 on 64bit */ +#define BITVEC_SZ 512 /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ diff --git a/src/btree.c b/src/btree.c index 4746b12..378a218 100644 --- a/src/btree.c +++ b/src/btree.c @@ -32,7 +32,16 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ # define TRACE(X) #endif - +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) #ifndef SQLITE_OMIT_SHARED_CACHE /* @@ -721,11 +730,16 @@ int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ ** Given a page number of a regular database page, return the page ** number for the pointer-map page that contains the entry for the ** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. */ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ int nPagesPerMapPage; Pgno iPtrMap, ret; assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; nPagesPerMapPage = (pBt->usableSize/5)+1; iPtrMap = (pgno-2)/nPagesPerMapPage; ret = (iPtrMap*nPagesPerMapPage) + 2; @@ -1154,7 +1168,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); if( gap>top ) return SQLITE_CORRUPT_BKPT; testcase( gap+2==top ); testcase( gap+1==top ); @@ -1164,7 +1178,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* Always defragment highly fragmented pages */ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); }else if( gap+2<=top ){ /* Search the freelist looking for a free slot big enough to satisfy ** the request. The allocation is made from the first free slot in @@ -1206,7 +1220,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( gap+2+nByte>top ){ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); } @@ -1372,10 +1386,10 @@ static int btreeInitPage(MemPage *pPage){ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ BtShared *pBt; /* The main btree structure */ - u16 usableSize; /* Amount of usable space on each page */ + int usableSize; /* Amount of usable space on each page */ u16 cellOffset; /* Offset from start of page to first cell pointer */ - u16 nFree; /* Number of unused bytes on the page */ - u16 top; /* First byte of the cell content area */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */ int iCellLast; /* Last possible cell or freeblock offset */ @@ -1384,12 +1398,12 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ @@ -1488,13 +1502,13 @@ static void zeroPage(MemPage *pPage, int flags){ memset(&data[hdr+1], 0, 4); data[hdr+7] = 0; put2byte(&data[hdr+5], pBt->usableSize); - pPage->nFree = pBt->usableSize - first; + pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->hdrOffset = hdr; pPage->cellOffset = first; pPage->nOverflow = 0; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nCell = 0; pPage->isInit = 1; } @@ -1802,7 +1816,7 @@ int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->secureDelete = 1; #endif - pBt->pageSize = get2byte(&zDbHeader[16]); + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ pBt->pageSize = 0; @@ -2004,7 +2018,7 @@ int sqlite3BtreeClose(Btree *p){ if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } - sqlite3_free(pBt->pSchema); + sqlite3DbFree(0, pBt->pSchema); freeTempSpace(pBt); sqlite3_free(pBt); } @@ -2116,7 +2130,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pPage1 && !pBt->pCursor ); - pBt->pageSize = (u16)pageSize; + pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); @@ -2251,15 +2265,13 @@ static int lockBtree(BtShared *pBt){ ** a valid database file. */ nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - if( (rc = sqlite3PagerPagecount(pBt->pPager, &nPageFile))!=SQLITE_OK ){; - goto page1_init_failed; - } + sqlite3PagerPagecount(pBt->pPager, &nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; } if( nPage>0 ){ - int pageSize; - int usableSize; + u32 pageSize; + u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; if( memcmp(page1, zMagicHeader, 16)!=0 ){ @@ -2310,15 +2322,16 @@ static int lockBtree(BtShared *pBt){ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } - pageSize = get2byte(&page1[16]); - if( ((pageSize-1)&pageSize)!=0 || pageSize<512 || - (SQLITE_MAX_PAGE_SIZE<32768 && pageSize>SQLITE_MAX_PAGE_SIZE) + pageSize = (page1[16]<<8) | (page1[17]<<16); + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 ){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); usableSize = pageSize - page1[20]; - if( pageSize!=pBt->pageSize ){ + if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size ** of BtShared.pageSize, we have discovered that the page-size is ** actually pageSize. Unlock the database, leave pBt->pPage1 at @@ -2326,8 +2339,8 @@ static int lockBtree(BtShared *pBt){ ** again with the correct page-size. */ releasePage(pPage1); - pBt->usableSize = (u16)usableSize; - pBt->pageSize = (u16)pageSize; + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); @@ -2340,8 +2353,8 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } - pBt->pageSize = (u16)pageSize; - pBt->usableSize = (u16)usableSize; + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); @@ -2357,14 +2370,14 @@ static int lockBtree(BtShared *pBt){ ** 9-byte nKey value ** 4-byte nData value ** 4-byte overflow page pointer - ** So a cell consists of a 2-byte poiner, a header which is as much as + ** So a cell consists of a 2-byte pointer, a header which is as much as ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow ** page pointer. */ - pBt->maxLocal = (pBt->usableSize-12)*64/255 - 23; - pBt->minLocal = (pBt->usableSize-12)*32/255 - 23; - pBt->maxLeaf = pBt->usableSize - 35; - pBt->minLeaf = (pBt->usableSize-12)*32/255 - 23; + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; pBt->nPage = nPage; @@ -2417,7 +2430,8 @@ static int newDatabase(BtShared *pBt){ if( rc ) return rc; memcpy(data, zMagicHeader, sizeof(zMagicHeader)); assert( sizeof(zMagicHeader)==16 ); - put2byte(&data[16], pBt->pageSize); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); data[18] = 1; data[19] = 1; assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); @@ -3328,9 +3342,11 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; rc = newDatabase(pBt); pBt->nPage = get4byte(28 + pBt->pPage1->aData); - if( pBt->nPage==0 ){ - sqlite3PagerPagecount(pBt->pPager, (int*)&pBt->nPage); - } + + /* The database size was written into the offset 28 of the header + ** when the transaction started, so we know that the value at offset + ** 28 is nonzero. */ + assert( pBt->nPage>0 ); } sqlite3BtreeLeave(p); } @@ -4426,9 +4442,9 @@ int sqlite3BtreeMovetoUnpacked( pCur->validNKey = 1; pCur->info.nKey = nCellKey; }else{ - /* The maximum supported page-size is 32768 bytes. This means that + /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree - ** page is at most 8198 bytes, which may be stored as a 2-byte + ** page is less than 16384 bytes and may be stored as a 2-byte ** varint. This information is used to attempt to avoid parsing ** the entire cell by checking for the cases where the record is ** stored entirely within the b-tree page by inspecting the first @@ -4791,6 +4807,10 @@ static int allocateBtreePage( if( !pPrevTrunk ){ memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); } }else{ @@ -5097,7 +5117,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ Pgno ovflPgno; int rc; int nOvfl; - u16 ovflPageSize; + u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); @@ -5322,7 +5342,7 @@ static int fillInCell( */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ int i; /* Loop counter */ - int pc; /* Offset to cell content of cell being deleted */ + u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ int rc; /* The return code */ @@ -5340,7 +5360,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ hdr = pPage->hdrOffset; testcase( pc==get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ *pRC = SQLITE_CORRUPT_BKPT; return; } @@ -5397,7 +5417,7 @@ static void insertCell( if( *pRC ) return; assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* The cell should normally be sized correctly. However, when moving a @@ -5477,12 +5497,12 @@ static void assemblePage( assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); /* Check that the page has just been zeroed by zeroPage() */ assert( pPage->nCell==0 ); - assert( get2byte(&data[hdr+5])==nUsable ); + assert( get2byteNotZero(&data[hdr+5])==nUsable ); pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; @@ -5548,6 +5568,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); + /* This error condition is now caught prior to reaching this function */ if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of @@ -5877,7 +5898,7 @@ static int balance_nonroot( ** is allocated. */ if( pBt->secureDelete ){ int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); - if( (iOff+szNew[i])>pBt->usableSize ){ + if( (iOff+szNew[i])>(int)pBt->usableSize ){ rc = SQLITE_CORRUPT_BKPT; memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -5956,7 +5977,7 @@ static int balance_nonroot( szCell[nCell] = sz; pTemp = &aSpace1[iSpace1]; iSpace1 += sz; - assert( sz<=pBt->pageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iSpace1<=pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); apCell[nCell] = pTemp+leafCorrection; @@ -6202,7 +6223,7 @@ static int balance_nonroot( } } iOvflSpace += sz; - assert( sz<=pBt->pageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace<=pBt->pageSize ); insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); if( rc!=SQLITE_OK ) goto balance_cleanup; @@ -7446,7 +7467,7 @@ static void checkList( checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); } #endif - if( n>pCheck->pBt->usableSize/4-2 ){ + if( n>(int)pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, zContext, "freelist leaf count too big on page %d", iPage); N--; @@ -7657,7 +7678,7 @@ static int checkTreePage( if( hit==0 ){ pCheck->mallocFailed = 1; }else{ - u16 contentOffset = get2byte(&data[hdr+5]); + int contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); @@ -7665,12 +7686,12 @@ static int checkTreePage( cellStart = hdr + 12 - 4*pPage->leaf; for(i=0; i=usableSize ){ + if( (int)(pc+size-1)>=usableSize ){ checkAppendMsg(pCheck, 0, "Corruption detected in cell %d on page %d",i,iPage); }else{ @@ -7763,6 +7784,7 @@ char *sqlite3BtreeIntegrityCheck( sCheck.anRef[i] = 1; } sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist */ @@ -7861,6 +7883,29 @@ int sqlite3BtreeIsInTrans(Btree *p){ return (p && (p->inTrans==TRANS_WRITE)); } +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +*/ +int sqlite3BtreeCheckpoint(Btree *p){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + /* ** Return non-zero if a read (or write) transaction is active. */ @@ -7900,7 +7945,7 @@ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ - pBt->pSchema = sqlite3MallocZero(nBytes); + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); pBt->xFreeSchema = xFree; } sqlite3BtreeLeave(p); diff --git a/src/btree.h b/src/btree.h index 584b463..c989307 100644 --- a/src/btree.h +++ b/src/btree.h @@ -201,6 +201,10 @@ int sqlite3BtreeCursorInfo(BtCursor*, int*, int); void sqlite3BtreeCursorList(Btree*); #endif +#ifndef SQLITE_OMIT_WAL + int sqlite3BtreeCheckpoint(Btree*); +#endif + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the diff --git a/src/btreeInt.h b/src/btreeInt.h index 8b68d7b..7b46bce 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -46,7 +46,7 @@ ** ** The file is divided into pages. The first page is called page 1, ** the second is page 2, and so forth. A page number of zero indicates -** "no such page". The page size can be any power of 2 between 512 and 32768. +** "no such page". The page size can be any power of 2 between 512 and 65536. ** Each page can be either a btree page, a freelist page, an overflow ** page, or a pointer-map page. ** @@ -413,14 +413,14 @@ struct BtShared { u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ #endif - u16 pageSize; /* Total number of bytes on a page */ - u16 usableSize; /* Number of usable bytes on each page */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ u8 inTransaction; /* Transaction state */ u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ int nTransaction; /* Number of open transactions (read + write) */ u32 nPage; /* Number of pages in the database */ void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ diff --git a/src/build.c b/src/build.c index d964177..636b8a6 100644 --- a/src/build.c +++ b/src/build.c @@ -345,33 +345,14 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ /* ** Reclaim the memory used by an index */ -static void freeIndex(Index *p){ - sqlite3 *db = p->pTable->dbMem; +static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE - sqlite3DeleteIndexSamples(p); + sqlite3DeleteIndexSamples(db, p); #endif sqlite3DbFree(db, p->zColAff); sqlite3DbFree(db, p); } -/* -** Remove the given index from the index hash table, and free -** its memory structures. -** -** The index is removed from the database hash tables but -** it is not unlinked from the Table that it indexes. -** Unlinking from the Table must be done by the calling function. -*/ -static void sqlite3DeleteIndex(Index *p){ - Index *pOld; - const char *zName = p->zName; - - pOld = sqlite3HashInsert(&p->pSchema->idxHash, zName, - sqlite3Strlen30(zName), 0); - assert( pOld==0 || pOld==p ); - freeIndex(p); -} - /* ** For the index called zIdxName which is found in the database iDb, ** unlike that index from its Table then remove the index from @@ -398,7 +379,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ p->pNext = pIndex->pNext; } } - freeIndex(pIndex); + freeIndex(db, pIndex); } db->flags |= SQLITE_InternChanges; } @@ -469,13 +450,12 @@ void sqlite3CommitInternalChanges(sqlite3 *db){ } /* -** Clear the column names from a table or view. +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). */ -static void sqliteResetColumnNames(Table *pTable){ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; - sqlite3 *db = pTable->dbMem; - testcase( db==0 ); assert( pTable!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ @@ -487,8 +467,6 @@ static void sqliteResetColumnNames(Table *pTable){ } sqlite3DbFree(db, pTable->aCol); } - pTable->aCol = 0; - pTable->nCol = 0; } /* @@ -500,42 +478,44 @@ static void sqliteResetColumnNames(Table *pTable){ ** memory structures of the indices and foreign keys associated with ** the table. */ -void sqlite3DeleteTable(Table *pTable){ +void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; - sqlite3 *db; - if( pTable==0 ) return; - db = pTable->dbMem; - testcase( db==0 ); + assert( !pTable || pTable->nRef>0 ); /* Do not delete the table until the reference count reaches zero. */ - pTable->nRef--; - if( pTable->nRef>0 ){ - return; - } - assert( pTable->nRef==0 ); + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - /* Delete all indices associated with this table - */ + /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema ); - sqlite3DeleteIndex(pIndex); + if( !db || db->pnBytesFreed==0 ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + ); + assert( pOld==pIndex || pOld==0 ); + } + freeIndex(db, pIndex); } /* Delete any foreign keys attached to this table. */ - sqlite3FkDelete(pTable); + sqlite3FkDelete(db, pTable); /* Delete the Table structure itself. */ - sqliteResetColumnNames(pTable); + sqliteDeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK sqlite3ExprDelete(db, pTable->pCheck); #endif - sqlite3VtabClear(pTable); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); +#endif sqlite3DbFree(db, pTable); } @@ -554,7 +534,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, sqlite3Strlen30(zTabName),0); - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -822,7 +802,6 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->dbMem = 0; assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -1374,7 +1353,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zEnd = "\n)"; } n += 35 + 6*p->nCol; - zStmt = sqlite3Malloc( n ); + zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ db->mallocFailed = 1; return 0; @@ -1555,7 +1534,7 @@ void sqlite3EndTable( p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); } } @@ -1799,7 +1778,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); pTable->pSchema->flags |= DB_UnresetViews; }else{ pTable->nCol = 0; @@ -1824,7 +1803,9 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteResetColumnNames(pTab); + sqliteDeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; } } DbClearProperty(db, idx, DB_UnresetViews); @@ -2821,7 +2802,7 @@ Index *sqlite3CreateIndex( /* Clean up before exiting */ exit_create_index: if( pIndex ){ - sqlite3_free(pIndex->zColAff); + sqlite3DbFree(db, pIndex->zColAff); sqlite3DbFree(db, pIndex); } sqlite3ExprListDelete(db, pList); @@ -3200,7 +3181,7 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); sqlite3DbFree(db, pItem->zIndex); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); sqlite3IdListDelete(db, pItem->pUsing); diff --git a/src/callback.c b/src/callback.c index c016959..fdcc9f9 100644 --- a/src/callback.c +++ b/src/callback.c @@ -422,8 +422,7 @@ void sqlite3SchemaFree(void *p){ sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - assert( pTab->dbMem==0 ); - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(0, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); @@ -440,7 +439,7 @@ Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ if( pBt ){ p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree); }else{ - p = (Schema *)sqlite3MallocZero(sizeof(Schema)); + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ db->mallocFailed = 1; diff --git a/src/delete.c b/src/delete.c index 9608dc2..bd7ac3d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -24,7 +24,7 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc==1 ); pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ pTab->nRef++; diff --git a/src/expr.c b/src/expr.c index 6a1d972..33864b2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -55,6 +55,18 @@ char sqlite3ExprAffinity(Expr *pExpr){ return pExpr->affinity; } +/* +** Set the explicit collating sequence for an expression to the +** collating sequence supplied in the second argument. +*/ +Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ + if( pExpr && pColl ){ + pExpr->pColl = pColl; + pExpr->flags |= EP_ExpCollate; + } + return pExpr; +} + /* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to the revised expression. @@ -62,18 +74,13 @@ char sqlite3ExprAffinity(Expr *pExpr){ ** flag. An explicit collating sequence will override implicit ** collating sequences. */ -Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pCollName){ +Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ char *zColl = 0; /* Dequoted name of collation sequence */ CollSeq *pColl; sqlite3 *db = pParse->db; zColl = sqlite3NameFromToken(db, pCollName); - if( pExpr && zColl ){ - pColl = sqlite3LocateCollSeq(pParse, zColl); - if( pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; - } - } + pColl = sqlite3LocateCollSeq(pParse, zColl); + sqlite3ExprSetColl(pExpr, pColl); sqlite3DbFree(db, zColl); return pExpr; } @@ -547,18 +554,19 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ }else if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ - int i = atoi((char*)&z[1]); + i64 i; + int bOk = sqlite3Atoi64(&z[1], &i); pExpr->iColumn = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); } if( i>pParse->nVar ){ - pParse->nVar = i; + pParse->nVar = (int)i; } }else{ /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable diff --git a/src/fkey.c b/src/fkey.c index e6f4bfc..f0ad40d 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -500,7 +500,8 @@ static void fkScanChildren( if( pIdx ){ Column *pCol; iCol = pIdx->aiColumn[i]; - pCol = &pIdx->pTable->aCol[iCol]; + pCol = &pTab->aCol[iCol]; + if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); @@ -1061,11 +1062,7 @@ static Trigger *fkActionTrigger( pWhere = 0; } - /* In the current implementation, pTab->dbMem==0 for all tables except - ** for temporary tables used to describe subqueries. And temporary - ** tables do not have foreign key constraints. Hence, pTab->dbMem - ** should always be 0 there. - */ + /* Disable lookaside memory allocation */ enableLookaside = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; @@ -1155,37 +1152,39 @@ void sqlite3FkActions( ** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash ** hash table. */ -void sqlite3FkDelete(Table *pTab){ +void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ - if( pFKey->pPrevTo ){ - pFKey->pPrevTo->pNextTo = pFKey->pNextTo; - }else{ - void *data = (void *)pFKey->pNextTo; - const char *z = (data ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), data); + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } } - if( pFKey->pNextTo ){ - pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; - } - - /* Delete any triggers created to implement actions for this FK. */ -#ifndef SQLITE_OMIT_TRIGGER - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[0]); - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[1]); -#endif /* EV: R-30323-21917 Each foreign key constraint in SQLite is ** classified as either immediate or deferred. */ assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif + pNext = pFKey->pNextFrom; - sqlite3DbFree(pTab->dbMem, pFKey); + sqlite3DbFree(db, pFKey); } } #endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ diff --git a/src/func.c b/src/func.c index 8b1b2f7..d461412 100644 --- a/src/func.c +++ b/src/func.c @@ -788,8 +788,10 @@ static void compileoptionusedFunc( const char *zOptName; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_used() C interface. */ + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); } @@ -810,8 +812,9 @@ static void compileoptiongetFunc( int n; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_get() C interface. */ + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ n = sqlite3_value_int(argv[0]); sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); } @@ -1010,14 +1013,14 @@ static void replaceFunc( testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); - sqlite3DbFree(db, zOut); + sqlite3_free(zOut); return; } zOld = zOut; zOut = sqlite3_realloc(zOut, (int)nOut); if( zOut==0 ){ sqlite3_result_error_nomem(context); - sqlite3DbFree(db, zOld); + sqlite3_free(zOld); return; } memcpy(&zOut[j], zRep, nRep); @@ -1378,7 +1381,7 @@ static void groupConcatStep( if( pAccum ){ sqlite3 *db = sqlite3_context_db_handle(context); int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 1; + pAccum->useMalloc = 2; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ diff --git a/src/insert.c b/src/insert.c index f6ad5ab..adf6ef2 100644 --- a/src/insert.c +++ b/src/insert.c @@ -67,7 +67,7 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ int n; Table *pTab = pIdx->pTable; sqlite3 *db = sqlite3VdbeDb(v); - pIdx->zColAff = (char *)sqlite3Malloc(pIdx->nColumn+2); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); if( !pIdx->zColAff ){ db->mallocFailed = 1; return 0; @@ -109,7 +109,7 @@ void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ int i; sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3Malloc(pTab->nCol+1); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ db->mallocFailed = 1; return; @@ -1220,6 +1220,7 @@ void sqlite3GenerateConstraintChecks( if( onError==OE_Ignore ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ sqlite3HaltConstraint(pParse, onError, 0, 0); } sqlite3VdbeResolveLabel(v, allOk); diff --git a/src/loadext.c b/src/loadext.c index 8fe35d9..835b8a8 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -376,13 +376,11 @@ static int sqlite3LoadExtension( handle = sqlite3OsDlOpen(pVfs, zFile); if( handle==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "unable to open shared library [%s]", zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } } return SQLITE_ERROR; @@ -391,13 +389,11 @@ static int sqlite3LoadExtension( sqlite3OsDlSym(pVfs, handle, zProc); if( xInit==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zProc,zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } sqlite3OsDlClose(pVfs, handle); } diff --git a/src/main.c b/src/main.c index c3828d5..b233c84 100644 --- a/src/main.c +++ b/src/main.c @@ -1308,16 +1308,7 @@ int sqlite3Checkpoint(sqlite3 *db, int iDb){ for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( sqlite3BtreeIsInReadTrans(pBt) ){ - rc = SQLITE_LOCKED; - }else{ - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); - } - } + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt); } } diff --git a/src/malloc.c b/src/malloc.c index c7f8a19..b36b44f 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -251,6 +251,7 @@ static int mallocWithAlarm(int n, void **pp){ if( p ){ nFull = sqlite3MallocSize(p); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; return nFull; @@ -368,12 +369,14 @@ void sqlite3ScratchFree(void *p){ || p=(void*)mem0.aScratchFree ){ assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); if( sqlite3GlobalConfig.bMemstat ){ int iSize = sqlite3MallocSize(p); sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -408,7 +411,7 @@ void sqlite3ScratchFree(void *p){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, void *p){ - return db && p && p>=db->lookaside.pStart && plookaside.pEnd; + return p && p>=db->lookaside.pStart && plookaside.pEnd; } #else #define isLookaside(A,B) 0 @@ -420,15 +423,17 @@ static int isLookaside(sqlite3 *db, void *p){ */ int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ + if( db && isLookaside(db, p) ){ return db->lookaside.sz; }else{ - assert( sqlite3MemdebugHasType(p, - db ? (MEMTYPE_DB|MEMTYPE_HEAP) : MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); return sqlite3GlobalConfig.m.xSize(p); } } @@ -438,10 +443,12 @@ int sqlite3DbMallocSize(sqlite3 *db, void *p){ */ void sqlite3_free(void *p){ if( p==0 ) return; + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -455,16 +462,24 @@ void sqlite3_free(void *p){ */ void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - db->lookaside.nOut--; - }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - sqlite3_free(p); + if( db ){ + if( db->pnBytesFreed ){ + *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + return; + } + if( isLookaside(db, p) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + db->lookaside.nOut--; + return; + } } + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); } /* @@ -496,6 +511,7 @@ void *sqlite3Realloc(void *pOld, int nBytes){ sqlite3MallocAlarm(nNew-nOld); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nBytes); @@ -568,6 +584,7 @@ void *sqlite3DbMallocZero(sqlite3 *db, int n){ void *sqlite3DbMallocRaw(sqlite3 *db, int n){ void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( db==0 || db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE if( db ){ LookasideSlot *pBuf; @@ -593,8 +610,8 @@ void *sqlite3DbMallocRaw(sqlite3 *db, int n){ if( !p && db ){ db->mallocFailed = 1; } - sqlite3MemdebugSetType(p, - (db && db->lookaside.bEnabled) ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(p, MEMTYPE_DB | + ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); return p; } @@ -620,14 +637,16 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc(p, n); if( !pNew ){ + sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; } - sqlite3MemdebugSetType(pNew, - db->lookaside.bEnabled ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; diff --git a/src/mem2.c b/src/mem2.c index a82fee8..83f12fd 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -378,7 +378,7 @@ void sqlite3MemSetDefault(void){ ** Set the "type" of an allocation. */ void sqlite3MemdebugSetType(void *p, u8 eType){ - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); @@ -397,23 +397,38 @@ void sqlite3MemdebugSetType(void *p, u8 eType){ */ int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ - assert( (pHdr->eType & (pHdr->eType-1))==0 ); /* Only one type bit set */ if( (pHdr->eType&eType)==0 ){ - void **pBt; - pBt = (void**)pHdr; - pBt -= pHdr->nBacktraceSlots; - backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(stderr)); - fprintf(stderr, "\n"); rc = 0; } } return rc; } - + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +*/ +int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)!=0 ){ + rc = 0; + } + } + return rc; +} /* ** Set the number of backtrace levels kept for each allocation. diff --git a/src/mutex_unix.c b/src/mutex_unix.c index a1ebc3b..196975e 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -235,6 +235,7 @@ static void pthreadMutexEnter(sqlite3_mutex *p){ */ pthread_mutex_lock(&p->mutex); #if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); p->owner = pthread_self(); p->nRef++; #endif @@ -307,6 +308,7 @@ static void pthreadMutexLeave(sqlite3_mutex *p){ assert( pthreadMutexHeld(p) ); #if SQLITE_MUTEX_NREF p->nRef--; + if( p->nRef==0 ) p->owner = 0; #endif assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 442a6b7..fba964a 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -220,7 +220,7 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); - assert( p->nRef==0 ); + assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); @@ -244,6 +244,7 @@ static void winMutexEnter(sqlite3_mutex *p){ #endif EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG + assert( p->nRef>0 || p->owner==0 ); p->owner = tid; p->nRef++; if( p->trace ){ @@ -297,6 +298,7 @@ static void winMutexLeave(sqlite3_mutex *p){ assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; + if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif LeaveCriticalSection(&p->mutex); diff --git a/src/notify.c b/src/notify.c index 4c8ab00..fcab5bf 100644 --- a/src/notify.c +++ b/src/notify.c @@ -255,7 +255,7 @@ void sqlite3ConnectionUnlocked(sqlite3 *db){ assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3DbMallocSize(db, aDyn)/sizeof(void*))) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) ){ /* The aArg[] array needs to grow. */ void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); diff --git a/src/os_unix.c b/src/os_unix.c index 9457516..e3ea4f2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -210,6 +210,7 @@ struct unixFile { int fileFlags; /* Miscellanous flags */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif @@ -2763,6 +2764,7 @@ static int unixWrite( } SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); + if( amt>0 ){ if( wrote<0 ){ /* lastErrno set by seekAndWrite */ @@ -2772,6 +2774,7 @@ static int unixWrite( return SQLITE_FULL; } } + return SQLITE_OK; } @@ -2973,12 +2976,23 @@ static int unixSync(sqlite3_file *id, int flags){ ** Truncate an open file to a specified size */ static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; int rc; - assert( id ); + assert( pFile ); SimulateIOError( return SQLITE_IOERR_TRUNCATE ); - rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = ftruncate(pFile->h, (off_t)nByte); if( rc ){ - ((unixFile*)id)->lastErrno = errno; + pFile->lastErrno = errno; return SQLITE_IOERR_TRUNCATE; }else{ #ifndef NDEBUG @@ -2989,8 +3003,8 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ** when restoring a database using the backup API from a zero-length ** source. */ - if( ((unixFile*)id)->inNormalWrite && nByte==0 ){ - ((unixFile*)id)->transCntrChng = 1; + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; } #endif @@ -3033,6 +3047,54 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ static int proxyFileControl(sqlite3_file*,int,void*); #endif +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. +** +** If the user has configured a chunk-size for this file, it could be +** that the file needs to be extended at this point. Otherwise, the +** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( fstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ + return SQLITE_IOERR_WRITE; + } +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + int nWrite; /* Return value from seekAndWrite() */ + + if( ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + do { + nWrite = seekAndWrite(pFile, iWrite, "", 1); + iWrite += nBlk; + } while( nWrite==1 && iWritelastErrno; return SQLITE_OK; } - case SQLITE_FCNTL_SIZE_HINT: { -#if 0 /* No performance advantage seen on Linux */ - sqlite3_int64 szFile = *(sqlite3_int64*)pArg; - unixFile *pFile = (unixFile*)id; - ftruncate(pFile->h, szFile); -#endif + case SQLITE_FCNTL_CHUNK_SIZE: { + ((unixFile*)id)->szChunk = *(int *)pArg; return SQLITE_OK; } + case SQLITE_FCNTL_SIZE_HINT: { + return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -5918,7 +5979,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } @@ -6109,9 +6170,9 @@ static int proxyClose(sqlite3_file *id) { if( rc ) return rc; sqlite3_free(conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); - sqlite3_free(pCtx->dbPath); + sqlite3DbFree(0, pCtx->dbPath); /* restore the original locking context and pMethod then close it */ pFile->lockingContext = pCtx->oldLockingContext; pFile->pMethod = pCtx->pOldMethod; diff --git a/src/os_win.c b/src/os_win.c index 0951312..562282f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -108,6 +108,7 @@ struct winFile { DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ @@ -620,6 +621,42 @@ static BOOL winceLockFileEx( ** by the sqlite3_io_methods object. ******************************************************************************/ +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occured, it is also necessary to call + ** GetLastError(). + */ + dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ + pFile->lastErrno = GetLastError(); + return 1; + } + + return 0; +} + /* ** Close a file. ** @@ -662,13 +699,6 @@ static int winClose(sqlite3_file *id){ return rc ? SQLITE_OK : SQLITE_IOERR; } -/* -** Some microsoft compilers lack this definition. -*/ -#ifndef INVALID_SET_FILE_POINTER -# define INVALID_SET_FILE_POINTER ((DWORD)-1) -#endif - /* ** Read data from a file into a buffer. Return SQLITE_OK if all ** bytes were read successfully and SQLITE_IOERR if anything goes @@ -680,32 +710,27 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD got; + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &got, 0) ){ + if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); return SQLITE_IOERR_READ; } - if( got==(DWORD)amt ){ - return SQLITE_OK; - }else{ + if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[got], 0, amt-got); + memset(&((char*)pBuf)[nRead], 0, amt-nRead); return SQLITE_IOERR_SHORT_READ; } + + return SQLITE_OK; } /* @@ -713,47 +738,42 @@ static int winRead( ** or some other error code on failure. */ static int winWrite( - sqlite3_file *id, /* File to write into */ - const void *pBuf, /* The bytes to be written */ - int amt, /* Number of bytes to write */ - sqlite3_int64 offset /* Offset into the file to begin writing at */ + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD wrote = 0; + int rc; /* True if error has occured, else false */ + winFile *pFile = (winFile*)id; /* File handle */ - assert( id!=0 ); + assert( amt>0 ); + assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); + OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; + + rc = seekWinFile(pFile, offset); + if( rc==0 ){ + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + + while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = GetLastError(); + rc = 1; } } - assert( amt>0 ); - while( - amt>0 - && (rc = WriteFile(pFile->h, pBuf, amt, &wrote, 0))!=0 - && wrote>0 - ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - if( !rc || amt>(int)wrote ){ - pFile->lastErrno = GetLastError(); + + if( rc ){ if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; } + return SQLITE_IOERR_WRITE; } return SQLITE_OK; } @@ -762,26 +782,32 @@ static int winWrite( ** Truncate an open file to a specified size */ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ - LONG upperBits = (LONG)((nByte>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(nByte & 0xffffffff); - DWORD dwRet; - winFile *pFile = (winFile*)id; - DWORD error; - int rc = SQLITE_OK; + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + + assert( pFile ); - assert( id!=0 ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( dwRet==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( seekWinFile(pFile, nByte) ){ rc = SQLITE_IOERR_TRUNCATE; - /* SetEndOfFile will fail if nByte is negative */ - }else if( !SetEndOfFile(pFile->h) ){ + }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); rc = SQLITE_IOERR_TRUNCATE; } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc==SQLITE_OK ? "ok" : "failed")); + + OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; } @@ -1146,6 +1172,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = (int)((winFile*)id)->lastErrno; return SQLITE_OK; } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((winFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } case SQLITE_FCNTL_SIZE_HINT: { sqlite3_int64 sz = *(sqlite3_int64*)pArg; SimulateIOErrorBenign(1); diff --git a/src/pager.c b/src/pager.c index dfb29e3..f0b5b77 100644 --- a/src/pager.c +++ b/src/pager.c @@ -22,8 +22,12 @@ #include "sqliteInt.h" #include "wal.h" -/* -******************** NOTES ON THE DESIGN OF THE PAGER ************************ + +/******************* NOTES ON THE DESIGN OF THE PAGER ************************ +** +** This comment block describes invariants that hold when using a rollback +** journal. These invariants do not apply for journal_mode=WAL, +** journal_mode=MEMORY, or journal_mode=OFF. ** ** Within this comment block, a page is deemed to have been synced ** automatically as soon as it is written when PRAGMA synchronous=OFF. @@ -57,7 +61,7 @@ ** transaction. ** ** (3) Writes to the database file are an integer multiple of the page size -** in length and are aligned to a page boundary. +** in length and are aligned on a page boundary. ** ** (4) Reads from the database file are either aligned on a page boundary and ** an integer multiple of the page size in length or are taken from the @@ -88,7 +92,8 @@ ** ** (9) Whenever the database file is modified, at least one bit in the range ** of bytes from 24 through 39 inclusive will be changed prior to releasing -** the EXCLUSIVE lock. +** the EXCLUSIVE lock, thus signaling other connections on the same +** database to flush their caches. ** ** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less ** than one billion transactions. @@ -101,7 +106,8 @@ ** ** (13) A SHARED lock is held on the database file while reading any ** content out of the database file. -*/ +** +******************************************************************************/ /* ** Macros for troubleshooting. Normally turned off @@ -126,58 +132,279 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define FILEHANDLEID(fd) ((int)fd) /* -** The page cache as a whole is always in one of the following -** states: +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. ** -** PAGER_UNLOCK The page cache is not currently reading or -** writing the database file. There is no -** data held in memory. This is the initial -** state. +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ ** -** PAGER_SHARED The page cache is reading the database. -** Writing is not permitted. There can be -** multiple readers accessing the same database -** file at the same time. ** -** PAGER_RESERVED This process has reserved the database for writing -** but has not yet made any changes. Only one process -** at a time can reserve the database. The original -** database file has not been modified so other -** processes may still be reading the on-disk -** database file. +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] ** -** PAGER_EXCLUSIVE The page cache is writing the database. -** Access is exclusive. No other processes or -** threads can be reading or writing while one -** process is writing. +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] ** -** PAGER_SYNCED The pager moves to this state from PAGER_EXCLUSIVE -** after all dirty pages have been written to the -** database file and the file has been synced to -** disk. All that remains to do is to remove or -** truncate the journal file and the transaction -** will be committed. +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** ** -** The page cache comes up in PAGER_UNLOCK. The first time a -** sqlite3PagerGet() occurs, the state transitions to PAGER_SHARED. -** After all pages have been released using sqlite_page_unref(), -** the state transitions back to PAGER_UNLOCK. The first time -** that sqlite3PagerWrite() is called, the state transitions to -** PAGER_RESERVED. (Note that sqlite3PagerWrite() can only be -** called on an outstanding page which means that the pager must -** be in PAGER_SHARED before it transitions to PAGER_RESERVED.) -** PAGER_RESERVED means that there is an open rollback journal. -** The transition to PAGER_EXCLUSIVE occurs before any changes -** are made to the database file, though writes to the rollback -** journal occurs with just PAGER_RESERVED. After an sqlite3PagerRollback() -** or sqlite3PagerCommitPhaseTwo(), the state can go back to PAGER_SHARED, -** or it can stay at PAGER_EXCLUSIVE if we are in exclusive access mode. +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occured, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). */ -#define PAGER_UNLOCK 0 -#define PAGER_SHARED 1 /* same as SHARED_LOCK */ -#define PAGER_RESERVED 2 /* same as RESERVED_LOCK */ -#define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ -#define PAGER_SYNCED 5 +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) /* ** A macro used for invoking the codec if there is one @@ -227,33 +454,28 @@ struct PagerSavepoint { }; /* -** A open page cache is an instance of the following structure. +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: ** -** errCode +** eState ** -** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or -** or SQLITE_FULL. Once one of the first three errors occurs, it persists -** and is returned as the result of every major pager API call. The -** SQLITE_FULL return code is slightly different. It persists only until the -** next successful rollback is performed on the pager cache. Also, -** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() -** APIs, they may still be used successfully. +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. ** -** dbSizeValid, dbSize, dbOrigSize, dbFileSize +** eLock ** -** Managing the size of the database file in pages is a little complicated. -** The variable Pager.dbSize contains the number of pages that the database -** image currently contains. As the database image grows or shrinks this -** variable is updated. The variable Pager.dbFileSize contains the number -** of pages in the database file. This may be different from Pager.dbSize -** if some pages have been appended to the database image but not yet written -** out from the cache to the actual file on disk. Or if the image has been -** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable -** contains the number of pages in the database image when the current -** transaction was opened. The contents of all three of these variables is -** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. ** -** TODO: Under what conditions is dbSizeValid set? Cleared? +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. ** ** changeCountDone ** @@ -272,67 +494,123 @@ struct PagerSavepoint { ** need only update the change-counter once, for the first transaction ** committed. ** -** dbModified -** -** The dbModified flag is set whenever a database page is dirtied. -** It is cleared at the end of each transaction. -** -** It is used when committing or otherwise ending a transaction. If -** the dbModified flag is clear then less work has to be done. -** -** journalStarted -** -** This flag is set whenever the the main journal is opened and -** initialized -** -** The point of this flag is that it must be set after the -** first journal header in a journal file has been synced to disk. -** After this has happened, new pages appended to the database -** do not need the PGHDR_NEED_SYNC flag set, as they do not need -** to wait for a journal sync before they can be written out to -** the database file (see function pager_write()). -** ** setMaster ** -** This variable is used to ensure that the master journal file name -** (if any) is only written into the journal file once. +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a master-journal name to be written into the +** journal file before it is synced to disk. ** -** When committing a transaction, the master journal file name (if any) -** may be written into the journal file while the pager is still in -** PAGER_RESERVED state (see CommitPhaseOne() for the action). It -** then attempts to upgrade to an exclusive lock. If this attempt -** fails, then SQLITE_BUSY may be returned to the user and the user -** may attempt to commit the transaction again later (calling -** CommitPhaseOne() again). This flag is used to ensure that the -** master journal name is only written to the journal file the first -** time CommitPhaseOne() is called. +** Whether or not a journal file contains a master-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a master-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain master journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** master journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setMaster flag +** is cleared anyway (and the pager will move to ERROR state). ** ** doNotSpill, doNotSyncSpill ** -** When enabled, cache spills are prohibited. The doNotSpill variable -** inhibits all cache spill and doNotSyncSpill inhibits those spills that -** would require a journal sync. The doNotSyncSpill is set and cleared -** by sqlite3PagerWrite() in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. The -** doNotSpill value set to prevent pagerStress() from trying to use -** the journal during a rollback. +** These two boolean variables control the behaviour of cache-spills +** (calls made by the pcache module to the pagerStress() routine to +** write cached data to the file-system in order to free up memory). ** -** needSync -** -** TODO: It might be easier to set this variable in writeJournalHdr() -** and writeMasterJournal() only. Change its meaning to "unsynced data -** has been written to the journal". +** When doNotSpill is non-zero, writing to the database from pagerStress() +** is disabled altogether. This is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). +** +** If doNotSyncSpill is non-zero, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** ** subjInMemory ** ** This is a boolean variable. If true, then any required sub-journal ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ - u8 journalMode; /* On of the PAGER_JOURNALMODE_* values */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noReadlock; /* Do not bother to obtain readlocks */ u8 noSync; /* Do not sync the journal if true */ @@ -342,30 +620,26 @@ struct Pager { u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ - /* The following block contains those class members that are dynamically - ** modified during normal operations. The other variables in this structure - ** are either constant throughout the lifetime of the pager, or else - ** used to store configuration parameters that affect the way the pager - ** operates. - ** - ** The 'state' variable is described in more detail along with the - ** descriptions of the values it may take - PAGER_UNLOCK etc. Many of the - ** other variables in this block are described in the comment directly - ** above this class definition. + /************************************************************************** + ** The following block contains those class members that change during + ** routine opertion. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. */ - u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ - u8 dbModified; /* True if there are any changes to the Db */ - u8 needSync; /* True if an fsync() is needed on the journal */ - u8 journalStarted; /* True if header of journal is synced */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ - u8 dbSizeValid; /* Set when dbSize is correct */ u8 subjInMemory; /* True to use in-memory sub-journals */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ int errCode; /* One of several kinds of errors */ int nRec; /* Pages journalled since last j-header written */ u32 cksumInit; /* Quasi-random value added to every checksum */ @@ -376,17 +650,21 @@ struct Pager { sqlite3_file *sjfd; /* File descriptor for sub-journal */ i64 journalOff; /* Current write offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ - i64 journalSizeLimit; /* Size limit for persistent journal files */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - u32 sectorSize; /* Assumed sector size during rollback */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ u16 nExtra; /* Add this many bytes to each in-memory page */ i16 nReserve; /* Number of unused bytes at end of each page */ u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ int pageSize; /* Number of bytes in a page */ Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ @@ -404,7 +682,6 @@ struct Pager { #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ - sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ #ifndef SQLITE_OMIT_WAL Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ @@ -483,22 +760,221 @@ static const unsigned char aJournalMagic[] = { */ #define PAGER_MAX_PGNO 2147483647 +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + #ifndef NDEBUG /* ** Usage: ** ** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. */ -static int assert_pager_state(Pager *pPager){ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; - /* A temp-file is always in PAGER_EXCLUSIVE or PAGER_SYNCED state. */ - assert( pPager->tempFile==0 || pPager->state>=PAGER_EXCLUSIVE ); + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); - /* The changeCountDone flag is always set for temp-files */ - assert( pPager->tempFile==0 || pPager->changeCountDone ); + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setMaster==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; + } return 1; } + +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +*/ +static char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; +} #endif /* @@ -551,6 +1027,7 @@ static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ */ #define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + /* ** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK ** on success or an error code is something goes wrong. @@ -562,27 +1039,53 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ } /* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. ** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. */ -#define isOpen(pFd) ((pFd)->pMethods) +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + return rc; +} /* -** If file pFd is open, call sqlite3OsUnlock() on it. +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. */ -static int osUnlock(sqlite3_file *pFd, int eLock){ - if( !isOpen(pFd) ){ - return SQLITE_OK; +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } } - return sqlite3OsUnlock(pFd, eLock); + return rc; } /* @@ -658,13 +1161,14 @@ static void pager_set_pagehash(PgHdr *pPage){ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( !pPg->pageHash || pPager->errCode - || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else #define pager_datahash(X,Y) 0 #define pager_pagehash(X) 0 +#define pager_set_pagehash(X) #define CHECK_PAGE(x) #endif /* SQLITE_CHECK_PAGES */ @@ -831,7 +1335,7 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ static int writeJournalHdr(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ - u32 nHeader = pPager->pageSize; /* Size of buffer pointed to by zHeader */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ u32 nWrite; /* Bytes of header sector written */ int ii; /* Loop counter */ @@ -874,7 +1378,7 @@ static int writeJournalHdr(Pager *pPager){ ** that garbage data is never appended to the journal file. */ assert( isOpen(pPager->fd) || pPager->noSync ); - if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); @@ -998,7 +1502,6 @@ static int readJournalHdr( if( pPager->journalOff==0 ){ u32 iPageSize; /* Page-size field of journal header */ u32 iSectorSize; /* Sector-size field of journal header */ - u16 iPageSize16; /* Copy of iPageSize in 16-bit variable */ /* Read the page-size and sector-size journal header fields. */ if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) @@ -1007,6 +1510,14 @@ static int readJournalHdr( return rc; } + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power ** of two greater than or equal to 512 or 32, and not greater than their @@ -1028,10 +1539,8 @@ static int readJournalHdr( ** Use a testcase() macro to make sure that malloc failure within ** PagerSetPagesize() is tested. */ - iPageSize16 = (u16)iPageSize; - rc = sqlite3PagerSetPagesize(pPager, &iPageSize16, -1); + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); testcase( rc!=SQLITE_OK ); - assert( rc!=SQLITE_OK || iPageSize16==(u16)iPageSize ); /* Update the assumed sector-size to match the value used by ** the process that created this journal. If this journal was @@ -1073,7 +1582,10 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ i64 jrnlSize; /* Size of journal file on disk */ u32 cksum = 0; /* Checksum of string zMaster */ - if( !zMaster || pPager->setMaster + assert( pPager->setMaster==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_OFF ){ @@ -1109,7 +1621,6 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } pPager->journalOff += (nMaster+20); - pPager->needSync = !pPager->noSync; /* If the pager is in peristent-journal mode, then the physical ** journal-file may extend past the end of the master-journal name @@ -1145,17 +1656,11 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ } /* -** Unless the pager is in error-state, discard all in-memory pages. If -** the pager is in error-state, then this call is a no-op. -** -** TODO: Why can we not reset the pager while in error state? +** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ - if( SQLITE_OK==pPager->errCode ){ - sqlite3BackupRestart(pPager->pBackup); - sqlite3PcacheClear(pPager->pPCache); - pPager->dbSizeValid = 0; - } + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); } /* @@ -1198,34 +1703,39 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ } /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. -*/ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); -} -#else -# define pagerUseWal(x) 0 -# define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 -# define pagerOpenWalIfPresent(z) SQLITE_OK -# define pagerBeginReadTransaction(z) SQLITE_OK -#endif - -/* -** Unlock the database file. This function is a no-op if the pager -** is in exclusive mode. +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. ** -** If the pager is currently in error state, discard the contents of -** the cache and reset the Pager structure internal state. If there is -** an open journal-file, then the next time a shared-lock is obtained -** on the pager file (by this or any other process), it will be -** treated as a hot-journal and rolled back. +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). */ static void pager_unlock(Pager *pPager){ - if( !pPager->exclusiveMode ){ - int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; /* If the operating system support deletion of open files, then @@ -1245,58 +1755,56 @@ static void pager_unlock(Pager *pPager){ sqlite3OsClose(pPager->jfd); } - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - releaseAllSavepoints(pPager); - - /* If the file is unlocked, somebody else might change it. The - ** values stored in Pager.dbSize etc. might become invalid if - ** this happens. One can argue that this doesn't need to be cleared - ** until the change-counter check fails in PagerSharedLock(). - ** Clearing the page size cache here is being conservative. + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. */ - pPager->dbSizeValid = 0; - - if( pagerUseWal(pPager) ){ - sqlite3WalEndReadTransaction(pPager->pWal); - }else{ - rc = osUnlock(pPager->fd, NO_LOCK); + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; } - if( rc ){ - pPager->errCode = rc; - } - IOTRACE(("UNLOCK %p\n", pPager)) - /* If Pager.errCode is set, the contents of the pager cache cannot be - ** trusted. Now that the pager file is unlocked, the contents of the - ** cache can be discarded and the error code safely cleared. + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. */ - if( pPager->errCode ){ - if( rc==SQLITE_OK ){ - pPager->errCode = SQLITE_OK; - } - pager_reset(pPager); - } - + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; - pPager->state = PAGER_UNLOCK; - pPager->dbModified = 0; + pPager->eState = PAGER_OPEN; } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; } /* -** This function should be called when an IOERR, CORRUPT or FULL error -** may have occurred. The first argument is a pointer to the pager -** structure, the second the error-code about to be returned by a pager -** API function. The value returned is a copy of the second argument -** to this function. +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** -** If the second argument is SQLITE_IOERR, SQLITE_CORRUPT, or SQLITE_FULL -** the error becomes persistent. Until the persistent error is cleared, -** subsequent API calls on this Pager will immediately return the same -** error code. +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. ** -** A persistent error indicates that the contents of the pager-cache +** The ERROR state indicates that the contents of the pager-cache ** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need @@ -1313,36 +1821,11 @@ static int pager_error(Pager *pPager, int rc){ ); if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; + pPager->eState = PAGER_ERROR; } return rc; } -/* -** Execute a rollback if a transaction is active and unlock the -** database file. -** -** If the pager has already entered the error state, do not attempt -** the rollback at this time. Instead, pager_unlock() is called. The -** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and clear the error state. If this means that -** there is a hot-journal left in the file-system, the next connection -** to obtain a shared lock on the pager (which may be this one) will -** roll it back. -** -** If the pager has not already entered the error state, but an IO or -** malloc error occurs during a rollback, then this will itself cause -** the pager to enter the error state. Which will be cleared by the -** call to pager_unlock(), as described above. -*/ -static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->errCode==SQLITE_OK && pPager->state>=PAGER_RESERVED ){ - sqlite3BeginBenignMalloc(); - sqlite3PagerRollback(pPager); - sqlite3EndBenignMalloc(); - } - pager_unlock(pPager); -} - /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1350,8 +1833,9 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** the journal file or writing the very first journal-header of a ** database transaction. ** -** If the pager is in PAGER_SHARED or PAGER_UNLOCK state when this -** routine is called, it is a no-op (returns SQLITE_OK). +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** @@ -1382,13 +1866,9 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** DELETE and the pager is in exclusive mode, the method described under ** journalMode==PERSIST is used instead. ** -** After the journal is finalized, if running in non-exclusive mode, the -** pager moves to PAGER_SHARED state (and downgrades the lock on the -** database file accordingly). -** -** If the pager is running in exclusive mode and is in PAGER_SYNCED state, -** it moves to PAGER_EXCLUSIVE. No locks are downgraded when running in -** exclusive mode. +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the @@ -1403,11 +1883,26 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - if( pPager->stateeState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); @@ -1423,14 +1918,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsTruncate(pPager->jfd, 0); } pPager->journalOff = 0; - pPager->journalStarted = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ rc = zeroJournalHdr(pPager, hasMaster); - pager_error(pPager, rc); pPager->journalOff = 0; - pPager->journalStarted = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal @@ -1446,49 +1938,77 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } + } #ifdef SQLITE_CHECK_PAGES - sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); -#endif + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnref(p); + } } +#endif + sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); - pPager->state = PAGER_SHARED; - - /* If the connection was in locking_mode=exclusive mode but is no longer, - ** drop the EXCLUSIVE lock held on the database file. - */ - if( !pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, 0) ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - } - }else if( !pPager->exclusiveMode ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - pPager->state = PAGER_SHARED; + } + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; - }else if( pPager->state==PAGER_SYNCED ){ - pPager->state = PAGER_EXCLUSIVE; } + pPager->eState = PAGER_READER; pPager->setMaster = 0; - pPager->needSync = 0; - pPager->dbModified = 0; - - /* TODO: Is this optimal? Why is the db size invalidated here - ** when the database file is not unlocked? */ - pPager->dbOrigSize = 0; - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } return (rc==SQLITE_OK?rc2:rc); } +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0); + } + } + pager_unlock(pPager); +} + /* ** Parameter aData must point to a buffer of pPager->pageSize bytes ** of data. Compute and return a checksum based ont the contents of the @@ -1539,9 +2059,8 @@ static void pagerReportSize(Pager *pPager){ ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The isMainJrnl flag is true if this is the main rollback journal and -** false for the statement journal. The main rollback journal uses -** checksums - the statement journal does not. +** The main rollback journal uses checksums - the statement journal does +** not. ** ** If the page number of the page record read from the (sub-)journal file ** is greater than the current value of Pager.dbSize, then playback is @@ -1595,6 +2114,17 @@ static int pager_playback_one_page( assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. */ @@ -1631,7 +2161,6 @@ static int pager_playback_one_page( if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE ); /* When playing back page 1, restore the nReserve setting */ @@ -1640,7 +2169,7 @@ static int pager_playback_one_page( pagerReportSize(pPager); } - /* If the pager is in RESERVED state, then there must be a copy of this + /* If the pager is in CACHEMOD state, then there must be a copy of this ** page in the pager cache. In this case just update the pager cache, ** not the database file. The page is left marked dirty in this case. ** @@ -1651,8 +2180,11 @@ static int pager_playback_one_page( ** either. So the condition described in the above paragraph is not ** assert()able. ** - ** If in EXCLUSIVE state, then we update the pager cache if it exists - ** and the main file. The page is then marked not dirty. + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. ** ** Ticket #1171: The statement journal might contain page content that is ** different from the page content at the start of the transaction. @@ -1678,6 +2210,7 @@ static int pager_playback_one_page( pPg = pager_lookup(pPager, pgno); } assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -1687,8 +2220,8 @@ static int pager_playback_one_page( }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->state>=PAGER_EXCLUSIVE) - && isOpen(pPager->fd) + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) && isSynced ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; @@ -1764,9 +2297,8 @@ static int pager_playback_one_page( assert( !pagerUseWal(pPager) ); sqlite3PcacheMakeClean(pPg); } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); + /* If this was page 1, then restore the value of Pager.dbFileVers. ** Do this before any decoding. */ if( pgno==1 ){ @@ -1918,10 +2450,10 @@ delmaster_out: ** file in the file-system. This only happens when committing a transaction, ** or rolling back a transaction (including rolling back a hot-journal). ** -** If the main database file is not open, or an exclusive lock is not -** held, this function is a no-op. Otherwise, the size of the file is -** changed to nPage pages (nPage*pPager->pageSize bytes). If the file -** on disk is currently larger than nPage pages, then use the VFS +** If the main database file is not open, or the pager is not in either +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** ** Or, it might might be the case that the file on disk is smaller than @@ -1935,8 +2467,14 @@ delmaster_out: */ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; - if( pPager->state>=PAGER_EXCLUSIVE && isOpen(pPager->fd) ){ + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ i64 currentSize, newSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); newSize = pPager->pageSize*(i64)nPage; @@ -2060,7 +2598,7 @@ static int pager_playback(Pager *pPager, int isHot){ */ assert( isOpen(pPager->jfd) ); rc = sqlite3OsFileSize(pPager->jfd, &szJ); - if( rc!=SQLITE_OK || szJ==0 ){ + if( rc!=SQLITE_OK ){ goto end_playback; } @@ -2094,7 +2632,7 @@ static int pager_playback(Pager *pPager, int isHot){ while( 1 ){ /* Read the next journal header from the journal file. If there are ** not enough bytes left in the journal file for a complete header, or - ** it is corrupted, then a process must of failed while writing it. + ** it is corrupted, then a process must have failed while writing it. ** This indicates nothing more needs to be rolled back. */ rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); @@ -2208,10 +2746,9 @@ end_playback: rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); - } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + if( rc==SQLITE_OK && !pPager->noSync + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } if( rc==SQLITE_OK ){ @@ -2253,7 +2790,7 @@ static int readDbPage(PgHdr *pPg){ int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ - assert( pPager->state>=PAGER_SHARED && !MEMDB ); + assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); if( NEVER(!isOpen(pPager->fd)) ){ @@ -2400,6 +2937,14 @@ static int pagerWalFrames( sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } } + +#ifdef SQLITE_CHECK_PAGES + { + PgHdr *p; + for(p=pList; p; p=p->pDirty) pager_set_pagehash(p); + } +#endif + return rc; } @@ -2416,6 +2961,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we @@ -2425,19 +2971,69 @@ static int pagerBeginReadTransaction(Pager *pPager){ sqlite3WalEndReadTransaction(pPager->pWal); rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); + if( rc==SQLITE_OK && changed ){ + pager_reset(pPager); } - pPager->state = PAGER_SHARED; return rc; } +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + } + nPage = (Pgno)(n / pPager->pageSize); + if( nPage==0 && n>0 ){ + nPage = 1; + } + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + + /* ** Check if the *-wal file that corresponds to the database opened by pPager ** exists if the database is not empy, or verify that the *-wal file does @@ -2450,8 +3046,6 @@ static int pagerBeginReadTransaction(Pager *pPager){ ** ** Return SQLITE_OK or an error code. ** -** If the WAL file is opened, also open a snapshot (read transaction). -** ** The caller must hold a SHARED lock on the database file to call this ** function. Because an EXCLUSIVE lock on the db file is required to delete ** a WAL on a none-empty database, this ensures there is no race condition @@ -2460,11 +3054,14 @@ static int pagerBeginReadTransaction(Pager *pPager){ */ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ - int nPage; /* Size of the database file */ - assert( pPager->state>=SHARED_LOCK ); - rc = sqlite3PagerPagecount(pPager, &nPage); + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); @@ -2476,11 +3073,8 @@ static int pagerOpenWalIfPresent(Pager *pPager){ } if( rc==SQLITE_OK ){ if( isWal ){ - pager_reset(pPager); + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); - if( rc==SQLITE_OK ){ - rc = pagerBeginReadTransaction(pPager); - } }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } @@ -2532,7 +3126,8 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ int rc = SQLITE_OK; /* Return code */ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - assert( pPager->state>=PAGER_SHARED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); /* Allocate a bitvec to use to store the set of pages rolled back */ if( pSavepoint ){ @@ -2671,7 +3266,6 @@ void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){ pPager->noSync = (level==1 || pPager->tempFile) ?1:0; pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); - if( pPager->noSync ) pPager->needSync = 0; } #endif @@ -2753,7 +3347,7 @@ void sqlite3PagerSetBusyhandler( ** ** If the pager is in the error state when this function is called, it ** is a no-op. The value returned is the error state error code (i.e. -** one of SQLITE_IOERR, SQLITE_CORRUPT or SQLITE_FULL). +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). ** ** Otherwise, if all of the following are true: ** @@ -2777,28 +3371,48 @@ void sqlite3PagerSetBusyhandler( ** function was called, or because the memory allocation attempt failed, ** then *pPageSize is set to the old, retained page size before returning. */ -int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ - int rc = pPager->errCode; +int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; - if( rc==SQLITE_OK ){ - u16 pageSize = *pPageSize; - assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); - if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=pPager->pageSize - ){ - char *pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ){ - rc = SQLITE_NOMEM; - }else{ - pager_reset(pPager); - pPager->pageSize = pageSize; - sqlite3PageFree(pPager->pTmpSpace); - pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); - } + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); } - *pPageSize = (u16)pPager->pageSize; + if( rc==SQLITE_OK ){ + pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + pPager->dbSize = (Pgno)(nByte/pageSize); + pPager->pageSize = pageSize; + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ if( nReserve<0 ) nReserve = pPager->nReserve; assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; @@ -2827,13 +3441,11 @@ void *sqlite3PagerTempSpace(Pager *pPager){ ** Regardless of mxPage, return the current maximum page count. */ int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ - int nPage; if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->state!=PAGER_UNLOCK ){ - sqlite3PagerPagecount(pPager, &nPage); - assert( (int)pPager->mxPgno>=nPage ); + if( pPager->eState!=PAGER_OPEN && pPager->mxPgnodbSize ){ + pPager->mxPgno = pPager->dbSize; } return pPager->mxPgno; } @@ -2898,66 +3510,16 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ } /* -** Return the total number of pages in the database file associated -** with pPager. Normally, this is calculated as (/). +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** ** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. -** -** If the pager is in error state when this function is called, then the -** error state error code is returned and *pnPage left unchanged. Or, -** if the file system has to be queried for the size of the file and -** the query attempt returns an IO error, the IO error code is returned -** and *pnPage is left unchanged. -** -** Otherwise, if everything is successful, then SQLITE_OK is returned -** and *pnPage is set to the number of pages in the database. */ -int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - Pgno nPage = 0; /* Value to return via *pnPage */ - - /* Determine the number of pages in the file. Store this in nPage. */ - if( pPager->dbSizeValid ){ - nPage = pPager->dbSize; - }else{ - int rc; /* Error returned by OsFileSize() */ - i64 n = 0; /* File size in bytes returned by OsFileSize() */ - - if( pagerUseWal(pPager) && pPager->state!=PAGER_UNLOCK ){ - sqlite3WalDbsize(pPager->pWal, &nPage); - } - - if( nPage==0 ){ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - if( SQLITE_OK!=(rc = sqlite3OsFileSize(pPager->fd, &n)) ){ - pager_error(pPager, rc); - return rc; - } - } - if( n>0 && npageSize ){ - nPage = 1; - }else{ - nPage = (Pgno)(n / pPager->pageSize); - } - } - if( pPager->state!=PAGER_UNLOCK ){ - pPager->dbSize = nPage; - pPager->dbFileSize = nPage; - pPager->dbSizeValid = 1; - } - } - - /* If the current number of pages in the file is greater than the - ** configured maximum pager number, increase the allowed limit so - ** that the file can be read. - */ - if( nPage>pPager->mxPgno ){ - pPager->mxPgno = (Pgno)nPage; - } - - /* Set the output variable and return SQLITE_OK */ - *pnPage = nPage; - return SQLITE_OK; +void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; } @@ -2978,38 +3540,19 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* The OS lock values must be the same as the Pager lock values */ - assert( PAGER_SHARED==SHARED_LOCK ); - assert( PAGER_RESERVED==RESERVED_LOCK ); - assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - - /* If the file is currently unlocked then the size must be unknown. It - ** must not have been modified at this point. - */ - assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); - assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 ); - /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ - assert( (pPager->state>=locktype) - || (pPager->state==PAGER_UNLOCK && locktype==PAGER_SHARED) - || (pPager->state==PAGER_RESERVED && locktype==PAGER_EXCLUSIVE) + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); - if( pPager->state>=locktype ){ - rc = SQLITE_OK; - }else{ - do { - rc = sqlite3OsLock(pPager->fd, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); - if( rc==SQLITE_OK ){ - pPager->state = (u8)locktype; - IOTRACE(("LOCK %p %d\n", pPager, locktype)) - } - } + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); return rc; } @@ -3054,9 +3597,8 @@ static void assertTruncateConstraint(Pager *pPager){ ** truncation will be done when the current transaction is committed. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; assertTruncateConstraint(pPager); } @@ -3106,7 +3648,7 @@ int sqlite3PagerClose(Pager *pPager){ disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pPager->errCode = 0; + /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, @@ -3119,14 +3661,19 @@ int sqlite3PagerClose(Pager *pPager){ if( MEMDB ){ pager_unlock(pPager); }else{ - /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() - ** call which may be made from within pagerUnlockAndRollback(). If it - ** is not -1, then the unsynced portion of an open journal file may - ** be played back into the database. If a power failure occurs while - ** this is happening, the database may become corrupt. + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. */ if( isOpen(pPager->jfd) ){ - pPager->errCode = pagerSyncHotJournal(pPager); + pager_error(pPager, pagerSyncHotJournal(pPager)); } pagerUnlockAndRollback(pPager); } @@ -3171,9 +3718,9 @@ void sqlite3PagerRef(DbPage *pPg){ ** been written to the journal have actually reached the surface of the ** disk and can be restored in the event of a hot-journal rollback. ** -** If the Pager.needSync flag is not set, then this function is a -** no-op. Otherwise, the actions required depend on the journal-mode -** and the device characteristics of the the file-system, as follows: +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need ** be taken. @@ -3197,18 +3744,25 @@ void sqlite3PagerRef(DbPage *pPg){ ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** The Pager.needSync flag is never be set for temporary files, or any -** file operating in no-sync mode (Pager.noSync set to non-zero). -** ** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ -static int syncJournal(Pager *pPager){ - if( pPager->needSync ){ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ assert( !pPager->tempFile ); - if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ - int rc; /* Return code */ + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( isOpen(pPager->jfd) ); @@ -3283,17 +3837,25 @@ static int syncJournal(Pager *pPager){ ); if( rc!=SQLITE_OK ) return rc; } - } - /* The journal file was just successfully synced. Set Pager.needSync - ** to zero and clear the PGHDR_NEED_SYNC flag on all pagess. - */ - pPager->needSync = 0; - pPager->journalStarted = 1; - pPager->journalHdr = pPager->journalOff; - sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } } + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); return SQLITE_OK; } @@ -3330,27 +3892,12 @@ static int syncJournal(Pager *pPager){ ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ - /* At this point there may be either a RESERVED or EXCLUSIVE lock on the - ** database file. If there is already an EXCLUSIVE lock, the following - ** call is a no-op. - ** - ** Moving the lock from RESERVED to EXCLUSIVE actually involves going - ** through an intermediate state PENDING. A PENDING lock prevents new - ** readers from attaching to the database but is unsufficient for us to - ** write. The idea of a PENDING lock is to prevent new readers from - ** coming in while we wait for existing readers to clear. - ** - ** While the pager is in the RESERVED state, the original database file - ** is unchanged and we can rollback without having to playback the - ** journal into the original database file. Once we transition to - ** EXCLUSIVE, it means the database file has been changed and any rollback - ** will require a journal playback. - */ + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->state>=PAGER_RESERVED ); - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -3365,9 +3912,10 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>(pPager->dbOrigSize+1) ){ + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; } while( rc==SQLITE_OK && pList ){ @@ -3385,6 +3933,8 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ char *pData; /* Data to write */ + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + /* Encode the database */ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); @@ -3413,9 +3963,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } -#ifdef SQLITE_CHECK_PAGES - pList->pageHash = pager_pagehash(pList); -#endif + pager_set_pagehash(pList); pList = pList->pDirty; } @@ -3527,9 +4075,14 @@ static int pagerStress(void *p, PgHdr *pPg){ ** The doNotSpill flag inhibits all cache spilling regardless of whether ** or not a sync is required. This is set during a rollback. ** - ** Spilling is also inhibited when in an error state. + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementaton it + ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. */ - if( pPager->errCode ) return SQLITE_OK; + if( NEVER(pPager->errCode) ) return SQLITE_OK; if( pPager->doNotSpill ) return SQLITE_OK; if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ return SQLITE_OK; @@ -3547,16 +4100,10 @@ static int pagerStress(void *p, PgHdr *pPg){ }else{ /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC ){ - assert( !pPager->noSync ); - rc = syncJournal(pPager); - if( rc==SQLITE_OK && - !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && - !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) - ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); - } + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); } /* If the page number of this page is larger than the current size of @@ -3594,6 +4141,7 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); rc = pager_write_pagelist(pPager, pPg); } } @@ -3604,7 +4152,7 @@ static int pagerStress(void *p, PgHdr *pPg){ sqlite3PcacheMakeClean(pPg); } - return pager_error(pPager, rc); + return pager_error(pPager, rc); } @@ -3659,7 +4207,7 @@ int sqlite3PagerOpen( int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ - u16 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). This @@ -3794,7 +4342,7 @@ int sqlite3PagerOpen( if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; }else{ - szPageDflt = (u16)pPager->sectorSize; + szPageDflt = (u32)pPager->sectorSize; } } #ifdef SQLITE_ENABLE_ATOMIC_WRITE @@ -3822,7 +4370,8 @@ int sqlite3PagerOpen( ** disk and uses an in-memory rollback journal. */ tempFile = 1; - pPager->state = PAGER_EXCLUSIVE; + pPager->eState = PAGER_READER; + pPager->eLock = EXCLUSIVE_LOCK; readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -3859,13 +4408,14 @@ int sqlite3PagerOpen( /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ - pPager->dbSizeValid = (u8)memDb; /* pPager->stmtSize = 0; */ /* pPager->stmtJSize = 0; */ /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ +#if 0 assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); +#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL @@ -3875,7 +4425,6 @@ int sqlite3PagerOpen( pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; - /* pPager->needSync = 0; */ assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; pPager->fullSync = pPager->noSync ?0:1; @@ -3940,10 +4489,10 @@ static int hasHotJournal(Pager *pPager, int *pExists){ int exists = 1; /* True if a journal file is present */ int jrnlOpen = !!isOpen(pPager->jfd); - assert( pPager!=0 ); assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->state <= PAGER_SHARED ); + assert( pPager->eState==PAGER_OPEN ); + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN )); @@ -3953,7 +4502,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); } if( rc==SQLITE_OK && exists ){ - int locked; /* True if some process holds a RESERVED lock */ + int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the ** the RESERVED lock and have a journal open at the sqlite3OsAccess() @@ -3965,7 +4514,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ */ rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); if( rc==SQLITE_OK && !locked ){ - int nPage; + Pgno nPage; /* Number of pages in database file */ /* Check the size of the database file. If it consists of 0 pages, ** then delete the journal file. See the header comment above for @@ -3973,13 +4522,13 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** a RESERVED lock to avoid race conditions and to avoid violating ** [H33020]. */ - rc = sqlite3PagerPagecount(pPager, &nPage); + rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); - if( sqlite3OsLock(pPager->fd, RESERVED_LOCK)==SQLITE_OK ){ + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + pagerUnlockDb(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ @@ -4032,7 +4581,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** ** The following operations are also performed by this function. ** -** 1) If the pager is currently in PAGER_UNLOCK state (no lock held +** 1) If the pager is currently in PAGER_OPEN state (no lock held ** on the database file), then an attempt is made to obtain a ** SHARED lock on the database file. Immediately after obtaining ** the SHARED lock, the file-system is checked for a hot-journal, @@ -4047,66 +4596,47 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** the contents of the page cache and rolling back any open journal ** file. ** -** If the operation described by (2) above is not attempted, and if the -** pager is in an error state other than SQLITE_FULL when this is called, -** the error state error code is returned. It is permitted to read the -** database when in SQLITE_FULL error state. -** -** Otherwise, if everything is successful, SQLITE_OK is returned. If an -** IO error occurs while locking the database, checking for a hot-journal -** file or rolling back a journal file, the IO error code is returned. +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. */ int sqlite3PagerSharedLock(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int isErrorReset = 0; /* True if recovering from error state */ /* This routine is only called from b-tree and only when there are no - ** outstanding pages */ + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } - /* If this database is in an error-state, now is a chance to clear - ** the error. Discard the contents of the pager-cache and rollback - ** any hot journal in the file-system. - */ - if( pPager->errCode ){ - if( isOpen(pPager->jfd) || pPager->zJournal ){ - isErrorReset = 1; - } - pPager->errCode = SQLITE_OK; - pager_reset(pPager); - } + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ - if( pagerUseWal(pPager) ){ - rc = pagerBeginReadTransaction(pPager); - }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ - sqlite3_vfs * const pVfs = pPager->pVfs; - int isHotJournal = 0; assert( !MEMDB ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ + assert( pPager->noReadlock==0 || pPager->readOnly ); + + if( pPager->noReadlock==0 ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; } } - assert( pPager->state>=SHARED_LOCK ); /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ - if( !isErrorReset ){ - assert( pPager->state <= PAGER_SHARED ); - rc = hasHotJournal(pPager, &isHotJournal); - if( rc!=SQLITE_OK ){ - goto failed; - } + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); } - if( isErrorReset || isHotJournal ){ + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -4118,58 +4648,45 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** other process attempting to access the database file will get to ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. */ - if( pPager->statefd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } - pPager->state = PAGER_EXCLUSIVE; - } - - /* Open the journal for read/write access. This is because in - ** exclusive-access mode the file descriptor will be kept open and - ** possibly used for a transaction later on. On some systems, the - ** OsTruncate() call used in exclusive-access mode also requires - ** a read/write file handle. - */ - if( !isOpen(pPager->jfd) ){ - int res; - rc = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS,&res); - if( rc==SQLITE_OK ){ - if( res ){ - int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - assert( !pPager->tempFile ); - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN_BKPT; - sqlite3OsClose(pPager->jfd); - } - }else{ - /* If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it - ** may mean that the pager was in the error-state when this - ** function was called and the journal file does not exist. */ - rc = pager_end_transaction(pPager, 0); - } - } - } + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ goto failed; } - - /* Reset the journal status fields to indicates that we have no - ** rollback journal at this time. */ - pPager->journalStarted = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - /* Make sure the journal file has been synced to disk. */ + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before @@ -4180,21 +4697,46 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** the journal before playing it back. */ if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ rc = pager_playback(pPager, 1); + pPager->eState = PAGER_OPEN; } - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); } - assert( (pPager->state==PAGER_SHARED) - || (pPager->exclusiveMode && pPager->state>PAGER_SHARED) + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); } - if( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 ){ + if( !pPager->tempFile + && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) + ){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4211,14 +4753,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** detected. The chance of an undetected change is so small that ** it can be neglected. */ - int nPage = 0; + Pgno nPage = 0; char dbFileVers[sizeof(pPager->dbFileVers)]; - sqlite3PagerPagecount(pPager, &nPage); - if( pPager->errCode ){ - rc = pPager->errCode; - goto failed; - } + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); @@ -4234,18 +4773,30 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } } - assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); /* If there is a WAL file in the file-system, open this database in WAL ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); + assert( pPager->pWal==0 || rc==SQLITE_OK ); + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); } failed: if( rc!=SQLITE_OK ){ - /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ + assert( !MEMDB ); pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; } return rc; } @@ -4259,9 +4810,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) - && (!pPager->exclusiveMode || pPager->journalOff>0) - ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -4325,8 +4874,8 @@ int sqlite3PagerAcquire( int rc; PgHdr *pPg; + assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( pPager->state>PAGER_UNLOCK ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -4334,7 +4883,7 @@ int sqlite3PagerAcquire( /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK && pPager->errCode!=SQLITE_FULL ){ + if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); @@ -4360,7 +4909,6 @@ int sqlite3PagerAcquire( }else{ /* The pager cache has created a new page. Its content needs to ** be initialized. */ - int nMax; PAGER_INCR(pPager->nMiss); pPg = *ppPage; @@ -4373,12 +4921,7 @@ int sqlite3PagerAcquire( goto pager_acquire_err; } - rc = sqlite3PagerPagecount(pPager, &nMax); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - - if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ + if( MEMDB || pPager->dbSizefd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -4408,9 +4951,7 @@ int sqlite3PagerAcquire( goto pager_acquire_err; } } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } return SQLITE_OK; @@ -4429,9 +4970,7 @@ pager_acquire_err: /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. Also, return 0 if the -** pager is in PAGER_UNLOCK state when this function is called, -** or if the pager is in an error state other than SQLITE_FULL. +** or 0 if the page is not in cache. ** ** See also sqlite3PagerGet(). The difference between this routine ** and sqlite3PagerGet() is that _get() will go to the disk and read @@ -4444,7 +4983,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->state > PAGER_UNLOCK ); + assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); return pPg; } @@ -4489,12 +5028,10 @@ void sqlite3PagerUnref(DbPage *pPg){ */ static int pager_open_journal(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int nPage; /* Size of database file */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ - assert( pPager->state>=PAGER_RESERVED ); - assert( pPager->useJournal ); - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF ); + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); /* If already in the error state, this function is a no-op. But on @@ -4502,56 +5039,56 @@ static int pager_open_journal(Pager *pPager){ ** an error state. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - testcase( pPager->dbSizeValid==0 ); - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; - pPager->pInJournal = sqlite3BitvecCreate(nPage); - if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; - } - - /* Open the journal file if it is not already open. */ - if( !isOpen(pPager->jfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ - sqlite3MemJournalOpen(pPager->jfd); - }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) - ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif + #else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); + #endif + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); } - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - } - - - /* Write the first journal header to the journal file and open - ** the sub-journal if necessary. - */ - if( rc==SQLITE_OK ){ - /* TODO: Check if all of these are really required. */ - pPager->dbOrigSize = pPager->dbSize; - pPager->journalStarted = 0; - pPager->needSync = 0; - pPager->nRec = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - rc = writeJournalHdr(pPager); } if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; } + return rc; } @@ -4564,14 +5101,6 @@ static int pager_open_journal(Pager *pPager){ ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** -** If this is not a temporary or in-memory file and, the journal file is -** opened if it has not been already. For a temporary file, the opening -** of the journal file is deferred until there is an actual need to -** write to the journal. TODO: Why handle temporary files differently? -** -** If the journal file is opened (or if it is already open), then a -** journal-header is written to the start of it. -** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when @@ -4582,20 +5111,20 @@ static int pager_open_journal(Pager *pPager){ */ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ int rc = SQLITE_OK; - assert( pPager->state!=PAGER_UNLOCK ); + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - if( pPager->state==PAGER_SHARED ){ + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); - assert( !MEMDB && !pPager->tempFile ); if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - pPager->state = PAGER_SHARED; + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } @@ -4606,52 +5135,40 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. - ** - ** WAL mode sets Pager.state to PAGER_RESERVED when it has an open - ** transaction, but never to PAGER_EXCLUSIVE. This is because in - ** PAGER_EXCLUSIVE state the code to roll back savepoint transactions - ** may copy data from the sub-journal into the database file as well - ** as into the page cache. Which would be incorrect in WAL mode. */ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); - if( rc==SQLITE_OK ){ - pPager->dbOrigSize = pPager->dbSize; - pPager->state = PAGER_RESERVED; - pPager->journalOff = 0; - } - - assert( rc!=SQLITE_OK || pPager->state==PAGER_RESERVED ); - assert( rc==SQLITE_OK || pPager->state==PAGER_SHARED ); }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } - /* No need to open the journal file at this time. It will be - ** opened before it is written to. If we defer opening the journal, - ** we might save the work of creating a file if the transaction - ** ends up being a no-op. - */ - - if( rc!=SQLITE_OK ){ - assert( !pPager->dbModified ); - /* Ignore any IO error that occurs within pager_end_transaction(). The - ** purpose of this call is to reset the internal state of the pager - ** sub-system. It doesn't matter if the journal-file is not properly - ** finalized at this point (since it is not a valid journal file anyway). + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. */ - pager_end_transaction(pPager, 0); + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); } PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); @@ -4670,22 +5187,24 @@ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a transaction has already been - ** started. + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. */ - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* If an error has been previously detected, report the same error - ** again. - */ + ** again. This should not happen, but the check provides robustness. */ if( NEVER(pPager->errCode) ) return pPager->errCode; /* Higher-level routines never call this function if database is not ** writable. But check anyway, just for robustness. */ if( NEVER(pPager->readOnly) ) return SQLITE_PERM; - assert( !pPager->setMaster ); - CHECK_PAGE(pPg); /* Mark the page as dirty. If the page has already been written @@ -4694,56 +5213,60 @@ static int pager_write(PgHdr *pPg){ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ assert( !pagerUseWal(pPager) ); - pPager->dbModified = 1; + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); }else{ /* If we get this far, it means that the page needs to be - ** written to the transaction journal or the ckeckpoint journal + ** written to the transaction journal or the checkpoint journal ** or both. ** - ** Higher level routines should have already started a transaction, - ** which means they have acquired the necessary locks but the rollback - ** journal might not yet be open. + ** Higher level routines have already obtained the necessary locks + ** to begin the write-transaction, but the rollback journal might not + ** yet be open. Open it now if this is the case. */ - assert( pPager->state>=RESERVED_LOCK ); - if( pPager->pInJournal==0 - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && !pagerUseWal(pPager) - ){ - assert( pPager->useJournal ); + if( pPager->eState==PAGER_WRITER_LOCKED ){ rc = pager_open_journal(pPager); if( rc!=SQLITE_OK ) return rc; } - pPager->dbModified = 1; + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ - assert( !pagerUseWal(pPager) ); - if( pPg->pgno<=pPager->dbOrigSize ){ + if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ u32 cksum; char *pData2; + i64 iOff = pPager->journalOff; /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - assert( pPager->journalHdr <= pPager->journalOff ); + assert( pPager->journalHdr<=pPager->journalOff ); CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); cksum = pager_cksum(pPager, (u8*)pData2); - rc = write32bits(pPager->jfd, pPager->journalOff, pPg->pgno); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, - pPager->journalOff + 4); - pPager->journalOff += pPager->pageSize+4; - } - if( rc==SQLITE_OK ){ - rc = write32bits(pPager->jfd, pPager->journalOff, cksum); - pPager->journalOff += 4; - } + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); @@ -4751,25 +5274,7 @@ static int pager_write(PgHdr *pPg){ PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - /* Even if an IO or diskfull error occurred while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - if( !pPager->noSync ){ - pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; - } - - /* An error has occurred writing to the journal file. The - ** transaction will be rolled back by the layer above. - */ - if( rc!=SQLITE_OK ){ - return rc; - } - + pPager->journalOff += 8 + pPager->pageSize; pPager->nRec++; assert( pPager->pInJournal!=0 ); rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); @@ -4781,9 +5286,8 @@ static int pager_write(PgHdr *pPg){ return rc; } }else{ - if( !pPager->journalStarted && !pPager->noSync ){ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", PAGERID(pPager), pPg->pgno, @@ -4803,7 +5307,6 @@ static int pager_write(PgHdr *pPg){ /* Update the database size and return. */ - assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -4831,6 +5334,10 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ @@ -4852,19 +5359,17 @@ int sqlite3PagerWrite(DbPage *pDbPage){ */ pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - rc = sqlite3PagerPagecount(pPager, (int *)&nPageCount); - if( rc==SQLITE_OK ){ - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); for(ii=0; iiflags&PGHDR_NEED_SYNC ){ needSync = 1; - assert(pPager->needSync); } sqlite3PagerUnref(pPage); } @@ -4896,7 +5400,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ ** before any of them can be written out to the database file. */ if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB && pPager->noSync==0 ); + assert( !MEMDB ); for(ii=0; iineedSync); } assert( pPager->doNotSyncSpill==1 ); @@ -4946,9 +5449,7 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } } @@ -4971,6 +5472,11 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + /* Declare and initialize constant integer 'isDirect'. If the ** atomic-write optimization is enabled in this build, then isDirect ** is initialized to the value passed as the isDirectMode parameter @@ -4989,7 +5495,6 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - assert( pPager->state>=PAGER_RESERVED ); if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ u32 change_counter; /* Initial value of change-counter field */ @@ -5061,6 +5566,30 @@ int sqlite3PagerSync(Pager *pPager){ return rc; } +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + return rc; +} + /* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual @@ -5094,22 +5623,29 @@ int sqlite3PagerCommitPhaseOne( ){ int rc = SQLITE_OK; /* Return code */ - /* The dbOrigSize is never set if journal_mode=OFF */ - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); /* If a prior error occurred, report that error again. */ - if( pPager->errCode ) return pPager->errCode; + if( NEVER(pPager->errCode) ) return pPager->errCode; PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); - if( MEMDB && pPager->dbModified ){ + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); - }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ + }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ @@ -5152,7 +5688,7 @@ int sqlite3PagerCommitPhaseOne( ); if( !zMaster && isOpen(pPager->jfd) && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbFileSize + && pPager->dbSize>=pPager->dbOrigSize && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) ){ /* Update the db file change counter via the direct-write method. The @@ -5182,13 +5718,10 @@ int sqlite3PagerCommitPhaseOne( ** that it took at the start of the transaction. Otherwise, the ** calls to sqlite3PagerGet() return zeroed pages instead of ** reading data from the database file. - ** - ** When journal_mode==OFF the dbOrigSize is always zero, so this - ** block never runs if journal_mode=OFF. */ #ifndef SQLITE_OMIT_AUTOVACUUM if( pPager->dbSizedbOrigSize - && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ Pgno i; /* Iterator variable */ const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ @@ -5215,14 +5748,20 @@ int sqlite3PagerCommitPhaseOne( rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Sync the journal file. If the atomic-update optimization is being - ** used, this call will not create the journal file or perform any - ** real IO. + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. */ - rc = syncJournal(pPager); + rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Write all dirty pages to the database file. */ rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); @@ -5235,7 +5774,7 @@ int sqlite3PagerCommitPhaseOne( */ if( pPager->dbSize!=pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->state>=PAGER_EXCLUSIVE ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } @@ -5246,11 +5785,12 @@ int sqlite3PagerCommitPhaseOne( } IOTRACE(("DBSYNC %p\n", pPager)) } - - pPager->state = PAGER_SYNCED; } commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } return rc; } @@ -5278,12 +5818,11 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - /* This function should not be called if the pager is not in at least - ** PAGER_RESERVED state. **FIXME**: Make it so that this test always - ** fails - make it so that we never reach this point if we do not hold - ** all necessary locks. - */ - if( NEVER(pPager->stateeState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); /* An optimization. If the database was not actually modified during ** this transaction, the pager is running in exclusive-mode and is @@ -5296,102 +5835,76 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->dbModified==0 && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; return SQLITE_OK; } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dbModified ); rc = pager_end_transaction(pPager, pPager->setMaster); return pager_error(pPager, rc); } /* -** Rollback all changes. The database falls back to PAGER_SHARED mode. +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. ** -** This function performs two tasks: +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: ** ** 1) It rolls back the journal file, restoring all database file and ** in-memory cache pages to the state they were in when the transaction ** was opened, and +** ** 2) It finalizes the journal file, so that it is not used for hot ** rollback at any point in the future. ** -** subject to the following qualifications: +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. ** -** * If the journal file is not yet open when this function is called, -** then only (2) is performed. In this case there is no journal file -** to roll back. -** -** * If in an error state other than SQLITE_FULL, then task (1) is -** performed. If successful, task (2). Regardless of the outcome -** of either, the error state error code is returned to the caller -** (i.e. either SQLITE_IOERR or SQLITE_CORRUPT). -** -** * If the pager is in PAGER_RESERVED state, then attempt (1). Whether -** or not (1) is successful, also attempt (2). If successful, return -** SQLITE_OK. Otherwise, enter the error state and return the first -** error code encountered. -** -** In this case there is no chance that the database was written to. -** So is safe to finalize the journal file even if the playback -** (operation 1) failed. However the pager must enter the error state -** as the contents of the in-memory cache are now suspect. -** -** * Finally, if in PAGER_EXCLUSIVE state, then attempt (1). Only -** attempt (2) if (1) is successful. Return SQLITE_OK if successful, -** otherwise enter the error state and return the error code from the -** failing operation. -** -** In this case the database file may have been written to. So if the -** playback operation did not succeed it would not be safe to finalize -** the journal file. It needs to be left in the file-system so that -** some other process can use it to restore the database state (by -** hot-journal rollback). +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. */ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + if( pagerUseWal(pPager) ){ int rc2; - rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; - rc = pager_error(pPager, rc); - }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){ - rc = pager_end_transaction(pPager, pPager->setMaster); - }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - if( pPager->state>=PAGER_EXCLUSIVE ){ - pager_playback(pPager, 0); - } - rc = pPager->errCode; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_end_transaction(pPager, 0); }else{ - if( pPager->state==PAGER_RESERVED ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ){ - rc = rc2; - } - }else{ - rc = pager_playback(pPager, 0); - } - - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } - - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - rc = pager_error(pPager, rc); + rc = pager_playback(pPager, 0); } - return rc; + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); } /* @@ -5414,9 +5927,11 @@ int sqlite3PagerRefcount(Pager *pPager){ ** used by the pager and its associated cache. */ int sqlite3PagerMemUsed(Pager *pPager){ - int perPageSize = pPager->pageSize + pPager->nExtra + 20; + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); return perPageSize*sqlite3PcachePagecount(pPager->pPCache) - + sqlite3MallocSize(pPager); + + sqlite3MallocSize(pPager) + + pPager->pageSize; } /* @@ -5435,8 +5950,8 @@ int *sqlite3PagerStats(Pager *pPager){ a[0] = sqlite3PcacheRefCount(pPager->pPCache); a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; - a[4] = pPager->state; + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; a[7] = pPager->nMiss; @@ -5468,13 +5983,12 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + if( nSavepoint>nCurrent && pPager->useJournal ){ int ii; /* Iterator variable */ PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - int nPage; /* Size of database file */ - - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM ** if the allocation fails. Otherwise, zero the new portion in case a @@ -5491,14 +6005,14 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ /* Populate the PagerSavepoint structures just allocated. */ for(ii=nCurrent; iidbSize; if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ aNew[ii].iOffset = pPager->journalOff; }else{ aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); } aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage); + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } @@ -5545,12 +6059,12 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = SQLITE_OK; + int rc = pPager->errCode; /* Return code */ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); - if( iSavepointnSavepoint ){ + if( rc==SQLITE_OK && iSavepointnSavepoint ){ int ii; /* Iterator variable */ int nNew; /* Number of remaining savepoints after this op. */ @@ -5586,8 +6100,8 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - } + return rc; } @@ -5685,6 +6199,10 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ Pgno origPgno; /* The original page number */ assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. @@ -5734,11 +6252,10 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ needSyncPgno = pPg->pgno; assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); - assert( pPager->needSync ); } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PgHdr.needSync was set for + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ @@ -5750,7 +6267,6 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ if( MEMDB ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ - assert( pPager->dbSizeValid ); sqlite3PcacheMove(pPgOld, pPager->dbSize+1); }else{ sqlite3PcacheDrop(pPgOld); @@ -5760,14 +6276,23 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ origPgno = pPg->pgno; sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); - pPager->dbModified = 1; + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); + } if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by - ** loading the page into the pager-cache and setting the PgHdr.needSync + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. ** ** If the attempt to load the page into the page-cache fails, (due @@ -5776,12 +6301,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** this transaction, it may be written to the database file before ** it is synced into the journal file. This way, it may end up in ** the journal file twice, but that is not a problem. - ** - ** The sqlite3PagerGet() call may cause the journal to sync. So make - ** sure the Pager.needSync flag is set too. */ PgHdr *pPgHdr; - assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ @@ -5790,23 +6311,11 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } return rc; } - pPager->needSync = 1; - assert( pPager->noSync==0 && !MEMDB ); pPgHdr->flags |= PGHDR_NEED_SYNC; sqlite3PcacheMakeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); } - /* - ** For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. Use pPgOld - ** as the original page since it has already been allocated. - */ - if( MEMDB ){ - sqlite3PcacheMove(pPgOld, origPgno); - sqlite3PagerUnref(pPgOld); - } - return SQLITE_OK; } #endif @@ -5872,6 +6381,13 @@ int sqlite3PagerLockingMode(Pager *pPager, int eMode){ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ +#ifdef SQLITE_DEBUG + /* The print_pager_state() routine is intended to be used by the debugger + ** only. We invoke it once here to suppress a compiler warning. */ + print_pager_state(pPager); +#endif + + /* The eMode parameter is always valid */ assert( eMode==PAGER_JOURNALMODE_DELETE || eMode==PAGER_JOURNALMODE_TRUNCATE @@ -5897,20 +6413,13 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( eMode!=eOld ){ - /* When changing between rollback modes, close the journal file prior - ** to the change. But when changing from a rollback mode to WAL, keep - ** the journal open since there is a rollback-style transaction in play - ** used to convert the version numbers in the btree header. - */ - if( isOpen(pPager->jfd) && eMode!=PAGER_JOURNALMODE_WAL ){ - sqlite3OsClose(pPager->jfd); - } /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; /* When transistioning from TRUNCATE or PERSIST to any other journal - ** mode except WAL (and we are not in locking_mode=EXCLUSIVE) then + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); @@ -5931,24 +6440,30 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ ** database file. This ensures that the journal file is not deleted ** while it is in use by some other client. */ - int rc = SQLITE_OK; - int state = pPager->state; - if( statestate==PAGER_SHARED ){ - assert( rc==SQLITE_OK ); - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - } - if( rc==SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); } - if( rc==SQLITE_OK && state==PAGER_SHARED ){ - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); - }else if( state==PAGER_UNLOCK ){ - pager_unlock(pPager); - } - assert( state==pPager->state ); } } @@ -5969,7 +6484,8 @@ int sqlite3PagerGetJournalMode(Pager *pPager){ ** is unmodified. */ int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ - if( pPager->dbModified ) return 0; + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; return 1; } @@ -6034,25 +6550,31 @@ int sqlite3PagerWalSupported(Pager *pPager){ ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, ** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pisOpen is +** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** ** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pisOpen to 1 and return SQLITE_OK +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK ** without doing anything. */ int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ - int *pisOpen /* OUT: Set to true if call is a no-op */ + int *pbOpen /* OUT: Set to true if call is a no-op */ ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->state>=PAGER_SHARED ); - assert( (pisOpen==0 && !pPager->tempFile && !pPager->pWal) || *pisOpen==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), unlock the database file and ** return an error code. @@ -6060,9 +6582,10 @@ int sqlite3PagerOpenWal( rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; } }else{ - *pisOpen = 1; + *pbOpen = 1; } return rc; @@ -6088,7 +6611,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ */ if( !pPager->pWal ){ int logexists = 0; - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); + rc = pagerLockDb(pPager, SHARED_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists @@ -6104,7 +6627,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, (pPager->noSync ? 0 : pPager->sync_flags), @@ -6114,7 +6637,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ }else{ /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock ** that we did get back to SHARED. */ - sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED); + pagerUnlockDb(pPager, SQLITE_LOCK_SHARED); } } return rc; diff --git a/src/pager.h b/src/pager.h index 6036827..0f100d5 100644 --- a/src/pager.h +++ b/src/pager.h @@ -99,7 +99,7 @@ int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); -int sqlite3PagerSetPagesize(Pager*, u16*, int); +int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); void sqlite3PagerSetSafetyLevel(Pager*,int,int); @@ -126,9 +126,10 @@ void *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ -int sqlite3PagerPagecount(Pager*, int*); +void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); +int sqlite3PagerExclusiveLock(Pager*); int sqlite3PagerSync(Pager *pPager); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); diff --git a/src/parse.y b/src/parse.y index c32f278..92abd5c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -780,7 +780,7 @@ expr(A) ::= VARIABLE(X). { spanSet(&A, &X, &X); } expr(A) ::= expr(E) COLLATE ids(C). { - A.pExpr = sqlite3ExprSetColl(pParse, E.pExpr, &C); + A.pExpr = sqlite3ExprSetCollByToken(pParse, E.pExpr, &C); A.zStart = E.zStart; A.zEnd = &C.z[C.n]; } @@ -1108,7 +1108,7 @@ idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { Expr *p = 0; if( C.n>0 ){ p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetColl(pParse, p, &C); + sqlite3ExprSetCollByToken(pParse, p, &C); } A = sqlite3ExprListAppend(pParse,X, p); sqlite3ExprListSetName(pParse,A,&Y,1); @@ -1119,7 +1119,7 @@ idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { Expr *p = 0; if( C.n>0 ){ p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetColl(pParse, p, &C); + sqlite3ExprSetCollByToken(pParse, p, &C); } A = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, A, &Y, 1); diff --git a/src/pcache1.c b/src/pcache1.c index 9f2b299..f9927b8 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -152,11 +152,11 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ static void *pcache1Alloc(int nByte){ void *p; assert( sqlite3_mutex_held(pcache1.mutex) ); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); if( nByte<=pcache1.szSlot && pcache1.pFree ){ assert( pcache1.isInit ); p = (PgHdr1 *)pcache1.pFree; pcache1.pFree = pcache1.pFree->pNext; - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); }else{ @@ -200,6 +200,25 @@ static void pcache1Free(void *p){ } } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** Return the size of a pache allocation +*/ +static int pcache1MemSize(void *p){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( p>=pcache1.pStart && ppTriggerPrg ){ TriggerPrg *pT = pParse->pTriggerPrg; pParse->pTriggerPrg = pT->pNext; - sqlite3VdbeProgramDelete(db, pT->pProgram, 0); sqlite3DbFree(db, pT); } diff --git a/src/printf.c b/src/printf.c index 6ab230b..da2fdf6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -772,7 +772,11 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ }else{ p->nAlloc = (int)szNew; } - zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + if( p->useMalloc==1 ){ + zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + }else{ + zNew = sqlite3_malloc(p->nAlloc); + } if( zNew ){ memcpy(zNew, p->zText, p->nChar); sqlite3StrAccumReset(p); @@ -797,7 +801,11 @@ char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ p->zText[p->nChar] = 0; if( p->useMalloc && p->zText==p->zBase ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( p->useMalloc==1 ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + }else{ + p->zText = sqlite3_malloc(p->nChar+1); + } if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); }else{ @@ -813,7 +821,11 @@ char *sqlite3StrAccumFinish(StrAccum *p){ */ void sqlite3StrAccumReset(StrAccum *p){ if( p->zText!=p->zBase ){ - sqlite3DbFree(p->db, p->zText); + if( p->useMalloc==1 ){ + sqlite3DbFree(p->db, p->zText); + }else{ + sqlite3_free(p->zText); + } } p->zText = 0; } @@ -895,6 +907,7 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + acc.useMalloc = 2; sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; diff --git a/src/select.c b/src/select.c index b03e506..17184b6 100644 --- a/src/select.c +++ b/src/select.c @@ -1298,16 +1298,15 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ return 0; } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled, so we might as well hard-code pTab->dbMem to NULL. */ + ** is disabled */ assert( db->lookaside.bEnabled==0 ); - pTab->dbMem = 0; pTab->nRef = 1; pTab->zName = 0; selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(db, pTab); return 0; } return pTab; @@ -1799,7 +1798,7 @@ multi_select_end: ** regReturn is the number of the register holding the subroutine ** return address. ** -** If regPrev>0 then it is a the first register in a vector that +** If regPrev>0 then it is the first register in a vector that ** records the previous output. mem[regPrev] is a flag that is false ** if there has been no previous output. If regPrev>0 then code is ** generated to suppress duplicates. pKeyInfo is used for comparing @@ -2496,12 +2495,13 @@ static void substSelect( ** (2) The subquery is not an aggregate or the outer query is not a join. ** ** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strenghtened by ticket #3300) +** (Originally ticket #306. Strengthened by ticket #3300) ** -** (4) The subquery is not DISTINCT or the outer query is not a join. +** (4) The subquery is not DISTINCT. ** -** (5) The subquery is not DISTINCT or the outer query does not use -** aggregates. +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. ** ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. @@ -2521,9 +2521,9 @@ static void substSelect( ** (**) Not implemented. Subsumed into restriction (3). Was previously ** a separate restriction deriving from ticket #350. ** -** (13) The subquery and outer query do not both use LIMIT +** (13) The subquery and outer query do not both use LIMIT. ** -** (14) The subquery does not use OFFSET +** (14) The subquery does not use OFFSET. ** ** (15) The outer query is not part of a compound select or the ** subquery does not have a LIMIT clause. @@ -2614,9 +2614,9 @@ static int flattenSubquery( return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( ((pSub->selFlags & SF_Distinct)!=0 || pSub->pLimit) - && (pSrc->nSrc>1 || isAgg) ){ /* Restrictions (4)(5)(8)(9) */ - return 0; + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ } if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ return 0; /* Restriction (6) */ @@ -3096,7 +3096,6 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3WalkSelect(pWalker, pSel); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; - pTab->dbMem = db->lookaside.bEnabled ? db : 0; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } diff --git a/src/shell.c b/src/shell.c index 23099b9..c5aa40c 100644 --- a/src/shell.c +++ b/src/shell.c @@ -398,6 +398,7 @@ struct previous_mode_data { struct callback_data { sqlite3 *db; /* The database */ int echoOn; /* True to echo input commands */ + int statsOn; /* True to display memory stats before each finalize */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ int mode; /* An output mode setting */ @@ -961,6 +962,86 @@ static char *save_err_msg( return zErrMsg; } +/* +** Display memory stats. +*/ +static int display_stats( + sqlite3 *db, /* Database to query */ + struct callback_data *pArg, /* Pointer to struct callback_data */ + int bReset /* True to reset the stats */ +){ + int iCur; + int iHiwtr; + + if( pArg && pArg->out ){ + + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Memory Used: %d (max %d) bytes\n", iCur, iHiwtr); + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Allocations: %d (max %d)\n", iCur, iHiwtr); +/* +** Not currently used by the CLI. +** iHiwtr = iCur = -1; +** sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); +** fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); +*/ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); +/* +** Not currently used by the CLI. +** iHiwtr = iCur = -1; +** sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); +** fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); +*/ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Largest Allocation: %d bytes\n", iHiwtr); + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", iHiwtr); + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", iHiwtr); +#ifdef YYTRACKMAXSTACKDEPTH + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", iCur, iHiwtr); +#endif + } + + if( pArg && pArg->out && db ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Pager Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); + } + + if( pArg && pArg->out && db && pArg->pStmt ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); + fprintf(pArg->out, "Fullscan Steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); + fprintf(pArg->out, "Sort Operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX, bReset); + fprintf(pArg->out, "Autoindex Inserts: %d\n", iCur); + } + + return 0; +} + /* ** Execute a statement or set of statements. Print ** any result rows/columns depending on the current mode @@ -1000,10 +1081,16 @@ static int shell_exec( continue; } + /* save off the prepared statment handle and reset row count */ + if( pArg ){ + pArg->pStmt = pStmt; + pArg->cnt = 0; + } + /* echo the sql statement if echo on */ - if( pArg->echoOn ){ + if( pArg && pArg->echoOn ){ const char *zStmtSql = sqlite3_sql(pStmt); - fprintf(pArg->out,"%s\n", zStmtSql ? zStmtSql : zSql); + fprintf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } /* perform the first step. this will tell us if we @@ -1029,11 +1116,6 @@ static int shell_exec( for(i=0; ipStmt = pStmt; - pArg->cnt = 0; - } do{ /* extract the data and data types */ for(i=0; ipStmt = NULL; - } } }else{ do{ @@ -1067,6 +1146,11 @@ static int shell_exec( } } + /* print usage stats if stats on */ + if( pArg && pArg->statsOn ){ + display_stats(db, pArg, 0); + } + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ @@ -1077,6 +1161,11 @@ static int shell_exec( }else if( pzErrMsg ){ *pzErrMsg = save_err_msg(db); } + + /* clear saved stmt handle */ + if( pArg ){ + pArg->pStmt = NULL; + } } } /* end while */ @@ -1256,6 +1345,7 @@ static char zHelp[] = " LIKE pattern TABLE.\n" ".separator STRING Change separator used by output mode and .import\n" ".show Show the current values for various settings\n" + ".stats ON|OFF Turn stats on or off\n" ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" @@ -2001,6 +2091,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ fprintf(p->out,"%9.9s: ", "separator"); output_c_string(p->out, p->separator); fprintf(p->out, "\n"); + fprintf(p->out,"%9.9s: %s\n","stats", p->statsOn ? "on" : "off"); fprintf(p->out,"%9.9s: ","width"); for (i=0;i<(int)ArraySize(p->colWidth) && p->colWidth[i] != 0;i++) { fprintf(p->out,"%d ",p->colWidth[i]); @@ -2008,6 +2099,10 @@ static int do_meta_command(char *zLine, struct callback_data *p){ fprintf(p->out,"\n"); }else + if( c=='s' && strncmp(azArg[0], "stats", n)==0 && nArg>1 && nArg<3 ){ + p->statsOn = booleanValue(azArg[1]); + }else + if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 && nArg<3 ){ char **azResult; int nRow; @@ -2391,6 +2486,7 @@ static const char zOptions[] = " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" " -separator 'x' set output field separator (|)\n" + " -stats print memory stats before each finalize\n" " -nullvalue 'text' set text string for NULL values\n" " -version show SQLite version\n" ; @@ -2555,6 +2651,8 @@ int main(int argc, char **argv){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ data.echoOn = 1; + }else if( strcmp(z,"-stats")==0 ){ + data.statsOn = 1; }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a8c6be2..39461aa 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -92,7 +92,7 @@ extern "C" { ** ** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management -** system. ^The SQLITE_SOURCE_ID macro evalutes to +** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and an SHA1 @@ -149,7 +149,7 @@ int sqlite3_libversion_number(void); ** compile time. ^The SQLITE_ prefix may be omitted from the ** option name passed to sqlite3_compileoption_used(). ** -** ^The sqlite3_compileoption_get() function allows interating +** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, ** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ @@ -157,7 +157,7 @@ int sqlite3_libversion_number(void); ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifing the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and @@ -263,7 +263,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** ** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. ** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfullly destroyed and all associated resources are deallocated. +** successfully destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with @@ -690,12 +690,21 @@ struct sqlite3_io_methods { ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 /* ** CAPI3REF: Mutex Handle @@ -2171,7 +2180,13 @@ int sqlite3_set_authorizer( ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time -** of how long that statement took to run. +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. */ void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, @@ -2652,7 +2667,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** ** In the templates above, NNN represents an integer literal, -** and VVV represents an alphanumeric identifer.)^ ^The values of these +** and VVV represents an alphanumeric identifier.)^ ^The values of these ** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** @@ -3431,7 +3446,7 @@ int sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context ** -** Implementions of aggregate SQL functions use this +** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called @@ -3703,7 +3718,7 @@ void sqlite3_result_zeroblob(sqlite3_context*, int n); ** ** A pointer to the user supplied routine must be passed as the fifth ** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). +** sequence (so that SQLite cannot call it any more). ** ^Each time the application supplied function is invoked, it is passed ** as its first parameter a copy of the void* passed as the fourth argument ** to sqlite3_create_collation() or sqlite3_create_collation16(). @@ -4320,7 +4335,8 @@ struct sqlite3_module { ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** -** The sqlite3_index_info structure and its substructures is used to +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to ** pass information into and receive the reply from the [xBestIndex] ** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its @@ -4328,10 +4344,12 @@ struct sqlite3_module { ** ** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -** 
column OP expr
+**
column OP expr
** ** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is -** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in ** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint ** is usable) and false if it cannot.)^ @@ -4391,6 +4409,15 @@ struct sqlite3_index_info { int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ }; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 @@ -4909,7 +4936,7 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** it is passed a NULL pointer). ** ** The xMutexInit() method must be threadsafe. ^It must be harmless to -** invoke xMutexInit() mutiple times within the same process and without +** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** @@ -5079,7 +5106,7 @@ int sqlite3_test_control(int op, ...); ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information -** about the preformance of SQLite, and optionally to reset various +** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes ** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ @@ -5131,6 +5158,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** +** ^(
SQLITE_STATUS_MALLOC_COUNT
+**
This parameter records the number of separate memory allocations.
)^ +** ** ^(
SQLITE_STATUS_PAGECACHE_USED
**
This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using @@ -5192,6 +5222,7 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 #define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status @@ -5201,7 +5232,7 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that -** determiness the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely ** to grow in future releases of SQLite. ** @@ -5231,16 +5262,33 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); **
This parameter returns the number of lookaside memory slots currently ** checked out.
)^ ** -**
SQLITE_DBSTATUS_CACHE_USED
-**
^This parameter returns the approximate number of of bytes of heap -** memory used by all pager caches associated with the database connection. +** ^(
SQLITE_DBSTATUS_CACHE_USED
+**
This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** ^(
SQLITE_DBSTATUS_SCHEMA_USED
+**
This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** ^(
SQLITE_DBSTATUS_STMT_USED
+**
This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 #define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_MAX 1 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ /* @@ -5606,7 +5654,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source databae file. +** up and the total number of pages in the source database file. ** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces ** retrieve these two values, respectively. ** @@ -5702,7 +5750,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -5784,7 +5832,7 @@ int sqlite3_unlock_notify( ** ** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a -** case-indendent fashion, using the same definition of case independence +** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ int sqlite3_strnicmp(const char *, const char *, int); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 15fedf5..c44c25f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -661,14 +661,6 @@ struct Db { /* ** An instance of the following structure stores a database schema. -** -** If there are no virtual tables configured in this schema, the -** Schema.db variable is set to NULL. After the first virtual table -** has been added, it is set to point to the database connection -** used to create the connection. Once a virtual table has been -** added to the Schema structure and the Schema.db variable populated, -** only that database connection may use the Schema to prepare -** statements. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ @@ -681,9 +673,6 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3 *db; /* "Owner" connection. See comment above */ -#endif }; /* @@ -871,6 +860,7 @@ struct sqlite3 { int nStatement; /* Number of nested statement-transactions */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ + int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER @@ -1228,7 +1218,6 @@ struct VTable { ** of a SELECT statement. */ struct Table { - sqlite3 *dbMem; /* DB connection used for lookaside allocations. */ char *zName; /* Name of the table or view */ int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ int nCol; /* Number of columns in this table */ @@ -1365,9 +1354,9 @@ struct FKey { */ struct KeyInfo { sqlite3 *db; /* The database connection */ - u8 enc; /* Text encoding - one of the TEXT_Utf* values */ + u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ u16 nField; /* Number of entries in aColl[] */ - u8 *aSortOrder; /* If defined an aSortOrder[i] is true, sort DESC */ + u8 *aSortOrder; /* Sort order for each column. May be NULL */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; @@ -2337,7 +2326,7 @@ struct StrAccum { int nAlloc; /* Amount of space allocated in zText */ int mxAlloc; /* Maximum allowed string length */ u8 mallocFailed; /* Becomes true if any memory allocation fails */ - u8 useMalloc; /* True if zText is enlargeable using realloc */ + u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ u8 tooBig; /* Becomes true if string size exceeds limits */ }; @@ -2635,7 +2624,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); #endif void sqlite3DropTable(Parse*, SrcList*, int, int); -void sqlite3DeleteTable(Table*); +void sqlite3DeleteTable(sqlite3*, Table*); #ifndef SQLITE_OMIT_AUTOINCREMENT void sqlite3AutoincrementBegin(Parse *pParse); void sqlite3AutoincrementEnd(Parse *pParse); @@ -2869,7 +2858,8 @@ int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -Expr *sqlite3ExprSetColl(Parse *pParse, Expr *, Token *); +Expr *sqlite3ExprSetColl(Expr*, CollSeq*); +Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*); int sqlite3CheckCollSeq(Parse *, CollSeq *); int sqlite3CheckObjectName(Parse *, const char *); void sqlite3VdbeSetChanges(sqlite3 *, int); @@ -2919,7 +2909,7 @@ int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); int sqlite3FindDbName(sqlite3 *, const char *); int sqlite3AnalysisLoad(sqlite3*,int iDB); -void sqlite3DeleteIndexSamples(Index*); +void sqlite3DeleteIndexSamples(sqlite3*,Index*); void sqlite3DefaultRowEst(Index*); void sqlite3RegisterLikeFunctions(sqlite3*, int); int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); @@ -2981,7 +2971,7 @@ void sqlite3AutoLoadExtensions(sqlite3*); # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) #else - void sqlite3VtabClear(Table*); + void sqlite3VtabClear(sqlite3 *db, Table*); int sqlite3VtabSync(sqlite3 *db, char **); int sqlite3VtabRollback(sqlite3 *db); int sqlite3VtabCommit(sqlite3 *db); @@ -3034,9 +3024,9 @@ int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #define sqlite3FkRequired(a,b,c,d) 0 #endif #ifndef SQLITE_OMIT_FOREIGN_KEY - void sqlite3FkDelete(Table*); + void sqlite3FkDelete(sqlite3 *, Table*); #else - #define sqlite3FkDelete(a) + #define sqlite3FkDelete(a,b) #endif @@ -3129,17 +3119,18 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); ** sqlite3MemdebugHasType() returns true if any of the bits in its second ** argument match the type set by the previous sqlite3MemdebugSetType(). ** sqlite3MemdebugHasType() is intended for use inside assert() statements. -** For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +** sqlite3MemdebugNoType() returns true if none of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). ** ** Perhaps the most important point is the difference between MEMTYPE_HEAP -** and MEMTYPE_DB. If an allocation is MEMTYPE_DB, that means it might have -** been allocated by lookaside, except the allocation was too large or -** lookaside was already full. It is important to verify that allocations -** that might have been satisfied by lookaside are not passed back to -** non-lookaside free() routines. Asserts such as the example above are -** placed on the non-lookaside free() routines to verify this constraint. +** and MEMTYPE_LOOKASIDE. If an allocation is MEMTYPE_LOOKASIDE, that means +** it might have been allocated by lookaside, except the allocation was +** too large or lookaside was already full. It is important to verify +** that allocations that might have been satisfied by lookaside are not +** passed back to non-lookaside free() routines. Asserts such as the +** example above are placed on the non-lookaside free() routines to verify +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. @@ -3147,13 +3138,16 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #ifdef SQLITE_MEMDEBUG void sqlite3MemdebugSetType(void*,u8); int sqlite3MemdebugHasType(void*,u8); + int sqlite3MemdebugNoType(void*,u8); #else # define sqlite3MemdebugSetType(X,Y) /* no-op */ # define sqlite3MemdebugHasType(X,Y) 1 +# define sqlite3MemdebugNoType(X,Y) 1 #endif -#define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_DB 0x02 /* Associated with a database connection */ -#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ -#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_HEAP 0x01 /* General heap allocations */ +#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ +#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ #endif /* _SQLITEINT_H_ */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 1dc5d67..62772cf 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -133,20 +133,21 @@ # define SQLITE_MAX_VARIABLE_NUMBER 999 #endif -/* Maximum page size. The upper bound on this value is 32768. This a limit -** imposed by the necessity of storing the value in a 2-byte unsigned integer -** and the fact that the page size must be a power of 2. +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. ** -** If this limit is changed, then the compiled library is technically -** incompatible with an SQLite library compiled with a different limit. If -** a process operating on a database with a page-size of 65536 bytes -** crashes, then an instance of SQLite compiled with the default page-size -** limit will not be able to rollback the aborted transaction. This could -** lead to database corruption. +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. */ -#ifndef SQLITE_MAX_PAGE_SIZE -# define SQLITE_MAX_PAGE_SIZE 32768 +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE #endif +#define SQLITE_MAX_PAGE_SIZE 65536 /* diff --git a/src/status.c b/src/status.c index f310f36..7f05f1c 100644 --- a/src/status.c +++ b/src/status.c @@ -14,14 +14,15 @@ ** functionality. */ #include "sqliteInt.h" +#include "vdbeInt.h" /* ** Variables in which to record status information. */ typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { - int nowValue[9]; /* Current value */ - int mxValue[9]; /* Maximum value */ + int nowValue[10]; /* Current value */ + int mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; @@ -103,6 +104,8 @@ int sqlite3_db_status( int *pHighwater, /* Write high-water mark here */ int resetFlag /* Reset high-water mark if true */ ){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { *pCurrent = db->lookaside.nOut; @@ -121,6 +124,7 @@ int sqlite3_db_status( case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; + sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ @@ -128,13 +132,78 @@ int sqlite3_db_status( totalUsed += sqlite3PagerMemUsed(pPager); } } + sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; *pHighwater = 0; break; } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store the schema for all databases (main, temp, and any ATTACHed + ** databases. *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_SCHEMA_USED: { + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( ALWAYS(pSchema!=0) ){ + HashElem *p; + + nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( + pSchema->tblHash.count + + pSchema->trigHash.count + + pSchema->idxHash.count + + pSchema->fkeyHash.count + ); + nByte += sqlite3MallocSize(pSchema->tblHash.ht); + nByte += sqlite3MallocSize(pSchema->trigHash.ht); + nByte += sqlite3MallocSize(pSchema->idxHash.ht); + nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + + for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); + } + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTable(db, (Table *)sqliteHashData(p)); + } + } + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store all prepared statements. + ** *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_STMT_USED: { + struct Vdbe *pVdbe; /* Used to iterate through VMs */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + sqlite3VdbeDeleteObject(db, pVdbe); + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + + break; + } + default: { - return SQLITE_ERROR; + rc = SQLITE_ERROR; } } - return SQLITE_OK; + sqlite3_mutex_leave(db->mutex); + return rc; } diff --git a/src/tclsqlite.c b/src/tclsqlite.c index a2d352c..5b5b4b3 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2571,7 +2571,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ int v; const char *zOp; if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 2, objv, "(step|sort)"); + Tcl_WrongNumArgs(interp, 2, objv, "(step|sort|autoindex)"); return TCL_ERROR; } zOp = Tcl_GetString(objv[2]); @@ -3015,7 +3015,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ ** used to open a new SQLite database. See the DbMain() routine above ** for additional information. */ -EXTERN int Sqlite3_Init(Tcl_Interp *interp){ +int Sqlite3_Init(Tcl_Interp *interp){ Tcl_InitStubs(interp, "8.4", 0); Tcl_CreateObjCommand(interp, "sqlite3", (Tcl_ObjCmdProc*)DbMain, 0, 0); Tcl_PkgProvide(interp, "sqlite3", PACKAGE_VERSION); @@ -3023,24 +3023,24 @@ EXTERN int Sqlite3_Init(Tcl_Interp *interp){ Tcl_PkgProvide(interp, "sqlite", PACKAGE_VERSION); return TCL_OK; } -EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } +int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } +int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } +int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #ifndef SQLITE_3_SUFFIX_ONLY -EXTERN int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } +int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } +int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } +int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } +int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } +int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #endif #ifdef TCLSH diff --git a/src/test1.c b/src/test1.c index 33ac7a1..f458cdb 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4613,6 +4613,44 @@ static int file_control_lasterrno_test( return TCL_OK; } +/* +** tclcmd: file_control_chunksize_test DB DBNAME SIZE +** +** This TCL command runs the sqlite3_file_control interface and +** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and +** SQLITE_SET_LOCKPROXYFILE verbs. +*/ +static int file_control_chunksize_test( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + int nSize; /* New chunk size */ + char *zDb; /* Db name ("main", "temp" etc.) */ + sqlite3 *db; /* Database handle */ + int rc; /* file_control() return code */ + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SIZE"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIntFromObj(interp, objv[3], &nSize) + ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = NULL; + + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_CHUNK_SIZE, (void *)&nSize); + if( rc ){ + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + return TCL_ERROR; + } + return TCL_OK; +} + /* ** tclcmd: file_control_lockproxy_test DB PWD ** @@ -4911,6 +4949,47 @@ static int test_wal_checkpoint( return TCL_OK; } +/* +** tclcmd: test_sqlite3_log ?SCRIPT? +*/ +static struct LogCallback { + Tcl_Interp *pInterp; + Tcl_Obj *pObj; +} logcallback = {0, 0}; +static void xLogcallback(void *unused, int err, char *zMsg){ + Tcl_Obj *pNew = Tcl_DuplicateObj(logcallback.pObj); + Tcl_IncrRefCount(pNew); + Tcl_ListObjAppendElement( + 0, pNew, Tcl_NewStringObj(sqlite3TestErrorName(err), -1) + ); + Tcl_ListObjAppendElement(0, pNew, Tcl_NewStringObj(zMsg, -1)); + Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT); + Tcl_DecrRefCount(pNew); +} +static int test_sqlite3_log( + ClientData clientData, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + if( objc>2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SCRIPT"); + return TCL_ERROR; + } + if( logcallback.pObj ){ + Tcl_DecrRefCount(logcallback.pObj); + logcallback.pObj = 0; + logcallback.pInterp = 0; + sqlite3_config(SQLITE_CONFIG_LOG, 0, 0); + } + if( objc>1 ){ + logcallback.pObj = objv[1]; + Tcl_IncrRefCount(logcallback.pObj); + logcallback.pInterp = interp; + sqlite3_config(SQLITE_CONFIG_LOG, xLogcallback, 0); + } + return TCL_OK; +} /* ** tcl_objproc COMMANDNAME ARGS... @@ -5106,6 +5185,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_test", file_control_test, 0 }, { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, + { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, /* Functions from os.h */ @@ -5133,6 +5213,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_unlock_notify", test_unlock_notify, 0 }, #endif { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0 }, + { "test_sqlite3_log", test_sqlite3_log, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test2.c b/src/test2.c index 520ad73..fa7dd76 100644 --- a/src/test2.c +++ b/src/test2.c @@ -72,7 +72,7 @@ static int pager_open( int argc, /* Number of arguments */ const char **argv /* Text of each argument */ ){ - u16 pageSize; + u32 pageSize; Pager *pPager; int nPage; int rc; diff --git a/src/test4.c b/src/test4.c index cb5087b..49447d3 100644 --- a/src/test4.c +++ b/src/test4.c @@ -142,7 +142,7 @@ static int tcl_thread_create( } threadset[i].busy = 1; sqlite3_free(threadset[i].zFilename); - threadset[i].zFilename = sqlite3DbStrDup(0, argv[2]); + threadset[i].zFilename = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum = 1; threadset[i].completed = 0; rc = pthread_create(&x, 0, thread_main, &threadset[i]); @@ -476,7 +476,7 @@ static int tcl_thread_compile( thread_wait(&threadset[i]); threadset[i].xOp = do_compile; sqlite3_free(threadset[i].zArg); - threadset[i].zArg = sqlite3DbStrDup(0, argv[2]); + threadset[i].zArg = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum++; return TCL_OK; } diff --git a/src/test5.c b/src/test5.c index dc7dafe..504fdb8 100644 --- a/src/test5.c +++ b/src/test5.c @@ -155,7 +155,7 @@ static int test_translate( if( enc_from==SQLITE_UTF8 ){ z = Tcl_GetString(objv[1]); if( objc==5 ){ - z = sqlite3DbStrDup(0, z); + z = sqlite3_mprintf("%s", z); } sqlite3ValueSetStr(pVal, -1, z, enc_from, xDel); }else{ diff --git a/src/test7.c b/src/test7.c index 77b3661..3383d86 100644 --- a/src/test7.c +++ b/src/test7.c @@ -164,7 +164,7 @@ static int tcl_client_create( } threadset[i].busy = 1; sqlite3_free(threadset[i].zFilename); - threadset[i].zFilename = sqlite3DbStrDup(0, argv[2]); + threadset[i].zFilename = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum = 1; threadset[i].completed = 0; rc = pthread_create(&x, 0, client_main, &threadset[i]); @@ -507,7 +507,7 @@ static int tcl_client_compile( client_wait(&threadset[i]); threadset[i].xOp = do_compile; sqlite3_free(threadset[i].zArg); - threadset[i].zArg = sqlite3DbStrDup(0, argv[2]); + threadset[i].zArg = sqlite3_mprintf("%s", argv[2]); threadset[i].opnum++; return TCL_OK; } diff --git a/src/test8.c b/src/test8.c index 110aa65..b3e6058 100644 --- a/src/test8.c +++ b/src/test8.c @@ -485,7 +485,7 @@ static int echoCreate( rc = sqlite3_exec(db, zSql, 0, 0, 0); sqlite3_free(zSql); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3DbStrDup(0, sqlite3_errmsg(db)); + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } } diff --git a/src/test_demovfs.c b/src/test_demovfs.c index 7cdae80..8c51192 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -127,6 +127,7 @@ #include #include #include +#include /* ** Size of the write buffer used by journal files in bytes. @@ -135,6 +136,11 @@ # define SQLITE_DEMOVFS_BUFFERSZ 8192 #endif +/* +** The maximum pathname length supported by this VFS. +*/ +#define MAXPATHNAME 512 + /* ** When using this VFS, the sqlite3_file* handles that SQLite uses are ** actually pointers to instances of type DemoFile. @@ -446,16 +452,19 @@ static int demoOpen( ** file has been synced to disk before returning. */ static int demoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - int rc; + int rc; /* Return code */ + rc = unlink(zPath); + if( rc!=0 && errno==ENOENT ) return SQLITE_OK; + if( rc==0 && dirSync ){ int dfd; /* File descriptor open on directory */ int i; /* Iterator variable */ - char zDir[pVfs->mxPathname+1];/* Name of directory containing file zPath */ + char zDir[MAXPATHNAME+1]; /* Name of directory containing file zPath */ /* Figure out the directory name from the path of the file deleted. */ - sqlite3_snprintf(pVfs->mxPathname, zDir, "%s", zPath); - zDir[pVfs->mxPathname] = '\0'; + sqlite3_snprintf(MAXPATHNAME, zDir, "%s", zPath); + zDir[MAXPATHNAME] = '\0'; for(i=strlen(zDir); i>1 && zDir[i]!='/'; i++); zDir[i] = '\0'; @@ -524,13 +533,13 @@ static int demoFullPathname( int nPathOut, /* Size of output buffer in bytes */ char *zPathOut /* Pointer to output buffer */ ){ - char zDir[pVfs->mxPathname+1]; + char zDir[MAXPATHNAME+1]; if( zPath[0]=='/' ){ zDir[0] = '\0'; }else{ getcwd(zDir, sizeof(zDir)); } - zDir[pVfs->mxPathname] = '\0'; + zDir[MAXPATHNAME] = '\0'; sqlite3_snprintf(nPathOut, zPathOut, "%s/%s", zDir, zPath); zPathOut[nPathOut-1] = '\0'; @@ -609,7 +618,7 @@ sqlite3_vfs *sqlite3_demovfs(void){ static sqlite3_vfs demovfs = { 1, /* iVersion */ sizeof(DemoFile), /* szOsFile */ - 512, /* mxPathname */ + MAXPATHNAME, /* mxPathname */ 0, /* pNext */ "demo", /* zName */ 0, /* pAppData */ diff --git a/src/test_journal.c b/src/test_journal.c index 97e0e1b..ca4c5c3 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -361,6 +361,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ sqlite3_file *p = pMain->pReal; int rc = SQLITE_OK; + closeTransaction(pMain); aData = sqlite3_malloc(pMain->nPagesize); pMain->pWritable = sqlite3BitvecCreate(pMain->nPage); pMain->aCksum = sqlite3_malloc(sizeof(u32) * (pMain->nPage + 1)); @@ -379,6 +380,15 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ ** leaf to the jt_file.pWritable bitvec. */ rc = sqlite3OsRead(p, aData, pMain->nPagesize, 0); + if( rc==SQLITE_OK ){ + u32 nDbsize = decodeUint32(&aData[28]); + if( nDbsize>0 && memcmp(&aData[24], &aData[92], 4)==0 ){ + u32 iPg; + for(iPg=nDbsize+1; iPg<=pMain->nPage; iPg++){ + sqlite3BitvecSet(pMain->pWritable, iPg); + } + } + } iTrunk = decodeUint32(&aData[32]); while( rc==SQLITE_OK && iTrunk>0 ){ u32 nLeaf; @@ -492,7 +502,6 @@ finish_rjf: return rc; } - /* ** Write data to an jt-file. */ diff --git a/src/test_malloc.c b/src/test_malloc.c index 1267f6e..08642c9 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -728,7 +728,10 @@ static int test_memdebug_settitle( return TCL_OK; } -#define MALLOC_LOG_FRAMES 10 +#define MALLOC_LOG_FRAMES 10 +#define MALLOC_LOG_KEYINTS ( \ + 10 * ((sizeof(int)>=sizeof(void*)) ? 1 : sizeof(void*)/sizeof(int)) \ +) static Tcl_HashTable aMallocLog; static int mallocLogEnabled = 0; @@ -745,8 +748,8 @@ static void test_memdebug_callback(int nByte, int nFrame, void **aFrame){ Tcl_HashEntry *pEntry; int isNew; - int aKey[MALLOC_LOG_FRAMES]; - int nKey = sizeof(int)*MALLOC_LOG_FRAMES; + int aKey[MALLOC_LOG_KEYINTS]; + int nKey = sizeof(int)*MALLOC_LOG_KEYINTS; memset(aKey, 0, nKey); if( (sizeof(void*)*nFrame)=sizeof(void*)); for( pEntry=Tcl_FirstHashEntry(&aMallocLog, &search); @@ -836,13 +839,13 @@ static int test_memdebug_log( ){ Tcl_Obj *apElem[MALLOC_LOG_FRAMES+2]; MallocLog *pLog = (MallocLog *)Tcl_GetHashValue(pEntry); - int *aKey = (int *)Tcl_GetHashKey(&aMallocLog, pEntry); + Tcl_WideInt *aKey = (Tcl_WideInt *)Tcl_GetHashKey(&aMallocLog, pEntry); int ii; apElem[0] = Tcl_NewIntObj(pLog->nCall); apElem[1] = Tcl_NewIntObj(pLog->nByte); for(ii=0; iiaPage[0].pPg==0 ){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ + int nPage; u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + sqlite3PagerPagecount(pPager, &nPage); + if( nPage==0 ){ + pCsr->isEof = 1; + return sqlite3_reset(pCsr->pStmt); + } rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg); pCsr->aPage[0].iPgno = iRoot; pCsr->aPage[0].iCell = 0; @@ -486,17 +492,9 @@ static int statFilter( int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ - sqlite3 *db = ((StatTable *)(pCursor->pVtab))->db; StatCursor *pCsr = (StatCursor *)pCursor; - int nPage = 0; - - statResetCsr((StatCursor *)pCursor); - sqlite3PagerPagecount(sqlite3BtreePager(db->aDb[0].pBt), &nPage); - if( nPage==0 ){ - pCsr->isEof = 1; - return SQLITE_OK; - } + statResetCsr(pCsr); return statNext(pCursor); } diff --git a/src/test_vfs.c b/src/test_vfs.c index f577bf5..c606cfb 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -1000,7 +1000,7 @@ static int testvfs_obj_cmd( int n; u8 *a = Tcl_GetByteArrayFromObj(objv[3], &n); int pgsz = pBuffer->pgsz; - if( pgsz==0 ) pgsz = 32768; + if( pgsz==0 ) pgsz = 65536; for(i=0; i*pgszaPage[i]; i++){ int pgsz = pBuffer->pgsz; - if( pgsz==0 ) pgsz = 32768; + if( pgsz==0 ) pgsz = 65536; Tcl_AppendObjToObj(pObj, Tcl_NewByteArrayObj(pBuffer->aPage[i], pgsz)); } Tcl_SetObjResult(interp, pObj); diff --git a/src/tokenize.c b/src/tokenize.c index 4b40770..c624efd 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -496,7 +496,7 @@ abort_parse: } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3DbFree(db, pParse->apVtabLock); + sqlite3_free(pParse->apVtabLock); #endif if( !IN_DECLARE_VTAB ){ @@ -504,7 +504,7 @@ abort_parse: ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); } sqlite3DeleteTrigger(db, pParse->pNewTrigger); @@ -518,7 +518,7 @@ abort_parse: while( pParse->pZombieTab ){ Table *p = pParse->pZombieTab; pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); } if( nErr>0 && pParse->rc==SQLITE_OK ){ pParse->rc = SQLITE_ERROR; diff --git a/src/trigger.c b/src/trigger.c index 27fc708..b1d43d0 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -800,6 +800,7 @@ static TriggerPrg *codeRowTrigger( int iEndTrigger = 0; /* Label to jump to if WHEN is false */ assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); /* Allocate the TriggerPrg and SubProgram objects. To ensure that they ** are freed if an error occurs, link them into the Parse.pTriggerPrg @@ -810,7 +811,7 @@ static TriggerPrg *codeRowTrigger( pTop->pTriggerPrg = pPrg; pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); if( !pProgram ) return 0; - pProgram->nRef = 1; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); pPrg->pTrigger = pTrigger; pPrg->orconf = orconf; pPrg->aColmask[0] = 0xffffffff; @@ -944,8 +945,9 @@ void sqlite3CodeRowTriggerDirect( /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - pPrg->pProgram->nRef++; sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); @@ -955,7 +957,7 @@ void sqlite3CodeRowTriggerDirect( ** invocation is disallowed if (a) the sub-program is really a trigger, ** not a foreign key action, and (b) the flag to enable recursive triggers ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)(p->zName && !(pParse->db->flags&SQLITE_RecTriggers))); + sqlite3VdbeChangeP5(v, (u8)bRecursive); } } diff --git a/src/update.c b/src/update.c index fe8344c..44f047b 100644 --- a/src/update.c +++ b/src/update.c @@ -8,7 +8,7 @@ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -sqlite************************************************************************* +************************************************************************* ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ diff --git a/src/vdbe.c b/src/vdbe.c index 6813837..1e7fc16 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -501,6 +501,20 @@ static int checkSavepointCount(sqlite3 *db){ } #endif +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +static void importVtabErrMsg(Vdbe *p, sqlite3_vtab *pVtab){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; +} + + /* ** Execute as much of a VDBE program as we can then return. ** @@ -4030,9 +4044,7 @@ case OP_Rowid: { /* out2-prerelease */ pModule = pVtab->pModule; assert( pModule->xRowid ); rc = pModule->xRowid(pC->pVtabCursor, &v); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( pC->pCursor!=0 ); @@ -4337,7 +4349,7 @@ case OP_IdxRowid: { /* out2-prerelease */ ** that if the key from register P3 is a prefix of the key in the cursor, ** the result is false whereas it would be true with IdxGT. */ -/* Opcode: IdxLT P1 P2 P3 * P5 +/* Opcode: IdxLT P1 P2 P3 P4 P5 ** ** The P4 register values beginning with P3 form an unpacked index ** key that omits the ROWID. Compare this key value against the index @@ -5375,11 +5387,7 @@ case OP_VBegin: { VTable *pVTab; pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, pVTab); - if( pVTab ){ - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVTab->pVtab->zErrMsg; - pVTab->pVtab->zErrMsg = 0; - } + if( pVTab ) importVtabErrMsg(p, pVTab->pVtab); break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -5429,9 +5437,7 @@ case OP_VOpen: { pModule = (sqlite3_module *)pVtab->pModule; assert(pVtab && pModule); rc = pModule->xOpen(pVtab, &pVtabCursor); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); if( SQLITE_OK==rc ){ /* Initialize sqlite3_vtab_cursor base class */ pVtabCursor->pVtab = pVtab; @@ -5508,9 +5514,7 @@ case OP_VFilter: { /* jump */ p->inVtabMethod = 1; rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); if( rc==SQLITE_OK ){ res = pModule->xEof(pVtabCursor); } @@ -5560,9 +5564,7 @@ case OP_VColumn: { MemSetTypeFlag(&sContext.s, MEM_Null); rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; } @@ -5615,9 +5617,7 @@ case OP_VNext: { /* jump */ p->inVtabMethod = 1; rc = pModule->xNext(pCur->pVtabCursor); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); if( rc==SQLITE_OK ){ res = pModule->xEof(pCur->pVtabCursor); } @@ -5647,9 +5647,7 @@ case OP_VRename: { REGISTER_TRACE(pOp->p1, pName); assert( pName->flags & MEM_Str ); rc = pVtab->pModule->xRename(pVtab, pName->z); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); break; } @@ -5701,9 +5699,7 @@ case OP_VUpdate: { pX++; } rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + importVtabErrMsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); db->lastRowid = rowid; diff --git a/src/vdbe.h b/src/vdbe.h index c234d51..71c871d 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -81,8 +81,8 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ - int nRef; /* Number of pointers to this structure */ void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ }; /* @@ -184,6 +184,7 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); +void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); int sqlite3VdbeFinalize(Vdbe*); void sqlite3VdbeResolveLabel(Vdbe*, int); @@ -201,7 +202,6 @@ sqlite3 *sqlite3VdbeDb(Vdbe*); void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); void sqlite3VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); -void sqlite3VdbeProgramDelete(sqlite3 *, SubProgram *, int); sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8); void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE @@ -212,6 +212,10 @@ UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +#ifndef SQLITE_OMIT_TRIGGER +void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +#endif + #ifndef NDEBUG void sqlite3VdbeComment(Vdbe*, const char*, ...); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index e8e1585..4a19f14 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -323,6 +323,7 @@ struct Vdbe { VdbeFrame *pFrame; /* Parent frame */ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ + SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2f5aaa3..afb4a1b 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -398,7 +398,7 @@ static int sqlite3Step(Vdbe *p){ if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ sqlite3_int64 iNow; sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, iNow - p->startTime); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); } #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index cf53f90..5a4ad84 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -573,15 +573,17 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ } } +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* ** Delete a P4 value if necessary. */ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ + assert( db ); switch( p4type ){ case P4_REAL: case P4_INT64: - case P4_MPRINTF: case P4_DYNAMIC: case P4_KEYINFO: case P4_INTARRAY: @@ -589,10 +591,14 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ sqlite3DbFree(db, p4); break; } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } case P4_VDBEFUNC: { VdbeFunc *pVdbeFunc = (VdbeFunc *)p4; freeEphemeralFunction(db, pVdbeFunc->pFunc); - sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); + if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); sqlite3DbFree(db, pVdbeFunc); break; } @@ -601,15 +607,17 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ break; } case P4_MEM: { - sqlite3ValueFree((sqlite3_value*)p4); + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } break; } case P4_VTAB : { - sqlite3VtabUnlock((VTable *)p4); - break; - } - case P4_SUBPROGRAM : { - sqlite3VdbeProgramDelete(db, (SubProgram *)p4, 1); + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } } @@ -635,35 +643,15 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ } /* -** Decrement the ref-count on the SubProgram structure passed as the -** second argument. If the ref-count reaches zero, free the structure. -** -** The array of VDBE opcodes stored as SubProgram.aOp is freed if -** either the ref-count reaches zero or parameter freeop is non-zero. -** -** Since the array of opcodes pointed to by SubProgram.aOp may directly -** or indirectly contain a reference to the SubProgram structure itself. -** By passing a non-zero freeop parameter, the caller may ensure that all -** SubProgram structures and their aOp arrays are freed, even when there -** are such circular references. +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. */ -void sqlite3VdbeProgramDelete(sqlite3 *db, SubProgram *p, int freeop){ - if( p ){ - assert( p->nRef>0 ); - if( freeop || p->nRef==1 ){ - Op *aOp = p->aOp; - p->aOp = 0; - vdbeFreeOpArray(db, aOp, p->nOp); - p->nOp = 0; - } - p->nRef--; - if( p->nRef==0 ){ - sqlite3DbFree(db, p); - } - } +void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; } - /* ** Change N opcodes starting at addr to No-ops. */ @@ -739,7 +727,7 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ nField = ((KeyInfo*)zP4)->nField; nByte = sizeof(*pKeyInfo) + (nField-1)*sizeof(pKeyInfo->aColl[0]) + nField; - pKeyInfo = sqlite3Malloc( nByte ); + pKeyInfo = sqlite3DbMallocRaw(0, nByte); pOp->p4.pKeyInfo = pKeyInfo; if( pKeyInfo ){ u8 *aSortOrder; @@ -1003,6 +991,12 @@ static void releaseMemArray(Mem *p, int N){ Mem *pEnd; sqlite3 *db = p->db; u8 malloc_failed = db->mallocFailed; + if( db->pnBytesFreed ){ + for(pEnd=&p[N]; pzMalloc); + } + return; + } for(pEnd=&p[N]; pzErrMsg); - if( rc!=SQLITE_OK ){ - return rc; - } /* This loop determines (a) if the commit hook should be invoked and ** (b) how many database files have open write transactions, not @@ -1662,13 +1653,17 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** one database file has an open write transaction, a master journal ** file is required for an atomic commit. */ - for(i=0; inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); } } + if( rc!=SQLITE_OK ){ + return rc; + } /* If there are any write-transactions at all, invoke the commit hook */ if( needXcommit && db->xCommitCallback ){ @@ -1807,6 +1802,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ sqlite3DbFree(db, zMaster); return rc; @@ -2065,8 +2061,17 @@ int sqlite3VdbeHalt(Vdbe *p){ isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; if( isSpecialError ){ - /* If the query was read-only, we need do no rollback at all. Otherwise, - ** proceed with the special handling. + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occured while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. */ if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ @@ -2335,6 +2340,30 @@ void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ } } +/* +** Free all memory associated with the Vdbe passed as the second argument. +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection. +*/ +void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aLabel); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p); +} + /* ** Delete an entire VDBE. */ @@ -2352,16 +2381,9 @@ void sqlite3VdbeDelete(Vdbe *p){ if( p->pNext ){ p->pNext->pPrev = p->pPrev; } - releaseMemArray(p->aVar, p->nVar); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aLabel); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); p->magic = VDBE_MAGIC_DEAD; - sqlite3DbFree(db, p->pFree); p->db = 0; - sqlite3DbFree(db, p); + sqlite3VdbeDeleteObject(db, p); } /* @@ -2387,11 +2409,8 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; p->lastRowid = p->movetoTarget; - p->rowidIsValid = ALWAYS(res==0) ?1:0; - if( NEVER(res<0) ){ - rc = sqlite3BtreeNext(p->pCursor, &res); - if( rc ) return rc; - } + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif diff --git a/src/vdbemem.c b/src/vdbemem.c index 622b617..e376a65 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1016,7 +1016,7 @@ int sqlite3ValueFromExpr( } op = pExpr->op; - /* op can only be TK_REGISTER is we have compiled with SQLITE_ENABLE_STAT2. + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. ** The ifdef here is to enable us to achieve 100% branch test coverage even ** when SQLITE_ENABLE_STAT2 is omitted. */ diff --git a/src/vtab.c b/src/vtab.c index 24e922e..5104eeb 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -221,14 +221,14 @@ void sqlite3VtabUnlockList(sqlite3 *db){ ** in the list are moved to the sqlite3.pDisconnect list of the associated ** database connection. */ -void sqlite3VtabClear(Table *p){ - vtabDisconnectAll(0, p); +void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->azModuleArg ){ int i; for(i=0; inModuleArg; i++){ - sqlite3DbFree(p->dbMem, p->azModuleArg[i]); + sqlite3DbFree(db, p->azModuleArg[i]); } - sqlite3DbFree(p->dbMem, p->azModuleArg); + sqlite3DbFree(db, p->azModuleArg); } } @@ -393,7 +393,6 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } - pSchema->db = pParse->db; pParse->pNewTable = 0; } } @@ -467,7 +466,7 @@ static int vtabCallConstructor( *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); }else { *pzErr = sqlite3MPrintf(db, "%s", zErr); - sqlite3DbFree(db, zErr); + sqlite3_free(zErr); } sqlite3DbFree(db, pVTable); }else if( ALWAYS(pVTable->pVtab) ){ @@ -682,7 +681,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ if( pParse->pVdbe ){ sqlite3VdbeFinalize(pParse->pVdbe); } - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); sqlite3StackFree(db, pParse); } @@ -769,8 +768,8 @@ int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ rc = x(pVtab); sqlite3DbFree(db, *pzErrmsg); - *pzErrmsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); } } db->aVTrans = aVTrans; diff --git a/src/wal.c b/src/wal.c index 5427464..22ce1fa 100644 --- a/src/wal.c +++ b/src/wal.c @@ -292,6 +292,10 @@ typedef struct WalCkptInfo WalCkptInfo; ** ** The actual header in the wal-index consists of two copies of this ** object. +** +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. */ struct WalIndexHdr { u32 iVersion; /* Wal-index version */ @@ -299,7 +303,7 @@ struct WalIndexHdr { u32 iChange; /* Counter incremented each transaction */ u8 isInit; /* 1 when initialized */ u8 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes */ + u16 szPage; /* Database page size in bytes. 1==64K */ u32 mxFrame; /* Index of last valid frame in the WAL */ u32 nPage; /* Size of database in pages */ u32 aFrameCksum[2]; /* Checksum of last frame in log */ @@ -410,7 +414,7 @@ struct Wal { u32 iCallback; /* Value to pass to log callback (or 0) */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ - u16 szPage; /* Database page size */ + u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ @@ -1081,7 +1085,7 @@ static int walIndexRecover(Wal *pWal){ goto finished; } pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); @@ -1131,7 +1135,9 @@ static int walIndexRecover(Wal *pWal){ if( nTruncate ){ pWal->hdr.mxFrame = iFrame; pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } @@ -1156,6 +1162,17 @@ finished: pInfo->nBackfill = 0; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; + + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + pWal->hdr.nPage, pWal->zWalName + ); + } } recovery_error: @@ -1516,14 +1533,18 @@ static int walCheckpoint( u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ - int szPage = pWal->hdr.szPage; /* Database page-size */ + int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; /* Allocate the iterator */ @@ -1534,7 +1555,7 @@ static int walCheckpoint( assert( pIter ); /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( pWal->hdr.szPage!=nBuf ){ + if( szPage!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; goto walcheckpoint_out; } @@ -1545,6 +1566,7 @@ static int walCheckpoint( ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; pInfo = walCkptInfo(pWal); for(i=1; iaReadMark[i]; @@ -1565,6 +1587,7 @@ static int walCheckpoint( if( pInfo->nBackfillnBackfill; /* Sync the WAL to disk */ @@ -1572,11 +1595,22 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } + /* If the database file may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue; + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); @@ -1683,7 +1717,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ /* The first page of the wal-index must be mapped at this point. */ assert( pWal->nWiData>0 && pWal->apWiData[0] ); - /* Read the header. This might happen currently with a write to the + /* Read the header. This might happen concurrently with a write to the ** same area of shared memory on a different CPU in a SMP, ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. @@ -1712,7 +1746,9 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ *pChanged = 1; memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); - pWal->szPage = pWal->hdr.szPage; + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); } /* The header was successfully read. Return zero. */ @@ -2031,6 +2067,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** read-lock. */ void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock>=0 ){ walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; @@ -2141,7 +2178,13 @@ int sqlite3WalRead( ** required page. Read and return data from the log file. */ if( iRead ){ - i64 iOffset = walFrameOffset(iRead, pWal->hdr.szPage) + WAL_FRAME_HDRSIZE; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); @@ -2153,11 +2196,13 @@ int sqlite3WalRead( /* -** Set *pPgno to the size of the database file (or zero, if unknown). +** Return the size of the database in pages (or zero, if unknown). */ -void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ - assert( pWal->readLock>=0 || pWal->lockError ); - *pPgno = pWal->hdr.nPage; +Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; + } + return 0; } @@ -2233,7 +2278,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){ */ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - if( pWal->writeLock ){ + if( ALWAYS(pWal->writeLock) ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; @@ -2422,7 +2467,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; pWal->hdr.aFrameCksum[1] = aCksum[1]; @@ -2517,7 +2562,9 @@ int sqlite3WalFrames( if( rc==SQLITE_OK ){ /* Update the private copy of the header. */ - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); pWal->hdr.mxFrame = iFrame; if( isCommit ){ pWal->hdr.iChange++; diff --git a/src/wal.h b/src/wal.h index f29d583..f20dfa8 100644 --- a/src/wal.h +++ b/src/wal.h @@ -25,7 +25,7 @@ # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 -# define sqlite3WalDbsize(y,z) +# define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 # define sqlite3WalUndo(x,y,z) 0 @@ -61,9 +61,8 @@ void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); -/* Return the size of the database as it existed at the beginning -** of the snapshot */ -void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno); +/* If the WAL is not empty, return the size of the database. */ +Pgno sqlite3WalDbsize(Wal *pWal); /* Obtain or release the WRITER lock. */ int sqlite3WalBeginWriteTransaction(Wal *pWal); diff --git a/src/where.c b/src/where.c index b2f75ea..9040ad0 100644 --- a/src/where.c +++ b/src/where.c @@ -327,6 +327,7 @@ static void whereClauseClear(WhereClause *pWC){ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ WhereTerm *pTerm; int idx; + testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pParse->db; @@ -472,6 +473,13 @@ static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are ** "=", "<", ">", "<=", ">=", and "IN". +** +** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be +** of one of the following forms: column = expression column > expression +** column >= expression column < expression column <= expression +** expression = column expression > column expression >= column +** expression < column expression <= column column IN +** (expression-list) column IN (subquery) column IS NULL */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GTdb; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ @@ -654,19 +661,6 @@ static int isLikeOrGlob( return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pColl = sqlite3ExprCollSeq(pParse, pLeft); - if( pColl==0 ) return 0; /* Happens when LHS has an undefined collation */ - if( (pColl->type!=SQLITE_COLL_BINARY || *pnoCase) && - (pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){ - /* IMP: R-09003-32046 For the GLOB operator, the column must use the - ** default BINARY collating sequence. - ** IMP: R-41408-28306 For the LIKE operator, if case_sensitive_like mode - ** is enabled then the column must use the default BINARY collating - ** sequence, or if case_sensitive_like mode is disabled then the column - ** must use the built-in NOCASE collating sequence. - */ - return 0; - } pRight = pList->a[0].pExpr; op = pRight->op; @@ -689,9 +683,9 @@ static int isLikeOrGlob( while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; } - if( cnt!=0 && c!=0 && 255!=(u8)z[cnt-1] ){ + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; - *pisComplete = z[cnt]==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0; pPrefix = sqlite3Expr(db, TK_STRING, z); if( pPrefix ) pPrefix->u.zToken[cnt] = 0; *ppPrefix = pPrefix; @@ -1030,6 +1024,8 @@ static void exprAnalyzeOrTerm( /* At this point, okToChngToIN is true if original pTerm satisfies ** case 1. In that case, construct a new virtual term that is ** pTerm converted into an IN operator. + ** + ** EV: R-00211-15100 */ if( okToChngToIN ){ Expr *pDup; /* A transient duplicate expression */ @@ -1246,6 +1242,7 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; + CollSeq *pColl; /* Collating sequence to use */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); @@ -1260,17 +1257,23 @@ static void exprAnalyze( ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ - if( c=='A'-1 ) isComplete = 0; + if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ + c = sqlite3UpperToLower[c]; } *pC = c + 1; } - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprDup(db,pLeft,0),pStr1,0); + pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprDup(db,pLeft,0),pStr2,0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); @@ -2022,7 +2025,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } - sqlite3DbFree(pParse->db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; for(i=0; inConstraint; i++){ @@ -2887,6 +2890,9 @@ static void bestIndex( ** in the ON clause. The term is disabled in (3) because it is not part ** of a LEFT OUTER JOIN. In (1), the term is not disabled. ** +** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are +** completely satisfied by indices. +** ** Disabling a term causes that term to not be tested in the inner loop ** of the join. Disabling is an optimization. When terms are satisfied ** by indices, we disable them to prevent redundant tests in the inner @@ -3098,6 +3104,7 @@ static int codeAllEqualityTerms( /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ @@ -3241,6 +3248,7 @@ static Bitmask codeOneLoopStart( assert( pTerm->pExpr!=0 ); assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); @@ -3281,6 +3289,7 @@ static Bitmask codeOneLoopStart( assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ pX = pStart->pExpr; assert( pX!=0 ); assert( pStart->leftCursor==iCur ); @@ -3298,6 +3307,7 @@ static Bitmask codeOneLoopStart( pX = pEnd->pExpr; assert( pX!=0 ); assert( pEnd->leftCursor==iCur ); + testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ memEndValue = ++pParse->nMem; sqlite3ExprCode(pParse, pX->pRight, memEndValue); if( pX->op==TK_LT || pX->op==TK_GT ){ @@ -3355,7 +3365,7 @@ static Bitmask codeOneLoopStart( ** constraints but an index is selected anyway, in order ** to force the output order to conform to an ORDER BY. */ - int aStartOp[] = { + static const u8 aStartOp[] = { 0, 0, OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ @@ -3365,12 +3375,12 @@ static Bitmask codeOneLoopStart( OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ }; - int aEndOp[] = { + static const u8 aEndOp[] = { OP_Noop, /* 0: (!end_constraints) */ OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLevel->plan.nEq; + int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ @@ -3380,10 +3390,10 @@ static Bitmask codeOneLoopStart( int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ char *zEndAff; /* Affinity for end of range constraint */ @@ -3465,6 +3475,7 @@ static Bitmask codeOneLoopStart( } } nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ }else if( isMinQuery ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); nConstraint++; @@ -3504,6 +3515,7 @@ static Bitmask codeOneLoopStart( } codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ } sqlite3DbFree(pParse->db, zStartAff); sqlite3DbFree(pParse->db, zEndAff); @@ -3710,11 +3722,15 @@ static Bitmask codeOneLoopStart( /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. + ** + ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through + ** the use of indices become tests that are evaluated against each row of + ** the relevant input tables. */ k = 0; for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ @@ -3742,7 +3758,7 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "record LEFT JOIN hit")); sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ @@ -3960,7 +3976,7 @@ WhereInfo *sqlite3WhereBegin( initMaskSet(pMaskSet); whereClauseInit(pWC, pParse, pMaskSet); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); + whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -4048,6 +4064,8 @@ WhereInfo *sqlite3WhereBegin( int bestJ = -1; /* The value of j */ Bitmask m; /* Bitmask value for j or bestJ */ int isOptimal; /* Iterator for optimal/non-optimal search */ + int nUnconstrained; /* Number tables without INDEXED BY */ + Bitmask notIndexed; /* Mask of tables that cannot use an index */ memset(&bestPlan, 0, sizeof(bestPlan)); bestPlan.rCost = SQLITE_BIG_DBL; @@ -4089,8 +4107,10 @@ WhereInfo *sqlite3WhereBegin( ** algorithm may choose to use t2 for the outer loop, which is a much ** costlier approach. */ + nUnconstrained = 0; + notIndexed = 0; for(isOptimal=(iFrom=0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ + Bitmask mask; /* Mask of tables not yet ready */ for(j=iFrom, pTabItem=&pTabList->a[j]; jpIndex==0 ) nUnconstrained++; assert( pTabItem->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -4118,9 +4139,43 @@ WhereInfo *sqlite3WhereBegin( } assert( isOptimal || (sCost.used¬Ready)==0 ); - if( (sCost.used¬Ready)==0 - && (bestJ<0 || sCost.rCostpIndex==0 + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sCost.plan.u.pIdx==pTabItem->pIndex ); + + if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + notIndexed |= m; + } + + /* Conditions under which this table becomes the best so far: + ** + ** (1) The table must not depend on other tables that have not + ** yet run. + ** + ** (2) A full-table-scan plan cannot supercede another plan unless + ** it is an "optimal" plan as defined above. + ** + ** (3) All tables have an INDEXED BY clause or this table lacks an + ** INDEXED BY clause or this table uses the specific + ** index specified by its INDEXED BY clause. This rule ensures + ** that a best-so-far is always selected even if an impossible + ** combination of INDEXED BY clauses are given. The error + ** will be detected and relayed back to the application later. + ** The NEVER() comes about because rule (2) above prevents + ** An indexable full-table-scan from reaching rule (3). + ** + ** (4) The plan cost must be lower than prior plans or else the + ** cost must be the same and the number of rows must be lower. + */ + if( (sCost.used¬Ready)==0 /* (1) */ + && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) + && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ + || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || sCost.rCost= "3.7.0" + && [code2 {sqlite3 -version}] >= "3.7.0" + } { + + do_test backcompat-2.1.1 { sql1 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES('I', 1); + INSERT INTO t1 VALUES('II', 2); + INSERT INTO t1 VALUES('III', 3); + SELECT * FROM t1; + } } {wal I 1 II 2 III 3} + do_test backcompat-2.1.2 { sql2 { + SELECT * FROM t1; + } } {I 1 II 2 III 3} + + set data [read_file_system] + code1 {db close} + code2 {db close} + write_file_system $data + code1 {sqlite3 db test.db} + code2 {sqlite3 db test.db} + + # The WAL file now in the file-system was created by the [code1] + # process. Check that the [code2] process can recover the log. + # + do_test backcompat-2.1.3 { sql2 { + SELECT * FROM t1; + } } {I 1 II 2 III 3} + do_test backcompat-2.1.4 { sql1 { + SELECT * FROM t1; + } } {I 1 II 2 III 3} + } +} + +finish_test diff --git a/test/conflict.test b/test/conflict.test index 44390d2..c612e3b 100644 --- a/test/conflict.test +++ b/test/conflict.test @@ -789,4 +789,26 @@ do_test conflict-12.4 { } {2 one} +# Ticket [c38baa3d969eab7946dc50ba9d9b4f0057a19437] +# REPLACE works like ABORT on a CHECK constraint. +# +do_test conflict-13.1 { + execsql { + CREATE TABLE t13(a CHECK(a!=2)); + BEGIN; + REPLACE INTO t13 VALUES(1); + } + catchsql { + REPLACE INTO t13 VALUES(2); + } +} {1 {constraint failed}} +do_test conflict-13.2 { + execsql { + REPLACE INTO t13 VALUES(3); + COMMIT; + SELECT * FROM t13; + } +} {1 3} + + finish_test diff --git a/test/dbstatus.test b/test/dbstatus.test index c7a6b58..5cb8701 100644 --- a/test/dbstatus.test +++ b/test/dbstatus.test @@ -33,7 +33,7 @@ do_test dbstatus-1.1 { set sz2 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_CACHE_USED 0] 1] set ::PAGESZ [expr {$sz2-$sz1}] set ::BASESZ [expr {$sz1-$::PAGESZ}] - expr {$::PAGESZ>1024 && $::PAGESZ<1200} + expr {$::PAGESZ>1024 && $::PAGESZ<1300} } {1} do_test dbstatus-1.2 { db eval { @@ -42,4 +42,301 @@ do_test dbstatus-1.2 { lindex [sqlite3_db_status db SQLITE_DBSTATUS_CACHE_USED 0] 1 } [expr {$BASESZ + 10*$PAGESZ}] + +proc lookaside {db} { + expr { $::lookaside_buffer_size * + [lindex [sqlite3_db_status $db SQLITE_DBSTATUS_LOOKASIDE_USED 0] 1] + } +} + +#--------------------------------------------------------------------------- +# Run the dbstatus-2 and dbstatus-3 tests with several of different +# lookaside buffer sizes. +# +foreach ::lookaside_buffer_size {0 64 120} { + + # Do not run any of these tests if there is SQL configured to run + # as part of the [sqlite3] command. This prevents the script from + # configuring the size of the lookaside buffer after [sqlite3] has + # returned. + if {[presql] != ""} break + + #------------------------------------------------------------------------- + # Tests for SQLITE_DBSTATUS_SCHEMA_USED. + # + # Each test in the following block works as follows. Each test uses a + # different database schema. + # + # 1. Open a connection to an empty database. Disable statement caching. + # + # 2. Execute the SQL to create the database schema. Measure the total + # heap and lookaside memory allocated by SQLite, and the memory + # allocated for the database schema according to sqlite3_db_status(). + # + # 3. Drop all tables in the database schema. Measure the total memory + # and the schema memory again. + # + # 4. Repeat step 2. + # + # 5. Repeat step 3. + # + # Then test that: + # + # a) The difference in schema memory quantities in steps 2 and 3 is the + # same as the difference in total memory in steps 2 and 3. + # + # b) Step 4 reports the same amount of schema and total memory used as + # in step 2. + # + # c) Step 5 reports the same amount of schema and total memory used as + # in step 3. + # + foreach {tn schema} { + 1 { CREATE TABLE t1(a, b) } + 2 { CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t1, c UNIQUE) } + 3 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + } + 4 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + CREATE TRIGGER AFTER INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(new.a, new.b); + SELECT * FROM t1, t2 WHERE a=c AND b=d GROUP BY b HAVING a>5 ORDER BY a; + END; + } + 5 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + CREATE VIEW v1 AS SELECT * FROM t1 UNION SELECT * FROM t2; + } + 6 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(a,b); + CREATE INDEX i3 ON t1(b,b); + INSERT INTO t1 VALUES(randomblob(20), randomblob(25)); + INSERT INTO t1 SELECT randomblob(20), randomblob(25) FROM t1; + INSERT INTO t1 SELECT randomblob(20), randomblob(25) FROM t1; + INSERT INTO t1 SELECT randomblob(20), randomblob(25) FROM t1; + ANALYZE; + } + 7 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + CREATE VIEW v1 AS + SELECT * FROM t1 + UNION + SELECT * FROM t2 + UNION ALL + SELECT c||b, d||a FROM t2 LEFT OUTER JOIN t1 GROUP BY c, d + ORDER BY 1, 2 + ; + CREATE TRIGGER tr1 INSTEAD OF INSERT ON v1 BEGIN + SELECT * FROM v1; + UPDATE t1 SET a=5, b=(SELECT c FROM t2); + END; + SELECT * FROM v1; + } + 8x { + CREATE TABLE t1(a, b, UNIQUE(a, b)); + CREATE VIRTUAL TABLE t2 USING echo(t1); + } + } { + set tn "$::lookaside_buffer_size-$tn" + + # Step 1. + db close + file delete -force test.db + sqlite3 db test.db + sqlite3_db_config_lookaside db 0 $::lookaside_buffer_size 500 + db cache size 0 + + catch { register_echo_module db } + ifcapable !vtab { if {[string match *x $tn]} continue } + + # Step 2. + execsql $schema + set nAlloc1 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc1 [lookaside db] + set nSchema1 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_SCHEMA_USED 0] 1] + + # Step 3. + drop_all_tables + set nAlloc2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc2 [lookaside db] + set nSchema2 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_SCHEMA_USED 0] 1] + + # Step 4. + execsql $schema + set nAlloc3 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc3 [lookaside db] + set nSchema3 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_SCHEMA_USED 0] 1] + + # Step 5. + drop_all_tables + set nAlloc4 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc4 [lookaside db] + set nSchema4 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_SCHEMA_USED 0] 1] + set nFree [expr {$nAlloc1-$nAlloc2}] + + # Tests for which the test name ends in an "x" report slightly less + # memory than is actually freed when all schema items are finalized. + # This is because memory allocated by virtual table implementations + # for any reason is not counted as "schema memory". + # + # Additionally, in auto-vacuum mode, dropping tables and indexes causes + # the page-cache to shrink. So the amount of memory freed is always + # much greater than just that reported by DBSTATUS_SCHEMA_USED in this + # case. + # + if {[string match *x $tn] || $AUTOVACUUM} { + do_test dbstatus-2.$tn.ax { expr {($nSchema1-$nSchema2)<=$nFree} } 1 + } else { + do_test dbstatus-2.$tn.a { expr {$nSchema1-$nSchema2} } $nFree + } + + do_test dbstatus-2.$tn.b { list $nAlloc1 $nSchema1 } "$nAlloc3 $nSchema3" + do_test dbstatus-2.$tn.c { list $nAlloc2 $nSchema2 } "$nAlloc4 $nSchema4" + } + + #------------------------------------------------------------------------- + # Tests for SQLITE_DBSTATUS_STMT_USED. + # + # Each test in the following block works as follows. Each test uses a + # different database schema. + # + # 1. Open a connection to an empty database. Initialized the database + # schema. + # + # 2. Prepare a bunch of SQL statements. Measure the total heap and + # lookaside memory allocated by SQLite, and the memory allocated + # for the prepared statements according to sqlite3_db_status(). + # + # 3. Finalize all prepared statements Measure the total memory + # and the prepared statement memory again. + # + # 4. Repeat step 2. + # + # 5. Repeat step 3. + # + # Then test that: + # + # a) The difference in schema memory quantities in steps 2 and 3 is the + # same as the difference in total memory in steps 2 and 3. + # + # b) Step 4 reports the same amount of schema and total memory used as + # in step 2. + # + # c) Step 5 reports the same amount of schema and total memory used as + # in step 3. + # + foreach {tn schema statements} { + 1 { CREATE TABLE t1(a, b) } { + SELECT * FROM t1; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 SELECT * FROM t1; + UPDATE t1 SET a=5; + DELETE FROM t1; + } + 2 { + PRAGMA recursive_triggers = 1; + CREATE TABLE t1(a, b); + CREATE TRIGGER tr1 AFTER INSERT ON t1 WHEN (new.a>0) BEGIN + INSERT INTO t1 VALUES(new.a-1, new.b); + END; + } { + INSERT INTO t1 VALUES(5, 'x'); + } + 3 { + PRAGMA recursive_triggers = 1; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TRIGGER tr1 AFTER INSERT ON t1 WHEN (new.a>0) BEGIN + INSERT INTO t2 VALUES(new.a-1, new.b); + END; + CREATE TRIGGER tr2 AFTER INSERT ON t1 WHEN (new.a>0) BEGIN + INSERT INTO t1 VALUES(new.a-1, new.b); + END; + } { + INSERT INTO t1 VALUES(10, 'x'); + } + 4 { + CREATE TABLE t1(a, b); + } { + SELECT count(*) FROM t1 WHERE upper(a)='ABC'; + } + 5x { + CREATE TABLE t1(a, b UNIQUE); + CREATE VIRTUAL TABLE t2 USING echo(t1); + } { + SELECT count(*) FROM t2; + SELECT * FROM t2 WHERE b>5; + SELECT * FROM t2 WHERE b='abcdefg'; + } + } { + set tn "$::lookaside_buffer_size-$tn" + + # Step 1. + db close + file delete -force test.db + sqlite3 db test.db + sqlite3_db_config_lookaside db 0 $::lookaside_buffer_size 500 + db cache size 1000 + + catch { register_echo_module db } + ifcapable !vtab { if {[string match *x $tn]} continue } + + execsql $schema + db cache flush + + # Step 2. + execsql $statements + set nAlloc1 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc1 [lookaside db] + set nStmt1 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_STMT_USED 0] 1] + execsql $statements + + # Step 3. + db cache flush + set nAlloc2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc2 [lookaside db] + set nStmt2 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_STMT_USED 0] 1] + + # Step 3. + execsql $statements + set nAlloc3 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc3 [lookaside db] + set nStmt3 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_STMT_USED 0] 1] + execsql $statements + + # Step 4. + db cache flush + set nAlloc4 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + incr nAlloc4 [lookaside db] + set nStmt4 [lindex [sqlite3_db_status db SQLITE_DBSTATUS_STMT_USED 0] 1] + + set nFree [expr {$nAlloc1-$nAlloc2}] + + do_test dbstatus-3.$tn.a { expr $nStmt2 } {0} + + # Tests for which the test name ends in an "x" report slightly less + # memory than is actually freed when all statements are finalized. + # This is because a small amount of memory allocated by a virtual table + # implementation using sqlite3_mprintf() is technically considered + # external and so is not counted as "statement memory". + # +#puts "$nStmt1 $nFree" + if {[string match *x $tn]} { + do_test dbstatus-3.$tn.bx { expr $nStmt1<=$nFree } {1} + } else { + do_test dbstatus-3.$tn.b { expr $nStmt1==$nFree } {1} + } + + do_test dbstatus-3.$tn.c { list $nAlloc1 $nStmt1 } [list $nAlloc3 $nStmt3] + do_test dbstatus-3.$tn.d { list $nAlloc2 $nStmt2 } [list $nAlloc4 $nStmt4] + } +} + finish_test diff --git a/test/e_expr.test b/test/e_expr.test index 2ca5ac7..a5c4457 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -329,4 +329,778 @@ foreach {n1 rhs} $literals { } } +#------------------------------------------------------------------------- +# Run some tests on the COLLATE "unary postfix operator". +# +# This collation sequence reverses both arguments before using +# [string compare] to compare them. For example, when comparing the +# strings 'one' and 'four', return the result of: +# +# string compare eno ruof +# +proc reverse_str {zStr} { + set out "" + foreach c [split $zStr {}] { set out "${c}${out}" } + set out +} +proc reverse_collate {zLeft zRight} { + string compare [reverse_str $zLeft] [reverse_str $zRight] +} +db collate reverse reverse_collate + +# EVIDENCE-OF: R-59577-33471 The COLLATE operator is a unary postfix +# operator that assigns a collating sequence to an expression. +# +# EVIDENCE-OF: R-23441-22541 The COLLATE operator has a higher +# precedence (binds more tightly) than any prefix unary operator or any +# binary operator. +# +do_execsql_test e_expr-9.1 { SELECT 'abcd' < 'bbbb' COLLATE reverse } 0 +do_execsql_test e_expr-9.2 { SELECT ('abcd' < 'bbbb') COLLATE reverse } 1 +do_execsql_test e_expr-9.3 { SELECT 'abcd' <= 'bbbb' COLLATE reverse } 0 +do_execsql_test e_expr-9.4 { SELECT ('abcd' <= 'bbbb') COLLATE reverse } 1 + +do_execsql_test e_expr-9.5 { SELECT 'abcd' > 'bbbb' COLLATE reverse } 1 +do_execsql_test e_expr-9.6 { SELECT ('abcd' > 'bbbb') COLLATE reverse } 0 +do_execsql_test e_expr-9.7 { SELECT 'abcd' >= 'bbbb' COLLATE reverse } 1 +do_execsql_test e_expr-9.8 { SELECT ('abcd' >= 'bbbb') COLLATE reverse } 0 + +do_execsql_test e_expr-9.10 { SELECT 'abcd' = 'ABCD' COLLATE nocase } 1 +do_execsql_test e_expr-9.11 { SELECT ('abcd' = 'ABCD') COLLATE nocase } 0 +do_execsql_test e_expr-9.12 { SELECT 'abcd' == 'ABCD' COLLATE nocase } 1 +do_execsql_test e_expr-9.13 { SELECT ('abcd' == 'ABCD') COLLATE nocase } 0 +do_execsql_test e_expr-9.14 { SELECT 'abcd' IS 'ABCD' COLLATE nocase } 1 +do_execsql_test e_expr-9.15 { SELECT ('abcd' IS 'ABCD') COLLATE nocase } 0 + +do_execsql_test e_expr-9.16 { SELECT 'abcd' != 'ABCD' COLLATE nocase } 0 +do_execsql_test e_expr-9.17 { SELECT ('abcd' != 'ABCD') COLLATE nocase } 1 +do_execsql_test e_expr-9.18 { SELECT 'abcd' <> 'ABCD' COLLATE nocase } 0 +do_execsql_test e_expr-9.19 { SELECT ('abcd' <> 'ABCD') COLLATE nocase } 1 +do_execsql_test e_expr-9.20 { SELECT 'abcd' IS NOT 'ABCD' COLLATE nocase } 0 +do_execsql_test e_expr-9.21 { SELECT ('abcd' IS NOT 'ABCD') COLLATE nocase } 1 + +do_execsql_test e_expr-9.22 { + SELECT 'bbb' BETWEEN 'AAA' AND 'CCC' COLLATE nocase +} 1 +do_execsql_test e_expr-9.23 { + SELECT ('bbb' BETWEEN 'AAA' AND 'CCC') COLLATE nocase +} 0 + +# EVIDENCE-OF: R-58731-25439 The collating sequence set by the COLLATE +# operator overrides the collating sequence determined by the COLLATE +# clause in a table column definition. +# +do_execsql_test e_expr-9.24 { + CREATE TABLE t24(a COLLATE NOCASE, b); + INSERT INTO t24 VALUES('aaa', 1); + INSERT INTO t24 VALUES('bbb', 2); + INSERT INTO t24 VALUES('ccc', 3); +} {} +do_execsql_test e_expr-9.25 { SELECT 'BBB' = a FROM t24 } {0 1 0} +do_execsql_test e_expr-9.25 { SELECT a = 'BBB' FROM t24 } {0 1 0} +do_execsql_test e_expr-9.25 { SELECT 'BBB' = a COLLATE binary FROM t24 } {0 0 0} +do_execsql_test e_expr-9.25 { SELECT a COLLATE binary = 'BBB' FROM t24 } {0 0 0} + +#------------------------------------------------------------------------- +# Test statements related to literal values. +# +# EVIDENCE-OF: R-31536-32008 Literal values may be integers, floating +# point numbers, strings, BLOBs, or NULLs. +# +do_execsql_test e_expr-10.1.1 { SELECT typeof(5) } {integer} +do_execsql_test e_expr-10.1.2 { SELECT typeof(5.1) } {real} +do_execsql_test e_expr-10.1.3 { SELECT typeof('5.1') } {text} +do_execsql_test e_expr-10.1.4 { SELECT typeof(X'ABCD') } {blob} +do_execsql_test e_expr-10.1.5 { SELECT typeof(NULL) } {null} + +# EVIDENCE-OF: R-26921-59298 Scientific notation is supported for +# floating point literal values. +# +do_execsql_test e_expr-10.2.1 { SELECT typeof(3.4e-02) } {real} +do_execsql_test e_expr-10.2.2 { SELECT typeof(3e+5) } {real} +do_execsql_test e_expr-10.2.3 { SELECT 3.4e-02 } {0.034} +do_execsql_test e_expr-10.2.4 { SELECT 3e+4 } {30000.0} + +# EVIDENCE-OF: R-35229-17830 A string constant is formed by enclosing +# the string in single quotes ('). +# +# EVIDENCE-OF: R-07100-06606 A single quote within the string can be +# encoded by putting two single quotes in a row - as in Pascal. +# +do_execsql_test e_expr-10.3.1 { SELECT 'is not' } {{is not}} +do_execsql_test e_expr-10.3.2 { SELECT typeof('is not') } {text} +do_execsql_test e_expr-10.3.3 { SELECT 'isn''t' } {isn't} +do_execsql_test e_expr-10.3.4 { SELECT typeof('isn''t') } {text} + +# EVIDENCE-OF: R-09593-03321 BLOB literals are string literals +# containing hexadecimal data and preceded by a single "x" or "X" +# character. +# +# EVIDENCE-OF: R-39344-59787 For example: X'53514C697465' +# +do_execsql_test e_expr-10.4.1 { SELECT typeof(X'0123456789ABCDEF') } blob +do_execsql_test e_expr-10.4.2 { SELECT typeof(x'0123456789ABCDEF') } blob +do_execsql_test e_expr-10.4.3 { SELECT typeof(X'0123456789abcdef') } blob +do_execsql_test e_expr-10.4.4 { SELECT typeof(x'0123456789abcdef') } blob +do_execsql_test e_expr-10.4.5 { SELECT typeof(X'53514C697465') } blob + +# EVIDENCE-OF: R-23914-51476 A literal value can also be the token +# "NULL". +# +do_execsql_test e_expr-10.5.1 { SELECT NULL } {{}} +do_execsql_test e_expr-10.5.2 { SELECT typeof(NULL) } {null} + +#------------------------------------------------------------------------- +# Test statements related to bound parameters +# + +proc parameter_test {tn sql params result} { + set stmt [sqlite3_prepare_v2 db $sql -1] + + foreach {number name} $params { + set nm [sqlite3_bind_parameter_name $stmt $number] + do_test $tn.name.$number [list set {} $nm] $name + sqlite3_bind_int $stmt $number [expr -1 * $number] + } + + sqlite3_step $stmt + + set res [list] + for {set i 0} {$i < [sqlite3_column_count $stmt]} {incr i} { + lappend res [sqlite3_column_text $stmt $i] + } + + set rc [sqlite3_finalize $stmt] + do_test $tn.rc [list set {} $rc] SQLITE_OK + do_test $tn.res [list set {} $res] $result +} + +# EVIDENCE-OF: R-33509-39458 A question mark followed by a number NNN +# holds a spot for the NNN-th parameter. NNN must be between 1 and +# SQLITE_MAX_VARIABLE_NUMBER. +# +set mvn $SQLITE_MAX_VARIABLE_NUMBER +parameter_test e_expr-11.1 " + SELECT ?1, ?123, ?$SQLITE_MAX_VARIABLE_NUMBER, ?123, ?4 +" "1 ?1 123 ?123 $mvn ?$mvn 4 ?4" "-1 -123 -$mvn -123 -4" + +set errmsg "variable number must be between ?1 and ?$SQLITE_MAX_VARIABLE_NUMBER" +foreach {tn param_number} [list \ + 2 0 \ + 3 [expr $SQLITE_MAX_VARIABLE_NUMBER+1] \ + 4 [expr $SQLITE_MAX_VARIABLE_NUMBER+2] \ + 5 12345678903456789034567890234567890 \ + 6 2147483648 \ + 7 2147483649 \ + 8 4294967296 \ + 9 4294967297 \ + 10 9223372036854775808 \ + 11 9223372036854775809 \ + 12 18446744073709551616 \ + 13 18446744073709551617 \ +] { + do_catchsql_test e_expr-11.1.$tn "SELECT ?$param_number" [list 1 $errmsg] +} + +# EVIDENCE-OF: R-33670-36097 A question mark that is not followed by a +# number creates a parameter with a number one greater than the largest +# parameter number already assigned. +# +# EVIDENCE-OF: R-42938-07030 If this means the parameter number is +# greater than SQLITE_MAX_VARIABLE_NUMBER, it is an error. +# +parameter_test e_expr-11.2.1 "SELECT ?" {1 {}} -1 +parameter_test e_expr-11.2.2 "SELECT ?, ?" {1 {} 2 {}} {-1 -2} +parameter_test e_expr-11.2.3 "SELECT ?5, ?" {5 ?5 6 {}} {-5 -6} +parameter_test e_expr-11.2.4 "SELECT ?, ?5" {1 {} 5 ?5} {-1 -5} +parameter_test e_expr-11.2.5 "SELECT ?, ?456, ?" { + 1 {} 456 ?456 457 {} +} {-1 -456 -457} +parameter_test e_expr-11.2.5 "SELECT ?, ?456, ?4, ?" { + 1 {} 456 ?456 4 ?4 457 {} +} {-1 -456 -4 -457} +foreach {tn sql} [list \ + 1 "SELECT ?$mvn, ?" \ + 2 "SELECT ?[expr $mvn-5], ?, ?, ?, ?, ?, ?" \ + 3 "SELECT ?[expr $mvn], ?5, ?6, ?" \ +] { + do_catchsql_test e_expr-11.3.$tn $sql [list 1 {too many SQL variables}] +} + +# EVIDENCE-OF: R-11620-22743 A colon followed by an identifier name +# holds a spot for a named parameter with the name :AAAA. +# +# Identifiers in SQLite consist of alphanumeric, '_' and '$' characters, +# and any UTF characters with codepoints larger than 127 (non-ASCII +# characters). +# +parameter_test e_expr-11.2.1 {SELECT :AAAA} {1 :AAAA} -1 +parameter_test e_expr-11.2.2 {SELECT :123} {1 :123} -1 +parameter_test e_expr-11.2.3 {SELECT :__} {1 :__} -1 +parameter_test e_expr-11.2.4 {SELECT :_$_} {1 :_$_} -1 +parameter_test e_expr-11.2.5 " + SELECT :\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25 +" "1 :\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25" -1 +parameter_test e_expr-11.2.6 "SELECT :\u0080" "1 :\u0080" -1 + +# EVIDENCE-OF: R-49783-61279 An "at" sign works exactly like a colon, +# except that the name of the parameter created is @AAAA. +# +parameter_test e_expr-11.3.1 {SELECT @AAAA} {1 @AAAA} -1 +parameter_test e_expr-11.3.2 {SELECT @123} {1 @123} -1 +parameter_test e_expr-11.3.3 {SELECT @__} {1 @__} -1 +parameter_test e_expr-11.3.4 {SELECT @_$_} {1 @_$_} -1 +parameter_test e_expr-11.3.5 " + SELECT @\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25 +" "1 @\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25" -1 +parameter_test e_expr-11.3.6 "SELECT @\u0080" "1 @\u0080" -1 + +# EVIDENCE-OF: R-62610-51329 A dollar-sign followed by an identifier +# name also holds a spot for a named parameter with the name $AAAA. +# +# EVIDENCE-OF: R-55025-21042 The identifier name in this case can +# include one or more occurrences of "::" and a suffix enclosed in +# "(...)" containing any text at all. +# +# Note: Looks like an identifier cannot consist entirely of "::" +# characters or just a suffix. Also, the other named variable characters +# (: and @) work the same way internally. Why not just document it that way? +# +parameter_test e_expr-11.4.1 {SELECT $AAAA} {1 $AAAA} -1 +parameter_test e_expr-11.4.2 {SELECT $123} {1 $123} -1 +parameter_test e_expr-11.4.3 {SELECT $__} {1 $__} -1 +parameter_test e_expr-11.4.4 {SELECT $_$_} {1 $_$_} -1 +parameter_test e_expr-11.4.5 " + SELECT \$\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25 +" "1 \$\u0e40\u0e2d\u0e28\u0e02\u0e39\u0e40\u0e2d\u0e25" -1 +parameter_test e_expr-11.4.6 "SELECT \$\u0080" "1 \$\u0080" -1 + +parameter_test e_expr-11.5.1 {SELECT $::::a(++--++)} {1 $::::a(++--++)} -1 +parameter_test e_expr-11.5.2 {SELECT $::a()} {1 $::a()} -1 +parameter_test e_expr-11.5.3 {SELECT $::1(::#$)} {1 $::1(::#$)} -1 + +# EVIDENCE-OF: R-11370-04520 Named parameters are also numbered. The +# number assigned is one greater than the largest parameter number +# already assigned. +# +# EVIDENCE-OF: R-42620-22184 If this means the parameter would be +# assigned a number greater than SQLITE_MAX_VARIABLE_NUMBER, it is an +# error. +# +parameter_test e_expr-11.6.1 "SELECT ?, @abc" {1 {} 2 @abc} {-1 -2} +parameter_test e_expr-11.6.2 "SELECT ?123, :a1" {123 ?123 124 :a1} {-123 -124} +parameter_test e_expr-11.6.3 {SELECT $a, ?8, ?, $b, ?2, $c} { + 1 $a 8 ?8 9 {} 10 $b 2 ?2 11 $c +} {-1 -8 -9 -10 -2 -11} +foreach {tn sql} [list \ + 1 "SELECT ?$mvn, \$::a" \ + 2 "SELECT ?$mvn, ?4, @a1" \ + 3 "SELECT ?[expr $mvn-2], :bag, @123, \$x" \ +] { + do_catchsql_test e_expr-11.7.$tn $sql [list 1 {too many SQL variables}] +} + +# EVIDENCE-OF: R-14068-49671 Parameters that are not assigned values +# using sqlite3_bind() are treated as NULL. +# +do_test e_expr-11.7.1 { + set stmt [sqlite3_prepare_v2 db { SELECT ?, :a, @b, $d } -1] + sqlite3_step $stmt + + list [sqlite3_column_type $stmt 0] \ + [sqlite3_column_type $stmt 1] \ + [sqlite3_column_type $stmt 2] \ + [sqlite3_column_type $stmt 3] +} {NULL NULL NULL NULL} +do_test e_expr-11.7.1 { sqlite3_finalize $stmt } SQLITE_OK + +#------------------------------------------------------------------------- +# "Test" the syntax diagrams in lang_expr.html. +# +# EVIDENCE-OF: R-04177-20688 -- syntax diagram signed-number +# +do_execsql_test e_expr-12.1.1 { SELECT 0, +0, -0 } {0 0 0} +do_execsql_test e_expr-12.1.2 { SELECT 1, +1, -1 } {1 1 -1} +do_execsql_test e_expr-12.1.3 { SELECT 2, +2, -2 } {2 2 -2} +do_execsql_test e_expr-12.1.4 { + SELECT 1.4, +1.4, -1.4 +} {1.4 1.4 -1.4} +do_execsql_test e_expr-12.1.5 { + SELECT 1.5e+5, +1.5e+5, -1.5e+5 +} {150000.0 150000.0 -150000.0} +do_execsql_test e_expr-12.1.6 { + SELECT 0.0001, +0.0001, -0.0001 +} {0.0001 0.0001 -0.0001} + +# EVIDENCE-OF: R-30740-26723 -- syntax diagram literal-value +# +set sqlite_current_time 1 +do_execsql_test e_expr-12.2.1 {SELECT 123} {123} +do_execsql_test e_expr-12.2.2 {SELECT 123.4e05} {12340000.0} +do_execsql_test e_expr-12.2.3 {SELECT 'abcde'} {abcde} +do_execsql_test e_expr-12.2.4 {SELECT X'414243'} {ABC} +do_execsql_test e_expr-12.2.5 {SELECT NULL} {{}} +do_execsql_test e_expr-12.2.6 {SELECT CURRENT_TIME} {00:00:01} +do_execsql_test e_expr-12.2.7 {SELECT CURRENT_DATE} {1970-01-01} +do_execsql_test e_expr-12.2.8 {SELECT CURRENT_TIMESTAMP} {{1970-01-01 00:00:01}} +set sqlite_current_time 0 + +# EVIDENCE-OF: R-57598-59332 -- syntax diagram expr +# +file delete -force test.db2 +execsql { + ATTACH 'test.db2' AS dbname; + CREATE TABLE dbname.tblname(cname); +} + +proc glob {args} {return 1} +db function glob glob +db function match glob +db function regexp glob + +foreach {tn expr} { + 1 123 + 2 123.4e05 + 3 'abcde' + 4 X'414243' + 5 NULL + 6 CURRENT_TIME + 7 CURRENT_DATE + 8 CURRENT_TIMESTAMP + + 9 ? + 10 ?123 + 11 @hello + 12 :world + 13 $tcl + 14 $tcl(array) + + 15 cname + 16 tblname.cname + 17 dbname.tblname.cname + + 18 "+ EXPR" + 19 "- EXPR" + 20 "NOT EXPR" + 21 "~ EXPR" + + 22 "EXPR1 || EXPR2" + 23 "EXPR1 * EXPR2" + 24 "EXPR1 / EXPR2" + 25 "EXPR1 % EXPR2" + 26 "EXPR1 + EXPR2" + 27 "EXPR1 - EXPR2" + 28 "EXPR1 << EXPR2" + 29 "EXPR1 >> EXPR2" + 30 "EXPR1 & EXPR2" + 31 "EXPR1 | EXPR2" + 32 "EXPR1 < EXPR2" + 33 "EXPR1 <= EXPR2" + 34 "EXPR1 > EXPR2" + 35 "EXPR1 >= EXPR2" + 36 "EXPR1 = EXPR2" + 37 "EXPR1 == EXPR2" + 38 "EXPR1 != EXPR2" + 39 "EXPR1 <> EXPR2" + 40 "EXPR1 IS EXPR2" + 41 "EXPR1 IS NOT EXPR2" + 42 "EXPR1 AND EXPR2" + 43 "EXPR1 OR EXPR2" + + 44 "count(*)" + 45 "count(DISTINCT EXPR)" + 46 "substr(EXPR, 10, 20)" + 47 "changes()" + + 48 "( EXPR )" + + 49 "CAST ( EXPR AS integer )" + 50 "CAST ( EXPR AS 'abcd' )" + 51 "CAST ( EXPR AS 'ab$ $cd' )" + + 52 "EXPR COLLATE nocase" + 53 "EXPR COLLATE binary" + + 54 "EXPR1 LIKE EXPR2" + 55 "EXPR1 LIKE EXPR2 ESCAPE EXPR" + 56 "EXPR1 GLOB EXPR2" + 57 "EXPR1 GLOB EXPR2 ESCAPE EXPR" + 58 "EXPR1 REGEXP EXPR2" + 59 "EXPR1 REGEXP EXPR2 ESCAPE EXPR" + 60 "EXPR1 MATCH EXPR2" + 61 "EXPR1 MATCH EXPR2 ESCAPE EXPR" + 62 "EXPR1 NOT LIKE EXPR2" + 63 "EXPR1 NOT LIKE EXPR2 ESCAPE EXPR" + 64 "EXPR1 NOT GLOB EXPR2" + 65 "EXPR1 NOT GLOB EXPR2 ESCAPE EXPR" + 66 "EXPR1 NOT REGEXP EXPR2" + 67 "EXPR1 NOT REGEXP EXPR2 ESCAPE EXPR" + 68 "EXPR1 NOT MATCH EXPR2" + 69 "EXPR1 NOT MATCH EXPR2 ESCAPE EXPR" + + 70 "EXPR ISNULL" + 71 "EXPR NOTNULL" + 72 "EXPR NOT NULL" + + 73 "EXPR1 IS EXPR2" + 74 "EXPR1 IS NOT EXPR2" + + 75 "EXPR NOT BETWEEN EXPR1 AND EXPR2" + 76 "EXPR BETWEEN EXPR1 AND EXPR2" + + 77 "EXPR NOT IN (SELECT cname FROM tblname)" + 78 "EXPR NOT IN (1)" + 79 "EXPR NOT IN (1, 2, 3)" + 80 "EXPR NOT IN tblname" + 81 "EXPR NOT IN dbname.tblname" + 82 "EXPR IN (SELECT cname FROM tblname)" + 83 "EXPR IN (1)" + 84 "EXPR IN (1, 2, 3)" + 85 "EXPR IN tblname" + 86 "EXPR IN dbname.tblname" + + 87 "EXISTS (SELECT cname FROM tblname)" + 88 "NOT EXISTS (SELECT cname FROM tblname)" + + 89 "CASE EXPR WHEN EXPR1 THEN EXPR2 ELSE EXPR END" + 90 "CASE EXPR WHEN EXPR1 THEN EXPR2 END" + 91 "CASE EXPR WHEN EXPR1 THEN EXPR2 WHEN EXPR THEN EXPR1 ELSE EXPR2 END" + 92 "CASE EXPR WHEN EXPR1 THEN EXPR2 WHEN EXPR THEN EXPR1 END" + 93 "CASE WHEN EXPR1 THEN EXPR2 ELSE EXPR END" + 94 "CASE WHEN EXPR1 THEN EXPR2 END" + 95 "CASE WHEN EXPR1 THEN EXPR2 WHEN EXPR THEN EXPR1 ELSE EXPR2 END" + 96 "CASE WHEN EXPR1 THEN EXPR2 WHEN EXPR THEN EXPR1 END" +} { + + # If the expression string being parsed contains "EXPR2", then replace + # string "EXPR1" and "EXPR2" with arbitrary SQL expressions. If it + # contains "EXPR", then replace EXPR with an arbitrary SQL expression. + # + set elist [list $expr] + if {[string match *EXPR2* $expr]} { + set elist [list] + foreach {e1 e2} { cname "34+22" } { + lappend elist [string map [list EXPR1 $e1 EXPR2 $e2] $expr] + } + } + if {[string match *EXPR* $expr]} { + set elist2 [list] + foreach el $elist { + foreach e { cname "34+22" } { + lappend elist2 [string map [list EXPR $e] $el] + } + } + set elist $elist2 + } + + set x 0 + foreach e $elist { + incr x + do_test e_expr-12.3.$tn.$x { + set rc [catch { execsql "SELECT $e FROM tblname" } msg] + } {0} + } +} + +# EVIDENCE-OF: R-49462-56079 -- syntax diagram raise-function +# +foreach {tn raiseexpr} { + 1 "RAISE(IGNORE)" + 2 "RAISE(ROLLBACK, 'error message')" + 3 "RAISE(ABORT, 'error message')" + 4 "RAISE(FAIL, 'error message')" +} { + do_execsql_test e_expr-12.4.$tn " + CREATE TRIGGER dbname.tr$tn BEFORE DELETE ON tblname BEGIN + SELECT $raiseexpr ; + END; + " {} +} + +#------------------------------------------------------------------------- +# Test the statements related to the BETWEEN operator. +# +# EVIDENCE-OF: R-40079-54503 The BETWEEN operator is logically +# equivalent to a pair of comparisons. "x BETWEEN y AND z" is equivalent +# to "x>=y AND x<=z" except that with BETWEEN, the x expression is +# only evaluated once. +# +db func x x +proc x {} { incr ::xcount ; return [expr $::x] } +foreach {tn x expr res nEval} { + 1 10 "x() >= 5 AND x() <= 15" 1 2 + 2 10 "x() BETWEEN 5 AND 15" 1 1 + + 3 5 "x() >= 5 AND x() <= 5" 1 2 + 4 5 "x() BETWEEN 5 AND 5" 1 1 +} { + do_test e_expr-13.1.$tn { + set ::xcount 0 + set a [execsql "SELECT $expr"] + list $::xcount $a + } [list $nEval $res] +} + +# EVIDENCE-OF: R-05155-34454 The precedence of the BETWEEN operator is +# the same as the precedence as operators == and != and LIKE and groups +# left to right. +# +# Therefore, BETWEEN groups more tightly than operator "AND", but less +# so than "<". +# +do_execsql_test e_expr-13.2.1 { SELECT 1 == 10 BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.2 { SELECT (1 == 10) BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.3 { SELECT 1 == (10 BETWEEN 0 AND 2) } 0 +do_execsql_test e_expr-13.2.4 { SELECT 6 BETWEEN 4 AND 8 == 1 } 1 +do_execsql_test e_expr-13.2.5 { SELECT (6 BETWEEN 4 AND 8) == 1 } 1 +do_execsql_test e_expr-13.2.6 { SELECT 6 BETWEEN 4 AND (8 == 1) } 0 + +do_execsql_test e_expr-13.2.7 { SELECT 5 BETWEEN 0 AND 0 != 1 } 1 +do_execsql_test e_expr-13.2.8 { SELECT (5 BETWEEN 0 AND 0) != 1 } 1 +do_execsql_test e_expr-13.2.9 { SELECT 5 BETWEEN 0 AND (0 != 1) } 0 +do_execsql_test e_expr-13.2.10 { SELECT 1 != 0 BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.11 { SELECT (1 != 0) BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.12 { SELECT 1 != (0 BETWEEN 0 AND 2) } 0 + +do_execsql_test e_expr-13.2.13 { SELECT 1 LIKE 10 BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.14 { SELECT (1 LIKE 10) BETWEEN 0 AND 2 } 1 +do_execsql_test e_expr-13.2.15 { SELECT 1 LIKE (10 BETWEEN 0 AND 2) } 0 +do_execsql_test e_expr-13.2.16 { SELECT 6 BETWEEN 4 AND 8 LIKE 1 } 1 +do_execsql_test e_expr-13.2.17 { SELECT (6 BETWEEN 4 AND 8) LIKE 1 } 1 +do_execsql_test e_expr-13.2.18 { SELECT 6 BETWEEN 4 AND (8 LIKE 1) } 0 + +do_execsql_test e_expr-13.2.19 { SELECT 0 AND 0 BETWEEN 0 AND 1 } 0 +do_execsql_test e_expr-13.2.20 { SELECT 0 AND (0 BETWEEN 0 AND 1) } 0 +do_execsql_test e_expr-13.2.21 { SELECT (0 AND 0) BETWEEN 0 AND 1 } 1 +do_execsql_test e_expr-13.2.22 { SELECT 0 BETWEEN -1 AND 1 AND 0 } 0 +do_execsql_test e_expr-13.2.23 { SELECT (0 BETWEEN -1 AND 1) AND 0 } 0 +do_execsql_test e_expr-13.2.24 { SELECT 0 BETWEEN -1 AND (1 AND 0) } 1 + +do_execsql_test e_expr-13.2.25 { SELECT 2 < 3 BETWEEN 0 AND 1 } 1 +do_execsql_test e_expr-13.2.26 { SELECT (2 < 3) BETWEEN 0 AND 1 } 1 +do_execsql_test e_expr-13.2.27 { SELECT 2 < (3 BETWEEN 0 AND 1) } 0 +do_execsql_test e_expr-13.2.28 { SELECT 2 BETWEEN 1 AND 2 < 3 } 0 +do_execsql_test e_expr-13.2.29 { SELECT 2 BETWEEN 1 AND (2 < 3) } 0 +do_execsql_test e_expr-13.2.30 { SELECT (2 BETWEEN 1 AND 2) < 3 } 1 + +#------------------------------------------------------------------------- +# Test the statements related to the LIKE and GLOB operators. +# +# EVIDENCE-OF: R-16584-60189 The LIKE operator does a pattern matching +# comparison. +# +# EVIDENCE-OF: R-11295-04657 The operand to the right of the LIKE +# operator contains the pattern and the left hand operand contains the +# string to match against the pattern. +# +do_execsql_test e_expr-14.1.1 { SELECT 'abc%' LIKE 'abcde' } 0 +do_execsql_test e_expr-14.1.2 { SELECT 'abcde' LIKE 'abc%' } 1 + +# EVIDENCE-OF: R-55406-38524 A percent symbol ("%") in the LIKE pattern +# matches any sequence of zero or more characters in the string. +# +do_execsql_test e_expr-14.2.1 { SELECT 'abde' LIKE 'ab%de' } 1 +do_execsql_test e_expr-14.2.2 { SELECT 'abXde' LIKE 'ab%de' } 1 +do_execsql_test e_expr-14.2.3 { SELECT 'abABCde' LIKE 'ab%de' } 1 + +# EVIDENCE-OF: R-30433-25443 An underscore ("_") in the LIKE pattern +# matches any single character in the string. +# +do_execsql_test e_expr-14.3.1 { SELECT 'abde' LIKE 'ab_de' } 0 +do_execsql_test e_expr-14.3.2 { SELECT 'abXde' LIKE 'ab_de' } 1 +do_execsql_test e_expr-14.3.3 { SELECT 'abABCde' LIKE 'ab_de' } 0 + +# EVIDENCE-OF: R-59007-20454 Any other character matches itself or its +# lower/upper case equivalent (i.e. case-insensitive matching). +# +do_execsql_test e_expr-14.4.1 { SELECT 'abc' LIKE 'aBc' } 1 +do_execsql_test e_expr-14.4.2 { SELECT 'aBc' LIKE 'aBc' } 1 +do_execsql_test e_expr-14.4.3 { SELECT 'ac' LIKE 'aBc' } 0 + +# EVIDENCE-OF: R-23648-58527 SQLite only understands upper/lower case +# for ASCII characters by default. +# +# EVIDENCE-OF: R-04532-11527 The LIKE operator is case sensitive by +# default for unicode characters that are beyond the ASCII range. +# +# EVIDENCE-OF: R-44381-11669 the expression +# 'a' LIKE 'A' is TRUE but +# 'æ' LIKE 'Æ' is FALSE. +# +do_execsql_test e_expr-14.5.1 { SELECT 'A' LIKE 'a' } 1 +do_execsql_test e_expr-14.5.2 "SELECT '\u00c6' LIKE '\u00e6'" 0 + +# EVIDENCE-OF: R-56683-13731 If the optional ESCAPE clause is present, +# then the expression following the ESCAPE keyword must evaluate to a +# string consisting of a single character. +# +do_catchsql_test e_expr-14.6.1 { + SELECT 'A' LIKE 'a' ESCAPE '12' +} {1 {ESCAPE expression must be a single character}} +do_catchsql_test e_expr-14.6.2 { + SELECT 'A' LIKE 'a' ESCAPE '' +} {1 {ESCAPE expression must be a single character}} +do_catchsql_test e_expr-14.6.3 { SELECT 'A' LIKE 'a' ESCAPE 'x' } {0 1} +do_catchsql_test e_expr-14.6.4 "SELECT 'A' LIKE 'a' ESCAPE '\u00e6'" {0 1} + +# EVIDENCE-OF: R-02045-23762 This character may be used in the LIKE +# pattern to include literal percent or underscore characters. +# +# EVIDENCE-OF: R-13345-31830 The escape character followed by a percent +# symbol (%), underscore (_), or a second instance of the escape +# character itself matches a literal percent symbol, underscore, or a +# single escape character, respectively. +# +do_execsql_test e_expr-14.7.1 { SELECT 'abc%' LIKE 'abcX%' ESCAPE 'X' } 1 +do_execsql_test e_expr-14.7.2 { SELECT 'abc5' LIKE 'abcX%' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.3 { SELECT 'abc' LIKE 'abcX%' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.4 { SELECT 'abcX%' LIKE 'abcX%' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.5 { SELECT 'abc%%' LIKE 'abcX%' ESCAPE 'X' } 0 + +do_execsql_test e_expr-14.7.6 { SELECT 'abc_' LIKE 'abcX_' ESCAPE 'X' } 1 +do_execsql_test e_expr-14.7.7 { SELECT 'abc5' LIKE 'abcX_' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.8 { SELECT 'abc' LIKE 'abcX_' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.9 { SELECT 'abcX_' LIKE 'abcX_' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.10 { SELECT 'abc__' LIKE 'abcX_' ESCAPE 'X' } 0 + +do_execsql_test e_expr-14.7.11 { SELECT 'abcX' LIKE 'abcXX' ESCAPE 'X' } 1 +do_execsql_test e_expr-14.7.12 { SELECT 'abc5' LIKE 'abcXX' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.13 { SELECT 'abc' LIKE 'abcXX' ESCAPE 'X' } 0 +do_execsql_test e_expr-14.7.14 { SELECT 'abcXX' LIKE 'abcXX' ESCAPE 'X' } 0 + +# EVIDENCE-OF: R-51359-17496 The infix LIKE operator is implemented by +# calling the application-defined SQL functions like(Y,X) or like(Y,X,Z). +# +proc likefunc {args} { + eval lappend ::likeargs $args + return 1 +} +db func like likefunc +set ::likeargs [list] +do_execsql_test e_expr-15.1.1 { SELECT 'abc' LIKE 'def' } 1 +do_test e_expr-15.1.2 { set likeargs } {def abc} +set ::likeargs [list] +do_execsql_test e_expr-15.1.3 { SELECT 'abc' LIKE 'def' ESCAPE 'X' } 1 +do_test e_expr-15.1.4 { set likeargs } {def abc X} +db close +sqlite3 db test.db + +# EVIDENCE-OF: R-22868-25880 The LIKE operator can be made case +# sensitive using the case_sensitive_like pragma. +# +do_execsql_test e_expr-16.1.1 { SELECT 'abcxyz' LIKE 'ABC%' } 1 +do_execsql_test e_expr-16.1.2 { PRAGMA case_sensitive_like = 1 } {} +do_execsql_test e_expr-16.1.3 { SELECT 'abcxyz' LIKE 'ABC%' } 0 +do_execsql_test e_expr-16.1.4 { SELECT 'ABCxyz' LIKE 'ABC%' } 1 +do_execsql_test e_expr-16.1.5 { PRAGMA case_sensitive_like = 0 } {} +do_execsql_test e_expr-16.1.6 { SELECT 'abcxyz' LIKE 'ABC%' } 1 +do_execsql_test e_expr-16.1.7 { SELECT 'ABCxyz' LIKE 'ABC%' } 1 + +# EVIDENCE-OF: R-52087-12043 The GLOB operator is similar to LIKE but +# uses the Unix file globbing syntax for its wildcards. +# +# EVIDENCE-OF: R-09813-17279 Also, GLOB is case sensitive, unlike LIKE. +# +do_execsql_test e_expr-17.1.1 { SELECT 'abcxyz' GLOB 'abc%' } 0 +do_execsql_test e_expr-17.1.2 { SELECT 'abcxyz' GLOB 'abc*' } 1 +do_execsql_test e_expr-17.1.3 { SELECT 'abcxyz' GLOB 'abc___' } 0 +do_execsql_test e_expr-17.1.4 { SELECT 'abcxyz' GLOB 'abc???' } 1 + +do_execsql_test e_expr-17.1.5 { SELECT 'abcxyz' GLOB 'abc*' } 1 +do_execsql_test e_expr-17.1.6 { SELECT 'ABCxyz' GLOB 'abc*' } 0 +do_execsql_test e_expr-17.1.7 { SELECT 'abcxyz' GLOB 'ABC*' } 0 + +# EVIDENCE-OF: R-39616-20555 Both GLOB and LIKE may be preceded by the +# NOT keyword to invert the sense of the test. +# +do_execsql_test e_expr-17.2.1 { SELECT 'abcxyz' NOT GLOB 'ABC*' } 1 +do_execsql_test e_expr-17.2.2 { SELECT 'abcxyz' NOT GLOB 'abc*' } 0 +do_execsql_test e_expr-17.2.3 { SELECT 'abcxyz' NOT LIKE 'ABC%' } 0 +do_execsql_test e_expr-17.2.4 { SELECT 'abcxyz' NOT LIKE 'abc%' } 0 +do_execsql_test e_expr-17.2.5 { SELECT 'abdxyz' NOT LIKE 'abc%' } 1 + +db nullvalue null +do_execsql_test e_expr-17.2.6 { SELECT 'abcxyz' NOT GLOB NULL } null +do_execsql_test e_expr-17.2.7 { SELECT 'abcxyz' NOT LIKE NULL } null +do_execsql_test e_expr-17.2.8 { SELECT NULL NOT GLOB 'abc*' } null +do_execsql_test e_expr-17.2.9 { SELECT NULL NOT LIKE 'ABC%' } null +db nullvalue {} + +# EVIDENCE-OF: R-39414-35489 The infix GLOB operator is implemented by +# calling the function glob(Y,X) and can be modified by overriding that +# function. +proc globfunc {args} { + eval lappend ::globargs $args + return 1 +} +db func glob -argcount 2 globfunc +set ::globargs [list] +do_execsql_test e_expr-17.3.1 { SELECT 'abc' GLOB 'def' } 1 +do_test e_expr-17.3.2 { set globargs } {def abc} +set ::globargs [list] +do_execsql_test e_expr-17.3.3 { SELECT 'X' NOT GLOB 'Y' } 0 +do_test e_expr-17.3.4 { set globargs } {Y X} +sqlite3 db test.db + +# EVIDENCE-OF: R-41650-20872 No regexp() user function is defined by +# default and so use of the REGEXP operator will normally result in an +# error message. +# +do_catchsql_test e_expr-18.1.1 { + SELECT regexp('abc', 'def') +} {1 {no such function: regexp}} +do_catchsql_test e_expr-18.1.2 { + SELECT 'abc' REGEXP 'def' +} {1 {no such function: REGEXP}} + +# EVIDENCE-OF: R-33693-50180 The REGEXP operator is a special syntax for +# the regexp() user function. +# +# EVIDENCE-OF: R-57289-13578 If a application-defined SQL function named +# "regexp" is added at run-time, that function will be called in order +# to implement the REGEXP operator. +# +proc regexpfunc {args} { + eval lappend ::regexpargs $args + return 1 +} +db func regexp -argcount 2 regexpfunc +set ::regexpargs [list] +do_execsql_test e_expr-18.2.1 { SELECT 'abc' REGEXP 'def' } 1 +do_test e_expr-18.2.2 { set regexpargs } {def abc} +set ::regexpargs [list] +do_execsql_test e_expr-18.2.3 { SELECT 'X' NOT REGEXP 'Y' } 0 +do_test e_expr-18.2.4 { set regexpargs } {Y X} +sqlite3 db test.db + +# EVIDENCE-OF: R-42037-37826 The default match() function implementation +# raises an exception and is not really useful for anything. +# +do_catchsql_test e_expr-19.1.1 { + SELECT 'abc' MATCH 'def' +} {1 {unable to use function MATCH in the requested context}} +do_catchsql_test e_expr-19.1.2 { + SELECT match('abc', 'def') +} {1 {unable to use function MATCH in the requested context}} + +# EVIDENCE-OF: R-37916-47407 The MATCH operator is a special syntax for +# the match() application-defined function. +# +# EVIDENCE-OF: R-06021-09373 But extensions can override the match() +# function with more helpful logic. +# +proc matchfunc {args} { + eval lappend ::matchargs $args + return 1 +} +db func match -argcount 2 matchfunc +set ::matchargs [list] +do_execsql_test e_expr-19.2.1 { SELECT 'abc' MATCH 'def' } 1 +do_test e_expr-19.2.2 { set matchargs } {def abc} +set ::matchargs [list] +do_execsql_test e_expr-19.2.3 { SELECT 'X' NOT MATCH 'Y' } 0 +do_test e_expr-19.2.4 { set matchargs } {Y X} +sqlite3 db test.db + + finish_test diff --git a/test/exclusive.test b/test/exclusive.test index 4f20a6b..a4ac9ea 100644 --- a/test/exclusive.test +++ b/test/exclusive.test @@ -253,6 +253,11 @@ db2 close # from being inspected externally. # if {$tcl_platform(platform) != "windows"} { + + # Return a list of two booleans (either 0 or 1). The first is true + # if the named file exists. The second is true only if the file + # exists and the first 28 bytes contain at least one non-zero byte. + # proc filestate {fname} { set exists 0 set content 0 @@ -263,6 +268,7 @@ if {$tcl_platform(platform) != "windows"} { } list $exists $content } + do_test exclusive-3.0 { filestate test.db-journal } {0 0} diff --git a/test/fallocate.test b/test/fallocate.test new file mode 100644 index 0000000..05aa2a8 --- /dev/null +++ b/test/fallocate.test @@ -0,0 +1,146 @@ +# 2010 July 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +file_control_chunksize_test db main [expr 1024*1024] + +do_test fallocate-1.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b); + } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.2 { + execsql { INSERT INTO t1 VALUES(1, zeroblob(1024*900)) } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.3 { + execsql { INSERT INTO t1 VALUES(2, zeroblob(1024*900)) } + file size test.db +} [expr 2*1024*1024] + +do_test fallocate-1.4 { + execsql { DELETE FROM t1 WHERE a = 1 } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.5 { + execsql { DELETE FROM t1 WHERE a = 2 } + file size test.db +} [expr 1*1024*1024] + +do_test fallocate-1.6 { + execsql { PRAGMA freelist_count } +} {0} + +# Start a write-transaction and read the "database file size" field from +# the journal file. This field should be set to the number of pages in +# the database file based on the size of the file on disk, not the actual +# logical size of the database within the file. +# +# We need to check this to verify that if in the unlikely event a rollback +# causes a database file to grow, the database grows to its previous size +# on disk, not to the minimum size required to hold the database image. +# +do_test fallocate-1.7 { + execsql { BEGIN; INSERT INTO t1 VALUES(1, 2); } + if {[permutation] != "inmemory_journal"} { + hexio_get_int [hexio_read test.db-journal 16 4] + } else { + set {} 1024 + } +} {1024} +do_test fallocate-1.8 { execsql { COMMIT } } {} + + +#------------------------------------------------------------------------- +# The following tests - fallocate-2.* - test that things work in WAL +# mode as well. +# +set skipwaltests [expr { + [permutation]=="journaltest" || [permutation]=="inmemory_journal" +}] +ifcapable !wal { set skipwaltests 1 } + +if {!$skipwaltests} { + db close + file delete -force test.db + sqlite3 db test.db + file_control_chunksize_test db main [expr 32*1024] + + do_test fallocate-2.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + } + file size test.db + } [expr 32*1024] + + do_test fallocate-2.2 { + execsql { INSERT INTO t1 VALUES(1, zeroblob(35*1024)) } + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.3 { + execsql { DELETE FROM t1 } + execsql { VACUUM } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.4 { + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 32*1024] + + do_test fallocate-2.5 { + execsql { + INSERT INTO t1 VALUES(2, randomblob(35*1024)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(3, randomblob(128)); + DELETE FROM t1 WHERE a = 2; + VACUUM; + } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.6 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT count(a) FROM t1 } db2 + execsql { + INSERT INTO t1 VALUES(4, randomblob(128)); + PRAGMA wal_checkpoint; + } + file size test.db + } [expr 64*1024] + + do_test fallocate-2.7 { + execsql { SELECT count(b) FROM t1 } db2 + } {1} + + do_test fallocate-2.8 { + execsql { COMMIT } db2 + execsql { PRAGMA wal_checkpoint } + file size test.db + } [expr 32*1024] +} + + +finish_test + diff --git a/test/fkey2.test b/test/fkey2.test index c530e9f..07c47e4 100644 --- a/test/fkey2.test +++ b/test/fkey2.test @@ -1936,5 +1936,50 @@ do_test fkey2-dd08e5.1.6 { } } {1 {foreign key constraint failed}} +#------------------------------------------------------------------------- +# Verify that ticket ce7c133ea6cc9ccdc1a60d80441f80b6180f5eba +# fixed. +# +do_test fkey2-ce7c13.1.1 { + execsql { + CREATE TABLE tce71(a INTEGER PRIMARY KEY, b); + CREATE UNIQUE INDEX ice71 ON tce71(a,b); + INSERT INTO tce71 VALUES(100,200); + CREATE TABLE tce72(w, x, y, FOREIGN KEY(x,y) REFERENCES tce71(a,b)); + INSERT INTO tce72 VALUES(300,100,200); + UPDATE tce71 set b = 200 where a = 100; + SELECT * FROM tce71, tce72; + } +} {100 200 300 100 200} +do_test fkey2-ce7c13.1.2 { + catchsql { + UPDATE tce71 set b = 201 where a = 100; + } +} {1 {foreign key constraint failed}} +do_test fkey2-ce7c13.1.3 { + catchsql { + UPDATE tce71 set a = 101 where a = 100; + } +} {1 {foreign key constraint failed}} +do_test fkey2-ce7c13.1.4 { + execsql { + CREATE TABLE tce73(a INTEGER PRIMARY KEY, b, UNIQUE(a,b)); + INSERT INTO tce73 VALUES(100,200); + CREATE TABLE tce74(w, x, y, FOREIGN KEY(x,y) REFERENCES tce73(a,b)); + INSERT INTO tce74 VALUES(300,100,200); + UPDATE tce73 set b = 200 where a = 100; + SELECT * FROM tce73, tce74; + } +} {100 200 300 100 200} +do_test fkey2-ce7c13.1.5 { + catchsql { + UPDATE tce73 set b = 201 where a = 100; + } +} {1 {foreign key constraint failed}} +do_test fkey2-ce7c13.1.6 { + catchsql { + UPDATE tce73 set a = 101 where a = 100; + } +} {1 {foreign key constraint failed}} finish_test diff --git a/test/icu.test b/test/icu.test index 46596e4..73cb9b9 100644 --- a/test/icu.test +++ b/test/icu.test @@ -114,4 +114,23 @@ do_test icu-4.3 { } } {apricot cherry chokecherry yamot peach plum} +#------------------------------------------------------------------------- +# Test that it is not possible to call the ICU regex() function with +# anything other than exactly two arguments. See also: +# +# http://src.chromium.org/viewvc/chrome/trunk/src/third_party/sqlite/icu-regexp.patch?revision=34807&view=markup +# +do_catchsql_test icu-5.1 { SELECT regexp('a[abc]c.*', 'abc') } {0 1} +do_catchsql_test icu-5.2 { + SELECT regexp('a[abc]c.*') +} {1 {wrong number of arguments to function regexp()}} +do_catchsql_test icu-5.3 { + SELECT regexp('a[abc]c.*', 'abc', 'c') +} {1 {wrong number of arguments to function regexp()}} +do_catchsql_test icu-5.4 { + SELECT 'abc' REGEXP 'a[abc]c.*' +} {0 1} +do_catchsql_test icu-5.4 { SELECT 'abc' REGEXP } {1 {near " ": syntax error}} +do_catchsql_test icu-5.5 { SELECT 'abc' REGEXP, 1 } {1 {near ",": syntax error}} + finish_test diff --git a/test/indexedby.test b/test/indexedby.test index 62a2a5f..70ab9f9 100644 --- a/test/indexedby.test +++ b/test/indexedby.test @@ -123,6 +123,16 @@ do_test indexedby-4.1 { do_test indexedby-4.2 { EQP { SELECT * FROM t1 INDEXED BY i1, t2 WHERE a = c } } {0 1 {TABLE t2} 1 0 {TABLE t1 WITH INDEX i1}} +do_test indexedby-4.3 { + catchsql { + SELECT * FROM t1 INDEXED BY i1, t2 INDEXED BY i3 WHERE a=c + } +} {1 {cannot use index: i1}} +do_test indexedby-4.4 { + catchsql { + SELECT * FROM t2 INDEXED BY i3, t1 INDEXED BY i1 WHERE a=c + } +} {1 {cannot use index: i3}} # Test embedding an INDEXED BY in a CREATE VIEW statement. This block # also tests that nothing bad happens if an index refered to by diff --git a/test/ioerr5.test b/test/ioerr5.test index 6e90145..a430f53 100644 --- a/test/ioerr5.test +++ b/test/ioerr5.test @@ -91,6 +91,15 @@ foreach locking_mode {normal exclusive} { INSERT INTO a VALUES(1, 'ABCDEFGHIJKLMNOP'); } } {} + + # Open a read-only cursor on table "a". If the COMMIT below is + # interrupted by a persistent IO error, the pager will transition to + # PAGER_ERROR state. If there are no other read-only cursors open, + # from there the pager immediately discards all cached data and + # switches to PAGER_OPEN state. This read-only cursor stops that + # from happening, leaving the pager stuck in PAGER_ERROR state. + # + set channel [db incrblob -readonly a Name [db last_insert_rowid]] # Now try to commit the transaction. Cause an IO error to occur # within this operation, which moves the pager into the error state. @@ -118,25 +127,11 @@ foreach locking_mode {normal exclusive} { # with the dirty page. # do_test ioerr5-1.$locking_mode-$iFail.3 { - set bt [btree_from_db db] sqlite3_soft_heap_limit 1024 compilesql16 "SELECT 10" - array set stats [btree_pager_stats $bt] + } {SQLITE_OK} - # If the pager made it all the way to PAGER_SYNCED state, then - # both in-memory pages are clean. Following the calls to - # release_memory() that were made as part of the [compilesql16] - # above, there will be zero pages left in the cache. - # - # If the pager did not make it as far as PAGER_SYNCED, the two - # in memory pages are still dirty. So there will be 2 pages left - # in the cache following the release_memory() calls. - # - if {$stats(state)==5} { - set nPage 0 - } - expr {$stats(page)==$nPage} - } {1} + close $channel # Ensure that nothing was written to the database while reclaiming # memory from the pager in error state. @@ -148,7 +143,7 @@ foreach locking_mode {normal exclusive} { close $fd expr {$zDatabase eq $zDatabase2} } {1} - + if {$rc eq [list 0 {}]} { do_test ioerr5.1-$locking_mode-$iFail.3 { execsql { SELECT count(*) FROM a } diff --git a/test/jrnlmode.test b/test/jrnlmode.test index f1e9c78..eab74d6 100644 --- a/test/jrnlmode.test +++ b/test/jrnlmode.test @@ -524,4 +524,35 @@ ifcapable pragma { do_test jrnlmode-7.2 { file size test.db } {1024} } +do_execsql_test jrnlmode-8.1 { PRAGMA locking_mode=EXCLUSIVE } {exclusive} +do_execsql_test jrnlmode-8.2 { CREATE TABLE t1(x) } {} +do_execsql_test jrnlmode-8.3 { INSERT INTO t1 VALUES(123) } {} +do_execsql_test jrnlmode-8.4 { SELECT * FROM t1 } {123} +do_execsql_test jrnlmode-8.5 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.6 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.7 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.8 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.9 { CREATE TABLE t2(y) } {} +do_execsql_test jrnlmode-8.10 { INSERT INTO t2 VALUES(456) } {} +do_execsql_test jrnlmode-8.11 { SELECT * FROM t1, t2 } {123 456} +do_execsql_test jrnlmode-8.12 { PRAGMA locking_mode=NORMAL } {normal} +do_execsql_test jrnlmode-8.13 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.14 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.15 { PRAGMA journal_mode=PERSIST } {persist} +do_execsql_test jrnlmode-8.16 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.17 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.18 { PRAGMA locking_mode=EXCLUSIVE } {exclusive} +do_execsql_test jrnlmode-8.19 { CREATE TABLE t3(z) } {} +do_execsql_test jrnlmode-8.20 { BEGIN IMMEDIATE } {} +do_execsql_test jrnlmode-8.21 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.22 { COMMIT } {} +do_execsql_test jrnlmode-8.23 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.24 { PRAGMA journal_mode=TRUNCATE } {truncate} +do_execsql_test jrnlmode-8.25 { PRAGMA locking_mode=NORMAL } {normal} +do_execsql_test jrnlmode-8.26 { CREATE TABLE t4(w) } {} +do_execsql_test jrnlmode-8.27 { BEGIN IMMEDIATE } {} +do_execsql_test jrnlmode-8.28 { PRAGMA journal_mode=DELETE } {delete} +do_execsql_test jrnlmode-8.29 { COMMIT } {} +do_execsql_test jrnlmode-8.30 { PRAGMA journal_mode=DELETE } {delete} + finish_test diff --git a/test/like.test b/test/like.test index a2de686..229f386 100644 --- a/test/like.test +++ b/test/like.test @@ -194,6 +194,31 @@ do_test like-3.4 { set sqlite_like_count } 0 +# The LIKE optimization still works when the RHS is a string with no +# wildcard. Ticket [e090183531fc2747] +# +do_test like-3.4.2 { + queryplan { + SELECT x FROM t1 WHERE x LIKE 'a' ORDER BY 1; + } +} {a nosort {} i1} +do_test like-3.4.3 { + queryplan { + SELECT x FROM t1 WHERE x LIKE 'ab' ORDER BY 1; + } +} {ab nosort {} i1} +do_test like-3.4.4 { + queryplan { + SELECT x FROM t1 WHERE x LIKE 'abcd' ORDER BY 1; + } +} {abcd nosort {} i1} +do_test like-3.4.5 { + queryplan { + SELECT x FROM t1 WHERE x LIKE 'abcde' ORDER BY 1; + } +} {nosort {} i1} + + # Partial optimization when the pattern does not end in '%' # do_test like-3.5 { @@ -309,6 +334,26 @@ do_test like-3.24 { set sqlite_like_count } 6 +# GLOB optimization when there is no wildcard. Ticket [e090183531fc2747] +# +do_test like-3.25 { + queryplan { + SELECT x FROM t1 WHERE x GLOB 'a' ORDER BY 1; + } +} {a nosort {} i1} +do_test like-3.26 { + queryplan { + SELECT x FROM t1 WHERE x GLOB 'abcd' ORDER BY 1; + } +} {abcd nosort {} i1} +do_test like-3.27 { + queryplan { + SELECT x FROM t1 WHERE x GLOB 'abcde' ORDER BY 1; + } +} {nosort {} i1} + + + # No optimization if the LHS of the LIKE is not a column name or # if the RHS is not a string. # @@ -733,5 +778,94 @@ do_test like-10.15 { } } {12 123 scan 5 like 6} +# LIKE and GLOB where the default collating sequence is not appropriate +# but an index with the appropriate collating sequence exists. +# +do_test like-11.0 { + execsql { + CREATE TABLE t11( + a INTEGER PRIMARY KEY, + b TEXT COLLATE nocase, + c TEXT COLLATE binary + ); + INSERT INTO t11 VALUES(1, 'a','a'); + INSERT INTO t11 VALUES(2, 'ab','ab'); + INSERT INTO t11 VALUES(3, 'abc','abc'); + INSERT INTO t11 VALUES(4, 'abcd','abcd'); + INSERT INTO t11 VALUES(5, 'A','A'); + INSERT INTO t11 VALUES(6, 'AB','AB'); + INSERT INTO t11 VALUES(7, 'ABC','ABC'); + INSERT INTO t11 VALUES(8, 'ABCD','ABCD'); + INSERT INTO t11 VALUES(9, 'x','x'); + INSERT INTO t11 VALUES(10, 'yz','yz'); + INSERT INTO t11 VALUES(11, 'X','X'); + INSERT INTO t11 VALUES(12, 'YZ','YZ'); + SELECT count(*) FROM t11; + } +} {12} +do_test like-11.1 { + queryplan { + PRAGMA case_sensitive_like=OFF; + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd ABC ABCD nosort t11 *} +do_test like-11.2 { + queryplan { + PRAGMA case_sensitive_like=ON; + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd nosort t11 *} +do_test like-11.3 { + queryplan { + PRAGMA case_sensitive_like=OFF; + CREATE INDEX t11b ON t11(b); + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd ABC ABCD sort {} t11b} +do_test like-11.4 { + queryplan { + PRAGMA case_sensitive_like=ON; + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd nosort t11 *} +do_test like-11.5 { + queryplan { + PRAGMA case_sensitive_like=OFF; + DROP INDEX t11b; + CREATE INDEX t11bnc ON t11(b COLLATE nocase); + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd ABC ABCD sort {} t11bnc} +do_test like-11.6 { + queryplan { + CREATE INDEX t11bb ON t11(b COLLATE binary); + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd ABC ABCD sort {} t11bnc} +do_test like-11.7 { + queryplan { + PRAGMA case_sensitive_like=ON; + SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; + } +} {abc abcd sort {} t11bb} +do_test like-11.8 { + queryplan { + PRAGMA case_sensitive_like=OFF; + SELECT b FROM t11 WHERE b GLOB 'abc*' ORDER BY a; + } +} {abc abcd sort {} t11bb} +do_test like-11.9 { + queryplan { + CREATE INDEX t11cnc ON t11(c COLLATE nocase); + CREATE INDEX t11cb ON t11(c COLLATE binary); + SELECT c FROM t11 WHERE c LIKE 'abc%' ORDER BY a; + } +} {abc abcd ABC ABCD sort {} t11cnc} +do_test like-11.10 { + queryplan { + SELECT c FROM t11 WHERE c GLOB 'abc*' ORDER BY a; + } +} {abc abcd sort {} t11cb} + finish_test diff --git a/test/lock_common.tcl b/test/lock_common.tcl index 4fe08e0..0b7c5f9 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -27,6 +27,8 @@ proc do_multiclient_test {varname script} { set tn 2 }] { faultsim_delete_and_reopen + + proc code1 {tcl} { uplevel #0 $tcl } # Open connections [db2] and [db3]. Depending on which iteration this # is, the connections may be created in this interpreter, or in @@ -63,15 +65,19 @@ proc do_multiclient_test {varname script} { # channel name is returned that may be passed as the first argument to proc # 'testfixture' to execute a command. The child testfixture process is shut # down by closing the channel. -proc launch_testfixture {} { +proc launch_testfixture {{prg ""}} { write_main_loop - set prg [info nameofexec] - if {$prg eq ""} { - set prg [file join . testfixture] - } + if {$prg eq ""} { set prg [info nameofexec] } + if {$prg eq ""} { set prg testfixture } + if {[file tail $prg]==$prg} { set prg [file join . $prg] } set chan [open "|$prg tf_main.tcl" r+] fconfigure $chan -buffering line - testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte" + set rc [catch { + testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte" + }] + if {$rc} { + testfixture $chan "set ::sqlite_pending_byte $::sqlite_pending_byte" + } return $chan } diff --git a/test/notify3.test b/test/notify3.test index 7c7d5ac..0d06ce1 100644 --- a/test/notify3.test +++ b/test/notify3.test @@ -92,50 +92,57 @@ set noerr {{0 {}}} # it is possible to use the unlock-notify mechanism to determine when # the ATTACH might succeed. # -foreach { - tn - db1_loaded - db2_loaded - enable_extended_errors - result - error1 error2 -} " - 0 0 0 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE - 1 0 0 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE - 2 0 1 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE - 3 0 1 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE - 4 1 0 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE - 5 1 0 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE - 6 1 1 0 $noerr SQLITE_OK SQLITE_OK - 7 1 1 1 $noerr SQLITE_OK SQLITE_OK -" { - - do_test notify3-2.$tn.1 { - catch { db1 close } - catch { db2 close } - sqlite3 db1 test.db - sqlite3 db2 test.db2 - - sqlite3_extended_result_codes db1 $enable_extended_errors - sqlite3_extended_result_codes db2 $enable_extended_errors - - if { $db1_loaded } { db1 eval "SELECT * FROM sqlite_master" } - if { $db2_loaded } { db2 eval "SELECT * FROM sqlite_master" } - - db2 eval "BEGIN EXCLUSIVE" - catchsql "ATTACH 'test.db2' AS two" db1 - } $result - - do_test notify3-2.$tn.2 { - list [sqlite3_errcode db1] [sqlite3_extended_errcode db1] - } [list $error1 $error2] - - do_test notify3-2.$tn.3 { - db1 unlock_notify {set invoked 1} - set invoked 0 - db2 eval commit - set invoked - } [lindex $result 0] +# This test does not work for test-permutations that specify SQL to +# be executed as part of the [sqlite3] command that opens the database. +# Executing such SQL causes SQLite to load the database schema into memory +# earlier than expected, causing test cases to fail. +# +if {[presql] == ""} { + foreach { + tn + db1_loaded + db2_loaded + enable_extended_errors + result + error1 error2 + } " + 0 0 0 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE + 1 0 0 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE + 2 0 1 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE + 3 0 1 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE + 4 1 0 0 $err SQLITE_LOCKED SQLITE_LOCKED_SHAREDCACHE + 5 1 0 1 $err SQLITE_LOCKED_SHAREDCACHE SQLITE_LOCKED_SHAREDCACHE + 6 1 1 0 $noerr SQLITE_OK SQLITE_OK + 7 1 1 1 $noerr SQLITE_OK SQLITE_OK + " { + + do_test notify3-2.$tn.1 { + catch { db1 close } + catch { db2 close } + sqlite3 db1 test.db + sqlite3 db2 test.db2 + + sqlite3_extended_result_codes db1 $enable_extended_errors + sqlite3_extended_result_codes db2 $enable_extended_errors + + if { $db1_loaded } { db1 eval "SELECT * FROM sqlite_master" } + if { $db2_loaded } { db2 eval "SELECT * FROM sqlite_master" } + + db2 eval "BEGIN EXCLUSIVE" + catchsql "ATTACH 'test.db2' AS two" db1 + } $result + + do_test notify3-2.$tn.2 { + list [sqlite3_errcode db1] [sqlite3_extended_errcode db1] + } [list $error1 $error2] + + do_test notify3-2.$tn.3 { + db1 unlock_notify {set invoked 1} + set invoked 0 + db2 eval commit + set invoked + } [lindex $result 0] + } } catch { db1 close } catch { db2 close } diff --git a/test/pager1.test b/test/pager1.test index 7f28002..4eab17c 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -15,7 +15,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl source $testdir/wal_common.tcl - + # Do not use a codec for tests in this file, as the database file is # manipulated directly using tcl scripts (using the [hexio_write] command). # @@ -502,7 +502,7 @@ foreach {tn ofst value result} { 7 24 511 {1 2 3 4} 8 24 513 {1 2 3 4} - 9 24 65536 {1 2 3 4} + 9 24 131072 {1 2 3 4} 10 32 65536 {1 2} } { @@ -1062,6 +1062,10 @@ do_execsql_test pager1-6.8 { } {11} do_execsql_test pager1-6.9 { COMMIT } {} +do_execsql_test pager1-6.10 { PRAGMA max_page_count = 10 } {10} +do_execsql_test pager1-6.11 { SELECT * FROM t11 } {1 2 3 4} +do_execsql_test pager1-6.12 { PRAGMA max_page_count } {11} + #------------------------------------------------------------------------- # The following tests work with "PRAGMA journal_mode=TRUNCATE" and @@ -1742,7 +1746,7 @@ do_multiclient_test tn { } } {1 2 3 4 5 6} do_test pager1-17.$tn.3.2 { - csql1 { INSERT INTO t1 VALUES(3, 4) } + csql1 { INSERT INTO t1 VALUES(3, 4) } } {1 {database is locked}} do_test pager1-17.$tn.3.3 { sql2 COMMIT } {} } @@ -2232,6 +2236,7 @@ db close tv delete #------------------------------------------------------------------------- +# do_test pager1.27.1 { faultsim_delete_and_reopen sqlite3_pager_refcounts db @@ -2243,4 +2248,104 @@ do_test pager1.27.1 { execsql COMMIT } {} +#------------------------------------------------------------------------- +# Test that attempting to open a write-transaction with +# locking_mode=exclusive in WAL mode fails if there are other clients on +# the same database. +# +catch { db close } +do_multiclient_test tn { + do_test pager1-28.$tn.1 { + sql1 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES('a', 'b'); + } + } {wal} + do_test pager1-28.$tn.2 { sql2 { SELECT * FROM t1 } } {a b} + + do_test pager1-28.$tn.3 { sql1 { PRAGMA locking_mode=exclusive } } {exclusive} + do_test pager1-28.$tn.4 { + csql1 { BEGIN; INSERT INTO t1 VALUES('c', 'd'); } + } {1 {database is locked}} + code2 { db2 close ; sqlite3 db2 test.db } + do_test pager1-28.$tn.4 { + sql1 { INSERT INTO t1 VALUES('c', 'd'); COMMIT } + } {} +} + +#------------------------------------------------------------------------- +# Normally, when changing from journal_mode=PERSIST to DELETE the pager +# attempts to delete the journal file. However, if it cannot obtain a +# RESERVED lock on the database file, this step is skipped. +# +do_multiclient_test tn { + do_test pager1-28.$tn.1 { + sql1 { + PRAGMA journal_mode = PERSIST; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES('a', 'b'); + } + } {persist} + do_test pager1-28.$tn.2 { file exists test.db-journal } 1 + do_test pager1-28.$tn.3 { sql1 { PRAGMA journal_mode = DELETE } } delete + do_test pager1-28.$tn.4 { file exists test.db-journal } 0 + + do_test pager1-28.$tn.5 { + sql1 { + PRAGMA journal_mode = PERSIST; + INSERT INTO t1 VALUES('c', 'd'); + } + } {persist} + do_test pager1-28.$tn.6 { file exists test.db-journal } 1 + do_test pager1-28.$tn.7 { + sql2 { BEGIN; INSERT INTO t1 VALUES('e', 'f'); } + } {} + do_test pager1-28.$tn.8 { file exists test.db-journal } 1 + do_test pager1-28.$tn.9 { sql1 { PRAGMA journal_mode = DELETE } } delete + do_test pager1-28.$tn.10 { file exists test.db-journal } 1 + + do_test pager1-28.$tn.11 { sql2 COMMIT } {} + do_test pager1-28.$tn.12 { file exists test.db-journal } 0 + + do_test pager1-28-$tn.13 { + code1 { set channel [db incrblob -readonly t1 a 2] } + sql1 { + PRAGMA journal_mode = PERSIST; + INSERT INTO t1 VALUES('g', 'h'); + } + } {persist} + do_test pager1-28.$tn.14 { file exists test.db-journal } 1 + do_test pager1-28.$tn.15 { + sql2 { BEGIN; INSERT INTO t1 VALUES('e', 'f'); } + } {} + do_test pager1-28.$tn.16 { sql1 { PRAGMA journal_mode = DELETE } } delete + do_test pager1-28.$tn.17 { file exists test.db-journal } 1 + + do_test pager1-28.$tn.17 { csql2 { COMMIT } } {1 {database is locked}} + do_test pager1-28-$tn.18 { code1 { read $channel } } c + do_test pager1-28-$tn.19 { code1 { close $channel } } {} + do_test pager1-28.$tn.20 { sql2 { COMMIT } } {} +} + +do_test pager1-29.1 { + faultsim_delete_and_reopen + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = full; + PRAGMA locking_mode=exclusive; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + file size test.db +} [expr 1024*3] +do_test pager1-29.2 { + execsql { + PRAGMA page_size = 4096; + VACUUM; + } + file size test.db +} [expr 4096*3] + + finish_test diff --git a/test/pager2.test b/test/pager2.test index 977d45f..06389f5 100644 --- a/test/pager2.test +++ b/test/pager2.test @@ -116,4 +116,36 @@ foreach code [list { db close tv delete + +#------------------------------------------------------------------------- +# +# pager2-2.1: Test a ROLLBACK with journal_mode=off. +# pager2-2.2: Test shrinking the database (auto-vacuum) with +# journal_mode=off +# +do_test pager2-2.1 { + faultsim_delete_and_reopen + execsql { + CREATE TABLE t1(a, b); + PRAGMA journal_mode = off; + BEGIN; + INSERT INTO t1 VALUES(1, 2); + ROLLBACK; + SELECT * FROM t1; + } +} {off 1 2} +do_test pager2-2.2 { + faultsim_delete_and_reopen + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA page_size = 1024; + PRAGMA journal_mode = off; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(zeroblob(5000), zeroblob(5000)); + DELETE FROM t1; + PRAGMA incremental_vacuum; + } + file size test.db +} {3072} + finish_test diff --git a/test/pager3.test b/test/pager3.test new file mode 100644 index 0000000..23435a7 --- /dev/null +++ b/test/pager3.test @@ -0,0 +1,33 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + + +foreach {tn sql res j} { + 1 "PRAGMA journal_mode = DELETE" delete 0 + 2 "CREATE TABLE t1(a, b)" {} 0 + 3 "PRAGMA locking_mode=EXCLUSIVE" {exclusive} 0 + 4 "INSERT INTO t1 VALUES(1, 2)" {} 1 + 5 "PRAGMA locking_mode=NORMAL" {normal} 1 + 6 "SELECT * FROM t1" {1 2} 0 +} { + do_execsql_test pager3-1.$tn.1 $sql $res + do_test pager3-1.$tn.2 { file exists test.db-journal } $j +} + + +finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index f9c82dd..c73bc42 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -52,7 +52,7 @@ do_faultsim_test pagerfault-1 -prep { } -body { execsql { SELECT count(*) FROM t1 } } -test { - faultsim_test_result {0 4} + faultsim_test_result {0 4} faultsim_integrity_check if {[db one { SELECT count(*) FROM t1 }] != 4} { error "Database content appears incorrect" @@ -1078,4 +1078,159 @@ do_faultsim_test pagerfault-22 -prep { faultsim_integrity_check } +#------------------------------------------------------------------------- +# Provoke an OOM error during a commit of multi-file transaction. One of +# the databases written during the transaction is an in-memory database. +# This test causes rollback of the in-memory database after CommitPhaseOne() +# has successfully returned. i.e. the series of calls for the aborted commit +# is: +# +# PagerCommitPhaseOne() -> SQLITE_OK +# PagerCommitPhaseOne() -> SQLITE_IOERR +# PagerRollback() +# PagerRollback() +# +do_faultsim_test pagerfault-23 -prep { + foreach f [glob -nocomplain test.db*] { file delete -force $f } + sqlite3 db :memory: + db eval { + ATTACH 'test.db2' AS aux; + CREATE TABLE t1(a, b); + CREATE TABLE aux.t2(a, b); + } +} -body { + execsql { + BEGIN; + INSERT INTO t1 VALUES(1,2); + INSERT INTO t2 VALUES(3,4); + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check +} + +do_faultsim_test pagerfault-24 -prep { + faultsim_delete_and_reopen + db eval { PRAGMA temp_store = file } + execsql { CREATE TABLE x(a, b) } +} -body { + execsql { CREATE TEMP TABLE t1(a, b) } +} -test { + faultsim_test_result {0 {}} \ + {1 {unable to open a temporary database file for storing temporary tables}} + set ic [db eval { PRAGMA temp.integrity_check }] + if {$ic != "ok"} { error "Integrity check: $ic" } +} + +proc lockrows {n} { + if {$n==0} { return "" } + db eval { SELECT * FROM t1 WHERE oid = $n } { + return [lockrows [expr {$n-1}]] + } +} + + +do_test pagerfault-25-pre1 { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(a_string(500)); + INSERT INTO t1 SELECT a_string(500) FROM t1; + INSERT INTO t1 SELECT a_string(500) FROM t1; + INSERT INTO t1 SELECT a_string(500) FROM t1; + INSERT INTO t1 SELECT a_string(500) FROM t1; + INSERT INTO t1 SELECT a_string(500) FROM t1; + } + faultsim_save_and_close +} {} +do_faultsim_test pagerfault-25 -prep { + faultsim_restore_and_reopen + db func a_string a_string + set ::channel [db incrblob -readonly t1 a 1] + execsql { + PRAGMA cache_size = 10; + BEGIN; + INSERT INTO t1 VALUES(a_string(3000)); + INSERT INTO t1 VALUES(a_string(3000)); + } +} -body { + lockrows 30 +} -test { + catch { lockrows 30 } + catch { db eval COMMIT } + close $::channel + faultsim_test_result {0 {}} +} + +do_faultsim_test pagerfault-26 -prep { + faultsim_delete_and_reopen + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = truncate; + PRAGMA auto_vacuum = full; + PRAGMA locking_mode=exclusive; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + PRAGMA page_size = 4096; + } +} -body { + execsql { + VACUUM; + } +} -test { + faultsim_test_result {0 {}} + + set contents [db eval {SELECT * FROM t1}] + if {$contents != "1 2"} { error "Bad database contents ($contents)" } + + set sz [file size test.db] + if {$testrc!=0 && $sz!=1024*3 && $sz!=4096*3} { + error "Expected file size to be 3072 or 12288 bytes - actual size $sz bytes" + } + if {$testrc==0 && $sz!=4096*3} { + error "Expected file size to be 12288 bytes - actual size $sz bytes" + } +} + +do_test pagerfault-27-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t1 VALUES (a_string(20000), a_string(20000)); + } + faultsim_save_and_close +} {} +do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { + faultsim_restore_and_reopen + db func a_string a_string + execsql { + PRAGMA cache_size = 10; + BEGIN EXCLUSIVE; + } + set ::channel [db incrblob t1 a 1] +} -body { + puts $::channel [string repeat abc 6000] + flush $::channel +} -test { + catchsql { UPDATE t2 SET a = a_string(800), b = a_string(800) } + catch { close $::channel } + catchsql { ROLLBACK } + faultsim_integrity_check +} + finish_test + diff --git a/test/permutations.test b/test/permutations.test index f1fb74f..086897c 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -129,7 +129,7 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] test_suite "quick" -prefix "" -description { diff --git a/test/selectC.test b/test/selectC.test index 45250ef..fd38405 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -210,4 +210,27 @@ do_test selectC-3.3 { } } {xxx abcxxx yyy defyyy} + +proc udf {} { incr ::udf } +set ::udf 0 +db function udf udf + +do_execsql_test selectC-4.1 { + create table t_distinct_bug (a, b, c); + insert into t_distinct_bug values ('1', '1', 'a'); + insert into t_distinct_bug values ('1', '2', 'b'); + insert into t_distinct_bug values ('1', '3', 'c'); + insert into t_distinct_bug values ('1', '1', 'd'); + insert into t_distinct_bug values ('1', '2', 'e'); + insert into t_distinct_bug values ('1', '3', 'f'); +} {} + +do_execsql_test selectC-4.2 { + select a from (select distinct a, b from t_distinct_bug) +} {1 1 1} + +do_execsql_test selectC-4.3 { + select a, udf() from (select distinct a, b from t_distinct_bug) +} {1 1 1 2 1 3} + finish_test diff --git a/test/shared.test b/test/shared.test index 9b04946..35ddf08 100644 --- a/test/shared.test +++ b/test/shared.test @@ -1018,6 +1018,44 @@ do_test shared-$av.14.3 { db close } {} +# Populate a database schema using connection [db]. Then drop it using +# [db2]. This is to try to find any points where shared-schema elements +# are allocated using the lookaside buffer of [db]. +# +# Mutexes are enabled for this test as that activates a couple of useful +# assert() statements in the C code. +# +do_test shared-$av-15.1 { + file delete -force test.db + sqlite3 db test.db -fullmutex 1 + sqlite3 db2 test.db -fullmutex 1 + execsql { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE VIEW v2 AS SELECT * FROM t1, v1 + WHERE t1.c=v1.c GROUP BY t1.a ORDER BY v1.b; + CREATE TRIGGER tr1 AFTER INSERT ON t1 + WHEN new.a!=1 + BEGIN + DELETE FROM t1 WHERE a=5; + INSERT INTO t1 VALUES(1, 2, 3); + UPDATE t1 SET c=c+1; + END; + + INSERT INTO t1 VALUES(5, 6, 7); + INSERT INTO t1 VALUES(8, 9, 10); + INSERT INTO t1 VALUES(11, 12, 13); + ANALYZE; + SELECT * FROM t1; + } +} {1 2 6 8 9 12 1 2 5 11 12 14 1 2 4} +do_test shared-$av-15.2 { + execsql { DROP TABLE t1 } db2 +} {} +db close +db2 close + } sqlite3_enable_shared_cache $::enable_shared_cache diff --git a/test/tester.tcl b/test/tester.tcl index 0b2299f..fb26561 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -70,6 +70,7 @@ # wal_set_journal_mode ?DB? # wal_check_journal_mode TESTNAME?DB? # permutation +# presql # # Set the precision of FP arithmatic used by the interpreter. And @@ -441,8 +442,10 @@ proc finalize_testing {} { puts "$sqlite_open_file_count files were left open" incr nErr } - if {[sqlite3_memory_used]>0} { - puts "Unfreed memory: [sqlite3_memory_used] bytes" + if {[lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]>0 || + [sqlite3_memory_used]>0} { + puts "Unfreed memory: [sqlite3_memory_used] bytes in\ + [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1] allocations" incr nErr ifcapable memdebug||mem5||(mem3&&debug) { puts "Writing unfreed memory log to \"./memleak.txt\"" @@ -489,6 +492,9 @@ proc show_memstats {} { set val [format {now %10d max %10d max-size %10d} \ [lindex $x 1] [lindex $x 2] [lindex $y 2]] puts "Memory used: $val" + set x [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] + set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] + puts "Allocation count: $val" set x [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] set y [sqlite3_status SQLITE_STATUS_PAGECACHE_SIZE 0] set val [format {now %10d max %10d max-size %10d} \ @@ -1145,6 +1151,11 @@ proc permutation {} { catch {set perm $::G(perm:name)} set perm } +proc presql {} { + set presql "" + catch {set presql $::G(perm:presql)} + set presql +} #------------------------------------------------------------------------- # diff --git a/test/threadtest3.c b/test/threadtest3.c new file mode 100644 index 0000000..6373926 --- /dev/null +++ b/test/threadtest3.c @@ -0,0 +1,1333 @@ + +/* +** The code in this file runs a few multi-threaded test cases using the +** SQLite library. It can be compiled to an executable on unix using the +** following command: +** +** gcc -O2 threadtest3.c sqlite3.c -ldl -lpthread -lm +** +** Then run the compiled program. The exit status is non-zero if any tests +** failed (hopefully there is also some output to stdout to clarify what went +** wrong). +** +** There are three parts to the code in this file, in the following order: +** +** 1. Code for the SQL aggregate function md5sum() copied from +** tclsqlite.c in the SQLite distribution. The names of all the +** types and functions in this section begin with "MD5" or "md5". +** +** 2. A set of utility functions that may be used to implement +** multi-threaded test cases. These are all called by test code +** via macros that help with error reporting. The macros are defined +** immediately below this comment. +** +** 3. The test code itself. And a main() routine to drive the test +** code. +*/ + +/************************************************************************* +** Start of test code/infrastructure interface macros. +** +** The following macros constitute the interface between the test +** programs and the test infrastructure. Test infrastructure code +** does not itself use any of these macros. Test code should not +** call any of the macroname_x() functions directly. +** +** See the header comments above the corresponding macroname_x() +** function for a description of each interface. +*/ + +/* Database functions */ +#define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) +#define closedb(y,z) (SEL(y), closedb_x(y,z)) + +/* Functions to execute SQL */ +#define sql_script(x,y,z) (SEL(x), sql_script_x(x,y,z)) +#define integrity_check(x,y) (SEL(x), integrity_check_x(x,y)) +#define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) +#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) +#define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) + +/* Thread functions */ +#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) +#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) + +/* Timer functions */ +#define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) +#define timetostop(z) (SEL(z), timetostop_x(z)) + +/* Report/clear errors. */ +#define test_error(z, ...) test_error_x(z, sqlite3_mprintf(__VA_ARGS__)) +#define clear_error(y,z) clear_error_x(y, z) + +/* File-system operations */ +#define filesize(y,z) (SEL(y), filesize_x(y,z)) +#define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) + +/* +** End of test code/infrastructure interface macros. +*************************************************************************/ + + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +/* + * If compiled on a machine that doesn't have a 32-bit integer, + * you just set "uint32" to the appropriate datatype for an + * unsigned 32-bit integer. For example: + * + * cc -Duint32='unsigned long' md5.c + * + */ +#ifndef uint32 +# define uint32 unsigned int +#endif + +struct MD5Context { + int isInit; + uint32 buf[4]; + uint32 bits[2]; + unsigned char in[64]; +}; +typedef struct MD5Context MD5Context; + +/* + * Note: this code is harmless on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32 t; + do { + t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32 *)buf = t; + buf += 4; + } while (--longs); +} +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32 buf[4], const uint32 in[16]){ + register uint32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +static void MD5Init(MD5Context *ctx){ + ctx->isInit = 1; + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +static +void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ + uint32 t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +static void MD5Final(unsigned char digest[16], MD5Context *ctx){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0]; + ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1]; + + MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(ctx)); /* In case it is sensitive */ +} + +/* +** Convert a 128-bit MD5 digest into a 32-digit base-16 number. +*/ +static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + +/* +** During testing, the special md5sum() aggregate function is available. +** inside SQLite. The following routines implement that function. +*/ +static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ + MD5Context *p; + int i; + if( argc<1 ) return; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + MD5Init(p); + } + for(i=0; izErr); + p->zErr = 0; + p->rc = 0; +} + +static void print_err(Error *p){ + if( p->rc!=SQLITE_OK ){ + printf("Error: (%d) \"%s\" at line %d\n", p->rc, p->zErr, p->iLine); + nGlobalErr++; + } +} + +static void print_and_free_err(Error *p){ + print_err(p); + free_err(p); +} + +static void system_error(Error *pErr, int iSys){ + pErr->rc = iSys; + pErr->zErr = (char *)sqlite3_malloc(512); + strerror_r(iSys, pErr->zErr, 512); + pErr->zErr[511] = '\0'; +} + +static void sqlite_error( + Error *pErr, + Sqlite *pDb, + const char *zFunc +){ + pErr->rc = sqlite3_errcode(pDb->db); + pErr->zErr = sqlite3_mprintf( + "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db), + sqlite3_extended_errcode(pDb->db) + ); +} + +static void test_error_x( + Error *pErr, + char *zErr +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = 1; + pErr->zErr = zErr; + }else{ + sqlite3_free(zErr); + } +} + +static void clear_error_x( + Error *pErr, + int rc +){ + if( pErr->rc==rc ){ + pErr->rc = SQLITE_OK; + sqlite3_free(pErr->zErr); + pErr->zErr = 0; + } +} + +static int busyhandler(void *pArg, int n){ + usleep(10*1000); + return 1; +} + +static void opendb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* OUT: Database handle */ + const char *zFile, /* Database file name */ + int bDelete /* True to delete db file before opening */ +){ + if( pErr->rc==SQLITE_OK ){ + int rc; + if( bDelete ) unlink(zFile); + rc = sqlite3_open(zFile, &pDb->db); + if( rc ){ + sqlite_error(pErr, pDb, "open"); + sqlite3_close(pDb->db); + pDb->db = 0; + }else{ + sqlite3_create_function( + pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize + ); + sqlite3_busy_handler(pDb->db, busyhandler, 0); + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); + } + } +} + +static void closedb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* OUT: Database handle */ +){ + int rc; + int i; + Statement *pIter; + Statement *pNext; + for(pIter=pDb->pCache; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_finalize(pIter->pStmt); + sqlite3_free(pIter); + } + for(i=0; inText; i++){ + sqlite3_free(pDb->aText[i]); + } + sqlite3_free(pDb->aText); + rc = sqlite3_close(pDb->db); + if( rc && pErr->rc==SQLITE_OK ){ + pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db)); + } + memset(pDb, 0, sizeof(Sqlite)); +} + +static void sql_script_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL script to execute */ +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + } +} + +static Statement *getSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL statement */ +){ + Statement *pRet; + int rc; + + for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){ + if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){ + return pRet; + } + } + + pRet = sqlite3_malloc(sizeof(Statement)); + rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite_error(pErr, pDb, "prepare_v2"); + return 0; + } + assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ); + + pRet->pNext = pDb->pCache; + pDb->pCache = pRet; + return pRet; +} + +static sqlite3_stmt *getAndBindSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + va_list ap /* SQL followed by parameters */ +){ + Statement *pStatement; /* The SQLite statement wrapper */ + sqlite3_stmt *pStmt; /* The SQLite statement to return */ + int i; /* Used to iterate through parameters */ + + pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *)); + if( !pStatement ) return 0; + pStmt = pStatement->pStmt; + for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){ + const char *zName = sqlite3_bind_parameter_name(pStmt, i); + void * pArg = va_arg(ap, void*); + + switch( zName[1] ){ + case 'i': + sqlite3_bind_int64(pStmt, i, *(i64 *)pArg); + break; + + default: + pErr->rc = 1; + pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName); + pStmt = 0; + break; + } + } + + return pStmt; +} + +static i64 execsql_i64_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + ... /* SQL and pointers to parameter values */ +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + int i; /* Used to iterate through parameters */ + va_start(ap, pDb); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int rc; + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + iRet = sqlite3_column_int64(pStmt, 0); + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + return iRet; +} + +static char * execsql_text_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + int iSlot, /* Db handle slot to store text in */ + ... /* SQL and pointers to parameter values */ +){ + char *zRet = 0; + + if( iSlot>=pDb->nText ){ + int nByte = sizeof(char *)*(iSlot+1); + pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte); + memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText)); + pDb->nText = iSlot+1; + } + + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + int i; /* Used to iterate through parameters */ + va_start(ap, iSlot); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int rc; + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + sqlite3_free(pDb->aText[iSlot]); + pDb->aText[iSlot] = zRet; + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + + return zRet; +} + +static void integrity_check_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* Database handle */ +){ + if( pErr->rc==SQLITE_OK ){ + Statement *pStatement; /* Statement to execute */ + int rc; /* Return code */ + char *zErr = 0; /* Integrity check error */ + + pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); + if( pStatement ){ + sqlite3_stmt *pStmt = pStatement->pStmt; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *z = sqlite3_column_text(pStmt, 0); + if( strcmp(z, "ok") ){ + if( zErr==0 ){ + zErr = sqlite3_mprintf("%s", z); + }else{ + zErr = sqlite3_mprintf("%z\n%s", zErr, z); + } + } + } + sqlite3_reset(pStmt); + + if( zErr ){ + pErr->zErr = zErr; + pErr->rc = 1; + } + } + } +} + +static void *launch_thread_main(void *pArg){ + Thread *p = (Thread *)pArg; + return (void *)p->xProc(p->iTid, p->iArg); +} + +static void launch_thread_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads, /* Thread set */ + char *(*xProc)(int, int), /* Proc to run */ + int iArg /* Argument passed to thread proc */ +){ + if( pErr->rc==SQLITE_OK ){ + int iTid = ++pThreads->iMaxTid; + Thread *p; + int rc; + + p = (Thread *)sqlite3_malloc(sizeof(Thread)); + memset(p, 0, sizeof(Thread)); + p->iTid = iTid; + p->iArg = iArg; + p->xProc = xProc; + + rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); + if( rc!=0 ){ + system_error(pErr, rc); + sqlite3_free(p); + }else{ + p->pNext = pThreads->pThread; + pThreads->pThread = p; + } + } +} + +static void join_all_threads_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads /* Thread set */ +){ + Thread *p; + Thread *pNext; + for(p=pThreads->pThread; p; p=pNext){ + void *ret; + pNext = p->pNext; + int rc; + rc = pthread_join(p->tid, &ret); + if( rc!=0 ){ + if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); + }else{ + printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); + } + sqlite3_free(p); + } + pThreads->pThread = 0; +} + +static i64 filesize_x( + Error *pErr, + const char *zFile +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + struct stat sStat; + if( stat(zFile, &sStat) ){ + iRet = -1; + }else{ + iRet = sStat.st_size; + } + } + return iRet; +} + +static void filecopy_x( + Error *pErr, + const char *zFrom, + const char *zTo +){ + if( pErr->rc==SQLITE_OK ){ + i64 nByte = filesize_x(pErr, zFrom); + if( nByte<0 ){ + test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom)); + }else{ + i64 iOff; + char aBuf[1024]; + int fd1; + int fd2; + unlink(zTo); + + fd1 = open(zFrom, O_RDONLY); + if( fd1<0 ){ + system_error(pErr, errno); + return; + } + fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644); + if( fd2<0 ){ + system_error(pErr, errno); + close(fd1); + return; + } + + iOff = 0; + while( iOffnByte ){ + nCopy = nByte - iOff; + } + if( nCopy!=read(fd1, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + if( nCopy!=write(fd2, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + iOff += nCopy; + } + + close(fd1); + close(fd2); + } + } +} + +/* +** Used by setstoptime() and timetostop(). +*/ +static double timelimit = 0.0; +static sqlite3_vfs *pTimelimitVfs = 0; + +static void setstoptime_x( + Error *pErr, /* IN/OUT: Error code */ + int nMs /* Milliseconds until "stop time" */ +){ + if( pErr->rc==SQLITE_OK ){ + double t; + int rc; + pTimelimitVfs = sqlite3_vfs_find(0); + rc = pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t); + if( rc!=SQLITE_OK ){ + pErr->rc = rc; + }else{ + timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0); + } + } +} + +static int timetostop_x( + Error *pErr /* IN/OUT: Error code */ +){ + int ret = 1; + if( pErr->rc==SQLITE_OK ){ + double t; + int rc; + rc = pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t); + if( rc!=SQLITE_OK ){ + pErr->rc = rc; + }else{ + ret = (t >= timelimit); + } + } + return ret; +} + +/* +** The "Set Error Line" macro. +*/ +#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) + + +/************************************************************************* +************************************************************************** +************************************************************************** +** End infrastructure. Begin tests. +*/ + +#define WALTHREAD1_NTHREAD 10 +#define WALTHREAD3_NTHREAD 6 + +static char *walthread1_thread(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int nIter = 0; /* Iterations so far */ + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + const char *azSql[] = { + "SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1)", + "SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1)", + }; + char *z1, *z2, *z3; + + execsql(&err, &db, "BEGIN"); + integrity_check(&err, &db); + z1 = execsql_text(&err, &db, 1, azSql[0]); + z2 = execsql_text(&err, &db, 2, azSql[1]); + z3 = execsql_text(&err, &db, 3, azSql[0]); + execsql(&err, &db, "COMMIT"); + + if( strcmp(z1, z2) || strcmp(z1, z3) ){ + test_error(&err, "Failed read: %s %s %s", z1, z2, z3); + } + + sql_script(&err, &db, + "BEGIN;" + "INSERT INTO t1 VALUES(randomblob(100));" + "INSERT INTO t1 VALUES(randomblob(100));" + "INSERT INTO t1 SELECT md5sum(x) FROM t1;" + "COMMIT;" + ); + nIter++; + } + closedb(&err, &db); + + print_and_free_err(&err); + return sqlite3_mprintf("%d iterations", nIter); +} + +static char *walthread1_ckpt_thread(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int nCkpt = 0; /* Checkpoints so far */ + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + usleep(500*1000); + execsql(&err, &db, "PRAGMA wal_checkpoint"); + if( err.rc==SQLITE_OK ) nCkpt++; + clear_error(&err, SQLITE_BUSY); + } + closedb(&err, &db); + + print_and_free_err(&err); + return sqlite3_mprintf("%d checkpoints", nCkpt); +} + +static void walthread1(int nMs){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + Threadset threads = {0}; /* Test threads */ + int i; /* Iterator variable */ + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "PRAGMA journal_mode = WAL;" + "CREATE TABLE t1(x PRIMARY KEY);" + "INSERT INTO t1 VALUES(randomblob(100));" + "INSERT INTO t1 VALUES(randomblob(100));" + "INSERT INTO t1 SELECT md5sum(x) FROM t1;" + ); + + setstoptime(&err, nMs); + for(i=0; i= 0); + wal_exists = (filesize(&err, "test.db-wal") >= 0); + if( (journal_exists+wal_exists)!=1 ){ + test_error(&err, "File system looks incorrect (%d, %d)", + journal_exists, wal_exists + ); + } + anTrans[journal_exists]++; + + sql_script(&err, &db, "COMMIT"); + integrity_check(&err, &db); + closedb(&err, &db); + } + + print_and_free_err(&err); + return sqlite3_mprintf("W %d R %d", anTrans[0], anTrans[1]); +} + +static void walthread2(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, "CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE)"); + closedb(&err, &db); + + setstoptime(&err, nMs); + launch_thread(&err, &threads, walthread2_thread, 0); + launch_thread(&err, &threads, walthread2_thread, 0); + launch_thread(&err, &threads, walthread2_thread, 1); + launch_thread(&err, &threads, walthread2_thread, 1); + join_all_threads(&err, &threads); + + print_and_free_err(&err); +} + +static char *walthread3_thread(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + i64 iNextWrite; /* Next value this thread will write */ + + opendb(&err, &db, "test.db", 0); + sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 10"); + + iNextWrite = iArg+1; + while( 1 ){ + i64 sum1; + i64 sum2; + int stop = 0; /* True to stop executing (test timed out) */ + + while( 0==(stop = timetostop(&err)) ){ + i64 iMax = execsql_i64(&err, &db, "SELECT max(cnt) FROM t1"); + if( iMax+1==iNextWrite ) break; + } + if( stop ) break; + + sum1 = execsql_i64(&err, &db, "SELECT sum(cnt) FROM t1"); + sum2 = execsql_i64(&err, &db, "SELECT sum(sum1) FROM t1"); + execsql_i64(&err, &db, + "INSERT INTO t1 VALUES(:iNextWrite, :iSum1, :iSum2)", + &iNextWrite, &sum1, &sum2 + ); + integrity_check(&err, &db); + + iNextWrite += WALTHREAD3_NTHREAD; + } + + closedb(&err, &db); + print_and_free_err(&err); + return 0; +} + +static void walthread3(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + int i; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "PRAGMA journal_mode = WAL;" + "CREATE TABLE t1(cnt PRIMARY KEY, sum1, sum2);" + "CREATE INDEX i1 ON t1(sum1);" + "CREATE INDEX i2 ON t1(sum2);" + "INSERT INTO t1 VALUES(0, 0, 0);" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + for(i=0; i2 ) goto usage; + if( argc==2 ){ + zTest = argv[1]; + nTest = strlen(zTest); + if( zTest[nTest-1]=='*' ){ + nTest--; + bPrefix = 1; + } + } + + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + + for(i=0; i0 ? 255 : 0); + + usage: + printf("Usage: %s [testname|testprefix*]\n", argv[0]); + printf("Available tests are:\n"); + for(i=0; i=8.5} { + # Set all zeroed slots in the first hash table to invalid values. + # + set blob [string range [tvfs shm $::filename] 0 16383] + set I [string range [tvfs shm $::filename] 16384 end] + binary scan $I t* L + set I [list] + foreach p $L { + lappend I [expr $p ? $p : 400] + } + append blob [binary format t* $I] + tvfs shm $::filename $blob + do_test wal2-11.2 { + catchsql { INSERT INTO t1 VALUES(10, 11, 12) } + } {1 {database disk image is malformed}} + + # Fill up the hash table on the first page of shared memory with 0x55 bytes. + # + set blob [string range [tvfs shm $::filename] 0 16383] + append blob [string repeat [binary format c 55] 16384] + tvfs shm $::filename $blob + do_test wal2-11.3 { + catchsql { SELECT * FROM t1 } db2 + } {1 {database disk image is malformed}} } -append blob [binary format t* $I] -tvfs shm $::filename $blob -do_test wal2-11.2 { - catchsql { INSERT INTO t1 VALUES(10, 11, 12) } -} {1 {database disk image is malformed}} - -# Fill up the hash table on the first page of shared memory with 0x55 bytes. -# -set blob [string range [tvfs shm $::filename] 0 16383] -append blob [string repeat [binary format c 55] 16384] -tvfs shm $::filename $blob -do_test wal2-11.3 { - catchsql { SELECT * FROM t1 } db2 -} {1 {database disk image is malformed}} db close db2 close @@ -1148,4 +1150,3 @@ if {$::tcl_platform(platform) == "unix"} { } finish_test - diff --git a/test/wal3.test b/test/wal3.test index b344c14..b53b844 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -110,6 +110,11 @@ for {set i 1} {$i < 50} {incr i} { db2 close } +proc byte_is_zero {file offset} { + if {[file size test.db] <= $offset} { return 1 } + expr { [hexio_read $file $offset 1] == "00" } +} + do_multiclient_test i { set testname(1) multiproc @@ -158,22 +163,24 @@ do_multiclient_test i { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 4*1024 : 3*1024] + byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024] + } {1} do_test wal3-2.$tn.5 { sql2 { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 5*1024 : 4*1024] + list [byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024]] \ + [byte_is_zero test.db [expr $AUTOVACUUM ? 5*1024 : 4*1024]] + } {0 1} do_test wal3-2.$tn.6 { sql3 { COMMIT; PRAGMA wal_checkpoint; } - file size test.db - } [expr $AUTOVACUUM ? 5*1024 : 4*1024] + list [byte_is_zero test.db [expr $AUTOVACUUM ? 4*1024 : 3*1024]] \ + [byte_is_zero test.db [expr $AUTOVACUUM ? 5*1024 : 4*1024]] + } {0 1} } catch {db close} @@ -700,6 +707,7 @@ do_test wal3-9.0 { file delete -force test.db test.db-journal test.db wal sqlite3 db test.db execsql { + PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; CREATE TABLE whoami(x); INSERT INTO whoami VALUES('nobody'); @@ -722,15 +730,19 @@ for {set i 0} {$i < 50} {incr i} { execsql { SELECT * FROM whoami } $c } $c } + +set sz [expr 1024 * (2+$AUTOVACUUM)] do_test wal3-9.3 { for {set i 0} {$i < 49} {incr i} { db$i close } execsql { PRAGMA wal_checkpoint } - set sz1 [file size test.db] + byte_is_zero test.db [expr $sz-1024] +} {1} +do_test wal3-9.4 { db49 close execsql { PRAGMA wal_checkpoint } set sz2 [file size test.db] - expr {$sz2 > $sz1} -} {1} + byte_is_zero test.db [expr $sz-1024] +} {0} db close diff --git a/test/walmode.test b/test/walmode.test index 1a54277..2506c90 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -377,4 +377,11 @@ sqlite3 db2 test.db2 do_test walmode-8.19 { execsql { PRAGMA main.journal_mode } db2 } {wal} db2 close +do_execsql_test walmode-8.20 { PRAGMA journal_mode = DELETE } {delete} +do_execsql_test walmode-8.21 { PRAGMA main.journal_mode } {delete} +do_execsql_test walmode-8.22 { PRAGMA two.journal_mode } {delete} +do_execsql_test walmode-8.21 { PRAGMA journal_mode = WAL } {wal} +do_execsql_test walmode-8.21 { PRAGMA main.journal_mode } {wal} +do_execsql_test walmode-8.22 { PRAGMA two.journal_mode } {wal} + finish_test diff --git a/test/walshared.test b/test/walshared.test new file mode 100644 index 0000000..658a25f --- /dev/null +++ b/test/walshared.test @@ -0,0 +1,60 @@ +# 2010 August 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode with shared-cache turned on. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +db close +set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test walshared-1.0 { + execsql { + PRAGMA cache_size = 10; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(randomblob(100), randomblob(200)); + } +} {wal} + +do_test walshared-1.1 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(randomblob(100), randomblob(200)); + INSERT INTO t1 SELECT randomblob(100), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(100), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(100), randomblob(200) FROM t1; + } +} {} + +do_test walshared-1.2 { + catchsql { PRAGMA wal_checkpoint } +} {1 {database table is locked}} + +do_test walshared-1.3 { + catchsql { PRAGMA wal_checkpoint } db2 +} {1 {database table is locked}} + +do_test walshared-1.4 { + execsql { COMMIT } + execsql { PRAGMA integrity_check } db2 +} {ok} + + + +sqlite3_enable_shared_cache $::enable_shared_cache +finish_test + diff --git a/test/where3.test b/test/where3.test index 13f9473..9fa4cf9 100644 --- a/test/where3.test +++ b/test/where3.test @@ -212,5 +212,53 @@ do_test where3-2.7 { } } {tB {} tC * tA * tD *} +# Ticket [13f033c865f878953] +# If the outer loop must be a full table scan, do not let ANALYZE trick +# the planner into use a table for the outer loop that might be indexable +# if held until an inner loop. +# +do_test where3-3.0 { + execsql { + CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); + CREATE INDEX t301c ON t301(c); + INSERT INTO t301 VALUES(1,2,3); + CREATE TABLE t302(x, y); + ANALYZE; + explain query plan + SELECT * FROM t302, t301 WHERE t302.x=5 AND t301.a=t302.y; + } +} {0 0 {TABLE t302} 1 1 {TABLE t301 USING PRIMARY KEY}} +do_test where3-3.1 { + execsql { + explain query plan + SELECT * FROM t301, t302 WHERE t302.x=5 AND t301.a=t302.y; + } +} {0 1 {TABLE t302} 1 0 {TABLE t301 USING PRIMARY KEY}} + +# Verify that when there are multiple tables in a join which must be +# full table scans that the query planner attempts put the table with +# the fewest number of output rows as the outer loop. +# +do_test where3-4.0 { + execsql { + CREATE TABLE t400(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t401(p INTEGER PRIMARY KEY, q, r); + CREATE TABLE t402(x INTEGER PRIMARY KEY, y, z); + EXPLAIN QUERY PLAN + SELECT * FROM t400, t401, t402 WHERE t402.z GLOB 'abc*'; + } +} {0 2 {TABLE t402} 1 0 {TABLE t400} 2 1 {TABLE t401}} +do_test where3-4.1 { + execsql { + EXPLAIN QUERY PLAN + SELECT * FROM t400, t401, t402 WHERE t401.r GLOB 'abc*'; + } +} {0 1 {TABLE t401} 1 0 {TABLE t400} 2 2 {TABLE t402}} +do_test where3-4.2 { + execsql { + EXPLAIN QUERY PLAN + SELECT * FROM t400, t401, t402 WHERE t400.c GLOB 'abc*'; + } +} {0 0 {TABLE t400} 1 1 {TABLE t401} 2 2 {TABLE t402}} finish_test diff --git a/tool/shell1.test b/tool/shell1.test index dcdac75..f21601a 100644 --- a/tool/shell1.test +++ b/tool/shell1.test @@ -178,6 +178,11 @@ do_test shell1-1.14.3 { [regexp {Error: missing argument for option: -separator} $res] } {1 1} +# -stats print memory stats before each finalize +do_test shell1-1.14b.1 { + catchcmd "-stats test.db" "" +} {0 {}} + # -nullvalue 'text' set text string for NULL values do_test shell1-1.15.1 { catchcmd "-nullvalue 'x' test.db" "" @@ -195,7 +200,7 @@ do_test shell1-1.15.3 { # -version show SQLite version do_test shell1-1.16.1 { catchcmd "-version test.db" "" -} {0 3.7.0} +} {0 3.7.1} #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. @@ -621,13 +626,29 @@ do_test shell1-3.23.1 { [regexp {nullvalue:} $res] \ [regexp {output:} $res] \ [regexp {separator:} $res] \ + [regexp {stats:} $res] \ [regexp {width:} $res] -} {1 1 1 1 1 1 1 1} +} {1 1 1 1 1 1 1 1 1} do_test shell1-3.23.2 { # too many arguments catchcmd "test.db" ".show BAD" } {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} +# .stats ON|OFF Turn stats on or off +do_test shell1-3.23b.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell1-3.23b.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell1-3.23b.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell1-3.23b.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + # .tables ?TABLE? List names of tables # If TABLE specified, only list tables matching # LIKE pattern TABLE. diff --git a/tool/shell4.test b/tool/shell4.test new file mode 100644 index 0000000..085c279 --- /dev/null +++ b/tool/shell4.test @@ -0,0 +1,129 @@ +# 2010 July 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .stats command. +# +# $Id: shell4.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell4-1.*: Basic tests specific to the "stats" command. +# + +set CLI "./sqlite3" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal + +#---------------------------------------------------------------------------- +# Test cases shell4-1.*: Tests specific to the "stats" command. +# + +# should default to off +do_test shell4-1.1.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.1.2 { + set res [catchcmd "test.db" ".show"] + list [regexp {stats: on} $res] +} {0} + +# -stats should turn it on +do_test shell4-1.2.1 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.2.2 { + set res [catchcmd "-stats test.db" ".show"] + list [regexp {stats: off} $res] +} {0} + +# .stats ON|OFF Turn stats on or off +do_test shell4-1.3.1 { + catchcmd "test.db" ".stats" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} +do_test shell4-1.3.2 { + catchcmd "test.db" ".stats ON" +} {0 {}} +do_test shell4-1.3.3 { + catchcmd "test.db" ".stats OFF" +} {0 {}} +do_test shell4-1.3.4 { + # too many arguments + catchcmd "test.db" ".stats OFF BAD" +} {1 {Error: unknown command or invalid arguments: "stats". Enter ".help" for help}} + +# NB. whitespace is important +do_test shell4-1.4.1 { + set res [catchcmd "test.db" {.show}] + list [regexp {stats: off} $res] +} {1} + +do_test shell4-1.4.2 { + set res [catchcmd "test.db" {.stats ON +.show +}] + list [regexp {stats: on} $res] +} {1} + +do_test shell4-1.4.3 { + set res [catchcmd "test.db" {.stats OFF +.show +}] + list [regexp {stats: off} $res] +} {1} + +# make sure stats not present when off +do_test shell4-1.5.1 { + set res [catchcmd "test.db" {SELECT 1;}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {0 0 0} + +# make sure stats are present when on +do_test shell4-1.5.2 { + set res [catchcmd "test.db" {.stats ON +SELECT 1; +}] + list [regexp {Memory Used} $res] \ + [regexp {Heap Usage} $res] \ + [regexp {Autoindex Inserts} $res] +} {1 1 1} + +puts "CLI tests completed successfully" diff --git a/tool/shell5.test b/tool/shell5.test new file mode 100644 index 0000000..a82f979 --- /dev/null +++ b/tool/shell5.test @@ -0,0 +1,243 @@ +# 2010 August 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# These tests are specific to the .import command. +# +# $Id: shell5.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell5-1.*: Basic tests specific to the ".import" command. +# + +set CLI "./sqlite3" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc catchcmd {db {cmd ""}} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal + +#---------------------------------------------------------------------------- +# Test cases shell5-1.*: Basic handling of the .import and .separator commands. +# + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell5-1.1.1 { + catchcmd "test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell5-1.1.2 { + catchcmd "test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell5-1.1.3 { + # too many arguments + catchcmd "test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-1.2.1 { + catchcmd "test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-1.2.2 { + catchcmd "test.db" ".separator FOO" +} {0 {}} +do_test shell1-1.2.3 { + # too many arguments + catchcmd "test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# separator should default to "|" +do_test shell5-1.3.1 { + set res [catchcmd "test.db" ".show"] + list [regexp {separator: \"\|\"} $res] +} {1} + +# set separator to different value. +# check that .show reports new value +do_test shell5-1.3.2 { + set res [catchcmd "test.db" {.separator , +.show}] + list [regexp {separator: \",\"} $res] +} {1} + +# import file doesn't exist +do_test shell5-1.4.1 { + file delete -force FOO + set res [catchcmd "test.db" {CREATE TABLE t1(a, b); +.import FOO t1}] +} {1 {Error: cannot open "FOO"}} + +# empty import file +do_test shell5-1.4.2 { + file delete -force shell5.csv + set in [open shell5.csv w] + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 0} + +# import file with 1 row, 1 column (expecting 2 cols) +do_test shell5-1.4.3 { + set in [open shell5.csv w] + puts $in "1" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}} + +# import file with 1 row, 3 columns (expecting 2 cols) +do_test shell5-1.4.4 { + set in [open shell5.csv w] + puts $in "1|2|3" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1}] +} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}} + +# import file with 1 row, 2 columns +do_test shell5-1.4.5 { + set in [open shell5.csv w] + puts $in "1|2" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 1} + +# import file with 2 rows, 2 columns +# note we end up with 3 rows because of the 1 row +# imported above. +do_test shell5-1.4.6 { + set in [open shell5.csv w] + puts $in "2|3" + puts $in "3|4" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 3} + +# import file with 1 row, 2 columns, using a comma +do_test shell5-1.4.7 { + set in [open shell5.csv w] + puts $in "4,5" + close $in + set res [catchcmd "test.db" {.separator , +.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 4} + +# import file with 1 row, 2 columns, text data +do_test shell5-1.4.8.1 { + set in [open shell5.csv w] + puts $in "5|Now is the time for all good men to come to the aid of their country." + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 5} + +do_test shell5-1.4.8.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} +} {0 {Now is the time for all good men to come to the aid of their country.}} + +# import file with 1 row, 2 columns, quoted text data +# note that currently sqlite doesn't support quoted fields, and +# imports the entire field, quotes and all. +do_test shell5-1.4.9.1 { + set in [open shell5.csv w] + puts $in "6|'Now is the time for all good men to come to the aid of their country.'" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 6} + +do_test shell5-1.4.9.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} +} {0 {'Now is the time for all good men to come to the aid of their country.'}} + +# import file with 1 row, 2 columns, quoted text data +do_test shell5-1.4.10.1 { + set in [open shell5.csv w] + puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT COUNT(*) FROM t1;}] +} {0 7} + +do_test shell5-1.4.10.2 { + catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} +} {0 {"Now is the time for all good men to come to the aid of their country."}} + +# check importing very long field +do_test shell5-1.5.1 { + set str [string repeat X 999] + set in [open shell5.csv w] + puts $in "8|$str" + close $in + set res [catchcmd "test.db" {.import shell5.csv t1 +SELECT length(b) FROM t1 WHERE a='8';}] +} {0 999} + +# try importing into a table with a large number of columns. +# This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999. +set cols 999 +do_test shell5-1.6.1 { + set sql {CREATE TABLE t2(} + set data {} + for {set i 1} {$i<$cols} {incr i} { + append sql "c$i," + append data "$i|" + } + append sql "c$cols);" + append data "$cols" + catchcmd "test.db" $sql + set in [open shell5.csv w] + puts $in $data + close $in + set res [catchcmd "test.db" {.import shell5.csv t2 +SELECT COUNT(*) FROM t2;}] +} {0 1} + +# try importing a large number of rows +set rows 999999 +do_test shell5-1.7.1 { + set in [open shell5.csv w] + for {set i 1} {$i<=$rows} {incr i} { + puts $in $i + } + close $in + set res [catchcmd "test.db" {CREATE TABLE t3(a); +.import shell5.csv t3 +SELECT COUNT(*) FROM t3;}] +} [list 0 $rows] + + +puts "CLI tests completed successfully" diff --git a/tool/showdb.c b/tool/showdb.c index a54eea8..f313b1a 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -35,6 +35,13 @@ static int decodeVarint(const unsigned char *z, i64 *pVal){ return 9; } +/* +** Extract a big-endian 32-bit integer +*/ +static unsigned int decodeInt32(const unsigned char *z){ + return (z[0]<<24) + (z[1]<<16) + (z[2]<<8) + z[3]; +} + /* Report an out-of-memory error and die. */ static void out_of_memory(void){ @@ -246,11 +253,64 @@ static void decode_btree_page(unsigned char *a, int pgno, int hdrSize){ } } +/* +** Decode a freelist trunk page. +*/ +static void decode_trunk_page( + int pgno, /* The page number */ + int pagesize, /* Size of each page */ + int detail, /* Show leaf pages if true */ + int recursive /* Follow the trunk change if true */ +){ + int n, i, k; + unsigned char *a; + while( pgno>0 ){ + a = getContent((pgno-1)*pagesize, pagesize); + printf("Decode of freelist trunk page %d:\n", pgno); + print_decode_line(a, 0, 4, "Next freelist trunk page"); + print_decode_line(a, 4, 4, "Number of entries on this page"); + if( detail ){ + n = (int)decodeInt32(&a[4]); + for(i=0; i