From 090d3d3b9b924410a1a0584db01180d25006324c Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Sat, 19 Dec 2009 15:56:40 -0500 Subject: [PATCH] update to 3.6.21 --- Makefile.in | 23 +- VERSION | 2 +- configure | 18 +- ext/async/sqlite3async.c | 2 +- ext/fts3/fts3.c | 7909 ++++++++---------------------------- ext/fts3/fts3Int.h | 279 ++ ext/fts3/fts3_expr.c | 51 +- ext/fts3/fts3_expr.h | 96 - ext/fts3/fts3_hash.c | 60 +- ext/fts3/fts3_hash.h | 26 +- ext/fts3/fts3_porter.c | 27 +- ext/fts3/fts3_snippet.c | 734 ++++ ext/fts3/fts3_tokenizer.c | 153 +- ext/fts3/fts3_tokenizer1.c | 12 +- ext/fts3/fts3_write.c | 2171 ++++++++++ ext/rtree/rtree.c | 2 - ext/rtree/rtree1.test | 2 - ext/rtree/rtree2.test | 2 - ext/rtree/rtree3.test | 3 - ext/rtree/rtree4.test | 2 - ext/rtree/rtree5.test | 2 - ext/rtree/rtree6.test | 4 +- ext/rtree/rtree_perf.tcl | 2 - ext/rtree/rtree_util.tcl | 3 - ext/rtree/tkt3363.test | 4 - ext/rtree/viewrtree.tcl | 1 - main.mk | 20 +- manifest | 349 +- manifest.uuid | 2 +- mkopcodeh.awk | 11 +- src/alter.c | 2 - src/analyze.c | 2 - src/attach.c | 2 - src/auth.c | 2 - src/backup.c | 2 - src/bitvec.c | 2 - src/btmutex.c | 2 - src/btree.c | 70 +- src/btree.h | 3 +- src/btreeInt.h | 2 - src/build.c | 2 - src/callback.c | 2 - src/complete.c | 9 +- src/date.c | 2 - src/delete.c | 7 +- src/expr.c | 491 ++- src/fault.c | 4 - src/fkey.c | 2 +- src/func.c | 34 +- src/global.c | 52 +- src/hash.c | 2 - src/hash.h | 2 - src/hwtime.h | 2 - src/insert.c | 2 - src/journal.c | 8 +- src/legacy.c | 2 - src/loadext.c | 2 - src/malloc.c | 2 - src/mem0.c | 2 - src/mem1.c | 2 - src/mem2.c | 2 - src/mem3.c | 2 - src/memjournal.c | 2 - src/mutex.c | 3 - src/mutex.h | 2 - src/mutex_noop.c | 2 - src/mutex_os2.c | 68 +- src/mutex_unix.c | 2 - src/mutex_w32.c | 2 - src/notify.c | 2 - src/os.c | 3 +- src/os.h | 2 - src/os_common.h | 2 - src/os_os2.c | 2 - src/os_unix.c | 47 +- src/pager.c | 83 +- src/pager.h | 2 - src/parse.y | 23 +- src/parse.y.ex1 | 1346 ------ src/pcache.c | 2 - src/pcache.h | 2 - src/pcache1.c | 12 +- src/pragma.c | 7 +- src/prepare.c | 2 - src/printf.c | 29 +- src/random.c | 2 - src/resolve.c | 14 +- src/rowset.c | 2 - src/select.c | 30 +- src/shell.c | 104 +- src/sqlite.h.in | 17 +- src/sqlite3ext.h | 2 - src/sqliteInt.h | 38 +- src/sqliteLimit.h | 6 - src/status.c | 2 - src/table.c | 2 - src/tclsqlite.c | 21 +- src/test1.c | 2 - src/test2.c | 2 - src/test3.c | 2 - src/test4.c | 2 - src/test5.c | 2 - src/test6.c | 2 - src/test7.c | 2 - src/test8.c | 2 - src/test9.c | 2 - src/test_async.c | 2 - src/test_autoext.c | 2 - src/test_backup.c | 2 +- src/test_btree.c | 2 - src/test_config.c | 2 - src/test_devsym.c | 2 - src/test_func.c | 2 - src/test_hexio.c | 45 +- src/test_init.c | 1 - src/test_intarray.c | 381 ++ src/test_intarray.h | 114 + src/test_journal.c | 2 - src/test_loadext.c | 2 - src/test_malloc.c | 2 - src/test_mutex.c | 3 +- src/test_onefile.c | 2 - src/test_osinst.c | 2 - src/test_pcache.c | 2 - src/test_schema.c | 2 - src/test_server.c | 2 - src/test_tclvar.c | 11 +- src/test_thread.c | 2 - src/test_wsd.c | 2 - src/tokenize.c | 13 +- src/trigger.c | 47 +- src/update.c | 67 +- src/utf.c | 2 - src/util.c | 2188 +++++----- src/vacuum.c | 2 - src/vdbe.c | 508 +-- src/vdbe.h | 8 +- src/vdbeInt.h | 3 - src/vdbeapi.c | 10 +- src/vdbeaux.c | 118 +- src/vdbeblob.c | 2 - src/vdbemem.c | 7 +- src/vdbetrace.c | 142 + src/vtab.c | 2 - src/walker.c | 2 - src/where.c | 128 +- test/analyze3.test | 1 - test/attach.test | 27 + test/coalesce.test | 84 + test/corrupt.test | 54 + test/e_fkey.test | 538 +-- test/e_fts3.test | 469 +++ test/fts3.test | 2 + test/fts3_common.tcl | 413 ++ test/fts3aa.test | 2 + test/fts3ab.test | 1 + test/fts3ad.test | 41 + test/fts3ae.test | 2 +- test/fts3ag.test | 1 + test/fts3an.test | 2 +- test/fts3b.test | 18 +- test/fts3c.test | 116 +- test/fts3d.test | 55 +- test/fts3malloc.test | 293 ++ test/fts3rnd.test | 292 ++ test/func2.test | 511 +++ test/intarray.test | 109 + test/printf.test | 37 + test/quick.test | 2 + test/tester.tcl | 2 +- test/tkt-94c04eaadb.test | 1 + test/trace.test | 66 + test/triggerC.test | 70 +- test/vtabE.test | 48 + test/where8.test | 1 + tool/mksqlite3c.tcl | 5 +- tool/shell1.test | 707 ++++ tool/shell2.test | 85 + 178 files changed, 11938 insertions(+), 10556 deletions(-) create mode 100644 ext/fts3/fts3Int.h delete mode 100644 ext/fts3/fts3_expr.h create mode 100644 ext/fts3/fts3_snippet.c create mode 100644 ext/fts3/fts3_write.c delete mode 100644 src/parse.y.ex1 create mode 100644 src/test_intarray.c create mode 100644 src/test_intarray.h create mode 100644 src/vdbetrace.c create mode 100644 test/coalesce.test create mode 100644 test/e_fts3.test create mode 100644 test/fts3_common.tcl create mode 100644 test/fts3malloc.test create mode 100644 test/fts3rnd.test create mode 100644 test/func2.test create mode 100644 test/intarray.test create mode 100644 test/vtabE.test create mode 100644 tool/shell1.test create mode 100644 tool/shell2.test diff --git a/Makefile.in b/Makefile.in index 27687f9..3bc5d8f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -175,7 +175,7 @@ OBJS0 = alter.lo analyze.lo attach.lo auth.lo backup.lo bitvec.lo btmutex.lo \ random.lo resolve.lo rowset.lo select.lo status.lo \ table.lo tokenize.lo trigger.lo update.lo \ util.lo vacuum.lo \ - vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo \ + vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \ walker.lo where.lo utf.lo vtab.lo # Object files for the amalgamation. @@ -272,6 +272,7 @@ SRC = \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbeblob.c \ $(TOP)/src/vdbemem.c \ + $(TOP)/src/vdbetrace.c \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vtab.c \ $(TOP)/src/walker.c \ @@ -311,15 +312,17 @@ SRC += \ SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ + $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_expr.h \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ + $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_tokenizer1.c + $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_write.c SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c @@ -359,6 +362,7 @@ TESTSRC2 = \ $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbemem.c \ + $(TOP)/src/vdbetrace.c \ $(TOP)/src/where.c \ parse.c @@ -383,6 +387,7 @@ TESTSRC = \ $(TOP)/src/test_func.c \ $(TOP)/src/test_hexio.c \ $(TOP)/src/test_init.c \ + $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ $(TOP)/src/test_mutex.c \ @@ -427,7 +432,7 @@ HDR += \ $(TOP)/ext/fts2/fts2_tokenizer.h HDR += \ $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3_expr.h \ + $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_tokenizer.h HDR += \ @@ -456,13 +461,6 @@ Makefile: $(TOP)/Makefile.in sqlite3.pc: $(TOP)/sqlite3.pc.in ./config.status -# Generate the file "last_change" which contains the date of change -# of the most recently modified source code file -# -last_change: $(SRC) - cat $(SRC) | grep '$$Id: ' | sort -k 5 | tail -1 \ - | $(NAWK) '{print $$5,$$6}' >last_change - libsqlite3.la: $(LIBOBJ) $(LTLINK) -o $@ $(LIBOBJ) $(TLIBS) \ ${ALLOWRELEASE} -rpath "$(libdir)" -version-info "8:6:8" @@ -733,6 +731,9 @@ vdbeblob.lo: $(TOP)/src/vdbeblob.c $(HDR) vdbemem.lo: $(TOP)/src/vdbemem.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbemem.c +vdbetrace.lo: $(TOP)/src/vdbetrace.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbetrace.c + vtab.lo: $(TOP)/src/vtab.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vtab.c diff --git a/VERSION b/VERSION index f9fb0e4..1bff231 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.20 +3.6.21 diff --git a/configure b/configure index 08c383b..b922d17 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.6.20. +# Generated by GNU Autoconf 2.62 for sqlite 3.6.21. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.6.20' -PACKAGE_STRING='sqlite 3.6.20' +PACKAGE_VERSION='3.6.21' +PACKAGE_STRING='sqlite 3.6.21' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1487,7 +1487,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.6.20 to adapt to many kinds of systems. +\`configure' configures sqlite 3.6.21 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1552,7 +1552,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.6.20:";; + short | recursive ) echo "Configuration of sqlite 3.6.21:";; esac cat <<\_ACEOF @@ -1670,7 +1670,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.6.20 +sqlite configure 3.6.21 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1684,7 +1684,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.6.20, which was +It was created by sqlite $as_me 3.6.21, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -13972,7 +13972,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.6.20, which was +This file was extended by sqlite $as_me 3.6.21, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14025,7 +14025,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.6.20 +sqlite config.status 3.6.21 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/ext/async/sqlite3async.c b/ext/async/sqlite3async.c index 0cfc7f6..127942b 100644 --- a/ext/async/sqlite3async.c +++ b/ext/async/sqlite3async.c @@ -1231,8 +1231,8 @@ static int asyncFullPathname( */ if( rc==SQLITE_OK ){ int i, j; - int n = nPathOut; char *z = zPathOut; + int n = strlen(z); while( n>1 && z[n-1]=='/' ){ n--; } for(i=j=0; i #include +#include #include #include -#include #include "fts3.h" -#include "fts3_expr.h" -#include "fts3_hash.h" -#include "fts3_tokenizer.h" #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif + /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it ** would be nice to order the file better, perhaps something along the ** lines of: @@ -313,25 +313,6 @@ # define FTSTRACE(A) #endif -/* It is not safe to call isspace(), tolower(), or isalnum() on -** hi-bit-set characters. This is the same solution used in the -** tokenizer. -*/ -/* TODO(shess) The snippet-generation code should be using the -** tokenizer-generated tokens rather than doing its own local -** tokenization. -*/ -/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ -static int safe_isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; -} -static int safe_tolower(char c){ - return (c&0x80)==0 ? tolower(c) : c; -} -static int safe_isalnum(char c){ - return (c&0x80)==0 ? isalnum(c) : 0; -} - typedef enum DocListType { DL_DOCIDS, /* docids only */ DL_POSITIONS, /* docids + positions */ @@ -357,11 +338,6 @@ enum { POS_BASE }; -/* MERGE_COUNT controls how often we merge segments (see comment at -** top of file). -*/ -#define MERGE_COUNT 16 - /* utility functions */ /* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single @@ -380,13 +356,12 @@ enum { # define SCRAMBLE(b) #endif -/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */ -#define VARINT_MAX 10 - -/* Write a 64-bit variable-length integer to memory starting at p[0]. - * The length of data written will be between 1 and VARINT_MAX bytes. - * The number of bytes written is returned. */ -static int fts3PutVarint(char *p, sqlite_int64 v){ +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; sqlite_uint64 vu = v; do{ @@ -394,20 +369,22 @@ static int fts3PutVarint(char *p, sqlite_int64 v){ vu >>= 7; }while( vu!=0 ); q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= VARINT_MAX ); + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); return (int) (q - (unsigned char *)p); } -/* Read a 64-bit variable-length integer from memory starting at p[0]. - * Return the number of bytes read, or 0 on error. - * The value is stored in *v. */ -static int fts3GetVarint(const char *p, sqlite_int64 *v){ +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; while( (*q & 0x80) == 0x80 ){ x += y * (*q++ & 0x7f); y <<= 7; - if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ + if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){ /* bad data */ assert( 0 ); return 0; } @@ -417,2184 +394,29 @@ static int fts3GetVarint(const char *p, sqlite_int64 *v){ return (int) (q - (unsigned char *)p); } -static int fts3GetVarint32(const char *p, int *pi){ +/* +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. +*/ +int sqlite3Fts3GetVarint32(const char *p, int *pi){ sqlite_int64 i; - int ret = fts3GetVarint(p, &i); + int ret = sqlite3Fts3GetVarint(p, &i); *pi = (int) i; assert( *pi==i ); return ret; } -/*******************************************************************/ -/* DataBuffer is used to collect data into a buffer in piecemeal -** fashion. It implements the usual distinction between amount of -** data currently stored (nData) and buffer capacity (nCapacity). -** -** dataBufferInit - create a buffer with given initial capacity. -** dataBufferReset - forget buffer's data, retaining capacity. -** dataBufferDestroy - free buffer's data. -** dataBufferSwap - swap contents of two buffers. -** dataBufferExpand - expand capacity without adding data. -** dataBufferAppend - append data. -** dataBufferAppend2 - append two pieces of data at once. -** dataBufferReplace - replace buffer's data. -*/ -typedef struct DataBuffer { - char *pData; /* Pointer to malloc'ed buffer. */ - int nCapacity; /* Size of pData buffer. */ - int nData; /* End of data loaded into pData. */ -} DataBuffer; - -static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ - assert( nCapacity>=0 ); - pBuffer->nData = 0; - pBuffer->nCapacity = nCapacity; - pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); -} -static void dataBufferReset(DataBuffer *pBuffer){ - pBuffer->nData = 0; -} -static void dataBufferDestroy(DataBuffer *pBuffer){ - if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData); - SCRAMBLE(pBuffer); -} -static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){ - DataBuffer tmp = *pBuffer1; - *pBuffer1 = *pBuffer2; - *pBuffer2 = tmp; -} -static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ - assert( nAddCapacity>0 ); - /* TODO(shess) Consider expanding more aggressively. Note that the - ** underlying malloc implementation may take care of such things for - ** us already. - */ - if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ - pBuffer->nCapacity = pBuffer->nData+nAddCapacity; - pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); - } -} -static void dataBufferAppend(DataBuffer *pBuffer, - const char *pSource, int nSource){ - assert( nSource>0 && pSource!=NULL ); - dataBufferExpand(pBuffer, nSource); - memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource); - pBuffer->nData += nSource; -} -static void dataBufferAppend2(DataBuffer *pBuffer, - const char *pSource1, int nSource1, - const char *pSource2, int nSource2){ - assert( nSource1>0 && pSource1!=NULL ); - assert( nSource2>0 && pSource2!=NULL ); - dataBufferExpand(pBuffer, nSource1+nSource2); - memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1); - memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2); - pBuffer->nData += nSource1+nSource2; -} -static void dataBufferReplace(DataBuffer *pBuffer, - const char *pSource, int nSource){ - dataBufferReset(pBuffer); - dataBufferAppend(pBuffer, pSource, nSource); -} - -/* StringBuffer is a null-terminated version of DataBuffer. */ -typedef struct StringBuffer { - DataBuffer b; /* Includes null terminator. */ -} StringBuffer; - -static void initStringBuffer(StringBuffer *sb){ - dataBufferInit(&sb->b, 100); - dataBufferReplace(&sb->b, "", 1); -} -static int stringBufferLength(StringBuffer *sb){ - return sb->b.nData-1; -} -static char *stringBufferData(StringBuffer *sb){ - return sb->b.pData; -} -static void stringBufferDestroy(StringBuffer *sb){ - dataBufferDestroy(&sb->b); -} - -static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){ - assert( sb->b.nData>0 ); - if( nFrom>0 ){ - sb->b.nData--; - dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1); - } -} -static void append(StringBuffer *sb, const char *zFrom){ - nappend(sb, zFrom, strlen(zFrom)); -} - -/* Append a list of strings separated by commas. */ -static void appendList(StringBuffer *sb, int nString, char **azString){ - int i; - for(i=0; i0 ) append(sb, ", "); - append(sb, azString[i]); - } -} - -static int endsInWhiteSpace(StringBuffer *p){ - return stringBufferLength(p)>0 && - safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]); -} - -/* If the StringBuffer ends in something other than white space, add a -** single space character to the end. -*/ -static void appendWhiteSpace(StringBuffer *p){ - if( stringBufferLength(p)==0 ) return; - if( !endsInWhiteSpace(p) ) append(p, " "); -} - -/* Remove white space from the end of the StringBuffer */ -static void trimWhiteSpace(StringBuffer *p){ - while( endsInWhiteSpace(p) ){ - p->b.pData[--p->b.nData-1] = '\0'; - } -} - -/*******************************************************************/ -/* DLReader is used to read document elements from a doclist. The -** current docid is cached, so dlrDocid() is fast. DLReader does not -** own the doclist buffer. -** -** dlrAtEnd - true if there's no more data to read. -** dlrDocid - docid of current document. -** dlrDocData - doclist data for current document (including docid). -** dlrDocDataBytes - length of same. -** dlrAllDataBytes - length of all remaining data. -** dlrPosData - position data for current document. -** dlrPosDataLen - length of pos data for current document (incl POS_END). -** dlrStep - step to current document. -** dlrInit - initial for doclist of given type against given data. -** dlrDestroy - clean up. -** -** Expected usage is something like: -** -** DLReader reader; -** dlrInit(&reader, pData, nData); -** while( !dlrAtEnd(&reader) ){ -** // calls to dlrDocid() and kin. -** dlrStep(&reader); -** } -** dlrDestroy(&reader); -*/ -typedef struct DLReader { - DocListType iType; - const char *pData; - int nData; - - sqlite_int64 iDocid; - int nElement; -} DLReader; - -static int dlrAtEnd(DLReader *pReader){ - assert( pReader->nData>=0 ); - return pReader->nData==0; -} -static sqlite_int64 dlrDocid(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->iDocid; -} -static const char *dlrDocData(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->pData; -} -static int dlrDocDataBytes(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->nElement; -} -static int dlrAllDataBytes(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->nData; -} -/* TODO(shess) Consider adding a field to track iDocid varint length -** to make these two functions faster. This might matter (a tiny bit) -** for queries. -*/ -static const char *dlrPosData(DLReader *pReader){ - sqlite_int64 iDummy; - int n = fts3GetVarint(pReader->pData, &iDummy); - assert( !dlrAtEnd(pReader) ); - return pReader->pData+n; -} -static int dlrPosDataLen(DLReader *pReader){ - sqlite_int64 iDummy; - int n = fts3GetVarint(pReader->pData, &iDummy); - assert( !dlrAtEnd(pReader) ); - return pReader->nElement-n; -} -static void dlrStep(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - - /* Skip past current doclist element. */ - assert( pReader->nElement<=pReader->nData ); - pReader->pData += pReader->nElement; - pReader->nData -= pReader->nElement; - - /* If there is more data, read the next doclist element. */ - if( pReader->nData!=0 ){ - sqlite_int64 iDocidDelta; - int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta); - pReader->iDocid += iDocidDelta; - if( pReader->iType>=DL_POSITIONS ){ - assert( nnData ); - while( 1 ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( n<=pReader->nData ); - if( iDummy==POS_END ) break; - if( iDummy==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( nnData ); - }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( nnData ); - } - } - } - pReader->nElement = n; - assert( pReader->nElement<=pReader->nData ); - } -} -static void dlrInit(DLReader *pReader, DocListType iType, - const char *pData, int nData){ - assert( pData!=NULL && nData!=0 ); - pReader->iType = iType; - pReader->pData = pData; - pReader->nData = nData; - pReader->nElement = 0; - pReader->iDocid = 0; - - /* Load the first element's data. There must be a first element. */ - dlrStep(pReader); -} -static void dlrDestroy(DLReader *pReader){ - SCRAMBLE(pReader); -} - -#ifndef NDEBUG -/* Verify that the doclist can be validly decoded. Also returns the -** last docid found because it is convenient in other assertions for -** DLWriter. -*/ -static void docListValidate(DocListType iType, const char *pData, int nData, - sqlite_int64 *pLastDocid){ - sqlite_int64 iPrevDocid = 0; - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - while( nData!=0 ){ - sqlite_int64 iDocidDelta; - int n = fts3GetVarint(pData, &iDocidDelta); - iPrevDocid += iDocidDelta; - if( iType>DL_DOCIDS ){ - int iDummy; - while( 1 ){ - n += fts3GetVarint32(pData+n, &iDummy); - if( iDummy==POS_END ) break; - if( iDummy==POS_COLUMN ){ - n += fts3GetVarint32(pData+n, &iDummy); - }else if( iType>DL_POSITIONS ){ - n += fts3GetVarint32(pData+n, &iDummy); - n += fts3GetVarint32(pData+n, &iDummy); - } - assert( n<=nData ); - } - } - assert( n<=nData ); - pData += n; - nData -= n; - } - if( pLastDocid ) *pLastDocid = iPrevDocid; -} -#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o) -#else -#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 ) -#endif - -/*******************************************************************/ -/* DLWriter is used to write doclist data to a DataBuffer. DLWriter -** always appends to the buffer and does not own it. -** -** dlwInit - initialize to write a given type doclistto a buffer. -** dlwDestroy - clear the writer's memory. Does not free buffer. -** dlwAppend - append raw doclist data to buffer. -** dlwCopy - copy next doclist from reader to writer. -** dlwAdd - construct doclist element and append to buffer. -** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter). -*/ -typedef struct DLWriter { - DocListType iType; - DataBuffer *b; - sqlite_int64 iPrevDocid; -#ifndef NDEBUG - int has_iPrevDocid; -#endif -} DLWriter; - -static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){ - pWriter->b = b; - pWriter->iType = iType; - pWriter->iPrevDocid = 0; -#ifndef NDEBUG - pWriter->has_iPrevDocid = 0; -#endif -} -static void dlwDestroy(DLWriter *pWriter){ - SCRAMBLE(pWriter); -} -/* iFirstDocid is the first docid in the doclist in pData. It is -** needed because pData may point within a larger doclist, in which -** case the first item would be delta-encoded. -** -** iLastDocid is the final docid in the doclist in pData. It is -** needed to create the new iPrevDocid for future delta-encoding. The -** code could decode the passed doclist to recreate iLastDocid, but -** the only current user (docListMerge) already has decoded this -** information. -*/ -/* TODO(shess) This has become just a helper for docListMerge. -** Consider a refactor to make this cleaner. -*/ -static void dlwAppend(DLWriter *pWriter, - const char *pData, int nData, - sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ - sqlite_int64 iDocid = 0; - char c[VARINT_MAX]; - int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */ -#ifndef NDEBUG - sqlite_int64 iLastDocidDelta; -#endif - - /* Recode the initial docid as delta from iPrevDocid. */ - nFirstOld = fts3GetVarint(pData, &iDocid); - assert( nFirstOldiType==DL_DOCIDS) ); - nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid); - - /* Verify that the incoming doclist is valid AND that it ends with - ** the expected docid. This is essential because we'll trust this - ** docid in future delta-encoding. - */ - ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta); - assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta ); - - /* Append recoded initial docid and everything else. Rest of docids - ** should have been delta-encoded from previous initial docid. - */ - if( nFirstOldb, c, nFirstNew, - pData+nFirstOld, nData-nFirstOld); - }else{ - dataBufferAppend(pWriter->b, c, nFirstNew); - } - pWriter->iPrevDocid = iLastDocid; -} -static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ - dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), - dlrDocid(pReader), dlrDocid(pReader)); -} -static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid); - - /* Docids must ascend. */ - assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid ); - assert( pWriter->iType==DL_DOCIDS ); - - dataBufferAppend(pWriter->b, c, n); - pWriter->iPrevDocid = iDocid; -#ifndef NDEBUG - pWriter->has_iPrevDocid = 1; -#endif -} - -/*******************************************************************/ -/* PLReader is used to read data from a document's position list. As -** the caller steps through the list, data is cached so that varints -** only need to be decoded once. -** -** plrInit, plrDestroy - create/destroy a reader. -** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors -** plrAtEnd - at end of stream, only call plrDestroy once true. -** plrStep - step to the next element. -*/ -typedef struct PLReader { - /* These refer to the next position's data. nData will reach 0 when - ** reading the last position, so plrStep() signals EOF by setting - ** pData to NULL. - */ - const char *pData; - int nData; - - DocListType iType; - int iColumn; /* the last column read */ - int iPosition; /* the last position read */ - int iStartOffset; /* the last start offset read */ - int iEndOffset; /* the last end offset read */ -} PLReader; - -static int plrAtEnd(PLReader *pReader){ - return pReader->pData==NULL; -} -static int plrColumn(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iColumn; -} -static int plrPosition(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iPosition; -} -static int plrStartOffset(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iStartOffset; -} -static int plrEndOffset(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iEndOffset; -} -static void plrStep(PLReader *pReader){ - int i, n; - - assert( !plrAtEnd(pReader) ); - - if( pReader->nData==0 ){ - pReader->pData = NULL; - return; - } - - n = fts3GetVarint32(pReader->pData, &i); - if( i==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn); - pReader->iPosition = 0; - pReader->iStartOffset = 0; - n += fts3GetVarint32(pReader->pData+n, &i); - } - /* Should never see adjacent column changes. */ - assert( i!=POS_COLUMN ); - - if( i==POS_END ){ - pReader->nData = 0; - pReader->pData = NULL; - return; - } - - pReader->iPosition += i-POS_BASE; - if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &i); - pReader->iStartOffset += i; - n += fts3GetVarint32(pReader->pData+n, &i); - pReader->iEndOffset = pReader->iStartOffset+i; - } - assert( n<=pReader->nData ); - pReader->pData += n; - pReader->nData -= n; -} - -static void plrInit(PLReader *pReader, DLReader *pDLReader){ - pReader->pData = dlrPosData(pDLReader); - pReader->nData = dlrPosDataLen(pDLReader); - pReader->iType = pDLReader->iType; - pReader->iColumn = 0; - pReader->iPosition = 0; - pReader->iStartOffset = 0; - pReader->iEndOffset = 0; - plrStep(pReader); -} -static void plrDestroy(PLReader *pReader){ - SCRAMBLE(pReader); -} - -/*******************************************************************/ -/* PLWriter is used in constructing a document's position list. As a -** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op. -** PLWriter writes to the associated DLWriter's buffer. -** -** plwInit - init for writing a document's poslist. -** plwDestroy - clear a writer. -** plwAdd - append position and offset information. -** plwCopy - copy next position's data from reader to writer. -** plwTerminate - add any necessary doclist terminator. -** -** Calling plwAdd() after plwTerminate() may result in a corrupt -** doclist. -*/ -/* TODO(shess) Until we've written the second item, we can cache the -** first item's information. Then we'd have three states: -** -** - initialized with docid, no positions. -** - docid and one position. -** - docid and multiple positions. -** -** Only the last state needs to actually write to dlw->b, which would -** be an improvement in the DLCollector case. -*/ -typedef struct PLWriter { - DLWriter *dlw; - - int iColumn; /* the last column written */ - int iPos; /* the last position written */ - int iOffset; /* the last start offset written */ -} PLWriter; - -/* TODO(shess) In the case where the parent is reading these values -** from a PLReader, we could optimize to a copy if that PLReader has -** the same type as pWriter. -*/ -static void plwAdd(PLWriter *pWriter, int iColumn, int iPos, - int iStartOffset, int iEndOffset){ - /* Worst-case space for POS_COLUMN, iColumn, iPosDelta, - ** iStartOffsetDelta, and iEndOffsetDelta. - */ - char c[5*VARINT_MAX]; - int n = 0; - - /* Ban plwAdd() after plwTerminate(). */ - assert( pWriter->iPos!=-1 ); - - if( pWriter->dlw->iType==DL_DOCIDS ) return; - - if( iColumn!=pWriter->iColumn ){ - n += fts3PutVarint(c+n, POS_COLUMN); - n += fts3PutVarint(c+n, iColumn); - pWriter->iColumn = iColumn; - pWriter->iPos = 0; - pWriter->iOffset = 0; - } - assert( iPos>=pWriter->iPos ); - n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos)); - pWriter->iPos = iPos; - if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){ - assert( iStartOffset>=pWriter->iOffset ); - n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset); - pWriter->iOffset = iStartOffset; - assert( iEndOffset>=iStartOffset ); - n += fts3PutVarint(c+n, iEndOffset-iStartOffset); - } - dataBufferAppend(pWriter->dlw->b, c, n); -} -static void plwCopy(PLWriter *pWriter, PLReader *pReader){ - plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader), - plrStartOffset(pReader), plrEndOffset(pReader)); -} -static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){ - char c[VARINT_MAX]; - int n; - - pWriter->dlw = dlw; - - /* Docids must ascend. */ - assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid ); - n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid); - dataBufferAppend(pWriter->dlw->b, c, n); - pWriter->dlw->iPrevDocid = iDocid; -#ifndef NDEBUG - pWriter->dlw->has_iPrevDocid = 1; -#endif - - pWriter->iColumn = 0; - pWriter->iPos = 0; - pWriter->iOffset = 0; -} -/* TODO(shess) Should plwDestroy() also terminate the doclist? But -** then plwDestroy() would no longer be just a destructor, it would -** also be doing work, which isn't consistent with the overall idiom. -** Another option would be for plwAdd() to always append any necessary -** terminator, so that the output is always correct. But that would -** add incremental work to the common case with the only benefit being -** API elegance. Punt for now. -*/ -static void plwTerminate(PLWriter *pWriter){ - if( pWriter->dlw->iType>DL_DOCIDS ){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, POS_END); - dataBufferAppend(pWriter->dlw->b, c, n); - } -#ifndef NDEBUG - /* Mark as terminated for assert in plwAdd(). */ - pWriter->iPos = -1; -#endif -} -static void plwDestroy(PLWriter *pWriter){ - SCRAMBLE(pWriter); -} - -/*******************************************************************/ -/* DLCollector wraps PLWriter and DLWriter to provide a -** dynamically-allocated doclist area to use during tokenization. -** -** dlcNew - malloc up and initialize a collector. -** dlcDelete - destroy a collector and all contained items. -** dlcAddPos - append position and offset information. -** dlcAddDoclist - add the collected doclist to the given buffer. -** dlcNext - terminate the current document and open another. -*/ -typedef struct DLCollector { - DataBuffer b; - DLWriter dlw; - PLWriter plw; -} DLCollector; - -/* TODO(shess) This could also be done by calling plwTerminate() and -** dataBufferAppend(). I tried that, expecting nominal performance -** differences, but it seemed to pretty reliably be worth 1% to code -** it this way. I suspect it is the incremental malloc overhead (some -** percentage of the plwTerminate() calls will cause a realloc), so -** this might be worth revisiting if the DataBuffer implementation -** changes. -*/ -static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){ - if( pCollector->dlw.iType>DL_DOCIDS ){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, POS_END); - dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n); - }else{ - dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData); - } -} -static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){ - plwTerminate(&pCollector->plw); - plwDestroy(&pCollector->plw); - plwInit(&pCollector->plw, &pCollector->dlw, iDocid); -} -static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos, - int iStartOffset, int iEndOffset){ - plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset); -} - -static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){ - DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector)); - dataBufferInit(&pCollector->b, 0); - dlwInit(&pCollector->dlw, iType, &pCollector->b); - plwInit(&pCollector->plw, &pCollector->dlw, iDocid); - return pCollector; -} -static void dlcDelete(DLCollector *pCollector){ - plwDestroy(&pCollector->plw); - dlwDestroy(&pCollector->dlw); - dataBufferDestroy(&pCollector->b); - SCRAMBLE(pCollector); - sqlite3_free(pCollector); -} - - -/* Copy the doclist data of iType in pData/nData into *out, trimming -** unnecessary data as we go. Only columns matching iColumn are -** copied, all columns copied if iColumn is -1. Elements with no -** matching columns are dropped. The output is an iOutType doclist. -*/ -/* NOTE(shess) This code is only valid after all doclists are merged. -** If this is run before merges, then doclist items which represent -** deletion will be trimmed, and will thus not effect a deletion -** during the merge. -*/ -static void docListTrim(DocListType iType, const char *pData, int nData, - int iColumn, DocListType iOutType, DataBuffer *out){ - DLReader dlReader; - DLWriter dlWriter; - - assert( iOutType<=iType ); - - dlrInit(&dlReader, iType, pData, nData); - dlwInit(&dlWriter, iOutType, out); - - while( !dlrAtEnd(&dlReader) ){ - PLReader plReader; - PLWriter plWriter; - int match = 0; - - plrInit(&plReader, &dlReader); - - while( !plrAtEnd(&plReader) ){ - if( iColumn==-1 || plrColumn(&plReader)==iColumn ){ - if( !match ){ - plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader)); - match = 1; - } - plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader), - plrStartOffset(&plReader), plrEndOffset(&plReader)); - } - plrStep(&plReader); - } - if( match ){ - plwTerminate(&plWriter); - plwDestroy(&plWriter); - } - - plrDestroy(&plReader); - dlrStep(&dlReader); - } - dlwDestroy(&dlWriter); - dlrDestroy(&dlReader); -} - -/* Used by docListMerge() to keep doclists in the ascending order by -** docid, then ascending order by age (so the newest comes first). -*/ -typedef struct OrderedDLReader { - DLReader *pReader; - - /* TODO(shess) If we assume that docListMerge pReaders is ordered by - ** age (which we do), then we could use pReader comparisons to break - ** ties. - */ - int idx; -} OrderedDLReader; - -/* Order eof to end, then by docid asc, idx desc. */ -static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){ - if( dlrAtEnd(r1->pReader) ){ - if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */ - return 1; /* Only r1 atEnd(). */ - } - if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */ - - if( dlrDocid(r1->pReader)pReader) ) return -1; - if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1; - - /* Descending on idx. */ - return r2->idx-r1->idx; -} - -/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that -** p[1..n-1] is already sorted. -*/ -/* TODO(shess) Is this frequent enough to warrant a binary search? -** Before implementing that, instrument the code to check. In most -** current usage, I expect that p[0] will be less than p[1] a very -** high proportion of the time. -*/ -static void orderedDLReaderReorder(OrderedDLReader *p, int n){ - while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){ - OrderedDLReader tmp = p[0]; - p[0] = p[1]; - p[1] = tmp; - n--; - p++; - } -} - -/* Given an array of doclist readers, merge their doclist elements -** into out in sorted order (by docid), dropping elements from older -** readers when there is a duplicate docid. pReaders is assumed to be -** ordered by age, oldest first. -*/ -/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably -** be fixed. -*/ -static void docListMerge(DataBuffer *out, - DLReader *pReaders, int nReaders){ - OrderedDLReader readers[MERGE_COUNT]; - DLWriter writer; - int i, n; - const char *pStart = 0; - int nStart = 0; - sqlite_int64 iFirstDocid = 0, iLastDocid = 0; - - assert( nReaders>0 ); - if( nReaders==1 ){ - dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders)); - return; - } - - assert( nReaders<=MERGE_COUNT ); - n = 0; - for(i=0; i0 ){ - orderedDLReaderReorder(readers+i, nReaders-i); - } - - dlwInit(&writer, pReaders[0].iType, out); - while( !dlrAtEnd(readers[0].pReader) ){ - sqlite_int64 iDocid = dlrDocid(readers[0].pReader); - - /* If this is a continuation of the current buffer to copy, extend - ** that buffer. memcpy() seems to be more efficient if it has a - ** lots of data to copy. - */ - if( dlrDocData(readers[0].pReader)==pStart+nStart ){ - nStart += dlrDocDataBytes(readers[0].pReader); - }else{ - if( pStart!=0 ){ - dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); - } - pStart = dlrDocData(readers[0].pReader); - nStart = dlrDocDataBytes(readers[0].pReader); - iFirstDocid = iDocid; - } - iLastDocid = iDocid; - dlrStep(readers[0].pReader); - - /* Drop all of the older elements with the same docid. */ - for(i=1; i0 ){ - orderedDLReaderReorder(readers+i, nReaders-i); - } - } - - /* Copy over any remaining elements. */ - if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); - dlwDestroy(&writer); -} - -/* Helper function for posListUnion(). Compares the current position -** between left and right, returning as standard C idiom of <0 if -** left0 if left>right, and 0 if left==right. "End" always -** compares greater. -*/ -static int posListCmp(PLReader *pLeft, PLReader *pRight){ - assert( pLeft->iType==pRight->iType ); - if( pLeft->iType==DL_DOCIDS ) return 0; - - if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1; - if( plrAtEnd(pRight) ) return -1; - - if( plrColumn(pLeft)plrColumn(pRight) ) return 1; - - if( plrPosition(pLeft)plrPosition(pRight) ) return 1; - if( pLeft->iType==DL_POSITIONS ) return 0; - - if( plrStartOffset(pLeft)plrStartOffset(pRight) ) return 1; - - if( plrEndOffset(pLeft)plrEndOffset(pRight) ) return 1; - - return 0; -} - -/* Write the union of position lists in pLeft and pRight to pOut. -** "Union" in this case meaning "All unique position tuples". Should -** work with any doclist type, though both inputs and the output -** should be the same type. -*/ -static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ - PLReader left, right; - PLWriter writer; - - assert( dlrDocid(pLeft)==dlrDocid(pRight) ); - assert( pLeft->iType==pRight->iType ); - assert( pLeft->iType==pOut->iType ); - - plrInit(&left, pLeft); - plrInit(&right, pRight); - plwInit(&writer, pOut, dlrDocid(pLeft)); - - while( !plrAtEnd(&left) || !plrAtEnd(&right) ){ - int c = posListCmp(&left, &right); - if( c<0 ){ - plwCopy(&writer, &left); - plrStep(&left); - }else if( c>0 ){ - plwCopy(&writer, &right); - plrStep(&right); - }else{ - plwCopy(&writer, &left); - plrStep(&left); - plrStep(&right); - } - } - - plwTerminate(&writer); - plwDestroy(&writer); - plrDestroy(&left); - plrDestroy(&right); -} - -/* Write the union of doclists in pLeft and pRight to pOut. For -** docids in common between the inputs, the union of the position -** lists is written. Inputs and outputs are always type DL_DEFAULT. -*/ -static void docListUnion( - const char *pLeft, int nLeft, - const char *pRight, int nRight, - DataBuffer *pOut /* Write the combined doclist here */ -){ - DLReader left, right; - DLWriter writer; - - if( nLeft==0 ){ - if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); - return; - } - if( nRight==0 ){ - dataBufferAppend(pOut, pLeft, nLeft); - return; - } - - dlrInit(&left, DL_DEFAULT, pLeft, nLeft); - dlrInit(&right, DL_DEFAULT, pRight, nRight); - dlwInit(&writer, DL_DEFAULT, pOut); - - while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ - if( dlrAtEnd(&right) ){ - dlwCopy(&writer, &left); - dlrStep(&left); - }else if( dlrAtEnd(&left) ){ - dlwCopy(&writer, &right); - dlrStep(&right); - }else if( dlrDocid(&left)dlrDocid(&right) ){ - dlwCopy(&writer, &right); - dlrStep(&right); - }else{ - posListUnion(&left, &right, &writer); - dlrStep(&left); - dlrStep(&right); - } - } - - dlrDestroy(&left); - dlrDestroy(&right); - dlwDestroy(&writer); -} - -/* -** This function is used as part of the implementation of phrase and -** NEAR matching. -** -** pLeft and pRight are DLReaders positioned to the same docid in -** lists of type DL_POSITION. This function writes an entry to the -** DLWriter pOut for each position in pRight that is less than -** (nNear+1) greater (but not equal to or smaller) than a position -** in pLeft. For example, if nNear is 0, and the positions contained -** by pLeft and pRight are: -** -** pLeft: 5 10 15 20 -** pRight: 6 9 17 21 -** -** then the docid is added to pOut. If pOut is of type DL_POSITIONS, -** then a positionids "6" and "21" are also added to pOut. -** -** If boolean argument isSaveLeft is true, then positionids are copied -** from pLeft instead of pRight. In the example above, the positions "5" -** and "20" would be added instead of "6" and "21". -*/ -static void posListPhraseMerge( - DLReader *pLeft, - DLReader *pRight, - int nNear, - int isSaveLeft, - DLWriter *pOut -){ - PLReader left, right; - PLWriter writer; - int match = 0; - - assert( dlrDocid(pLeft)==dlrDocid(pRight) ); - assert( pOut->iType!=DL_POSITIONS_OFFSETS ); - - plrInit(&left, pLeft); - plrInit(&right, pRight); - - while( !plrAtEnd(&left) && !plrAtEnd(&right) ){ - if( plrColumn(&left)plrColumn(&right) ){ - plrStep(&right); - }else if( plrPosition(&left)>=plrPosition(&right) ){ - plrStep(&right); - }else{ - if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){ - if( !match ){ - plwInit(&writer, pOut, dlrDocid(pLeft)); - match = 1; - } - if( !isSaveLeft ){ - plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0); - }else{ - plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0); - } - plrStep(&right); - }else{ - plrStep(&left); - } - } - } - - if( match ){ - plwTerminate(&writer); - plwDestroy(&writer); - } - - plrDestroy(&left); - plrDestroy(&right); -} - /* -** Compare the values pointed to by the PLReaders passed as arguments. -** Return -1 if the value pointed to by pLeft is considered less than -** the value pointed to by pRight, +1 if it is considered greater -** than it, or 0 if it is equal. i.e. -** -** (*pLeft - *pRight) -** -** A PLReader that is in the EOF condition is considered greater than -** any other. If neither argument is in EOF state, the return value of -** plrColumn() is used. If the plrColumn() values are equal, the -** comparison is on the basis of plrPosition(). +** Return the number of bytes required to store the value passed as the +** first argument in varint form. */ -static int plrCompare(PLReader *pLeft, PLReader *pRight){ - assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight)); - - if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){ - return (plrAtEnd(pRight) ? -1 : 1); - } - if( plrColumn(pLeft)!=plrColumn(pRight) ){ - return ((plrColumn(pLeft)0) -** and write the results into pOut. -** -** A phrase intersection means that two documents only match -** if pLeft.iPos+1==pRight.iPos. -** -** A NEAR intersection means that two documents only match if -** (abs(pLeft.iPos-pRight.iPos) 0", - /* SEGDIR_DELETE */ "delete from %_segdir where level = ?", - - /* NOTE(shess): The first three results of the following two - ** statements must match. - */ - /* SEGDIR_SELECT_SEGMENT */ - "select start_block, leaves_end_block, root from %_segdir " - " where level = ? and idx = ?", - /* SEGDIR_SELECT_ALL */ - "select start_block, leaves_end_block, root from %_segdir " - " order by level desc, idx asc", - /* SEGDIR_DELETE_ALL */ "delete from %_segdir", - /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir", -}; - -/* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. -*/ -struct fulltext_vtab { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of columns in virtual table */ - char **azColumn; /* column names. malloced */ - char **azContentColumn; /* column names in content table; malloced */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - - /* Precompiled statements which we keep as long as the table is - ** open. - */ - sqlite3_stmt *pFulltextStatements[MAX_STMT]; - - /* Precompiled statements used for segment merges. We run a - ** separate select across the leaf level of each tree being merged. - */ - sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT]; - /* The statement used to prepare pLeafSelectStmts. */ -#define LEAF_SELECT \ - "select block from %_segments where blockid between ? and ? order by blockid" - - /* These buffer pending index updates during transactions. - ** nPendingData estimates the memory size of the pending data. It - ** doesn't include the hash-bucket overhead, nor any malloc - ** overhead. When nPendingData exceeds kPendingThreshold, the - ** buffer is flushed even before the transaction closes. - ** pendingTerms stores the data, and is only valid when nPendingData - ** is >=0 (nPendingData<0 means pendingTerms has not been - ** initialized). iPrevDocid is the last docid written, used to make - ** certain we're inserting in sorted order. - */ - int nPendingData; -#define kPendingThreshold (1*1024*1024) - sqlite_int64 iPrevDocid; - fts3Hash pendingTerms; -}; - -/* -** When the core wants to do a query, it create a cursor using a -** call to xOpen. This structure is an instance of a cursor. It -** is destroyed by xClose. -*/ -typedef struct fulltext_cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - int eof; /* True if at End Of Results */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - Snippet snippet; /* Cached snippet for the current row */ - int iColumn; /* Column being searched */ - DataBuffer result; /* Doclist results from fulltextQuery */ - DLReader reader; /* Result reader if result not empty */ -} fulltext_cursor; - -static fulltext_vtab *cursor_vtab(fulltext_cursor *c){ - return (fulltext_vtab *) c->base.pVtab; -} - -static const sqlite3_module fts3Module; /* forward declaration */ - -/* Return a dynamically generated statement of the form - * insert into %_content (docid, ...) values (?, ...) - */ -static const char *contentInsertStatement(fulltext_vtab *v){ - StringBuffer sb; - int i; - - initStringBuffer(&sb); - append(&sb, "insert into %_content (docid, "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, ") values (?"); - for(i=0; inColumn; ++i) - append(&sb, ", ?"); - append(&sb, ")"); - return stringBufferData(&sb); -} - -/* Return a dynamically generated statement of the form - * select from %_content where docid = ? - */ -static const char *contentSelectStatement(fulltext_vtab *v){ - StringBuffer sb; - initStringBuffer(&sb); - append(&sb, "SELECT "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, " FROM %_content WHERE docid = ?"); - return stringBufferData(&sb); -} - -/* Return a dynamically generated statement of the form - * update %_content set [col_0] = ?, [col_1] = ?, ... - * where docid = ? - */ -static const char *contentUpdateStatement(fulltext_vtab *v){ - StringBuffer sb; - int i; - - initStringBuffer(&sb); - append(&sb, "update %_content set "); - for(i=0; inColumn; ++i) { - if( i>0 ){ - append(&sb, ", "); - } - append(&sb, v->azContentColumn[i]); - append(&sb, " = ?"); - } - append(&sb, " where docid = ?"); - return stringBufferData(&sb); -} - -/* Puts a freshly-prepared statement determined by iStmt in *ppStmt. -** If the indicated statement has never been prepared, it is prepared -** and cached, otherwise the cached version is reset. -*/ -static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, - sqlite3_stmt **ppStmt){ - assert( iStmtpFulltextStatements[iStmt]==NULL ){ - const char *zStmt; - int rc; - switch( iStmt ){ - case CONTENT_INSERT_STMT: - zStmt = contentInsertStatement(v); break; - case CONTENT_SELECT_STMT: - zStmt = contentSelectStatement(v); break; - case CONTENT_UPDATE_STMT: - zStmt = contentUpdateStatement(v); break; - default: - zStmt = fulltext_zStatement[iStmt]; - } - rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt], - zStmt); - if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt); - if( rc!=SQLITE_OK ) return rc; - } else { - int rc = sqlite3_reset(v->pFulltextStatements[iStmt]); - if( rc!=SQLITE_OK ) return rc; - } - - *ppStmt = v->pFulltextStatements[iStmt]; - return SQLITE_OK; -} - -/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and -** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE, -** where we expect no results. -*/ -static int sql_single_step(sqlite3_stmt *s){ - int rc = sqlite3_step(s); - return (rc==SQLITE_DONE) ? SQLITE_OK : rc; -} - -/* Like sql_get_statement(), but for special replicated LEAF_SELECT -** statements. idx -1 is a special case for an uncached version of -** the statement (used in the optimize implementation). -*/ -/* TODO(shess) Write version for generic statements and then share -** that between the cached-statement functions. -*/ -static int sql_get_leaf_statement(fulltext_vtab *v, int idx, - sqlite3_stmt **ppStmt){ - assert( idx>=-1 && idxdb, v->zDb, v->zName, ppStmt, LEAF_SELECT); - }else if( v->pLeafSelectStmts[idx]==NULL ){ - int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx], - LEAF_SELECT); - if( rc!=SQLITE_OK ) return rc; - }else{ - int rc = sqlite3_reset(v->pLeafSelectStmts[idx]); - if( rc!=SQLITE_OK ) return rc; - } - - *ppStmt = v->pLeafSelectStmts[idx]; - return SQLITE_OK; -} - -/* insert into %_content (docid, ...) values ([docid], [pValues]) -** If the docid contains SQL NULL, then a unique docid will be -** generated. -*/ -static int content_insert(fulltext_vtab *v, sqlite3_value *docid, - sqlite3_value **pValues){ - sqlite3_stmt *s; - int i; - int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_value(s, 1, docid); - if( rc!=SQLITE_OK ) return rc; - - for(i=0; inColumn; ++i){ - rc = sqlite3_bind_value(s, 2+i, pValues[i]); - if( rc!=SQLITE_OK ) return rc; - } - - return sql_single_step(s); -} - -/* update %_content set col0 = pValues[0], col1 = pValues[1], ... - * where docid = [iDocid] */ -static int content_update(fulltext_vtab *v, sqlite3_value **pValues, - sqlite_int64 iDocid){ - sqlite3_stmt *s; - int i; - int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - for(i=0; inColumn; ++i){ - rc = sqlite3_bind_value(s, 1+i, pValues[i]); - if( rc!=SQLITE_OK ) return rc; - } - - rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -static void freeStringArray(int nString, const char **pString){ - int i; - - for (i=0 ; i < nString ; ++i) { - if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]); - } - sqlite3_free((void *) pString); -} - -/* select * from %_content where docid = [iDocid] - * The caller must delete the returned array and all strings in it. - * null fields will be NULL in the returned array. - * - * TODO: Perhaps we should return pointer/length strings here for consistency - * with other code which uses pointer/length. */ -static int content_select(fulltext_vtab *v, sqlite_int64 iDocid, - const char ***pValues){ - sqlite3_stmt *s; - const char **values; - int i; - int rc; - - *pValues = NULL; - - rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iDocid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc!=SQLITE_ROW ) return rc; - - values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *)); - for(i=0; inColumn; ++i){ - if( sqlite3_column_type(s, i)==SQLITE_NULL ){ - values[i] = NULL; - }else{ - values[i] = string_dup((char*)sqlite3_column_text(s, i)); - } - } - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ){ - *pValues = values; - return SQLITE_OK; - } - - freeStringArray(v->nColumn, values); - return rc; -} - -/* delete from %_content where docid = [iDocid ] */ -static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iDocid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if -** no rows exist, and any error in case of failure. -*/ -static int content_exists(fulltext_vtab *v){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc!=SQLITE_ROW ) return rc; - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_ROW; - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; -} - -/* insert into %_segments values ([pData]) -** returns assigned blockid in *piBlockid -*/ -static int block_insert(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 *piBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - - /* blockid column is an alias for rowid. */ - *piBlockid = sqlite3_last_insert_rowid(v->db); - return SQLITE_OK; -} - -/* delete from %_segments -** where blockid between [iStartBlockid] and [iEndBlockid] -** -** Deletes the range of blocks, inclusive, used to delete the blocks -** which form a segment. -*/ -static int block_delete(fulltext_vtab *v, - sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 2, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found -** at iLevel. Returns SQLITE_DONE if there are no segments at -** iLevel. Otherwise returns an error. -*/ -static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - /* Should always get at least one row due to how max() works. */ - if( rc==SQLITE_DONE ) return SQLITE_DONE; - if( rc!=SQLITE_ROW ) return rc; - - /* NULL means that there were no inputs to max(). */ - if( SQLITE_NULL==sqlite3_column_type(s, 0) ){ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; - } - - *pidx = sqlite3_column_int(s, 0); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - return SQLITE_ROW; -} - -/* insert into %_segdir values ( -** [iLevel], [idx], -** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid], -** [pRootData] -** ) -*/ -static int segdir_set(fulltext_vtab *v, int iLevel, int idx, - sqlite_int64 iStartBlockid, - sqlite_int64 iLeavesEndBlockid, - sqlite_int64 iEndBlockid, - const char *pRootData, int nRootData){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 2, idx); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 3, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 5, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Queries %_segdir for the block span of the segments in level -** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel, -** SQLITE_ROW if there are blocks, else an error. -*/ -static int segdir_span(fulltext_vtab *v, int iLevel, - sqlite_int64 *piStartBlockid, - sqlite_int64 *piEndBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */ - if( rc!=SQLITE_ROW ) return rc; - - /* This happens if all segments at this level are entirely inline. */ - if( SQLITE_NULL==sqlite3_column_type(s, 0) ){ - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - int rc2 = sqlite3_step(s); - if( rc2==SQLITE_ROW ) return SQLITE_ERROR; - return rc2; - } - - *piStartBlockid = sqlite3_column_int64(s, 0); - *piEndBlockid = sqlite3_column_int64(s, 1); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - return SQLITE_ROW; -} - -/* Delete the segment blocks and segment directory records for all -** segments at iLevel. -*/ -static int segdir_delete(fulltext_vtab *v, int iLevel){ - sqlite3_stmt *s; - sqlite_int64 iStartBlockid, iEndBlockid; - int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid); - if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc; - - if( rc==SQLITE_ROW ){ - rc = block_delete(v, iStartBlockid, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - } - - /* Delete the segment directory itself. */ - rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Delete entire fts index, SQLITE_OK on success, relevant error on -** failure. -*/ -static int segdir_delete_all(fulltext_vtab *v){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sql_single_step(s); - if( rc!=SQLITE_OK ) return rc; - - rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_OK with *pnSegments set to the number of entries in -** %_segdir and *piMaxLevel set to the highest level which has a -** segment. Otherwise returns the SQLite error which caused failure. -*/ -static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - /* TODO(shess): This case should not be possible? Should stronger - ** measures be taken if it happens? - */ - if( rc==SQLITE_DONE ){ - *pnSegments = 0; - *piMaxLevel = 0; - return SQLITE_OK; - } - if( rc!=SQLITE_ROW ) return rc; - - *pnSegments = sqlite3_column_int(s, 0); - *piMaxLevel = sqlite3_column_int(s, 1); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_OK; - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; -} - -/* TODO(shess) clearPendingTerms() is far down the file because -** writeZeroSegment() is far down the file because LeafWriter is far -** down the file. Consider refactoring the code to move the non-vtab -** code above the vtab code so that we don't need this forward -** reference. -*/ -static int clearPendingTerms(fulltext_vtab *v); - -/* -** Free the memory used to contain a fulltext_vtab structure. -*/ -static void fulltext_vtab_destroy(fulltext_vtab *v){ - int iStmt, i; - - FTSTRACE(("FTS3 Destroy %p\n", v)); - for( iStmt=0; iStmtpFulltextStatements[iStmt]!=NULL ){ - sqlite3_finalize(v->pFulltextStatements[iStmt]); - v->pFulltextStatements[iStmt] = NULL; - } - } - - for( i=0; ipLeafSelectStmts[i]!=NULL ){ - sqlite3_finalize(v->pLeafSelectStmts[i]); - v->pLeafSelectStmts[i] = NULL; - } - } - - if( v->pTokenizer!=NULL ){ - v->pTokenizer->pModule->xDestroy(v->pTokenizer); - v->pTokenizer = NULL; - } - - clearPendingTerms(v); - - sqlite3_free(v->azColumn); - for(i = 0; i < v->nColumn; ++i) { - sqlite3_free(v->azContentColumn[i]); - } - sqlite3_free(v->azContentColumn); - sqlite3_free(v); -} - -/* -** Token types for parsing the arguments to xConnect or xCreate. -*/ -#define TOKEN_EOF 0 /* End of file */ -#define TOKEN_SPACE 1 /* Any kind of whitespace */ -#define TOKEN_ID 2 /* An identifier */ -#define TOKEN_STRING 3 /* A string literal */ -#define TOKEN_PUNCT 4 /* A single punctuation character */ - -/* -** If X is a character that can be used in an identifier then -** ftsIdChar(X) will be true. Otherwise it is false. -** -** For ASCII, any character with the high-order bit set is -** allowed in an identifier. For 7-bit characters, -** isFtsIdChar[X] must be 1. -** -** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. -** SQLite will allow '$' in identifiers for compatibility. -** But the feature is undocumented. -*/ -static const char isFtsIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ -}; -#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20])) - - -/* -** Return the length of the token that begins at z[0]. -** Store the token type in *tokenType before returning. -*/ -static int ftsGetToken(const char *z, int *tokenType){ - int i, c; - switch( *z ){ - case 0: { - *tokenType = TOKEN_EOF; - return 0; - } - case ' ': case '\t': case '\n': case '\f': case '\r': { - for(i=1; safe_isspace(z[i]); i++){} - *tokenType = TOKEN_SPACE; - return i; - } - case '`': - case '\'': - case '"': { - int delim = z[0]; - for(i=1; (c=z[i])!=0; i++){ - if( c==delim ){ - if( z[i+1]==delim ){ - i++; - }else{ - break; - } - } - } - *tokenType = TOKEN_STRING; - return i + (c!=0); - } - case '[': { - for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} - *tokenType = TOKEN_ID; - return i; - } - default: { - if( !ftsIdChar(*z) ){ - break; - } - for(i=1; ftsIdChar(z[i]); i++){} - *tokenType = TOKEN_ID; - return i; - } - } - *tokenType = TOKEN_PUNCT; - return 1; -} - -/* -** A token extracted from a string is an instance of the following -** structure. -*/ -typedef struct FtsToken { - const char *z; /* Pointer to token text. Not '\000' terminated */ - short int n; /* Length of the token text in bytes. */ -} FtsToken; - -/* -** Given a input string (which is really one of the argv[] parameters -** passed into xConnect or xCreate) split the string up into tokens. -** Return an array of pointers to '\000' terminated strings, one string -** for each non-whitespace token. -** -** The returned array is terminated by a single NULL pointer. -** -** Space to hold the returned array is obtained from a single -** malloc and should be freed by passing the return value to free(). -** The individual strings within the token list are all a part of -** the single memory allocation and will all be freed at once. -*/ -static char **tokenizeString(const char *z, int *pnToken){ - int nToken = 0; - FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) ); - int n = 1; - int e, i; - int totalSize = 0; - char **azToken; - char *zCopy; - while( n>0 ){ - n = ftsGetToken(z, &e); - if( e!=TOKEN_SPACE ){ - aToken[nToken].z = z; - aToken[nToken].n = n; - nToken++; - totalSize += n+1; - } - z += n; - } - azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize ); - zCopy = (char*)&azToken[nToken]; - nToken--; - for(i=0; i>= 7; + }while( v!=0 ); + return i; } /* @@ -2610,10 +432,10 @@ static char **tokenizeString(const char *z, int *pnToken){ ** [pqr] becomes pqr ** `mno` becomes mno */ -static void dequoteString(char *z){ +void sqlite3Fts3Dequote(char *z){ int quote; int i, j; - if( z==0 ) return; + quote = z[0]; switch( quote ){ case '\'': break; @@ -2625,7 +447,7 @@ static void dequoteString(char *z){ for(i=1, j=0; z[i]; i++){ if( z[i]==quote ){ if( z[i+1]==quote ){ - z[j++] = quote; + z[j++] = (char)quote; i++; }else{ z[j++] = 0; @@ -2637,4250 +459,1780 @@ static void dequoteString(char *z){ } } -/* -** The input azIn is a NULL-terminated list of tokens. Remove the first -** token and all punctuation tokens. Remove the quotes from -** around string literal tokens. -** -** Example: -** -** input: tokenize chinese ( 'simplifed' , 'mixed' ) -** output: chinese simplifed mixed -** -** Another example: -** -** input: delimiters ( '[' , ']' , '...' ) -** output: [ ] ... -*/ -static void tokenListToIdList(char **azIn){ - int i, j; - if( azIn ){ - for(i=0, j=-1; azIn[i]; i++){ - if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){ - dequoteString(azIn[i]); - if( j>=0 ){ - azIn[j] = azIn[i]; - } - j++; - } - } - azIn[j] = 0; +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + fts3GetDeltaVarint(pp, pVal); } } - /* -** Find the first alphanumeric token in the string zIn. Null-terminate -** this token. Remove any quotation marks. And return a pointer to -** the result. +** The xDisconnect() virtual table method. */ -static char *firstToken(char *zIn, char **pzTail){ - int n, ttype; - while(1){ - n = ftsGetToken(zIn, &ttype); - if( ttype==TOKEN_SPACE ){ - zIn += n; - }else if( ttype==TOKEN_EOF ){ - *pzTail = zIn; - return 0; - }else{ - zIn[n] = 0; - *pzTail = &zIn[1]; - dequoteString(zIn); - return zIn; - } +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); } - /*NOTREACHED*/ -} - -/* Return true if... -** -** * s begins with the string t, ignoring case -** * s is longer than t -** * The first character of s beyond t is not a alphanumeric -** -** Ignore leading space in *s. -** -** To put it another way, return true if the first token of -** s[] is t[]. -*/ -static int startsWith(const char *s, const char *t){ - while( safe_isspace(*s) ){ s++; } - while( *t ){ - if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0; + for(i=0; inLeavesStmt; i++){ + sqlite3_finalize(p->aLeavesStmt[i]); } - return *s!='_' && !safe_isalnum(*s); -} + sqlite3_free(p->zSelectLeaves); + sqlite3_free(p->aLeavesStmt); -/* -** An instance of this structure defines the "spec" of a -** full text index. This structure is populated by parseSpec -** and use by fulltextConnect and fulltextCreate. -*/ -typedef struct TableSpec { - const char *zDb; /* Logical database name */ - const char *zName; /* Name of the full-text index */ - int nColumn; /* Number of columns to be indexed */ - char **azColumn; /* Original names of columns to be indexed */ - char **azContentColumn; /* Column names for %_content */ - char **azTokenizer; /* Name of tokenizer and its arguments */ -} TableSpec; - -/* -** Reclaim all of the memory used by a TableSpec -*/ -static void clearTableSpec(TableSpec *p) { - sqlite3_free(p->azColumn); - sqlite3_free(p->azContentColumn); - sqlite3_free(p->azTokenizer); -} - -/* Parse a CREATE VIRTUAL TABLE statement, which looks like this: - * - * CREATE VIRTUAL TABLE email - * USING fts3(subject, body, tokenize mytokenizer(myarg)) - * - * We return parsed information in a TableSpec structure. - * - */ -static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv, - char**pzErr){ - int i, n; - char *z, *zDummy; - char **azArg; - const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */ - - assert( argc>=3 ); - /* Current interface: - ** argv[0] - module name - ** argv[1] - database name - ** argv[2] - table name - ** argv[3..] - columns, optionally followed by tokenizer specification - ** and snippet delimiters specification. - */ - - /* Make a copy of the complete argv[][] array in a single allocation. - ** The argv[][] array is read-only and transient. We can write to the - ** copy in order to modify things and the copy is persistent. - */ - CLEAR(pSpec); - for(i=n=0; izDb = azArg[1]; - pSpec->zName = azArg[2]; - pSpec->nColumn = 0; - pSpec->azColumn = azArg; - zTokenizer = "tokenize simple"; - for(i=3; inColumn] = firstToken(azArg[i], &zDummy); - pSpec->nColumn++; - } - } - if( pSpec->nColumn==0 ){ - azArg[0] = "content"; - pSpec->nColumn = 1; - } - - /* - ** Construct the list of content column names. - ** - ** Each content column name will be of the form cNNAAAA - ** where NN is the column number and AAAA is the sanitized - ** column name. "sanitized" means that special characters are - ** converted to "_". The cNN prefix guarantees that all column - ** names are unique. - ** - ** The AAAA suffix is not strictly necessary. It is included - ** for the convenience of people who might examine the generated - ** %_content table and wonder what the columns are used for. - */ - pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) ); - if( pSpec->azContentColumn==0 ){ - clearTableSpec(pSpec); - return SQLITE_NOMEM; - } - for(i=0; inColumn; i++){ - char *p; - pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]); - for (p = pSpec->azContentColumn[i]; *p ; ++p) { - if( !safe_isalnum(*p) ) *p = '_'; - } - } - - /* - ** Parse the tokenizer specification string. - */ - pSpec->azTokenizer = tokenizeString(zTokenizer, &n); - tokenListToIdList(pSpec->azTokenizer); + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + sqlite3_free(p); return SQLITE_OK; } /* -** Generate a CREATE TABLE statement that describes the schema of -** the virtual table. Return a pointer to this schema string. -** -** Space is obtained from sqlite3_mprintf() and should be freed -** using sqlite3_free(). +** The xDestroy() virtual table method. */ -static char *fulltextSchema( - int nColumn, /* Number of columns */ - const char *const* azColumn, /* List of columns */ - const char *zTableName /* Name of the table */ -){ - int i; - char *zSchema, *zNext; - const char *zSep = "("; - zSchema = sqlite3_mprintf("CREATE TABLE x"); - for(i=0; izDb, p->zName, p->zDb, p->zName, p->zDb, p->zName + ); + + /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to + ** execute the SQL script created above. + */ + if( zSql ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; } - zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName); - sqlite3_free(zSchema); - zSchema = zNext; - zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema); - sqlite3_free(zSchema); - return zNext; + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); } + /* -** Build a new sqlite3_vtab structure that will describe the -** fulltext index defined by spec. +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. */ -static int constructVtab( - sqlite3 *db, /* The SQLite database connection */ - fts3Hash *pHash, /* Hash table containing tokenizers */ - TableSpec *spec, /* Parsed spec information from parseSpec() */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ -){ - int rc; - int n; - fulltext_vtab *v = 0; - const sqlite3_tokenizer_module *m = NULL; - char *schema; +static int fts3DeclareVtab(Fts3Table *p){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ - char const *zTok; /* Name of tokenizer to use for this fts table */ - int nTok; /* Length of zTok, including nul terminator */ - - v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab)); - if( v==0 ) return SQLITE_NOMEM; - CLEAR(v); - /* sqlite will initialize v->base */ - v->db = db; - v->zDb = spec->zDb; /* Freed when azColumn is freed */ - v->zName = spec->zName; /* Freed when azColumn is freed */ - v->nColumn = spec->nColumn; - v->azContentColumn = spec->azContentColumn; - spec->azContentColumn = 0; - v->azColumn = spec->azColumn; - spec->azColumn = 0; - - if( spec->azTokenizer==0 ){ - return SQLITE_NOMEM; + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } - zTok = spec->azTokenizer[0]; - if( !zTok ){ - zTok = "simple"; - } - nTok = strlen(zTok)+1; + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + ); - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok); - if( !m ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]); - rc = SQLITE_ERROR; - goto err; - } - - for(n=0; spec->azTokenizer[n]; n++){} - if( n ){ - rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1], - &v->pTokenizer); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; }else{ - rc = m->xCreate(0, 0, &v->pTokenizer); + rc = sqlite3_declare_vtab(p->db, zSql); } - if( rc!=SQLITE_OK ) goto err; - v->pTokenizer->pModule = m; - /* TODO: verify the existence of backing tables foo_content, foo_term */ - - schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn, - spec->zName); - rc = sqlite3_declare_vtab(db, schema); - sqlite3_free(schema); - if( rc!=SQLITE_OK ) goto err; - - memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements)); - - /* Indicate that the buffer is not live. */ - v->nPendingData = -1; - - *ppVTab = &v->base; - FTSTRACE(("FTS3 Connect %p\n", v)); - - return rc; - -err: - fulltext_vtab_destroy(v); + sqlite3_free(zSql); + sqlite3_free(zCols); return rc; } -static int fulltextConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVTab, - char **pzErr -){ - TableSpec spec; - int rc = parseSpec(&spec, argc, argv, pzErr); - if( rc!=SQLITE_OK ) return rc; +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc; /* Return code */ + int i; /* Iterator variable */ + char *zContentCols; /* Columns of %_content table */ + char *zSql; /* SQL script to create required tables */ - rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); - clearTableSpec(&spec); + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + + /* Create the whole SQL script */ + zSql = sqlite3_mprintf( + "CREATE TABLE %Q.'%q_content'(%s);" + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);" + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName + ); + + /* Unless a malloc() failure has occurred, execute the SQL script to + ** create the tables used to store data for this FTS3 virtual table. + */ + if( zContentCols==0 || zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + } + + sqlite3_free(zSql); + sqlite3_free(zContentCols); return rc; } -/* The %_content table holds the text of each document, with -** the docid column exposed as the SQLite rowid for the table. -*/ -/* TODO(shess) This comment needs elaboration to match the updated -** code. Work it into the top-of-file comment at that time. -*/ -static int fulltextCreate(sqlite3 *db, void *pAux, - int argc, const char * const *argv, - sqlite3_vtab **ppVTab, char **pzErr){ - int rc; - TableSpec spec; - StringBuffer schema; - FTSTRACE(("FTS3 Create\n")); - - rc = parseSpec(&spec, argc, argv, pzErr); - if( rc!=SQLITE_OK ) return rc; - - initStringBuffer(&schema); - append(&schema, "CREATE TABLE %_content("); - append(&schema, " docid INTEGER PRIMARY KEY,"); - appendList(&schema, spec.nColumn, spec.azContentColumn); - append(&schema, ")"); - rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema)); - stringBufferDestroy(&schema); - if( rc!=SQLITE_OK ) goto out; - - rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segments(" - " blockid INTEGER PRIMARY KEY," - " block blob" - ");" - ); - if( rc!=SQLITE_OK ) goto out; - - rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segdir(" - " level integer," - " idx integer," - " start_block integer," - " leaves_end_block integer," - " end_block integer," - " root blob," - " primary key(level, idx)" - ");"); - if( rc!=SQLITE_OK ) goto out; - - rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); - -out: - clearTableSpec(&spec); - return rc; -} - -/* Decide how to handle an SQL query. */ -static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - fulltext_vtab *v = (fulltext_vtab *)pVTab; - int i; - FTSTRACE(("FTS3 BestIndex\n")); - - for(i=0; inConstraint; ++i){ - const struct sqlite3_index_constraint *pConstraint; - pConstraint = &pInfo->aConstraint[i]; - if( pConstraint->usable ) { - if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) && - pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - pInfo->idxNum = QUERY_DOCID; /* lookup by docid */ - FTSTRACE(("FTS3 QUERY_DOCID\n")); - } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn && - pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* full-text search */ - pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn; - FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn)); - } else continue; - - pInfo->aConstraintUsage[i].argvIndex = 1; - pInfo->aConstraintUsage[i].omit = 1; - - /* An arbitrary value for now. - * TODO: Perhaps docid matches should be considered cheaper than - * full-text searches. */ - pInfo->estimatedCost = 1.0; - - return SQLITE_OK; - } - } - pInfo->idxNum = QUERY_GENERIC; - return SQLITE_OK; -} - -static int fulltextDisconnect(sqlite3_vtab *pVTab){ - FTSTRACE(("FTS3 Disconnect %p\n", pVTab)); - fulltext_vtab_destroy((fulltext_vtab *)pVTab); - return SQLITE_OK; -} - -static int fulltextDestroy(sqlite3_vtab *pVTab){ - fulltext_vtab *v = (fulltext_vtab *)pVTab; - int rc; - - FTSTRACE(("FTS3 Destroy %p\n", pVTab)); - rc = sql_exec(v->db, v->zDb, v->zName, - "drop table if exists %_content;" - "drop table if exists %_segments;" - "drop table if exists %_segdir;" - ); - if( rc!=SQLITE_OK ) return rc; - - fulltext_vtab_destroy((fulltext_vtab *)pVTab); - return SQLITE_OK; -} - -static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - fulltext_cursor *c; - - c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor)); - if( c ){ - memset(c, 0, sizeof(fulltext_cursor)); - /* sqlite will initialize c->base */ - *ppCursor = &c->base; - FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c)); - return SQLITE_OK; - }else{ - return SQLITE_NOMEM; - } -} - -/* Free all of the dynamically allocated memory held by the -** Snippet -*/ -static void snippetClear(Snippet *p){ - sqlite3_free(p->aMatch); - sqlite3_free(p->zOffset); - sqlite3_free(p->zSnippet); - CLEAR(p); -} - /* -** Append a single entry to the p->aMatch[] log. +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. */ -static void snippetAppendMatch( - Snippet *p, /* Append the entry to this snippet */ - int iCol, int iTerm, /* The column and query term */ - int iToken, /* Matching token in document */ - int iStart, int nByte /* Offset and size of the match */ +int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ ){ - int i; - struct snippetMatch *pMatch; - if( p->nMatch+1>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + 10; - p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); - if( p->aMatch==0 ){ - p->nMatch = 0; - p->nAlloc = 0; - return; - } - } - i = p->nMatch++; - pMatch = &p->aMatch[i]; - pMatch->iCol = iCol; - pMatch->iTerm = iTerm; - pMatch->iToken = iToken; - pMatch->iStart = iStart; - pMatch->nByte = nByte; -} - -/* -** Sizing information for the circular buffer used in snippetOffsetsOfColumn() -*/ -#define FTS3_ROTOR_SZ (32) -#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) - -/* -** Function to iterate through the tokens of a compiled expression. -** -** Except, skip all tokens on the right-hand side of a NOT operator. -** This function is used to find tokens as part of snippet and offset -** generation and we do nt want snippets and offsets to report matches -** for tokens on the RHS of a NOT. -*/ -static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ - Fts3Expr *p = *ppExpr; - int iToken = *piToken; - if( iToken<0 ){ - /* In this case the expression p is the root of an expression tree. - ** Move to the first token in the expression tree. - */ - while( p->pLeft ){ - p = p->pLeft; - } - iToken = 0; - }else{ - assert(p && p->eType==FTSQUERY_PHRASE ); - if( iToken<(p->pPhrase->nToken-1) ){ - iToken++; - }else{ - iToken = 0; - while( p->pParent && p->pParent->pLeft!=p ){ - assert( p->pParent->pRight==p ); - p = p->pParent; - } - p = p->pParent; - if( p ){ - assert( p->pRight!=0 ); - p = p->pRight; - while( p->pLeft ){ - p = p->pLeft; - } - } - } - } - - *ppExpr = p; - *piToken = iToken; - return p?1:0; -} - -/* -** Return TRUE if the expression node pExpr is located beneath the -** RHS of a NOT operator. -*/ -static int fts3ExprBeneathNot(Fts3Expr *p){ - Fts3Expr *pParent; - while( p ){ - pParent = p->pParent; - if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ - return 1; - } - p = pParent; - } - return 0; -} - -/* -** Add entries to pSnippet->aMatch[] for every match that occurs against -** document zDoc[0..nDoc-1] which is stored in column iColumn. -*/ -static void snippetOffsetsOfColumn( - fulltext_cursor *pCur, /* The fulltest search cursor */ - Snippet *pSnippet, /* The Snippet object to be filled in */ - int iColumn, /* Index of fulltext table column */ - const char *zDoc, /* Text of the fulltext table column */ - int nDoc /* Length of zDoc in bytes */ -){ - const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ - sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ - sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ - fulltext_vtab *pVtab; /* The full text index */ - int nColumn; /* Number of columns in the index */ - int i, j; /* Loop counters */ - int rc; /* Return code */ - unsigned int match, prevMatch; /* Phrase search bitmasks */ - const char *zToken; /* Next token from the tokenizer */ - int nToken; /* Size of zToken */ - int iBegin, iEnd, iPos; /* Offsets of beginning and end */ - - /* The following variables keep a circular buffer of the last - ** few tokens */ - unsigned int iRotor = 0; /* Index of current token */ - int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ - int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ - - pVtab = cursor_vtab(pCur); - nColumn = pVtab->nColumn; - pTokenizer = pVtab->pTokenizer; - pTModule = pTokenizer->pModule; - rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); - if( rc ) return; - pTCursor->pTokenizer = pTokenizer; - - prevMatch = 0; - while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){ - Fts3Expr *pIter = pCur->pExpr; - int iIter = -1; - iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; - iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; - match = 0; - for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ - int nPhrase; /* Number of tokens in current phrase */ - struct PhraseToken *pToken; /* Current token */ - int iCol; /* Column index */ - - if( fts3ExprBeneathNot(pIter) ) continue; - nPhrase = pIter->pPhrase->nToken; - pToken = &pIter->pPhrase->aToken[iIter]; - iCol = pIter->pPhrase->iColumn; - if( iCol>=0 && iColn>nToken ) continue; - if( !pToken->isPrefix && pToken->nn<=nToken ); - if( memcmp(pToken->z, zToken, pToken->n) ) continue; - if( iIter>0 && (prevMatch & (1<=0; j--){ - int k = (iRotor-j) & FTS3_ROTOR_MASK; - snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, - iRotorBegin[k], iRotorLen[k]); - } - } - } - prevMatch = match<<1; - iRotor++; - } - pTModule->xClose(pTCursor); -} - -/* -** Remove entries from the pSnippet structure to account for the NEAR -** operator. When this is called, pSnippet contains the list of token -** offsets produced by treating all NEAR operators as AND operators. -** This function removes any entries that should not be present after -** accounting for the NEAR restriction. For example, if the queried -** document is: -** -** "A B C D E A" -** -** and the query is: -** -** A NEAR/0 E -** -** then when this function is called the Snippet contains token offsets -** 0, 4 and 5. This function removes the "0" entry (because the first A -** is not near enough to an E). -** -** When this function is called, the value pointed to by parameter piLeft is -** the integer id of the left-most token in the expression tree headed by -** pExpr. This function increments *piLeft by the total number of tokens -** in the expression tree headed by pExpr. -** -** Return 1 if any trimming occurs. Return 0 if no trimming is required. -*/ -static int trimSnippetOffsets( - Fts3Expr *pExpr, /* The search expression */ - Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ - int *piLeft /* Index of left-most token in pExpr */ -){ - if( pExpr ){ - if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ - return 1; - } - - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: - *piLeft += pExpr->pPhrase->nToken; - break; - case FTSQUERY_NEAR: { - /* The right-hand-side of a NEAR operator is always a phrase. The - ** left-hand-side is either a phrase or an expression tree that is - ** itself headed by a NEAR operator. The following initializations - ** set local variable iLeft to the token number of the left-most - ** token in the right-hand phrase, and iRight to the right most - ** token in the same phrase. For example, if we had: - ** - ** MATCH '"abc def" NEAR/2 "ghi jkl"' - ** - ** then iLeft will be set to 2 (token number of ghi) and nToken will - ** be set to 4. - */ - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - int iLeft = *piLeft; - int nNear = pExpr->nNear; - int nToken = pRight->pPhrase->nToken; - int jj, ii; - if( pLeft->eType==FTSQUERY_NEAR ){ - pLeft = pLeft->pRight; - } - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - nToken += pLeft->pPhrase->nToken; - - for(ii=0; iinMatch; ii++){ - struct snippetMatch *p = &pSnippet->aMatch[ii]; - if( p->iTerm==iLeft ){ - int isOk = 0; - /* Snippet ii is an occurence of query term iLeft in the document. - ** It occurs at position (p->iToken) of the document. We now - ** search for an instance of token (iLeft-1) somewhere in the - ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within - ** the set of snippetMatch structures. If one is found, proceed. - ** If one cannot be found, then remove snippets ii..(ii+N-1) - ** from the matching snippets, where N is the number of tokens - ** in phrase pRight->pPhrase. - */ - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==(iLeft-1) ){ - if( p2->iToken>=(p->iToken-nNear-1) - && p2->iToken<(p->iToken+nNear+nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[kk+ii].iTerm = -2; - } - return 1; - } - } - if( p->iTerm==(iLeft-1) ){ - int isOk = 0; - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==iLeft ){ - if( p2->iToken<=(p->iToken+nNear+1) - && p2->iToken>(p->iToken-nNear-nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[ii-kk].iTerm = -2; - } - return 1; - } - } - } - break; - } - } - - if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ - return 1; - } - } - return 0; -} - -/* -** Compute all offsets for the current row of the query. -** If the offsets have already been computed, this routine is a no-op. -*/ -static void snippetAllOffsets(fulltext_cursor *p){ - int nColumn; - int iColumn, i; - int iFirst, iLast; - int iTerm = 0; - fulltext_vtab *pFts = cursor_vtab(p); - - if( p->snippet.nMatch || p->pExpr==0 ){ - return; - } - nColumn = pFts->nColumn; - iColumn = (p->iCursorType - QUERY_FULLTEXT); - if( iColumn<0 || iColumn>=nColumn ){ - /* Look for matches over all columns of the full-text index */ - iFirst = 0; - iLast = nColumn-1; - }else{ - /* Look for matches in the iColumn-th column of the index only */ - iFirst = iColumn; - iLast = iColumn; - } - for(i=iFirst; i<=iLast; i++){ - const char *zDoc; - int nDoc; - zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1); - nDoc = sqlite3_column_bytes(p->pStmt, i+1); - snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc); - } - - while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){ - iTerm = 0; - } -} - -/* -** Convert the information in the aMatch[] array of the snippet -** into the string zOffset[0..nOffset-1]. This string is used as -** the return of the SQL offsets() function. -*/ -static void snippetOffsetText(Snippet *p){ - int i; - int cnt = 0; - StringBuffer sb; - char zBuf[200]; - if( p->zOffset ) return; - initStringBuffer(&sb); - for(i=0; inMatch; i++){ - struct snippetMatch *pMatch = &p->aMatch[i]; - if( pMatch->iTerm>=0 ){ - /* If snippetMatch.iTerm is less than 0, then the match was - ** discarded as part of processing the NEAR operator (see the - ** trimSnippetOffsetsForNear() function for details). Ignore - ** it in this case - */ - zBuf[0] = ' '; - sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", - pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); - append(&sb, zBuf); - cnt++; - } - } - p->zOffset = stringBufferData(&sb); - p->nOffset = stringBufferLength(&sb); -} - -/* -** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set -** of matching words some of which might be in zDoc. zDoc is column -** number iCol. -** -** iBreak is suggested spot in zDoc where we could begin or end an -** excerpt. Return a value similar to iBreak but possibly adjusted -** to be a little left or right so that the break point is better. -*/ -static int wordBoundary( - int iBreak, /* The suggested break point */ - const char *zDoc, /* Document text */ - int nDoc, /* Number of bytes in zDoc[] */ - struct snippetMatch *aMatch, /* Matching words */ - int nMatch, /* Number of entries in aMatch[] */ - int iCol /* The column number for zDoc[] */ -){ - int i; - if( iBreak<=10 ){ - return 0; - } - if( iBreak>=nDoc-10 ){ - return nDoc; - } - for(i=0; i0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ - return aMatch[i-1].iStart; - } - } - for(i=1; i<=10; i++){ - if( safe_isspace(zDoc[iBreak-i]) ){ - return iBreak - i + 1; - } - if( safe_isspace(zDoc[iBreak+i]) ){ - return iBreak + i + 1; - } - } - return iBreak; -} - - - -/* -** Allowed values for Snippet.aMatch[].snStatus -*/ -#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ -#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ - -/* -** Generate the text of a snippet. -*/ -static void snippetText( - fulltext_cursor *pCursor, /* The cursor we need the snippet for */ - const char *zStartMark, /* Markup to appear before each match */ - const char *zEndMark, /* Markup to appear after each match */ - const char *zEllipsis /* Ellipsis mark */ -){ - int i, j; - struct snippetMatch *aMatch; - int nMatch; - int nDesired; - StringBuffer sb; - int tailCol; - int tailOffset; + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p; /* Pointer to allocated vtab */ + int rc; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ int iCol; - int nDoc; - const char *zDoc; - int iStart, iEnd; - int tailEllipsis = 0; - int iMatch; - + int nString = 0; + int nCol = 0; + char *zCsr; + int nDb; + int nName; - sqlite3_free(pCursor->snippet.zSnippet); - pCursor->snippet.zSnippet = 0; - aMatch = pCursor->snippet.aMatch; - nMatch = pCursor->snippet.nMatch; - initStringBuffer(&sb); + const char *zTokenizer = 0; /* Name of tokenizer to use */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - for(i=0; i0; i++){ - if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; - nDesired--; - iCol = aMatch[i].iCol; - zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); - nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); - iStart = aMatch[i].iStart - 40; - iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); - if( iStart<=10 ){ - iStart = 0; + if( nCol==0 ){ + nCol = 1; + } + + /* Allocate and populate the Fts3Table structure. */ + nByte = sizeof(Fts3Table) + /* Fts3Table */ + nCol * sizeof(char *) + /* azColumn */ + nName + /* zName */ + nDb + /* zDb */ + nString; /* Space for azColumn strings */ + p = (Fts3Table*)sqlite3_malloc(nByte); + if( p==0 ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + memset(p, 0, nByte); + + p->db = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nNodeSize = 1000; + zCsr = (char *)&p->azColumn[nCol]; + + fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + + /* Fill in the zName and zDb fields of the vtab structure. */ + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + iCol = 0; + for(i=3; iazColumn[iCol++] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); } - if( iCol==tailCol && iStart<=tailOffset+20 ){ - iStart = tailOffset; - } - if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ - trimWhiteSpace(&sb); - appendWhiteSpace(&sb); - append(&sb, zEllipsis); - appendWhiteSpace(&sb); - } - iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; - iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); - if( iEnd>=nDoc-10 ){ - iEnd = nDoc; - tailEllipsis = 0; + } + if( iCol==0 ){ + assert( nCol==1 ); + p->azColumn[0] = "content"; + } + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + if( rc!=SQLITE_OK ) goto fts3_init_out; + } + + rc = fts3DeclareVtab(p); + if( rc!=SQLITE_OK ) goto fts3_init_out; + +#ifdef SQLITE_TEST + if( zTestParam ){ + p->nNodeSize = atoi(&zTestParam[5]); + } +#endif + *ppVTab = &p->base; + +fts3_init_out: + assert( p || (pTokenizer && rc!=SQLITE_OK) ); + if( rc!=SQLITE_OK ){ + if( p ){ + fts3DisconnectMethod((sqlite3_vtab *)p); }else{ - tailEllipsis = 1; + pTokenizer->pModule->xDestroy(pTokenizer); } - while( iMatchsnippet.zSnippet = stringBufferData(&sb); - pCursor->snippet.nSnippet = stringBufferLength(&sb); + return rc; } +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 500000; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ) continue; + + /* A direct lookup on the rowid or docid column. This is the best + ** strategy in all cases. Assign a cost of 1.0 and return early. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) + ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + break; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the rowid/docid strategy. + */ + if( iCons<0 + && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + } + + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].omit = 1; + } + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/****************************************************************/ +/****************************************************************/ +/****************************************************************/ +/****************************************************************/ + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fulltextClose(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - FTSTRACE(("FTS3 Close %p\n", c)); - sqlite3_finalize(c->pStmt); - sqlite3Fts3ExprFree(c->pExpr); - snippetClear(&c->snippet); - if( c->result.nData!=0 ){ - dlrDestroy(&c->reader); - } - dataBufferDestroy(&c->result); - sqlite3_free(c); + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr); return SQLITE_OK; } -static int fulltextNext(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - int rc; - - FTSTRACE(("FTS3 Next %p\n", pCursor)); - snippetClear(&c->snippet); - if( c->iCursorType < QUERY_FULLTEXT ){ - /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ - rc = sqlite3_step(c->pStmt); - switch( rc ){ - case SQLITE_ROW: - c->eof = 0; - return SQLITE_OK; - case SQLITE_DONE: - c->eof = 1; - return SQLITE_OK; - default: - c->eof = 1; - return rc; - } - } else { /* full-text query */ - rc = sqlite3_reset(c->pStmt); - if( rc!=SQLITE_OK ) return rc; - - if( c->result.nData==0 || dlrAtEnd(&c->reader) ){ - c->eof = 1; +static int fts3CursorSeek(Fts3Cursor *pCsr){ + if( pCsr->isRequireSeek ){ + pCsr->isRequireSeek = 0; + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; + }else{ + int rc; + pCsr->isEof = 1; + if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){ + rc = SQLITE_ERROR; + } + return rc; } - rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader)); - dlrStep(&c->reader); - if( rc!=SQLITE_OK ) return rc; - /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ - rc = sqlite3_step(c->pStmt); - if( rc==SQLITE_ROW ){ /* the case we expect */ - c->eof = 0; - return SQLITE_OK; - } - /* an error occurred; abort */ - return rc==SQLITE_DONE ? SQLITE_ERROR : rc; + }else{ + return SQLITE_OK; } } +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ + int rc = SQLITE_OK; /* Return code */ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; -/* TODO(shess) If we pushed LeafReader to the top of the file, or to -** another file, term_select() could be pushed above -** docListOfTerm(). -*/ -static int termSelect(fulltext_vtab *v, int iColumn, - const char *pTerm, int nTerm, int isPrefix, - DocListType iType, DataBuffer *out); - -/* -** Return a DocList corresponding to the phrase *pPhrase. -** -** The resulting DL_DOCIDS doclist is stored in pResult, which is -** overwritten. -*/ -static int docListOfPhrase( - fulltext_vtab *pTab, /* The full text index */ - Fts3Phrase *pPhrase, /* Phrase to return a doclist corresponding to */ - DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */ - DataBuffer *pResult /* Write the result here */ -){ - int ii; - int rc = SQLITE_OK; - int iCol = pPhrase->iColumn; - DocListType eType = eListType; - assert( eType==DL_POSITIONS || eType==DL_DOCIDS ); - if( pPhrase->nToken>1 ){ - eType = DL_POSITIONS; - } - - /* This code should never be called with buffered updates. */ - assert( pTab->nPendingData<0 ); - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - DataBuffer tmp; - struct PhraseToken *p = &pPhrase->aToken[ii]; - rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp); - if( rc==SQLITE_OK ){ - if( ii==0 ){ - *pResult = tmp; - }else{ - DataBuffer res = *pResult; - dataBufferInit(pResult, 0); - if( ii==(pPhrase->nToken-1) ){ - eType = eListType; - } - docListPhraseMerge( - res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult - ); - dataBufferDestroy(&res); - dataBufferDestroy(&tmp); - } + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); } + }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + }else{ + sqlite3_reset(pCsr->pStmt); + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + pCsr->isRequireSeek = 1; } + return rc; +} + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains the +** root node of a b-tree segment. The segment is guaranteed to be at least +** one level high (i.e. the root node is not also a leaf). If successful, +** this function locates the leaf node of the segment that may contain the +** term specified by arguments zTerm and nTerm and writes its block number +** to *piLeaf. +** +** It is possible that the returned leaf node does not contain the specified +** term. However, if the segment does contain said term, it is stored on +** the identified leaf node. Because this function only inspects interior +** segment nodes (and never loads leaf nodes into memory), it is not possible +** to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + + while( 1 ){ + int isFirstTerm = 1; /* True when processing first term on page */ + int iHeight; /* Height of this node in tree */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + int nBlock; /* Size of child node in bytes */ + + zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + + while( zCsrnAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + sqlite3_free(zBuffer); + return SQLITE_NOMEM; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; + iChild++; + }; + + /* If (iHeight==1), the children of this interior node are leaves. The + ** specified term may be present on leaf node iChild. + */ + if( iHeight==1 ){ + *piLeaf = iChild; + break; + } + + /* Descend to interior node iChild. */ + rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); + if( rc!=SQLITE_OK ) break; + zEnd = &zCsr[nBlock]; + } + sqlite3_free(zBuffer); return rc; } /* -** Evaluate the full-text expression pExpr against fts3 table pTab. Write -** the results into pRes. +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + while( *pEnd | c ) c = *pEnd++ & 0x80; + pEnd++; + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ) c = *pEnd++ & 0x80; + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an offset-list. This is safe because +** it is not possible to have a document with 2^31 terms. +*/ +#define OFFSET_LIST_END 0x7fffffff + +/* +** This function is used to help parse offset-lists. When this function is +** called, *pp may point to the start of the next varint in the offset-list +** being parsed, or it may point to 1 byte past the end of the offset-list +** (in which case **pp will be 0x00 or 0x01). +** +** If *pp points past the end of the current offset list, set *pi to +** OFFSET_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into offset-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from offset-list */ +){ + if( **pp&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; + }else{ + *pi = OFFSET_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an 0x01 byte followed by the value of +** iCol encoded as a varint to *pp. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; + int iCol2; + + if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==0x00 ) iCol1 = OFFSET_LIST_END; + else iCol1 = 0; + + if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==0x00 ) iCol2 = OFFSET_LIST_END; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of offset-lists. + ** An offset-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a 0 + ** or 1 value (0x00 or 0x01). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (0x00 or 0x01) is written to + ** the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1iPos1 && iPos2<=iPos1+nToken ){ + sqlite3_int64 iSave; + if( !pp ){ + fts3PoslistCopy(0, &p2); + fts3PoslistCopy(0, &p1); + *pp1 = p1; + *pp2 = p2; + return 1; + } + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + + } + if( pSave && pp ){ + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p2++; + p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1 D */ +#define MERGE_AND 3 /* D + D -> D */ +#define MERGE_OR 4 /* D + D -> D */ +#define MERGE_POS_OR 5 /* P + P -> P */ +#define MERGE_PHRASE 6 /* P + P -> D */ +#define MERGE_POS_PHRASE 7 /* P + P -> P */ +#define MERGE_NEAR 8 /* P + P -> D */ +#define MERGE_POS_NEAR 9 /* P + P -> P */ + +/* +** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 +** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, +** which is guaranteed to be large enough to hold the results. The number +** of bytes written to aBuffer is stored in *pnBuffer before returning. +** +** If successful, SQLITE_OK is returned. Otherwise, if a malloc error +** occurs while allocating a temporary buffer as part of the merge operation, +** SQLITE_NOMEM is returned. +*/ +static int fts3DoclistMerge( + int mergetype, /* One of the MERGE_XXX constants */ + int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ + int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ + char *aBuffer, /* Pre-allocated output buffer */ + int *pnBuffer, /* OUT: Bytes written to aBuffer */ + char *a1, /* Buffer containing first doclist */ + int n1, /* Size of buffer a1 */ + char *a2, /* Buffer containing second doclist */ + int n2 /* Size of buffer a2 */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + + char *p = aBuffer; + char *p1 = a1; + char *p2 = a2; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + + assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR + || mergetype==MERGE_AND || mergetype==MERGE_NOT + || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE + || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR + ); + + if( !aBuffer ){ + return SQLITE_NOMEM; + } + if( n1==0 && n2==0 ){ + *pnBuffer = 0; + return SQLITE_OK; + } + + /* Read the first docid from each doclist */ + fts3GetDeltaVarint2(&p1, pEnd1, &i1); + fts3GetDeltaVarint2(&p2, pEnd2, &i2); + + switch( mergetype ){ + case MERGE_OR: + case MERGE_POS_OR: + while( p1 || p2 ){ + if( p2 && p1 && i1==i2 ){ + fts3PutDeltaVarint(&p, &iPrev, i1); + if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint2(&p1, pEnd1, &i1); + fts3GetDeltaVarint2(&p2, pEnd2, &i2); + }else if( !p2 || (p1 && i1nOutput + nDoclist; + char *aNew = sqlite3_malloc(nNew); + + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(zTerm); + UNUSED_PARAMETER(nTerm); + + if( !aNew ){ + return SQLITE_NOMEM; + } + + if( pTS->nOutput==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). TODO: Add a way to transfer control of the + ** aDoclist buffer from the caller so as to avoid the memcpy(). + */ + memcpy(aNew, aDoclist, nDoclist); + }else{ + /* The output buffer is not empty. Merge doclist aDoclist with the + ** existing output. This can only happen with prefix-searches (as + ** searches for exact terms return exactly one doclist). + */ + int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); + fts3DoclistMerge(mergetype, 0, 0, + aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist + ); + } + + sqlite3_free(pTS->aOutput); + pTS->aOutput = aNew; + pTS->nOutput = nNew; + + return SQLITE_OK; +} + +/* +** This function retreives the doclist for the specified term (or term +** prefix) from the database. +** +** The returned doclist may be in one of two formats, depending on the +** value of parameter isReqPos. If isReqPos is zero, then the doclist is +** a sorted list of delta-compressed docids. If isReqPos is non-zero, +** then the returned list is in the same format as is stored in the +** database without the found length specifier at the start of on-disk +** doclists. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + int iColumn, /* Column to query (or -ve for all columns) */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isReqPos, /* True to include position lists in output */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int i; + TermSelect tsc; + Fts3SegFilter filter; /* Segment term filter configuration */ + Fts3SegReader **apSegment = 0; /* Array of segments to read data from */ + int nSegment = 0; /* Size of apSegment array */ + int nAlloc = 0; /* Allocated size of segment array */ + int rc; /* Return code */ + sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */ + int iAge = 0; /* Used to assign ages to segments */ + + /* Loop through the entire %_segdir table. For each segment, create a + ** Fts3SegReader to iterate through the subset of the segment leaves + ** that may contain a term that matches zTerm/nTerm. For non-prefix + ** searches, this is always a single leaf. For prefix searches, this + ** may be a contiguous block of leaves. + ** + ** The code in this loop does not actually load any leaves into memory + ** (unless the root node happens to be a leaf). It simply examines the + ** b-tree structure to determine which leaves need to be inspected. + */ + rc = sqlite3Fts3AllSegdirs(p, &pStmt); + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pNew = 0; + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + if( sqlite3_column_int64(pStmt, 1)==0 ){ + /* The entire segment is stored on the root node (which must be a + ** leaf). Do not bother inspecting any data in this case, just + ** create a Fts3SegReader to scan the single leaf. + */ + rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); + }else{ + int rc2; /* Return value of sqlite3Fts3ReadBlock() */ + sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */ + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); + if( rc==SQLITE_OK ){ + sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); + rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); + } + + /* The following call to ReadBlock() serves to reset the SQL statement + ** used to retrieve blocks of data from the %_segments table. If it is + ** not reset here, then it may remain classified as an active statement + ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands + ** failing. + */ + rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + iAge++; + + /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ + assert( pNew!=0 || rc!=SQLITE_OK ); + if( pNew ){ + if( nSegment==nAlloc ){ + Fts3SegReader **pArray; + nAlloc += 16; + pArray = (Fts3SegReader **)sqlite3_realloc( + apSegment, nAlloc*sizeof(Fts3SegReader *) + ); + if( !pArray ){ + sqlite3Fts3SegReaderFree(p, pNew); + rc = SQLITE_NOMEM; + goto finished; + } + apSegment = pArray; + } + apSegment[nSegment++] = pNew; + } + } + if( rc!=SQLITE_DONE ){ + assert( rc!=SQLITE_OK ); + goto finished; + } + + memset(&tsc, 0, sizeof(TermSelect)); + tsc.isReqPos = isReqPos; + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = zTerm; + filter.nTerm = nTerm; + + rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, + fts3TermSelectCb, (void *)&tsc + ); + + if( rc==SQLITE_OK ){ + *ppOut = tsc.aOutput; + *pnOut = tsc.nOutput; + }else{ + sqlite3_free(tsc.aOutput); + } + +finished: + sqlite3_reset(pStmt); + for(i=0; iiColumn; + int isTermPos = (pPhrase->nToken>1 || isReqPos); + + assert( p->nPendingData==0 ); + + for(ii=0; iinToken; ii++){ + struct PhraseToken *pTok = &pPhrase->aToken[ii]; + char *z = pTok->z; /* Next token of the phrase */ + int n = pTok->n; /* Size of z in bytes */ + int isPrefix = pTok->isPrefix;/* True if token is a prefix */ + char *pList; /* Pointer to token doclist */ + int nList; /* Size of buffer at pList */ + + rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); + if( rc!=SQLITE_OK ) break; + + if( ii==0 ){ + pOut = pList; + nOut = nList; + }else{ + /* Merge the new term list and the current output. If this is the + ** last term in the phrase, and positions are not required in the + ** output of this function, the positions can be dropped as part + ** of this merge. Either way, the result of this merge will be + ** smaller than nList bytes. The code in fts3DoclistMerge() is written + ** so that it is safe to use pList as the output as well as an input + ** in this case. + */ + int mergetype = MERGE_POS_PHRASE; + if( ii==pPhrase->nToken-1 && !isReqPos ){ + mergetype = MERGE_PHRASE; + } + fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList); + sqlite3_free(pOut); + pOut = pList; + } + } + + if( rc==SQLITE_OK ){ + *paOut = pOut; + *pnOut = nOut; + }else{ + sqlite3_free(pOut); + } + return rc; +} + +/* +** Evaluate the full-text expression pExpr against fts3 table pTab. Store +** the resulting doclist in *paOut and *pnOut. */ static int evalFts3Expr( - fulltext_vtab *pTab, /* Fts3 Virtual table object */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - DataBuffer *pRes /* OUT: Write results of the expression here */ + Fts3Table *p, /* Virtual table handle */ + Fts3Expr *pExpr, /* Parsed fts3 expression */ + char **paOut, /* OUT: Pointer to malloc'd result buffer */ + int *pnOut /* OUT: Size of buffer at *paOut */ ){ - int rc = SQLITE_OK; + int rc = SQLITE_OK; /* Return code */ - /* Initialize the output buffer. If this is an empty query (pExpr==0), - ** this is all that needs to be done. Empty queries produce empty - ** result sets. - */ - dataBufferInit(pRes, 0); + /* Zero the output parameters. */ + *paOut = 0; + *pnOut = 0; if( pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ - DocListType eType = DL_DOCIDS; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - eType = DL_POSITIONS; - } - rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes); + int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR); + rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut); }else{ - DataBuffer lhs; - DataBuffer rhs; + char *aLeft; + char *aRight; + int nLeft; + int nRight; - dataBufferInit(&rhs, 0); - if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs)) - && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs)) + if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight)) + && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft)) ){ + assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT + ); switch( pExpr->eType ){ case FTSQUERY_NEAR: { - int nToken; Fts3Expr *pLeft; - DocListType eType = DL_DOCIDS; + Fts3Expr *pRight; + int mergetype = MERGE_NEAR; + int nParam1; + int nParam2; + char *aBuffer; + if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - eType = DL_POSITIONS; + mergetype = MERGE_POS_NEAR; } pLeft = pExpr->pLeft; while( pLeft->eType==FTSQUERY_NEAR ){ pLeft=pLeft->pRight; } - assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); + pRight = pExpr->pRight; + assert( pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->eType==FTSQUERY_PHRASE ); - nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken; - docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, - pExpr->nNear+1, nToken, eType, pRes + + nParam1 = pExpr->nNear+1; + nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2; + aBuffer = sqlite3_malloc(nLeft+nRight+1); + rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer, + pnOut, aLeft, nLeft, aRight, nRight ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aBuffer); + }else{ + *paOut = aBuffer; + } + sqlite3_free(aLeft); break; } - case FTSQUERY_NOT: { - docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); - break; - } - case FTSQUERY_AND: { - docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); - break; - } + case FTSQUERY_OR: { - docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + /* Allocate a buffer for the output. The maximum size is the + ** sum of the sizes of the two input buffers. The +1 term is + ** so that a buffer of zero bytes is never allocated - this can + ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. + */ + char *aBuffer = sqlite3_malloc(nRight+nLeft+1); + rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, + aLeft, nLeft, aRight, nRight + ); + *paOut = aBuffer; + sqlite3_free(aLeft); + break; + } + + default: { + assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); + fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, + aLeft, nLeft, aRight, nRight + ); + *paOut = aLeft; break; } } } - dataBufferDestroy(&lhs); - dataBufferDestroy(&rhs); + sqlite3_free(aRight); } } return rc; } -/* TODO(shess) Refactor the code to remove this forward decl. */ -static int flushPendingTerms(fulltext_vtab *v); - -/* Perform a full-text query using the search expression in -** zInput[0..nInput-1]. Return a list of matching documents -** in pResult. -** -** Queries must match column iColumn. Or if iColumn>=nColumn -** they are allowed to match against any column. -*/ -static int fulltextQuery( - fulltext_vtab *v, /* The full text index */ - int iColumn, /* Match against this column by default */ - const char *zInput, /* The query string */ - int nInput, /* Number of bytes in zInput[] */ - DataBuffer *pResult, /* Write the result doclist here */ - Fts3Expr **ppExpr /* Put parsed query string here */ -){ - int rc; - - /* TODO(shess) Instead of flushing pendingTerms, we could query for - ** the relevant term and merge the doclist into what we receive from - ** the database. Wait and see if this is a common issue, first. - ** - ** A good reason not to flush is to not generate update-related - ** error codes from here. - */ - - /* Flush any buffered updates before executing the query. */ - rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Parse the query passed to the MATCH operator. */ - rc = sqlite3Fts3ExprParse(v->pTokenizer, - v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr - ); - if( rc!=SQLITE_OK ){ - assert( 0==(*ppExpr) ); - return rc; - } - - return evalFts3Expr(v, *ppExpr, pResult); -} - /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. ** -** If idxNum==QUERY_GENERIC then do a full table scan against +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against ** the %_content table. ** -** If idxNum==QUERY_DOCID then do a docid lookup for a single entry +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry ** in the %_content table. ** -** If idxNum>=QUERY_FULLTEXT then use the full text index. The +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The ** column on the left-hand side of the MATCH operator is column -** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ /* TODO(shess) Upgrade the cursor initialization and destruction to -** account for fulltextFilter() being called multiple times on the -** same cursor. The current solution is very fragile. Apply fix to +** account for fts3FilterMethod() being called multiple times on the +** same cursor. The current solution is very fragile. Apply fix to ** fts3 as appropriate. */ -static int fulltextFilter( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, const char *idxStr, /* Which indexing scheme to use */ - int argc, sqlite3_value **argv /* Arguments for the indexing scheme */ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - fulltext_vtab *v = cursor_vtab(c); - int rc; + const char *azSql[] = { + "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */ + "SELECT * FROM %Q.'%q_content'", /* full-table-scan */ + }; + int rc; /* Return code */ + char *zSql; /* SQL statement used to access %_content */ + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - FTSTRACE(("FTS3 Filter %p\n",pCursor)); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); - /* If the cursor has a statement that was not prepared according to - ** idxNum, clear it. I believe all calls to fulltextFilter with a - ** given cursor will have the same idxNum , but in this case it's - ** easy to be safe. + assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( nVal==0 || nVal==1 ); + assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. */ - if( c->pStmt && c->iCursorType!=idxNum ){ - sqlite3_finalize(c->pStmt); - c->pStmt = NULL; - } - - /* Get a fresh statement appropriate to idxNum. */ - /* TODO(shess): Add a prepared-statement cache in the vt structure. - ** The cache must handle multiple open cursors. Easier to cache the - ** statement variants at the vt to reduce malloc/realloc/free here. - ** Or we could have a StringBuffer variant which allowed stack - ** construction for small values. - */ - if( !c->pStmt ){ - StringBuffer sb; - initStringBuffer(&sb); - append(&sb, "SELECT docid, "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, " FROM %_content"); - if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?"); - rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, - stringBufferData(&sb)); - stringBufferDestroy(&sb); - if( rc!=SQLITE_OK ) return rc; - c->iCursorType = idxNum; + zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); + if( !zSql ){ + rc = SQLITE_NOMEM; }else{ - sqlite3_reset(c->pStmt); - assert( c->iCursorType==idxNum ); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); } + if( rc!=SQLITE_OK ) return rc; + pCsr->eSearch = (i16)idxNum; - switch( idxNum ){ - case QUERY_GENERIC: - break; + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ + int iCol = idxNum-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); - case QUERY_DOCID: - rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0])); - if( rc!=SQLITE_OK ) return rc; - break; - - default: /* full-text search */ - { - int iCol = idxNum-QUERY_FULLTEXT; - const char *zQuery = (const char *)sqlite3_value_text(argv[0]); - assert( idxNum<=QUERY_FULLTEXT+v->nColumn); - assert( argc==1 ); - if( c->result.nData!=0 ){ - /* This case happens if the same cursor is used repeatedly. */ - dlrDestroy(&c->reader); - dataBufferReset(&c->result); - }else{ - dataBufferInit(&c->result, 0); - } - rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr); - if( rc!=SQLITE_OK ) return rc; - if( c->result.nData!=0 ){ - dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData); - } - break; + if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + return SQLITE_NOMEM; } + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, + iCol, zQuery, -1, &pCsr->pExpr + ); + if( rc!=SQLITE_OK ) return rc; + + rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist); + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; } - return fulltextNext(pCursor); + if( rc!=SQLITE_OK ) return rc; + return fts3NextMethod(pCursor); } -/* This is the xEof method of the virtual table. The SQLite core -** calls this routine to find out if it has reached the end of -** a query's results set. +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. */ -static int fulltextEof(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - return c->eof; +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; } -/* This is the xColumn method of the virtual table. The SQLite -** core calls this method during a query when it needs the value -** of a column from the virtual table. This method needs to use -** one of the sqlite3_result_*() routines to store the requested -** value back in the pContext. -*/ -static int fulltextColumn(sqlite3_vtab_cursor *pCursor, - sqlite3_context *pContext, int idxCol){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - fulltext_vtab *v = cursor_vtab(c); - - if( idxColnColumn ){ - sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1); - sqlite3_result_value(pContext, pVal); - }else if( idxCol==v->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor - */ - sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT); - }else if( idxCol==v->nColumn+1 ){ - /* The docid column, which is an alias for rowid. */ - sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0); - sqlite3_result_value(pContext, pVal); - } - return SQLITE_OK; -} - -/* This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ -static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - - *pRowid = sqlite3_column_int64(c->pStmt, 0); - return SQLITE_OK; -} - -/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0, -** we also store positions and offsets in the hash table using that -** column number. -*/ -static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid, - const char *zText, int iColumn){ - sqlite3_tokenizer *pTokenizer = v->pTokenizer; - sqlite3_tokenizer_cursor *pCursor; - const char *pToken; - int nTokenBytes; - int iStartOffset, iEndOffset, iPosition; - int rc; - - rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor); - if( rc!=SQLITE_OK ) return rc; - - pCursor->pTokenizer = pTokenizer; - while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor, - &pToken, &nTokenBytes, - &iStartOffset, &iEndOffset, - &iPosition)) ){ - DLCollector *p; - int nData; /* Size of doclist before our update. */ - - /* Positions can't be negative; we use -1 as a terminator - * internally. Token can't be NULL or empty. */ - if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){ - rc = SQLITE_ERROR; - break; - } - - p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes); - if( p==NULL ){ - nData = 0; - p = dlcNew(iDocid, DL_DEFAULT); - fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p); - - /* Overhead for our hash table entry, the key, and the value. */ - v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes; - }else{ - nData = p->b.nData; - if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid); - } - if( iColumn>=0 ){ - dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset); - } - - /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */ - v->nPendingData += p->b.nData-nData; - } - - /* TODO(shess) Check return? Should this be able to cause errors at - ** this point? Actually, same question about sqlite3_finalize(), - ** though one could argue that failure there means that the data is - ** not durable. *ponder* - */ - pTokenizer->pModule->xClose(pCursor); - if( SQLITE_DONE == rc ) return SQLITE_OK; - return rc; -} - -/* Add doclists for all terms in [pValues] to pendingTerms table. */ -static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid, - sqlite3_value **pValues){ - int i; - for(i = 0; i < v->nColumn ; ++i){ - char *zText = (char*)sqlite3_value_text(pValues[i]); - int rc = buildTerms(v, iDocid, zText, i); - if( rc!=SQLITE_OK ) return rc; - } - return SQLITE_OK; -} - -/* Add empty doclists for all terms in the given row's content to -** pendingTerms. -*/ -static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){ - const char **pValues; - int i, rc; - - /* TODO(shess) Should we allow such tables at all? */ - if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR; - - rc = content_select(v, iDocid, &pValues); - if( rc!=SQLITE_OK ) return rc; - - for(i = 0 ; i < v->nColumn; ++i) { - rc = buildTerms(v, iDocid, pValues[i], -1); - if( rc!=SQLITE_OK ) break; - } - - freeStringArray(v->nColumn, pValues); - return SQLITE_OK; -} - -/* TODO(shess) Refactor the code to remove this forward decl. */ -static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid); - -/* Insert a row into the %_content table; set *piDocid to be the ID of the -** new row. Add doclists for terms to pendingTerms. -*/ -static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid, - sqlite3_value **pValues, sqlite_int64 *piDocid){ - int rc; - - rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */ - if( rc!=SQLITE_OK ) return rc; - - /* docid column is an alias for rowid. */ - *piDocid = sqlite3_last_insert_rowid(v->db); - rc = initPendingTerms(v, *piDocid); - if( rc!=SQLITE_OK ) return rc; - - return insertTerms(v, *piDocid, pValues); -} - -/* Delete a row from the %_content table; add empty doclists for terms -** to pendingTerms. -*/ -static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){ - int rc = initPendingTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - rc = deleteTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - return content_delete(v, iRow); /* execute an SQL DELETE */ -} - -/* Update a row in the %_content table; add delete doclists to -** pendingTerms for old terms not in the new data, add insert doclists -** to pendingTerms for terms in the new data. -*/ -static int index_update(fulltext_vtab *v, sqlite_int64 iRow, - sqlite3_value **pValues){ - int rc = initPendingTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - /* Generate an empty doclist for each term that previously appeared in this - * row. */ - rc = deleteTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */ - if( rc!=SQLITE_OK ) return rc; - - /* Now add positions for terms which appear in the updated row. */ - return insertTerms(v, iRow, pValues); -} - -/*******************************************************************/ -/* InteriorWriter is used to collect terms and block references into -** interior nodes in %_segments. See commentary at top of file for -** format. -*/ - -/* How large interior nodes can grow. */ -#define INTERIOR_MAX 2048 - -/* Minimum number of terms per interior node (except the root). This -** prevents large terms from making the tree too skinny - must be >0 -** so that the tree always makes progress. Note that the min tree -** fanout will be INTERIOR_MIN_TERMS+1. -*/ -#define INTERIOR_MIN_TERMS 7 -#if INTERIOR_MIN_TERMS<1 -# error INTERIOR_MIN_TERMS must be greater than 0. -#endif - -/* ROOT_MAX controls how much data is stored inline in the segment -** directory. -*/ -/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's -** only here so that interiorWriterRootInfo() and leafWriterRootInfo() -** can both see it, but if the caller passed it in, we wouldn't even -** need a define. -*/ -#define ROOT_MAX 1024 -#if ROOT_MAXterm, 0); - dataBufferReplace(&block->term, pTerm, nTerm); - - n = fts3PutVarint(c, iHeight); - n += fts3PutVarint(c+n, iChildBlock); - dataBufferInit(&block->data, INTERIOR_MAX); - dataBufferReplace(&block->data, c, n); - } - return block; -} - -#ifndef NDEBUG -/* Verify that the data is readable as an interior node. */ -static void interiorBlockValidate(InteriorBlock *pBlock){ - const char *pData = pBlock->data.pData; - int nData = pBlock->data.nData; - int n, iDummy; - sqlite_int64 iBlockid; - - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - - /* Must lead with height of node as a varint(n), n>0 */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>0 ); - assert( n0 ); - assert( n<=nData ); - pData += n; - nData -= n; - - /* Zero or more terms of positive length */ - if( nData!=0 ){ - /* First term is not delta-encoded. */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>0 ); - assert( n+iDummy>0); - assert( n+iDummy<=nData ); - pData += n+iDummy; - nData -= n+iDummy; - - /* Following terms delta-encoded. */ - while( nData!=0 ){ - /* Length of shared prefix. */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>=0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0); - assert( n+iDummy<=nData ); - pData += n+iDummy; - nData -= n+iDummy; - } - } -} -#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x) -#else -#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 ) -#endif - -typedef struct InteriorWriter { - int iHeight; /* from 0 at leaves. */ - InteriorBlock *first, *last; - struct InteriorWriter *parentWriter; - - DataBuffer term; /* Last term written to block "last". */ - sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */ -#ifndef NDEBUG - sqlite_int64 iLastChildBlock; /* for consistency checks. */ -#endif -} InteriorWriter; - -/* Initialize an interior node where pTerm[nTerm] marks the leftmost -** term in the tree. iChildBlock is the leftmost child block at the -** next level down the tree. -*/ -static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm, - sqlite_int64 iChildBlock, - InteriorWriter *pWriter){ - InteriorBlock *block; - assert( iHeight>0 ); - CLEAR(pWriter); - - pWriter->iHeight = iHeight; - pWriter->iOpeningChildBlock = iChildBlock; -#ifndef NDEBUG - pWriter->iLastChildBlock = iChildBlock; -#endif - block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm); - pWriter->last = pWriter->first = block; - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); - dataBufferInit(&pWriter->term, 0); -} - -/* Append the child node rooted at iChildBlock to the interior node, -** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree. -*/ -static void interiorWriterAppend(InteriorWriter *pWriter, - const char *pTerm, int nTerm, - sqlite_int64 iChildBlock){ - char c[VARINT_MAX+VARINT_MAX]; - int n, nPrefix = 0; - - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); - - /* The first term written into an interior node is actually - ** associated with the second child added (the first child was added - ** in interiorWriterInit, or in the if clause at the bottom of this - ** function). That term gets encoded straight up, with nPrefix left - ** at 0. - */ - if( pWriter->term.nData==0 ){ - n = fts3PutVarint(c, nTerm); +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + if( pCsr->aDoclist ){ + *pRowid = pCsr->iPrevId; }else{ - while( nPrefixterm.nData && - pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ - nPrefix++; - } - - n = fts3PutVarint(c, nPrefix); - n += fts3PutVarint(c+n, nTerm-nPrefix); - } - -#ifndef NDEBUG - pWriter->iLastChildBlock++; -#endif - assert( pWriter->iLastChildBlock==iChildBlock ); - - /* Overflow to a new block if the new term makes the current block - ** too big, and the current block already has enough terms. - */ - if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX && - iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){ - pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock, - pTerm, nTerm); - pWriter->last = pWriter->last->next; - pWriter->iOpeningChildBlock = iChildBlock; - dataBufferReset(&pWriter->term); - }else{ - dataBufferAppend2(&pWriter->last->data, c, n, - pTerm+nPrefix, nTerm-nPrefix); - dataBufferReplace(&pWriter->term, pTerm, nTerm); - } - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); -} - -/* Free the space used by pWriter, including the linked-list of -** InteriorBlocks, and parentWriter, if present. -*/ -static int interiorWriterDestroy(InteriorWriter *pWriter){ - InteriorBlock *block = pWriter->first; - - while( block!=NULL ){ - InteriorBlock *b = block; - block = block->next; - dataBufferDestroy(&b->term); - dataBufferDestroy(&b->data); - sqlite3_free(b); - } - if( pWriter->parentWriter!=NULL ){ - interiorWriterDestroy(pWriter->parentWriter); - sqlite3_free(pWriter->parentWriter); - } - dataBufferDestroy(&pWriter->term); - SCRAMBLE(pWriter); - return SQLITE_OK; -} - -/* If pWriter can fit entirely in ROOT_MAX, return it as the root info -** directly, leaving *piEndBlockid unchanged. Otherwise, flush -** pWriter to %_segments, building a new layer of interior nodes, and -** recursively ask for their root into. -*/ -static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter, - char **ppRootInfo, int *pnRootInfo, - sqlite_int64 *piEndBlockid){ - InteriorBlock *block = pWriter->first; - sqlite_int64 iBlockid = 0; - int rc; - - /* If we can fit the segment inline */ - if( block==pWriter->last && block->data.nDatadata.pData; - *pnRootInfo = block->data.nData; - return SQLITE_OK; - } - - /* Flush the first block to %_segments, and create a new level of - ** interior node. - */ - ASSERT_VALID_INTERIOR_BLOCK(block); - rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - *piEndBlockid = iBlockid; - - pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter)); - interiorWriterInit(pWriter->iHeight+1, - block->term.pData, block->term.nData, - iBlockid, pWriter->parentWriter); - - /* Flush additional blocks and append to the higher interior - ** node. - */ - for(block=block->next; block!=NULL; block=block->next){ - ASSERT_VALID_INTERIOR_BLOCK(block); - rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - *piEndBlockid = iBlockid; - - interiorWriterAppend(pWriter->parentWriter, - block->term.pData, block->term.nData, iBlockid); - } - - /* Parent node gets the chance to be the root. */ - return interiorWriterRootInfo(v, pWriter->parentWriter, - ppRootInfo, pnRootInfo, piEndBlockid); -} - -/****************************************************************/ -/* InteriorReader is used to read off the data from an interior node -** (see comment at top of file for the format). -*/ -typedef struct InteriorReader { - const char *pData; - int nData; - - DataBuffer term; /* previous term, for decoding term delta. */ - - sqlite_int64 iBlockid; -} InteriorReader; - -static void interiorReaderDestroy(InteriorReader *pReader){ - dataBufferDestroy(&pReader->term); - SCRAMBLE(pReader); -} - -/* TODO(shess) The assertions are great, but what if we're in NDEBUG -** and the blob is empty or otherwise contains suspect data? -*/ -static void interiorReaderInit(const char *pData, int nData, - InteriorReader *pReader){ - int n, nTerm; - - /* Require at least the leading flag byte */ - assert( nData>0 ); - assert( pData[0]!='\0' ); - - CLEAR(pReader); - - /* Decode the base blockid, and set the cursor to the first term. */ - n = fts3GetVarint(pData+1, &pReader->iBlockid); - assert( 1+n<=nData ); - pReader->pData = pData+1+n; - pReader->nData = nData-(1+n); - - /* A single-child interior node (such as when a leaf node was too - ** large for the segment directory) won't have any terms. - ** Otherwise, decode the first term. - */ - if( pReader->nData==0 ){ - dataBufferInit(&pReader->term, 0); - }else{ - n = fts3GetVarint32(pReader->pData, &nTerm); - dataBufferInit(&pReader->term, nTerm); - dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); - assert( n+nTerm<=pReader->nData ); - pReader->pData += n+nTerm; - pReader->nData -= n+nTerm; - } -} - -static int interiorReaderAtEnd(InteriorReader *pReader){ - return pReader->term.nData==0; -} - -static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){ - return pReader->iBlockid; -} - -static int interiorReaderTermBytes(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - return pReader->term.nData; -} -static const char *interiorReaderTerm(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - return pReader->term.pData; -} - -/* Step forward to the next term in the node. */ -static void interiorReaderStep(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - - /* If the last term has been read, signal eof, else construct the - ** next term. - */ - if( pReader->nData==0 ){ - dataBufferReset(&pReader->term); - }else{ - int n, nPrefix, nSuffix; - - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); - - /* Truncate the current term and append suffix data. */ - pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); - - assert( n+nSuffix<=pReader->nData ); - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; - } - pReader->iBlockid++; -} - -/* Compare the current term to pTerm[nTerm], returning strcmp-style -** results. If isPrefix, equality means equal through nTerm bytes. -*/ -static int interiorReaderTermCmp(InteriorReader *pReader, - const char *pTerm, int nTerm, int isPrefix){ - const char *pReaderTerm = interiorReaderTerm(pReader); - int nReaderTerm = interiorReaderTermBytes(pReader); - int c, n = nReaderTerm0 ) return -1; - if( nTerm>0 ) return 1; - return 0; - } - - c = memcmp(pReaderTerm, pTerm, n); - if( c!=0 ) return c; - if( isPrefix && n==nTerm ) return 0; - return nReaderTerm - nTerm; -} - -/****************************************************************/ -/* LeafWriter is used to collect terms and associated doclist data -** into leaf blocks in %_segments (see top of file for format info). -** Expected usage is: -** -** LeafWriter writer; -** leafWriterInit(0, 0, &writer); -** while( sorted_terms_left_to_process ){ -** // data is doclist data for that term. -** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData); -** if( rc!=SQLITE_OK ) goto err; -** } -** rc = leafWriterFinalize(v, &writer); -**err: -** leafWriterDestroy(&writer); -** return rc; -** -** leafWriterStep() may write a collected leaf out to %_segments. -** leafWriterFinalize() finishes writing any buffered data and stores -** a root node in %_segdir. leafWriterDestroy() frees all buffers and -** InteriorWriters allocated as part of writing this segment. -** -** TODO(shess) Document leafWriterStepMerge(). -*/ - -/* Put terms with data this big in their own block. */ -#define STANDALONE_MIN 1024 - -/* Keep leaf blocks below this size. */ -#define LEAF_MAX 2048 - -typedef struct LeafWriter { - int iLevel; - int idx; - sqlite_int64 iStartBlockid; /* needed to create the root info */ - sqlite_int64 iEndBlockid; /* when we're done writing. */ - - DataBuffer term; /* previous encoded term */ - DataBuffer data; /* encoding buffer */ - - /* bytes of first term in the current node which distinguishes that - ** term from the last term of the previous node. - */ - int nTermDistinct; - - InteriorWriter parentWriter; /* if we overflow */ - int has_parent; -} LeafWriter; - -static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){ - CLEAR(pWriter); - pWriter->iLevel = iLevel; - pWriter->idx = idx; - - dataBufferInit(&pWriter->term, 32); - - /* Start out with a reasonably sized block, though it can grow. */ - dataBufferInit(&pWriter->data, LEAF_MAX); -} - -#ifndef NDEBUG -/* Verify that the data is readable as a leaf node. */ -static void leafNodeValidate(const char *pData, int nData){ - int n, iDummy; - - if( nData==0 ) return; - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - - /* Must lead with a varint(0) */ - n = fts3GetVarint32(pData, &iDummy); - assert( iDummy==0 ); - assert( n>0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy<=nData ); - ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL); - pData += n+iDummy; - nData -= n+iDummy; - - /* Verify that trailing terms and doclists also are readable. */ - while( nData!=0 ){ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>=0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy<=nData ); - ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL); - pData += n+iDummy; - nData -= n+iDummy; - } -} -#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n) -#else -#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 ) -#endif - -/* Flush the current leaf node to %_segments, and adding the resulting -** blockid and the starting term to the interior node which will -** contain it. -*/ -static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter, - int iData, int nData){ - sqlite_int64 iBlockid = 0; - const char *pStartingTerm; - int nStartingTerm, rc, n; - - /* Must have the leading varint(0) flag, plus at least some - ** valid-looking data. - */ - assert( nData>2 ); - assert( iData>=0 ); - assert( iData+nData<=pWriter->data.nData ); - ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData); - - rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - assert( iBlockid!=0 ); - - /* Reconstruct the first term in the leaf for purposes of building - ** the interior node. - */ - n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm); - pStartingTerm = pWriter->data.pData+iData+1+n; - assert( pWriter->data.nData>iData+1+n+nStartingTerm ); - assert( pWriter->nTermDistinct>0 ); - assert( pWriter->nTermDistinct<=nStartingTerm ); - nStartingTerm = pWriter->nTermDistinct; - - if( pWriter->has_parent ){ - interiorWriterAppend(&pWriter->parentWriter, - pStartingTerm, nStartingTerm, iBlockid); - }else{ - interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid, - &pWriter->parentWriter); - pWriter->has_parent = 1; - } - - /* Track the span of this segment's leaf nodes. */ - if( pWriter->iEndBlockid==0 ){ - pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid; - }else{ - pWriter->iEndBlockid++; - assert( iBlockid==pWriter->iEndBlockid ); - } - - return SQLITE_OK; -} -static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){ - int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData); - if( rc!=SQLITE_OK ) return rc; - - /* Re-initialize the output buffer. */ - dataBufferReset(&pWriter->data); - - return SQLITE_OK; -} - -/* Fetch the root info for the segment. If the entire leaf fits -** within ROOT_MAX, then it will be returned directly, otherwise it -** will be flushed and the root info will be returned from the -** interior node. *piEndBlockid is set to the blockid of the last -** interior or leaf node written to disk (0 if none are written at -** all). -*/ -static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter, - char **ppRootInfo, int *pnRootInfo, - sqlite_int64 *piEndBlockid){ - /* we can fit the segment entirely inline */ - if( !pWriter->has_parent && pWriter->data.nDatadata.pData; - *pnRootInfo = pWriter->data.nData; - *piEndBlockid = 0; - return SQLITE_OK; - } - - /* Flush remaining leaf data. */ - if( pWriter->data.nData>0 ){ - int rc = leafWriterFlush(v, pWriter); - if( rc!=SQLITE_OK ) return rc; - } - - /* We must have flushed a leaf at some point. */ - assert( pWriter->has_parent ); - - /* Tenatively set the end leaf blockid as the end blockid. If the - ** interior node can be returned inline, this will be the final - ** blockid, otherwise it will be overwritten by - ** interiorWriterRootInfo(). - */ - *piEndBlockid = pWriter->iEndBlockid; - - return interiorWriterRootInfo(v, &pWriter->parentWriter, - ppRootInfo, pnRootInfo, piEndBlockid); -} - -/* Collect the rootInfo data and store it into the segment directory. -** This has the effect of flushing the segment's leaf data to -** %_segments, and also flushing any interior nodes to %_segments. -*/ -static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){ - sqlite_int64 iEndBlockid; - char *pRootInfo; - int rc, nRootInfo; - - rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - /* Don't bother storing an entirely empty segment. */ - if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK; - - return segdir_set(v, pWriter->iLevel, pWriter->idx, - pWriter->iStartBlockid, pWriter->iEndBlockid, - iEndBlockid, pRootInfo, nRootInfo); -} - -static void leafWriterDestroy(LeafWriter *pWriter){ - if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter); - dataBufferDestroy(&pWriter->term); - dataBufferDestroy(&pWriter->data); -} - -/* Encode a term into the leafWriter, delta-encoding as appropriate. -** Returns the length of the new term which distinguishes it from the -** previous term, which can be used to set nTermDistinct when a node -** boundary is crossed. -*/ -static int leafWriterEncodeTerm(LeafWriter *pWriter, - const char *pTerm, int nTerm){ - char c[VARINT_MAX+VARINT_MAX]; - int n, nPrefix = 0; - - assert( nTerm>0 ); - while( nPrefixterm.nData && - pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ - nPrefix++; - /* Failing this implies that the terms weren't in order. */ - assert( nPrefixdata.nData==0 ){ - /* Encode the node header and leading term as: - ** varint(0) - ** varint(nTerm) - ** char pTerm[nTerm] - */ - n = fts3PutVarint(c, '\0'); - n += fts3PutVarint(c+n, nTerm); - dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm); - }else{ - /* Delta-encode the term as: - ** varint(nPrefix) - ** varint(nSuffix) - ** char pTermSuffix[nSuffix] - */ - n = fts3PutVarint(c, nPrefix); - n += fts3PutVarint(c+n, nTerm-nPrefix); - dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix); - } - dataBufferReplace(&pWriter->term, pTerm, nTerm); - - return nPrefix+1; -} - -/* Used to avoid a memmove when a large amount of doclist data is in -** the buffer. This constructs a node and term header before -** iDoclistData and flushes the resulting complete node using -** leafWriterInternalFlush(). -*/ -static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - int iDoclistData){ - char c[VARINT_MAX+VARINT_MAX]; - int iData, n = fts3PutVarint(c, 0); - n += fts3PutVarint(c+n, nTerm); - - /* There should always be room for the header. Even if pTerm shared - ** a substantial prefix with the previous term, the entire prefix - ** could be constructed from earlier data in the doclist, so there - ** should be room. - */ - assert( iDoclistData>=n+nTerm ); - - iData = iDoclistData-(n+nTerm); - memcpy(pWriter->data.pData+iData, c, n); - memcpy(pWriter->data.pData+iData+n, pTerm, nTerm); - - return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData); -} - -/* Push pTerm[nTerm] along with the doclist data to the leaf layer of -** %_segments. -*/ -static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - DLReader *pReaders, int nReaders){ - char c[VARINT_MAX+VARINT_MAX]; - int iTermData = pWriter->data.nData, iDoclistData; - int i, nData, n, nActualData, nActual, rc, nTermDistinct; - - ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData); - nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm); - - /* Remember nTermDistinct if opening a new node. */ - if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct; - - iDoclistData = pWriter->data.nData; - - /* Estimate the length of the merged doclist so we can leave space - ** to encode it. - */ - for(i=0, nData=0; idata, c, n); - - docListMerge(&pWriter->data, pReaders, nReaders); - ASSERT_VALID_DOCLIST(DL_DEFAULT, - pWriter->data.pData+iDoclistData+n, - pWriter->data.nData-iDoclistData-n, NULL); - - /* The actual amount of doclist data at this point could be smaller - ** than the length we encoded. Additionally, the space required to - ** encode this length could be smaller. For small doclists, this is - ** not a big deal, we can just use memmove() to adjust things. - */ - nActualData = pWriter->data.nData-(iDoclistData+n); - nActual = fts3PutVarint(c, nActualData); - assert( nActualData<=nData ); - assert( nActual<=n ); - - /* If the new doclist is big enough for force a standalone leaf - ** node, we can immediately flush it inline without doing the - ** memmove(). - */ - /* TODO(shess) This test matches leafWriterStep(), which does this - ** test before it knows the cost to varint-encode the term and - ** doclist lengths. At some point, change to - ** pWriter->data.nData-iTermData>STANDALONE_MIN. - */ - if( nTerm+nActualData>STANDALONE_MIN ){ - /* Push leaf node from before this term. */ - if( iTermData>0 ){ - rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); - if( rc!=SQLITE_OK ) return rc; - - pWriter->nTermDistinct = nTermDistinct; - } - - /* Fix the encoded doclist length. */ - iDoclistData += n - nActual; - memcpy(pWriter->data.pData+iDoclistData, c, nActual); - - /* Push the standalone leaf node. */ - rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData); - if( rc!=SQLITE_OK ) return rc; - - /* Leave the node empty. */ - dataBufferReset(&pWriter->data); - - return rc; - } - - /* At this point, we know that the doclist was small, so do the - ** memmove if indicated. - */ - if( nActualdata.pData+iDoclistData+nActual, - pWriter->data.pData+iDoclistData+n, - pWriter->data.nData-(iDoclistData+n)); - pWriter->data.nData -= n-nActual; - } - - /* Replace written length with actual length. */ - memcpy(pWriter->data.pData+iDoclistData, c, nActual); - - /* If the node is too large, break things up. */ - /* TODO(shess) This test matches leafWriterStep(), which does this - ** test before it knows the cost to varint-encode the term and - ** doclist lengths. At some point, change to - ** pWriter->data.nData>LEAF_MAX. - */ - if( iTermData+nTerm+nActualData>LEAF_MAX ){ - /* Flush out the leading data as a node */ - rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); - if( rc!=SQLITE_OK ) return rc; - - pWriter->nTermDistinct = nTermDistinct; - - /* Rebuild header using the current term */ - n = fts3PutVarint(pWriter->data.pData, 0); - n += fts3PutVarint(pWriter->data.pData+n, nTerm); - memcpy(pWriter->data.pData+n, pTerm, nTerm); - n += nTerm; - - /* There should always be room, because the previous encoding - ** included all data necessary to construct the term. - */ - assert( ndata.nData-iDoclistDatadata.pData+n, - pWriter->data.pData+iDoclistData, - pWriter->data.nData-iDoclistData); - pWriter->data.nData -= iDoclistData-n; - } - ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData); - - return SQLITE_OK; -} - -/* Push pTerm[nTerm] along with the doclist data to the leaf layer of -** %_segments. -*/ -/* TODO(shess) Revise writeZeroSegment() so that doclists are -** constructed directly in pWriter->data. -*/ -static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - const char *pData, int nData){ - int rc; - DLReader reader; - - dlrInit(&reader, DL_DEFAULT, pData, nData); - rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1); - dlrDestroy(&reader); - - return rc; -} - - -/****************************************************************/ -/* LeafReader is used to iterate over an individual leaf node. */ -typedef struct LeafReader { - DataBuffer term; /* copy of current term. */ - - const char *pData; /* data for current term. */ - int nData; -} LeafReader; - -static void leafReaderDestroy(LeafReader *pReader){ - dataBufferDestroy(&pReader->term); - SCRAMBLE(pReader); -} - -static int leafReaderAtEnd(LeafReader *pReader){ - return pReader->nData<=0; -} - -/* Access the current term. */ -static int leafReaderTermBytes(LeafReader *pReader){ - return pReader->term.nData; -} -static const char *leafReaderTerm(LeafReader *pReader){ - assert( pReader->term.nData>0 ); - return pReader->term.pData; -} - -/* Access the doclist data for the current term. */ -static int leafReaderDataBytes(LeafReader *pReader){ - int nData; - assert( pReader->term.nData>0 ); - fts3GetVarint32(pReader->pData, &nData); - return nData; -} -static const char *leafReaderData(LeafReader *pReader){ - int n, nData; - assert( pReader->term.nData>0 ); - n = fts3GetVarint32(pReader->pData, &nData); - return pReader->pData+n; -} - -static void leafReaderInit(const char *pData, int nData, - LeafReader *pReader){ - int nTerm, n; - - assert( nData>0 ); - assert( pData[0]=='\0' ); - - CLEAR(pReader); - - /* Read the first term, skipping the header byte. */ - n = fts3GetVarint32(pData+1, &nTerm); - dataBufferInit(&pReader->term, nTerm); - dataBufferReplace(&pReader->term, pData+1+n, nTerm); - - /* Position after the first term. */ - assert( 1+n+nTermpData = pData+1+n+nTerm; - pReader->nData = nData-1-n-nTerm; -} - -/* Step the reader forward to the next term. */ -static void leafReaderStep(LeafReader *pReader){ - int n, nData, nPrefix, nSuffix; - assert( !leafReaderAtEnd(pReader) ); - - /* Skip previous entry's data block. */ - n = fts3GetVarint32(pReader->pData, &nData); - assert( n+nData<=pReader->nData ); - pReader->pData += n+nData; - pReader->nData -= n+nData; - - if( !leafReaderAtEnd(pReader) ){ - /* Construct the new term using a prefix from the old term plus a - ** suffix from the leaf data. - */ - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); - assert( n+nSuffixnData ); - pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); - - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; - } -} - -/* strcmp-style comparison of pReader's current term against pTerm. -** If isPrefix, equality means equal through nTerm bytes. -*/ -static int leafReaderTermCmp(LeafReader *pReader, - const char *pTerm, int nTerm, int isPrefix){ - int c, n = pReader->term.nDataterm.nData : nTerm; - if( n==0 ){ - if( pReader->term.nData>0 ) return -1; - if(nTerm>0 ) return 1; - return 0; - } - - c = memcmp(pReader->term.pData, pTerm, n); - if( c!=0 ) return c; - if( isPrefix && n==nTerm ) return 0; - return pReader->term.nData - nTerm; -} - - -/****************************************************************/ -/* LeavesReader wraps LeafReader to allow iterating over the entire -** leaf layer of the tree. -*/ -typedef struct LeavesReader { - int idx; /* Index within the segment. */ - - sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */ - int eof; /* we've seen SQLITE_DONE from pStmt. */ - - LeafReader leafReader; /* reader for the current leaf. */ - DataBuffer rootData; /* root data for inline. */ -} LeavesReader; - -/* Access the current term. */ -static int leavesReaderTermBytes(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderTermBytes(&pReader->leafReader); -} -static const char *leavesReaderTerm(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderTerm(&pReader->leafReader); -} - -/* Access the doclist data for the current term. */ -static int leavesReaderDataBytes(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderDataBytes(&pReader->leafReader); -} -static const char *leavesReaderData(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderData(&pReader->leafReader); -} - -static int leavesReaderAtEnd(LeavesReader *pReader){ - return pReader->eof; -} - -/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus -** leaving the statement handle open, which locks the table. -*/ -/* TODO(shess) This "solution" is not satisfactory. Really, there -** should be check-in function for all statement handles which -** arranges to call sqlite3_reset(). This most likely will require -** modification to control flow all over the place, though, so for now -** just punt. -** -** Note the the current system assumes that segment merges will run to -** completion, which is why this particular probably hasn't arisen in -** this case. Probably a brittle assumption. -*/ -static int leavesReaderReset(LeavesReader *pReader){ - return sqlite3_reset(pReader->pStmt); -} - -static void leavesReaderDestroy(LeavesReader *pReader){ - /* If idx is -1, that means we're using a non-cached statement - ** handle in the optimize() case, so we need to release it. - */ - if( pReader->pStmt!=NULL && pReader->idx==-1 ){ - sqlite3_finalize(pReader->pStmt); - } - leafReaderDestroy(&pReader->leafReader); - dataBufferDestroy(&pReader->rootData); - SCRAMBLE(pReader); -} - -/* Initialize pReader with the given root data (if iStartBlockid==0 -** the leaf data was entirely contained in the root), or from the -** stream of blocks between iStartBlockid and iEndBlockid, inclusive. -*/ -static int leavesReaderInit(fulltext_vtab *v, - int idx, - sqlite_int64 iStartBlockid, - sqlite_int64 iEndBlockid, - const char *pRootData, int nRootData, - LeavesReader *pReader){ - CLEAR(pReader); - pReader->idx = idx; - - dataBufferInit(&pReader->rootData, 0); - if( iStartBlockid==0 ){ - /* Entire leaf level fit in root data. */ - dataBufferReplace(&pReader->rootData, pRootData, nRootData); - leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, - &pReader->leafReader); - }else{ - sqlite3_stmt *s; - int rc = sql_get_leaf_statement(v, idx, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 2, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ){ - pReader->eof = 1; - return SQLITE_OK; - } - if( rc!=SQLITE_ROW ) return rc; - - pReader->pStmt = s; - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; } -/* Step the current leaf forward to the next term. If we reach the -** end of the current leaf, step forward to the next leaf block. +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. */ -static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ - assert( !leavesReaderAtEnd(pReader) ); - leafReaderStep(&pReader->leafReader); - - if( leafReaderAtEnd(&pReader->leafReader) ){ - int rc; - if( pReader->rootData.pData ){ - pReader->eof = 1; - return SQLITE_OK; - } - rc = sqlite3_step(pReader->pStmt); - if( rc!=SQLITE_ROW ){ - pReader->eof = 1; - return rc==SQLITE_DONE ? SQLITE_OK : rc; - } - leafReaderDestroy(&pReader->leafReader); - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); - } - return SQLITE_OK; -} - -/* Order LeavesReaders by their term, ignoring idx. Readers at eof -** always sort to the end. -*/ -static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){ - if( leavesReaderAtEnd(lr1) ){ - if( leavesReaderAtEnd(lr2) ) return 0; - return 1; - } - if( leavesReaderAtEnd(lr2) ) return -1; - - return leafReaderTermCmp(&lr1->leafReader, - leavesReaderTerm(lr2), leavesReaderTermBytes(lr2), - 0); -} - -/* Similar to leavesReaderTermCmp(), with additional ordering by idx -** so that older segments sort before newer segments. -*/ -static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){ - int c = leavesReaderTermCmp(lr1, lr2); - if( c!=0 ) return c; - return lr1->idx-lr2->idx; -} - -/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its -** sorted position. -*/ -static void leavesReaderReorder(LeavesReader *pLr, int nLr){ - while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){ - LeavesReader tmp = pLr[0]; - pLr[0] = pLr[1]; - pLr[1] = tmp; - nLr--; - pLr++; - } -} - -/* Initializes pReaders with the segments from level iLevel, returning -** the number of segments in *piReaders. Leaves pReaders in sorted -** order. -*/ -static int leavesReadersInit(fulltext_vtab *v, int iLevel, - LeavesReader *pReaders, int *piReaders){ - sqlite3_stmt *s; - int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - i = 0; - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - sqlite_int64 iStart = sqlite3_column_int64(s, 0); - sqlite_int64 iEnd = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - int nRootData = sqlite3_column_bytes(s, 2); - - assert( i0 ){ - leavesReaderDestroy(&pReaders[i]); - } - return rc; - } - - *piReaders = i; - - /* Leave our results sorted by term, then age. */ - while( i-- ){ - leavesReaderReorder(pReaders+i, *piReaders-i); - } - return SQLITE_OK; -} - -/* Merge doclists from pReaders[nReaders] into a single doclist, which -** is written to pWriter. Assumes pReaders is ordered oldest to -** newest. -*/ -/* TODO(shess) Consider putting this inline in segmentMerge(). */ -static int leavesReadersMerge(fulltext_vtab *v, - LeavesReader *pReaders, int nReaders, - LeafWriter *pWriter){ - DLReader dlReaders[MERGE_COUNT]; - const char *pTerm = leavesReaderTerm(pReaders); - int i, nTerm = leavesReaderTermBytes(pReaders); - - assert( nReaders<=MERGE_COUNT ); - - for(i=0; i0 ){ - rc = leavesReaderStep(v, lrs+i); - if( rc!=SQLITE_OK ) goto err; - - /* Reorder by term, then by age. */ - leavesReaderReorder(lrs+i, MERGE_COUNT-i); - } - } - - for(i=0; i0 ); - - for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader); - rc=leavesReaderStep(v, pReader)){ - /* TODO(shess) Really want leavesReaderTermCmp(), but that name is - ** already taken to compare the terms of two LeavesReaders. Think - ** on a better name. [Meanwhile, break encapsulation rather than - ** use a confusing name.] - */ - int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); - if( c>0 ) break; /* Past any possible matches. */ - if( c==0 ){ - const char *pData = leavesReaderData(pReader); - int iBuffer, nData = leavesReaderDataBytes(pReader); - - /* Find the first empty buffer. */ - for(iBuffer=0; iBuffer0 ){ - assert(pBuffers!=NULL); - memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers)); - sqlite3_free(pBuffers); - } - pBuffers = p; - } - dataBufferInit(&(pBuffers[nBuffers]), 0); - nBuffers++; - } - - /* At this point, must have an empty at iBuffer. */ - assert(iBufferpData, p->nData); - - /* dataBufferReset() could allow a large doclist to blow up - ** our memory requirements. - */ - if( p->nCapacity<1024 ){ - dataBufferReset(p); - }else{ - dataBufferDestroy(p); - dataBufferInit(p, 0); - } - } - } - } - } - - /* Union all the doclists together into *out. */ - /* TODO(shess) What if *out is big? Sigh. */ - if( rc==SQLITE_OK && nBuffers>0 ){ - int iBuffer; - for(iBuffer=0; iBuffer0 ){ - if( out->nData==0 ){ - dataBufferSwap(out, &(pBuffers[iBuffer])); - }else{ - docListAccumulateUnion(out, pBuffers[iBuffer].pData, - pBuffers[iBuffer].nData); - } - } - } - } - - while( nBuffers-- ){ - dataBufferDestroy(&(pBuffers[nBuffers])); - } - if( pBuffers!=NULL ) sqlite3_free(pBuffers); - - return rc; -} - -/* Call loadSegmentLeavesInt() with pData/nData as input. */ -static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - LeavesReader reader; - int rc; - - assert( nData>1 ); - assert( *pData=='\0' ); - rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader); - if( rc!=SQLITE_OK ) return rc; - - rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out); - leavesReaderReset(&reader); - leavesReaderDestroy(&reader); - return rc; -} - -/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to -** iEndLeaf (inclusive) as input, and merge the resulting doclist into -** out. -*/ -static int loadSegmentLeaves(fulltext_vtab *v, - sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - int rc; - LeavesReader reader; - - assert( iStartLeaf<=iEndLeaf ); - rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader); - if( rc!=SQLITE_OK ) return rc; - - rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out); - leavesReaderReset(&reader); - leavesReaderDestroy(&reader); - return rc; -} - -/* Taking pData/nData as an interior node, find the sequence of child -** nodes which could include pTerm/nTerm/isPrefix. Note that the -** interior node terms logically come between the blocks, so there is -** one more blockid than there are terms (that block contains terms >= -** the last interior-node term). -*/ -/* TODO(shess) The calling code may already know that the end child is -** not worth calculating, because the end may be in a later sibling -** node. Consider whether breaking symmetry is worthwhile. I suspect -** it is not worthwhile. -*/ -static void getChildrenContaining(const char *pData, int nData, - const char *pTerm, int nTerm, int isPrefix, - sqlite_int64 *piStartChild, - sqlite_int64 *piEndChild){ - InteriorReader reader; - - assert( nData>1 ); - assert( *pData!='\0' ); - interiorReaderInit(pData, nData, &reader); - - /* Scan for the first child which could contain pTerm/nTerm. */ - while( !interiorReaderAtEnd(&reader) ){ - if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break; - interiorReaderStep(&reader); - } - *piStartChild = interiorReaderCurrentBlockid(&reader); - - /* Keep scanning to find a term greater than our term, using prefix - ** comparison if indicated. If isPrefix is false, this will be the - ** same blockid as the starting block. - */ - while( !interiorReaderAtEnd(&reader) ){ - if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break; - interiorReaderStep(&reader); - } - *piEndChild = interiorReaderCurrentBlockid(&reader); - - interiorReaderDestroy(&reader); - - /* Children must ascend, and if !prefix, both must be the same. */ - assert( *piEndChild>=*piStartChild ); - assert( isPrefix || *piStartChild==*piEndChild ); -} - -/* Read block at iBlockid and pass it with other params to -** getChildrenContaining(). -*/ -static int loadAndGetChildrenContaining( - fulltext_vtab *v, - sqlite_int64 iBlockid, - const char *pTerm, int nTerm, int isPrefix, - sqlite_int64 *piStartChild, sqlite_int64 *piEndChild +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ ){ - sqlite3_stmt *s = NULL; - int rc; + int rc; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; - assert( iBlockid!=0 ); - assert( pTerm!=NULL ); - assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */ - assert( piStartChild!=NULL ); - assert( piEndChild!=NULL ); + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+1 ); - rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_ERROR; - if( rc!=SQLITE_ROW ) return rc; - - getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0), - pTerm, nTerm, isPrefix, piStartChild, piEndChild); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain - * locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - - return SQLITE_OK; -} - -/* Traverse the tree represented by pData[nData] looking for -** pTerm[nTerm], placing its doclist into *out. This is internal to -** loadSegment() to make error-handling cleaner. -*/ -static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 iLeavesEnd, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - /* Special case where root is a leaf. */ - if( *pData=='\0' ){ - return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out); - }else{ - int rc; - sqlite_int64 iStartChild, iEndChild; - - /* Process pData as an interior node, then loop down the tree - ** until we find the set of leaf nodes to scan for the term. - */ - getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, - &iStartChild, &iEndChild); - while( iStartChild>iLeavesEnd ){ - sqlite_int64 iNextStart, iNextEnd; - rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix, - &iNextStart, &iNextEnd); - if( rc!=SQLITE_OK ) return rc; - - /* If we've branched, follow the end branch, too. */ - if( iStartChild!=iEndChild ){ - sqlite_int64 iDummy; - rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix, - &iDummy, &iNextEnd); - if( rc!=SQLITE_OK ) return rc; - } - - assert( iNextStart<=iNextEnd ); - iStartChild = iNextStart; - iEndChild = iNextEnd; - } - assert( iStartChild<=iLeavesEnd ); - assert( iEndChild<=iLeavesEnd ); - - /* Scan through the leaf segments for doclists. */ - return loadSegmentLeaves(v, iStartChild, iEndChild, - pTerm, nTerm, isPrefix, out); - } -} - -/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then -** merge its doclist over *out (any duplicate doclists read from the -** segment rooted at pData will overwrite those in *out). -*/ -/* TODO(shess) Consider changing this to determine the depth of the -** leaves using either the first characters of interior nodes (when -** ==1, we're one level above the leaves), or the first character of -** the root (which will describe the height of the tree directly). -** Either feels somewhat tricky to me. -*/ -/* TODO(shess) The current merge is likely to be slow for large -** doclists (though it should process from newest/smallest to -** oldest/largest, so it may not be that bad). It might be useful to -** modify things to allow for N-way merging. This could either be -** within a segment, with pairwise merges across segments, or across -** all segments at once. -*/ -static int loadSegment(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 iLeavesEnd, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - DataBuffer result; - int rc; - - assert( nData>1 ); - - /* This code should never be called with buffered updates. */ - assert( v->nPendingData<0 ); - - dataBufferInit(&result, 0); - rc = loadSegmentInt(v, pData, nData, iLeavesEnd, - pTerm, nTerm, isPrefix, &result); - if( rc==SQLITE_OK && result.nData>0 ){ - if( out->nData==0 ){ - DataBuffer tmp = *out; - *out = result; - result = tmp; + rc = fts3CursorSeek(pCsr); + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_int64 iRowid; + rc = fts3RowidMethod(pCursor, &iRowid); + sqlite3_result_int64(pContext, iRowid); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. + */ + sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else{ - DataBuffer merged; - DLReader readers[2]; - - dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); - dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); - dataBufferInit(&merged, out->nData+result.nData); - docListMerge(&merged, readers, 2); - dataBufferDestroy(out); - *out = merged; - dlrDestroy(&readers[0]); - dlrDestroy(&readers[1]); + sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } - dataBufferDestroy(&result); return rc; } -/* Scan the database and merge together the posting lists for the term -** into *out. +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. */ -static int termSelect( - fulltext_vtab *v, - int iColumn, - const char *pTerm, int nTerm, /* Term to query for */ - int isPrefix, /* True for a prefix search */ - DocListType iType, - DataBuffer *out /* Write results here */ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ - DataBuffer doclist; - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - /* This code should never be called with buffered updates. */ - assert( v->nPendingData<0 ); - - dataBufferInit(&doclist, 0); - dataBufferInit(out, 0); - - /* Traverse the segments from oldest to newest so that newer doclist - ** elements for given docids overwrite older elements. - */ - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - const char *pData = sqlite3_column_blob(s, 2); - const int nData = sqlite3_column_bytes(s, 2); - const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); - rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, - &doclist); - if( rc!=SQLITE_OK ) goto err; - } - if( rc==SQLITE_DONE ){ - if( doclist.nData!=0 ){ - /* TODO(shess) The old term_select_all() code applied the column - ** restrict as we merged segments, leading to smaller buffers. - ** This is probably worthwhile to bring back, once the new storage - ** system is checked in. - */ - if( iColumn==v->nColumn) iColumn = -1; - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - iColumn, iType, out); - } - rc = SQLITE_OK; - } - - err: - dataBufferDestroy(&doclist); - return rc; + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); } -/****************************************************************/ -/* Used to hold hashtable data for sorting. */ -typedef struct TermData { - const char *pTerm; - int nTerm; - DLCollector *pCollector; -} TermData; - -/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0 -** for equal, >0 for greater-than). +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. */ -static int termDataCmp(const void *av, const void *bv){ - const TermData *a = (const TermData *)av; - const TermData *b = (const TermData *)bv; - int n = a->nTermnTerm ? a->nTerm : b->nTerm; - int c = memcmp(a->pTerm, b->pTerm, n); - if( c!=0 ) return c; - return a->nTerm-b->nTerm; +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); } -/* Order pTerms data by term, then write a new level 0 segment using -** LeafWriter. +/* +** Implementation of xBegin() method. This is a no-op. */ -static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){ - fts3HashElem *e; - int idx, rc, i, n; - TermData *pData; - LeafWriter writer; - DataBuffer dl; - - /* Determine the next index at level 0, merging as necessary. */ - rc = segdirNextIndex(v, 0, &idx); - if( rc!=SQLITE_OK ) return rc; - - n = fts3HashCount(pTerms); - pData = sqlite3_malloc(n*sizeof(TermData)); - - for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){ - assert( i1 ) qsort(pData, n, sizeof(*pData), termDataCmp); - - /* TODO(shess) Refactor so that we can write directly to the segment - ** DataBuffer, as happens for segment merges. - */ - leafWriterInit(0, idx, &writer); - dataBufferInit(&dl, 0); - for(i=0; inPendingData>=0 ){ - fts3HashElem *e; - for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){ - dlcDelete(fts3HashData(e)); - } - fts3HashClear(&v->pendingTerms); - v->nPendingData = -1; - } +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + UNUSED_PARAMETER(pVtab); + assert( ((Fts3Table *)pVtab)->nPendingData==0 ); return SQLITE_OK; } -/* If pendingTerms has data, flush it to a level-zero segment, and -** free it. +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). */ -static int flushPendingTerms(fulltext_vtab *v){ - if( v->nPendingData>=0 ){ - int rc = writeZeroSegment(v, &v->pendingTerms); - if( rc==SQLITE_OK ) clearPendingTerms(v); - return rc; - } +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + UNUSED_PARAMETER(pVtab); + assert( ((Fts3Table *)pVtab)->nPendingData==0 ); return SQLITE_OK; } -/* If pendingTerms is "too big", or docid is out of order, flush it. -** Regardless, be certain that pendingTerms is initialized for use. +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. */ -static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){ - /* TODO(shess) Explore whether partially flushing the buffer on - ** forced-flush would provide better performance. I suspect that if - ** we ordered the doclists by size and flushed the largest until the - ** buffer was half empty, that would let the less frequent terms - ** generate longer doclists. - */ - if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){ - int rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ) return rc; - } - if( v->nPendingData<0 ){ - fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1); - v->nPendingData = 0; - } - v->iPrevDocid = iDocid; +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); return SQLITE_OK; } -/* This function implements the xUpdate callback; it is the top-level entry - * point for inserting, deleting or updating a row in a full-text table. */ -static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg, - sqlite_int64 *pRowid){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - int rc; - - FTSTRACE(("FTS3 Update %p\n", pVtab)); - - if( nArg<2 ){ - rc = index_delete(v, sqlite3_value_int64(ppArg[0])); - if( rc==SQLITE_OK ){ - /* If we just deleted the last row in the table, clear out the - ** index data. - */ - rc = content_exists(v); - if( rc==SQLITE_ROW ){ - rc = SQLITE_OK; - }else if( rc==SQLITE_DONE ){ - /* Clear the pending terms so we don't flush a useless level-0 - ** segment when the transaction closes. - */ - rc = clearPendingTerms(v); - if( rc==SQLITE_OK ){ - rc = segdir_delete_all(v); - } - } - } - } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){ - /* An update: - * ppArg[0] = old rowid - * ppArg[1] = new rowid - * ppArg[2..2+v->nColumn-1] = values - * ppArg[2+v->nColumn] = value for magic column (we ignore this) - * ppArg[2+v->nColumn+1] = value for docid - */ - sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]); - if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER || - sqlite3_value_int64(ppArg[1]) != rowid ){ - rc = SQLITE_ERROR; /* we don't allow changing the rowid */ - }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER || - sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){ - rc = SQLITE_ERROR; /* we don't allow changing the docid */ - }else{ - assert( nArg==2+v->nColumn+2); - rc = index_update(v, rowid, &ppArg[2]); - } - } else { - /* An insert: - * ppArg[1] = requested rowid - * ppArg[2..2+v->nColumn-1] = values - * ppArg[2+v->nColumn] = value for magic column (we ignore this) - * ppArg[2+v->nColumn+1] = value for docid - */ - sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1]; - assert( nArg==2+v->nColumn+2); - if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) && - SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){ - /* TODO(shess) Consider allowing this to work if the values are - ** identical. I'm inclined to discourage that usage, though, - ** given that both rowid and docid are special columns. Better - ** would be to define one or the other as the default winner, - ** but should it be fts3-centric (docid) or SQLite-centric - ** (rowid)? - */ - rc = SQLITE_ERROR; - }else{ - if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){ - pRequestDocid = ppArg[1]; - } - rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid); - } +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; } - - return rc; -} - -static int fulltextSync(sqlite3_vtab *pVtab){ - FTSTRACE(("FTS3 xSync()\n")); - return flushPendingTerms((fulltext_vtab *)pVtab); -} - -static int fulltextBegin(sqlite3_vtab *pVtab){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - FTSTRACE(("FTS3 xBegin()\n")); - - /* Any buffered updates should have been cleared by the previous - ** transaction. - */ - assert( v->nPendingData<0 ); - return clearPendingTerms(v); -} - -static int fulltextCommit(sqlite3_vtab *pVtab){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - FTSTRACE(("FTS3 xCommit()\n")); - - /* Buffered updates should have been cleared by fulltextSync(). */ - assert( v->nPendingData<0 ); - return clearPendingTerms(v); -} - -static int fulltextRollback(sqlite3_vtab *pVtab){ - FTSTRACE(("FTS3 xRollback()\n")); - return clearPendingTerms((fulltext_vtab *)pVtab); + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; } /* ** Implementation of the snippet() function for FTS3 */ -static void snippetFunc( - sqlite3_context *pContext, - int argc, - sqlite3_value **argv +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ ){ - fulltext_cursor *pCursor; - if( argc<1 ) return; - if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1); - }else{ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - if( argc>=2 ){ - zStart = (const char*)sqlite3_value_text(argv[1]); - if( argc>=3 ){ - zEnd = (const char*)sqlite3_value_text(argv[2]); - if( argc>=4 ){ - zEllipsis = (const char*)sqlite3_value_text(argv[3]); - } - } - } - snippetAllOffsets(pCursor); - snippetText(pCursor, zStart, zEnd, zEllipsis); - sqlite3_result_text(pContext, pCursor->snippet.zSnippet, - pCursor->snippet.nSnippet, SQLITE_STATIC); + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>4 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis); } /* ** Implementation of the offsets() function for FTS3 */ -static void snippetOffsetsFunc( - sqlite3_context *pContext, - int argc, - sqlite3_value **argv +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ ){ - fulltext_cursor *pCursor; - if( argc<1 ) return; - if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to offsets",-1); - }else{ - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - snippetAllOffsets(pCursor); - snippetOffsetText(&pCursor->snippet); - sqlite3_result_text(pContext, - pCursor->snippet.zOffset, pCursor->snippet.nOffset, - SQLITE_STATIC); - } + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + sqlite3Fts3Offsets(pContext, pCsr); } -/* OptLeavesReader is nearly identical to LeavesReader, except that -** where LeavesReader is geared towards the merging of complete -** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader -** is geared towards implementation of the optimize() function, and -** can merge all segments simultaneously. This version may be -** somewhat less efficient than LeavesReader because it merges into an -** accumulator rather than doing an N-way merge, but since segment -** size grows exponentially (so segment count logrithmically) this is -** probably not an immediate problem. +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. */ -/* TODO(shess): Prove that assertion, or extend the merge code to -** merge tree fashion (like the prefix-searching code does). -*/ -/* TODO(shess): OptLeavesReader and LeavesReader could probably be -** merged with little or no loss of performance for LeavesReader. The -** merged code would need to handle >MERGE_COUNT segments, and would -** also need to be able to optionally optimize away deletes. -*/ -typedef struct OptLeavesReader { - /* Segment number, to order readers by age. */ - int segment; - LeavesReader reader; -} OptLeavesReader; - -static int optLeavesReaderAtEnd(OptLeavesReader *pReader){ - return leavesReaderAtEnd(&pReader->reader); -} -static int optLeavesReaderTermBytes(OptLeavesReader *pReader){ - return leavesReaderTermBytes(&pReader->reader); -} -static const char *optLeavesReaderData(OptLeavesReader *pReader){ - return leavesReaderData(&pReader->reader); -} -static int optLeavesReaderDataBytes(OptLeavesReader *pReader){ - return leavesReaderDataBytes(&pReader->reader); -} -static const char *optLeavesReaderTerm(OptLeavesReader *pReader){ - return leavesReaderTerm(&pReader->reader); -} -static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){ - return leavesReaderStep(v, &pReader->reader); -} -static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ - return leavesReaderTermCmp(&lr1->reader, &lr2->reader); -} -/* Order by term ascending, segment ascending (oldest to newest), with -** exhausted readers to the end. -*/ -static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ - int c = optLeavesReaderTermCmp(lr1, lr2); - if( c!=0 ) return c; - return lr1->segment-lr2->segment; -} -/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that -** pLr[1..nLr-1] is already sorted. -*/ -static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){ - while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){ - OptLeavesReader tmp = pLr[0]; - pLr[0] = pLr[1]; - pLr[1] = tmp; - nLr--; - pLr++; - } -} - -/* optimize() helper function. Put the readers in order and iterate -** through them, merging doclists for matching terms into pWriter. -** Returns SQLITE_OK on success, or the SQLite error code which -** prevented success. -*/ -static int optimizeInternal(fulltext_vtab *v, - OptLeavesReader *readers, int nReaders, - LeafWriter *pWriter){ - int i, rc = SQLITE_OK; - DataBuffer doclist, merged, tmp; - - /* Order the readers. */ - i = nReaders; - while( i-- > 0 ){ - optLeavesReaderReorder(&readers[i], nReaders-i); - } - - dataBufferInit(&doclist, LEAF_MAX); - dataBufferInit(&merged, LEAF_MAX); - - /* Exhausted readers bubble to the end, so when the first reader is - ** at eof, all are at eof. - */ - while( !optLeavesReaderAtEnd(&readers[0]) ){ - - /* Figure out how many readers share the next term. */ - for(i=1; i 0 ){ - dlrDestroy(&dlReaders[nReaders]); - } - - /* Accumulated doclist to reader 0 for next pass. */ - dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); - } - - /* Destroy reader that was left in the pipeline. */ - dlrDestroy(&dlReaders[0]); - - /* Trim deletions from the doclist. */ - dataBufferReset(&merged); - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - -1, DL_DEFAULT, &merged); - } - - /* Only pass doclists with hits (skip if all hits deleted). */ - if( merged.nData>0 ){ - rc = leafWriterStep(v, pWriter, - optLeavesReaderTerm(&readers[0]), - optLeavesReaderTermBytes(&readers[0]), - merged.pData, merged.nData); - if( rc!=SQLITE_OK ) goto err; - } - - /* Step merged readers to next term and reorder. */ - while( i-- > 0 ){ - rc = optLeavesReaderStep(v, &readers[i]); - if( rc!=SQLITE_OK ) goto err; - - optLeavesReaderReorder(&readers[i], nReaders-i); - } - } - - err: - dataBufferDestroy(&doclist); - dataBufferDestroy(&merged); - return rc; -} - -/* Implement optimize() function for FTS3. optimize(t) merges all -** segments in the fts index into a single segment. 't' is the magic -** table-named column. -*/ -static void optimizeFunc(sqlite3_context *pContext, - int argc, sqlite3_value **argv){ - fulltext_cursor *pCursor; - if( argc>1 ){ - sqlite3_result_error(pContext, "excess arguments to optimize()",-1); - }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to optimize",-1); - }else{ - fulltext_vtab *v; - int i, rc, iMaxLevel; - OptLeavesReader *readers; - int nReaders; - LeafWriter writer; - sqlite3_stmt *s; - - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - v = cursor_vtab(pCursor); - - /* Flush any buffered updates before optimizing. */ - rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ) goto err; - - rc = segdir_count(v, &nReaders, &iMaxLevel); - if( rc!=SQLITE_OK ) goto err; - if( nReaders==0 || nReaders==1 ){ - sqlite3_result_text(pContext, "Index already optimal", -1, - SQLITE_STATIC); - return; - } - - rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); - if( rc!=SQLITE_OK ) goto err; - - readers = sqlite3_malloc(nReaders*sizeof(readers[0])); - if( readers==NULL ) goto err; - - /* Note that there will already be a segment at this position - ** until we call segdir_delete() on iMaxLevel. - */ - leafWriterInit(iMaxLevel, 0, &writer); - - i = 0; - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - sqlite_int64 iStart = sqlite3_column_int64(s, 0); - sqlite_int64 iEnd = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - int nRootData = sqlite3_column_bytes(s, 2); - - assert( i 0 ){ - leavesReaderDestroy(&readers[i].reader); - } - sqlite3_free(readers); - - /* If we've successfully gotten to here, delete the old segments - ** and flush the interior structure of the new segment. - */ - if( rc==SQLITE_OK ){ - for( i=0; i<=iMaxLevel; i++ ){ - rc = segdir_delete(v, i); - if( rc!=SQLITE_OK ) break; - } - - if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer); - } - - leafWriterDestroy(&writer); - - if( rc!=SQLITE_OK ) goto err; - - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - return; - - /* TODO(shess): Error-handling needs to be improved along the - ** lines of the dump_ functions. - */ - err: - { - char buf[512]; - sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s", - sqlite3_errmsg(sqlite3_context_db_handle(pContext))); - sqlite3_result_error(pContext, buf, -1); - } - } -} - -#ifdef SQLITE_TEST -/* Generate an error of the form ": ". If msg is NULL, -** pull the error from the context's db handle. -*/ -static void generateError(sqlite3_context *pContext, - const char *prefix, const char *msg){ - char buf[512]; - if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext)); - sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg); - sqlite3_result_error(pContext, buf, -1); -} - -/* Helper function to collect the set of terms in the segment into -** pTerms. The segment is defined by the leaf nodes between -** iStartBlockid and iEndBlockid, inclusive, or by the contents of -** pRootData if iStartBlockid is 0 (in which case the entire segment -** fit in a leaf). -*/ -static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, - fts3Hash *pTerms){ - const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0); - const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - const int nRootData = sqlite3_column_bytes(s, 2); - LeavesReader reader; - int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, - pRootData, nRootData, &reader); - if( rc!=SQLITE_OK ) return rc; - - while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ - const char *pTerm = leavesReaderTerm(&reader); - const int nTerm = leavesReaderTermBytes(&reader); - void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm); - void *newValue = (void *)((char *)oldValue+1); - - /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c, - ** the data value passed is returned in case of malloc failure. - */ - if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){ - rc = SQLITE_NOMEM; - }else{ - rc = leavesReaderStep(v, &reader); - } - } - - leavesReaderDestroy(&reader); - return rc; -} - -/* Helper function to build the result string for dump_terms(). */ -static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){ - int iTerm, nTerms, nResultBytes, iByte; - char *result; - TermData *pData; - fts3HashElem *e; - - /* Iterate pTerms to generate an array of terms in pData for - ** sorting. - */ - nTerms = fts3HashCount(pTerms); - assert( nTerms>0 ); - pData = sqlite3_malloc(nTerms*sizeof(TermData)); - if( pData==NULL ) return SQLITE_NOMEM; - - nResultBytes = 0; - for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){ - nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */ - assert( iTerm0 ); /* nTerms>0, nResultsBytes must be, too. */ - result = sqlite3_malloc(nResultBytes); - if( result==NULL ){ - sqlite3_free(pData); - return SQLITE_NOMEM; - } - - if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp); - - /* Read the terms in order to build the result. */ - iByte = 0; - for(iTerm=0; iTermbase.pVtab; + assert( p ); - if( rc!=SQLITE_OK ){ - generateError(pContext, "dump_terms", NULL); - return; - } + rc = sqlite3Fts3Optimize(p); - /* Collect the terms for each segment. */ - sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1); - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - rc = collectSegmentTerms(v, s, &terms); - if( rc!=SQLITE_OK ) break; - } - - if( rc!=SQLITE_DONE ){ - sqlite3_reset(s); - generateError(pContext, "dump_terms", NULL); - }else{ - const int nTerms = fts3HashCount(&terms); - if( nTerms>0 ){ - rc = generateTermsResult(pContext, &terms); - if( rc==SQLITE_NOMEM ){ - generateError(pContext, "dump_terms", "out of memory"); - }else{ - assert( rc==SQLITE_OK ); - } - }else if( argc==3 ){ - /* The specific segment asked for could not be found. */ - generateError(pContext, "dump_terms", "segment not found"); - }else{ - /* No segments found. */ - /* TODO(shess): It should be impossible to reach this. This - ** case can only happen for an empty table, in which case - ** SQLite has no rows to call this function on. - */ - sqlite3_result_null(pContext); - } - } - sqlite3Fts3HashClear(&terms); + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; } } -/* Expand the DL_DEFAULT doclist in pData into a text result in -** pContext. -*/ -static void createDoclistResult(sqlite3_context *pContext, - const char *pData, int nData){ - DataBuffer dump; - DLReader dlReader; - - assert( pData!=NULL && nData>0 ); - - dataBufferInit(&dump, 0); - dlrInit(&dlReader, DL_DEFAULT, pData, nData); - for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ - char buf[256]; - PLReader plReader; - - plrInit(&plReader, &dlReader); - if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ - sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); - dataBufferAppend(&dump, buf, strlen(buf)); - }else{ - int iColumn = plrColumn(&plReader); - - sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[", - dlrDocid(&dlReader), iColumn); - dataBufferAppend(&dump, buf, strlen(buf)); - - for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ - if( plrColumn(&plReader)!=iColumn ){ - iColumn = plrColumn(&plReader); - sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dataBufferAppend(&dump, buf, strlen(buf)); - } - if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){ - sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ", - plrPosition(&plReader), - plrStartOffset(&plReader), plrEndOffset(&plReader)); - }else if( DL_DEFAULT==DL_POSITIONS ){ - sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader)); - }else{ - assert( NULL=="Unhandled DL_DEFAULT value"); - } - dataBufferAppend(&dump, buf, strlen(buf)); - } - plrDestroy(&plReader); - - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dataBufferAppend(&dump, "]] ", 3); - } - } - dlrDestroy(&dlReader); - - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dump.pData[dump.nData] = '\0'; - assert( dump.nData>0 ); - - /* Passes ownership of dump's buffer to pContext. */ - sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); - dump.pData = NULL; - dump.nData = dump.nCapacity = 0; -} - -/* Implements dump_doclist() for use in inspecting the fts3 index from -** tests. TEXT result containing a string representation of the -** doclist for the indicated term. dump_doclist(t, term, level, idx) -** dumps the doclist for term from the segment specified by level, idx -** (in %_segdir), while dump_doclist(t, term) dumps the logical -** doclist for the term across all segments. The per-segment doclist -** can contain deletions, while the full-index doclist will not -** (deletions are omitted). -** -** Result formats differ with the setting of DL_DEFAULTS. Examples: -** -** DL_DOCIDS: [1] [3] [7] -** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]] -** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]] -** -** In each case the number after the outer '[' is the docid. In the -** latter two cases, the number before the inner '[' is the column -** associated with the values within. For DL_POSITIONS the numbers -** within are the positions, for DL_POSITIONS_OFFSETS they are the -** position, the start offset, and the end offset. -*/ -static void dumpDoclistFunc( - sqlite3_context *pContext, - int argc, sqlite3_value **argv -){ - fulltext_cursor *pCursor; - if( argc!=2 && argc!=4 ){ - generateError(pContext, "dump_doclist", "incorrect arguments"); - }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - generateError(pContext, "dump_doclist", "illegal first argument"); - }else if( sqlite3_value_text(argv[1])==NULL || - sqlite3_value_text(argv[1])[0]=='\0' ){ - generateError(pContext, "dump_doclist", "empty second argument"); - }else{ - const char *pTerm = (const char *)sqlite3_value_text(argv[1]); - const int nTerm = strlen(pTerm); - fulltext_vtab *v; - int rc; - DataBuffer doclist; - - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - v = cursor_vtab(pCursor); - - dataBufferInit(&doclist, 0); - - /* termSelect() yields the same logical doclist that queries are - ** run against. - */ - if( argc==2 ){ - rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist); - }else{ - sqlite3_stmt *s = NULL; - - /* Get our specific segment's information. */ - rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2])); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3])); - } - } - - if( rc==SQLITE_OK ){ - rc = sqlite3_step(s); - - if( rc==SQLITE_DONE ){ - dataBufferDestroy(&doclist); - generateError(pContext, "dump_doclist", "segment not found"); - return; - } - - /* Found a segment, load it into doclist. */ - if( rc==SQLITE_ROW ){ - const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); - const char *pData = sqlite3_column_blob(s, 2); - const int nData = sqlite3_column_bytes(s, 2); - - /* loadSegment() is used by termSelect() to load each - ** segment's data. - */ - rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0, - &doclist); - if( rc==SQLITE_OK ){ - rc = sqlite3_step(s); - - /* Should not have more than one matching segment. */ - if( rc!=SQLITE_DONE ){ - sqlite3_reset(s); - dataBufferDestroy(&doclist); - generateError(pContext, "dump_doclist", "invalid segdir"); - return; - } - rc = SQLITE_OK; - } - } - } - - sqlite3_reset(s); - } - - if( rc==SQLITE_OK ){ - if( doclist.nData>0 ){ - createDoclistResult(pContext, doclist.pData, doclist.nData); - }else{ - /* TODO(shess): This can happen if the term is not present, or - ** if all instances of the term have been deleted and this is - ** an all-index dump. It may be interesting to distinguish - ** these cases. - */ - sqlite3_result_text(pContext, "", 0, SQLITE_STATIC); - } - }else if( rc==SQLITE_NOMEM ){ - /* Handle out-of-memory cases specially because if they are - ** generated in fts3 code they may not be reflected in the db - ** handle. - */ - /* TODO(shess): Handle this more comprehensively. - ** sqlite3ErrStr() has what I need, but is internal. - */ - generateError(pContext, "dump_doclist", "out of memory"); - }else{ - generateError(pContext, "dump_doclist", NULL); - } - - dataBufferDestroy(&doclist); - } -} -#endif - /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ -static int fulltextFindFunction( - sqlite3_vtab *pVtab, - int nArg, - const char *zName, - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), - void **ppArg +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ ){ - if( strcmp(zName,"snippet")==0 ){ - *pxFunc = snippetFunc; - return 1; - }else if( strcmp(zName,"offsets")==0 ){ - *pxFunc = snippetOffsetsFunc; - return 1; - }else if( strcmp(zName,"optimize")==0 ){ - *pxFunc = optimizeFunc; - return 1; -#ifdef SQLITE_TEST - /* NOTE(shess): These functions are present only for testing - ** purposes. No particular effort is made to optimize their - ** execution or how they build their results. - */ - }else if( strcmp(zName,"dump_terms")==0 ){ - /* fprintf(stderr, "Found dump_terms\n"); */ - *pxFunc = dumpTermsFunc; - return 1; - }else if( strcmp(zName,"dump_doclist")==0 ){ - /* fprintf(stderr, "Found dump_doclist\n"); */ - *pxFunc = dumpDoclistFunc; - return 1; -#endif + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; ipLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_init.c */ +int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); +int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, + sqlite3_vtab **, char **); + +/* fts3_write.c */ +int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +int sqlite3Fts3PendingTermsFlush(Fts3Table *); +void sqlite3Fts3PendingTermsClear(Fts3Table *); +int sqlite3Fts3Optimize(Fts3Table *); +int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); +int sqlite3Fts3SegReaderIterate( + Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, + int (*)(Fts3Table *, void *, char *, int, char *, int), void * +); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); +int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +/* fts3.c */ +int sqlite3Fts3PutVarint(char *, sqlite3_int64); +int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +int sqlite3Fts3GetVarint32(const char *, int *); +int sqlite3Fts3VarintLen(sqlite3_uint64); +void sqlite3Fts3Dequote(char *); + +/* fts3_tokenizer.c */ +const char *sqlite3Fts3NextToken(const char *, int *); +int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, + const char *, sqlite3_tokenizer **, const char **, char ** +); + +/* fts3_snippet.c */ +void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*, + const char *, const char *, const char * +); + +/* fts3_expr.c */ +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, + char **, int, int, const char *, int, Fts3Expr ** +); +void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +#endif + +#endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 49bea2f..5b955f9 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -13,8 +13,7 @@ ** This module contains code that implements a parser for fts3 query strings ** (the right-hand argument to the MATCH operator). Because the supported ** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. The public interface to this module is declared in source -** code file "fts3_expr.h". +** hand-coded. */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -40,7 +39,29 @@ ** to zero causes the module to use the old syntax. If it is set to ** non-zero the new syntax is activated. This is so both syntaxes can ** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". */ + #ifdef SQLITE_TEST int sqlite3_fts3_enable_parentheses = 0; #else @@ -56,8 +77,7 @@ int sqlite3_fts3_enable_parentheses = 0; */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 -#include "fts3_expr.h" -#include "sqlite3.h" +#include "fts3Int.h" #include #include #include @@ -232,7 +252,7 @@ static int getNextString( if( rc==SQLITE_DONE ){ int jj; - char *zNew; + char *zNew = NULL; int nNew = 0; int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken); @@ -291,7 +311,7 @@ static int getNextNode( int *pnConsumed /* OUT: Number of bytes consumed */ ){ static const struct Fts3Keyword { - char z[4]; /* Keyword text */ + char *z; /* Keyword text */ unsigned char n; /* Length of the keyword */ unsigned char parenOnly; /* Only valid in paren mode */ unsigned char eType; /* Keyword code */ @@ -354,11 +374,14 @@ static int getNextNode( || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; - *pnConsumed = (zInput - z) + nKey; + *pnConsumed = (int)((zInput - z) + nKey); return SQLITE_OK; } @@ -378,14 +401,14 @@ static int getNextNode( if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } - *pnConsumed = (zInput - z) + 1 + nConsumed; + *pnConsumed = (int)((zInput - z) + 1 + nConsumed); return rc; } /* Check for a close bracket. */ if( *zInput==')' ){ pParse->nNest--; - *pnConsumed = (zInput - z) + 1; + *pnConsumed = (int)((zInput - z) + 1); return SQLITE_DONE; } } @@ -397,7 +420,7 @@ static int getNextNode( */ if( *zInput=='"' ){ for(ii=1; iinCol; ii++){ const char *zStr = pParse->azCol[ii]; - int nStr = strlen(zStr); + int nStr = (int)strlen(zStr); if( nInput>nStr && zInput[nStr]==':' && sqlite3_strnicmp(zStr, zInput, nStr)==0 ){ iCol = ii; - iColLen = ((zInput - z) + nStr + 1); + iColLen = (int)((zInput - z) + nStr + 1); break; } } @@ -691,7 +714,7 @@ int sqlite3Fts3ExprParse( return SQLITE_OK; } if( n<0 ){ - n = strlen(z); + n = (int)strlen(z); } rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); @@ -746,7 +769,7 @@ static int queryTestTokenizer( sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } diff --git a/ext/fts3/fts3_expr.h b/ext/fts3/fts3_expr.h deleted file mode 100644 index a48dee6..0000000 --- a/ext/fts3/fts3_expr.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -** 2008 Nov 28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -*/ - -#include "fts3_tokenizer.h" -#include "sqlite3.h" - -/* -** The following describes the syntax supported by the fts3 MATCH -** operator in a similar format to that used by the lemon parser -** generator. This module does not use actually lemon, it uses a -** custom parser. -** -** query ::= andexpr (OR andexpr)*. -** -** andexpr ::= notexpr (AND? notexpr)*. -** -** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. -** notexpr ::= LP query RP. -** -** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. -** -** distance_opt ::= . -** distance_opt ::= / INTEGER. -** -** phrase ::= TOKEN. -** phrase ::= COLUMN:TOKEN. -** phrase ::= "TOKEN TOKEN TOKEN...". -*/ - -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; - -/* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in "...", nToken will be the number -** of tokens in the string. -*/ -struct Fts3Phrase { - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ - struct PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer pointed to by z */ - int isPrefix; /* True if token ends in with a "*" character */ - } aToken[1]; /* One entry for each token in the phrase */ -}; - -/* -** A tree of these objects forms the RHS of a MATCH operator. -*/ -struct Fts3Expr { - int eType; /* One of the FTSQUERY_XXX values defined below */ - int nNear; /* Valid if eType==FTSQUERY_NEAR */ - Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ -}; - -int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, - const char *, int, Fts3Expr **); -void sqlite3Fts3ExprFree(Fts3Expr *); - -/* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: -** -** "a OR b AND c NOT d NEAR e" -** -** is equivalent to: -** -** "a OR (b AND (c NOT (d NEAR e)))" -*/ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - -#ifdef SQLITE_TEST -void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -#endif diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index ee30117..3cb91f3 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -56,7 +56,7 @@ static void fts3HashFree(void *p){ ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. */ -void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){ +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ assert( pNew!=0 ); assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); pNew->keyClass = keyClass; @@ -71,8 +71,8 @@ void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){ ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. */ -void sqlite3Fts3HashClear(fts3Hash *pH){ - fts3HashElem *elem; /* For looping over all elements of the table */ +void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ assert( pH!=0 ); elem = pH->first; @@ -81,7 +81,7 @@ void sqlite3Fts3HashClear(fts3Hash *pH){ pH->ht = 0; pH->htsize = 0; while( elem ){ - fts3HashElem *next_elem = elem->next; + Fts3HashElem *next_elem = elem->next; if( pH->copyKey && elem->pKey ){ fts3HashFree(elem->pKey); } @@ -164,11 +164,11 @@ static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ /* Link an element into the hash table */ static void fts3HashInsertElement( - fts3Hash *pH, /* The complete hash table */ + Fts3Hash *pH, /* The complete hash table */ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ - fts3HashElem *pNew /* The element to be inserted */ + Fts3HashElem *pNew /* The element to be inserted */ ){ - fts3HashElem *pHead; /* First element already in pEntry */ + Fts3HashElem *pHead; /* First element already in pEntry */ pHead = pEntry->chain; if( pHead ){ pNew->next = pHead; @@ -190,15 +190,17 @@ static void fts3HashInsertElement( /* Resize the hash table so that it cantains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. */ -static void fts3Rehash(fts3Hash *pH, int new_size){ +static int fts3Rehash(Fts3Hash *pH, int new_size){ struct _fts3ht *new_ht; /* The new hash table */ - fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ int (*xHash)(const void*,int); /* The hash function */ assert( (new_size & (new_size-1))==0 ); new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); - if( new_ht==0 ) return; + if( new_ht==0 ) return 1; fts3HashFree(pH->ht); pH->ht = new_ht; pH->htsize = new_size; @@ -208,19 +210,20 @@ static void fts3Rehash(fts3Hash *pH, int new_size){ next_elem = elem->next; fts3HashInsertElement(pH, &new_ht[h], elem); } + return 0; } /* This function (for internal use only) locates an element in an ** hash table that matches the given key. The hash for this key has ** already been computed and is passed as the 4th parameter. */ -static fts3HashElem *fts3FindElementByHash( - const fts3Hash *pH, /* The pH to be searched */ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ const void *pKey, /* The key we are searching for */ int nKey, int h /* The hash for this key. */ ){ - fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ int (*xCompare)(const void*,int,const void*,int); /* comparison function */ @@ -243,8 +246,8 @@ static fts3HashElem *fts3FindElementByHash( ** element and a hash on the element's key. */ static void fts3RemoveElementByHash( - fts3Hash *pH, /* The pH containing "elem" */ - fts3HashElem* elem, /* The element to be removed from the pH */ + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ int h /* Hash value for the element */ ){ struct _fts3ht *pEntry; @@ -280,9 +283,9 @@ static void fts3RemoveElementByHash( ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){ +void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ int h; /* A hash on key */ - fts3HashElem *elem; /* The element that matches key */ + Fts3HashElem *elem; /* The element that matches key */ int (*xHash)(const void*,int); /* The hash function */ if( pH==0 || pH->ht==0 ) return 0; @@ -310,15 +313,15 @@ void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){ ** element corresponding to "key" is removed from the hash table. */ void *sqlite3Fts3HashInsert( - fts3Hash *pH, /* The hash table to insert into */ + Fts3Hash *pH, /* The hash table to insert into */ const void *pKey, /* The key */ int nKey, /* Number of bytes in the key */ void *data /* The data */ ){ int hraw; /* Raw hash value of the key */ int h; /* the hash of the key modulo hash table size */ - fts3HashElem *elem; /* Used to loop thru the element list */ - fts3HashElem *new_elem; /* New element added to the pH */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ int (*xHash)(const void*,int); /* The hash function */ assert( pH!=0 ); @@ -338,14 +341,14 @@ void *sqlite3Fts3HashInsert( return old_data; } if( data==0 ) return 0; - if( pH->htsize==0 ){ - fts3Rehash(pH,8); - if( pH->htsize==0 ){ - pH->count = 0; - return data; - } + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; } - new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) ); + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); if( new_elem==0 ) return data; if( pH->copyKey && pKey!=0 ){ new_elem->pKey = fts3HashMalloc( nKey ); @@ -359,9 +362,6 @@ void *sqlite3Fts3HashInsert( } new_elem->nKey = nKey; pH->count++; - if( pH->count > pH->htsize ){ - fts3Rehash(pH,pH->htsize*2); - } assert( pH->htsize>0 ); assert( (pH->htsize & (pH->htsize-1))==0 ); h = hraw & (pH->htsize-1); diff --git a/ext/fts3/fts3_hash.h b/ext/fts3/fts3_hash.h index e01954e..b00f365 100644 --- a/ext/fts3/fts3_hash.h +++ b/ext/fts3/fts3_hash.h @@ -18,8 +18,8 @@ #define _FTS3_HASH_H_ /* Forward declarations of structures. */ -typedef struct fts3Hash fts3Hash; -typedef struct fts3HashElem fts3HashElem; +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; /* A complete hash table is an instance of the following structure. ** The internals of this structure are intended to be opaque -- client @@ -29,15 +29,15 @@ typedef struct fts3HashElem fts3HashElem; ** accessing this structure are really macros, so we can't really make ** this structure opaque. */ -struct fts3Hash { +struct Fts3Hash { char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ char copyKey; /* True if copy of key made on insert */ int count; /* Number of entries in this table */ - fts3HashElem *first; /* The first element of the array */ + Fts3HashElem *first; /* The first element of the array */ int htsize; /* Number of buckets in the hash table */ struct _fts3ht { /* the hash table */ int count; /* Number of entries with this hash */ - fts3HashElem *chain; /* Pointer to first entry with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; @@ -47,8 +47,8 @@ struct fts3Hash { ** Again, this structure is intended to be opaque, but it can't really ** be opaque because it is used by macros. */ -struct fts3HashElem { - fts3HashElem *next, *prev; /* Next and previous elements in the table */ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ void *pKey; int nKey; /* Key associated with this element */ }; @@ -71,10 +71,10 @@ struct fts3HashElem { /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey); -void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData); -void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey); -void sqlite3Fts3HashClear(fts3Hash*); +void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +void sqlite3Fts3HashClear(Fts3Hash*); /* ** Shorthand for the functions above @@ -88,8 +88,8 @@ void sqlite3Fts3HashClear(fts3Hash*); ** Macros for looping over all elements of a hash table. The idiom is ** like this: ** -** fts3Hash h; -** fts3HashElem *p; +** Fts3Hash h; +** Fts3HashElem *p; ** ... ** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ ** SomeStructure *pData = fts3HashData(p); diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 6ff67a9..4e68087 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -24,6 +24,7 @@ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#include "fts3Int.h" #include #include @@ -54,10 +55,6 @@ typedef struct porter_tokenizer_cursor { } porter_tokenizer_cursor; -/* Forward declaration */ -static const sqlite3_tokenizer_module porterTokenizerModule; - - /* ** Create a new tokenizer instance. */ @@ -66,6 +63,10 @@ static int porterCreate( sqlite3_tokenizer **ppTokenizer ){ porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); if( t==NULL ) return SQLITE_NOMEM; memset(t, 0, sizeof(*t)); @@ -94,6 +95,8 @@ static int porterOpen( ){ porter_tokenizer_cursor *c; + UNUSED_PARAMETER(pTokenizer); + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; @@ -234,7 +237,7 @@ static int hasVowel(const char *z){ ** the first two characters of z[]. */ static int doubleConsonant(const char *z){ - return isConsonant(z) && z[0]==z[1] && isConsonant(z+1); + return isConsonant(z) && z[0]==z[1]; } /* @@ -247,10 +250,10 @@ static int doubleConsonant(const char *z){ */ static int star_oh(const char *z){ return - z[0]!=0 && isConsonant(z) && + isConsonant(z) && z[0]!='w' && z[0]!='x' && z[0]!='y' && - z[1]!=0 && isVowel(z+1) && - z[2]!=0 && isConsonant(z+2); + isVowel(z+1) && + isConsonant(z+2); } /* @@ -294,7 +297,7 @@ static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ int i, mx, j; int hasDigit = 0; for(i=0; i='A' && c<='Z' ){ zOut[i] = c - 'A' + 'a'; }else{ @@ -338,7 +341,7 @@ static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ ** no chance of overflowing the zOut buffer. */ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ - int i, j, c; + int i, j; char zReverse[28]; char *z, *z2; if( nIn<3 || nIn>=sizeof(zReverse)-7 ){ @@ -348,7 +351,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ return; } for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ zReverse[j] = c + 'a' - 'A'; }else if( c>='a' && c<='z' ){ @@ -547,7 +550,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ /* z[] is now the stemmed word in reverse order. Flip it back ** around into forward order and return. */ - *pnOut = i = strlen(z); + *pnOut = i = (int)strlen(z); zOut[i] = 0; while( *z ){ zOut[--i] = *(z++); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c new file mode 100644 index 0000000..cdc55af --- /dev/null +++ b/ext/fts3/fts3_snippet.c @@ -0,0 +1,734 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "fts3Int.h" +#include +#include +#include + +typedef struct Snippet Snippet; + +/* +** An instance of the following structure keeps track of generated +** matching-word offset information and snippets. +*/ +struct Snippet { + int nMatch; /* Total number of matches */ + int nAlloc; /* Space allocated for aMatch[] */ + struct snippetMatch { /* One entry for each matching term */ + char snStatus; /* Status flag for use while constructing snippets */ + short int nByte; /* Number of bytes in the term */ + short int iCol; /* The column that contains the match */ + short int iTerm; /* The index in Query.pTerms[] of the matching term */ + int iToken; /* The index of the matching document token */ + int iStart; /* The offset to the first character of the term */ + } *aMatch; /* Points to space obtained from malloc */ + char *zOffset; /* Text rendering of aMatch[] */ + int nOffset; /* strlen(zOffset) */ + char *zSnippet; /* Snippet text */ + int nSnippet; /* strlen(zSnippet) */ +}; + + +/* It is not safe to call isspace(), tolower(), or isalnum() on +** hi-bit-set characters. This is the same solution used in the +** tokenizer. +*/ +static int fts3snippetIsspace(char c){ + return (c&0x80)==0 ? isspace(c) : 0; +} + + +/* +** A StringBuffer object holds a zero-terminated string that grows +** arbitrarily by appending. Space to hold the string is obtained +** from sqlite3_malloc(). After any memory allocation failure, +** StringBuffer.z is set to NULL and no further allocation is attempted. +*/ +typedef struct StringBuffer { + char *z; /* Text of the string. Space from malloc. */ + int nUsed; /* Number bytes of z[] used, not counting \000 terminator */ + int nAlloc; /* Bytes allocated for z[] */ +} StringBuffer; + + +/* +** Initialize a new StringBuffer. +*/ +static void fts3SnippetSbInit(StringBuffer *p){ + p->nAlloc = 100; + p->nUsed = 0; + p->z = sqlite3_malloc( p->nAlloc ); +} + +/* +** Append text to the string buffer. +*/ +static void fts3SnippetAppend(StringBuffer *p, const char *zNew, int nNew){ + if( p->z==0 ) return; + if( nNew<0 ) nNew = (int)strlen(zNew); + if( p->nUsed + nNew >= p->nAlloc ){ + int nAlloc; + char *zNew; + + nAlloc = p->nUsed + nNew + p->nAlloc; + zNew = sqlite3_realloc(p->z, nAlloc); + if( zNew==0 ){ + sqlite3_free(p->z); + p->z = 0; + return; + } + p->z = zNew; + p->nAlloc = nAlloc; + } + memcpy(&p->z[p->nUsed], zNew, nNew); + p->nUsed += nNew; + p->z[p->nUsed] = 0; +} + +/* If the StringBuffer ends in something other than white space, add a +** single space character to the end. +*/ +static void fts3SnippetAppendWhiteSpace(StringBuffer *p){ + if( p->z && p->nUsed && !fts3snippetIsspace(p->z[p->nUsed-1]) ){ + fts3SnippetAppend(p, " ", 1); + } +} + +/* Remove white space from the end of the StringBuffer */ +static void fts3SnippetTrimWhiteSpace(StringBuffer *p){ + if( p->z ){ + while( p->nUsed && fts3snippetIsspace(p->z[p->nUsed-1]) ){ + p->nUsed--; + } + p->z[p->nUsed] = 0; + } +} + +/* +** Release all memory associated with the Snippet structure passed as +** an argument. +*/ +static void fts3SnippetFree(Snippet *p){ + if( p ){ + sqlite3_free(p->aMatch); + sqlite3_free(p->zOffset); + sqlite3_free(p->zSnippet); + sqlite3_free(p); + } +} + +/* +** Append a single entry to the p->aMatch[] log. +*/ +static int snippetAppendMatch( + Snippet *p, /* Append the entry to this snippet */ + int iCol, int iTerm, /* The column and query term */ + int iToken, /* Matching token in document */ + int iStart, int nByte /* Offset and size of the match */ +){ + int i; + struct snippetMatch *pMatch; + if( p->nMatch+1>=p->nAlloc ){ + struct snippetMatch *pNew; + p->nAlloc = p->nAlloc*2 + 10; + pNew = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); + if( pNew==0 ){ + p->aMatch = 0; + p->nMatch = 0; + p->nAlloc = 0; + return SQLITE_NOMEM; + } + p->aMatch = pNew; + } + i = p->nMatch++; + pMatch = &p->aMatch[i]; + pMatch->iCol = (short)iCol; + pMatch->iTerm = (short)iTerm; + pMatch->iToken = iToken; + pMatch->iStart = iStart; + pMatch->nByte = (short)nByte; + return SQLITE_OK; +} + +/* +** Sizing information for the circular buffer used in snippetOffsetsOfColumn() +*/ +#define FTS3_ROTOR_SZ (32) +#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) + +/* +** Function to iterate through the tokens of a compiled expression. +** +** Except, skip all tokens on the right-hand side of a NOT operator. +** This function is used to find tokens as part of snippet and offset +** generation and we do nt want snippets and offsets to report matches +** for tokens on the RHS of a NOT. +*/ +static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ + Fts3Expr *p = *ppExpr; + int iToken = *piToken; + if( iToken<0 ){ + /* In this case the expression p is the root of an expression tree. + ** Move to the first token in the expression tree. + */ + while( p->pLeft ){ + p = p->pLeft; + } + iToken = 0; + }else{ + assert(p && p->eType==FTSQUERY_PHRASE ); + if( iToken<(p->pPhrase->nToken-1) ){ + iToken++; + }else{ + iToken = 0; + while( p->pParent && p->pParent->pLeft!=p ){ + assert( p->pParent->pRight==p ); + p = p->pParent; + } + p = p->pParent; + if( p ){ + assert( p->pRight!=0 ); + p = p->pRight; + while( p->pLeft ){ + p = p->pLeft; + } + } + } + } + + *ppExpr = p; + *piToken = iToken; + return p?1:0; +} + +/* +** Return TRUE if the expression node pExpr is located beneath the +** RHS of a NOT operator. +*/ +static int fts3ExprBeneathNot(Fts3Expr *p){ + Fts3Expr *pParent; + while( p ){ + pParent = p->pParent; + if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ + return 1; + } + p = pParent; + } + return 0; +} + +/* +** Add entries to pSnippet->aMatch[] for every match that occurs against +** document zDoc[0..nDoc-1] which is stored in column iColumn. +*/ +static int snippetOffsetsOfColumn( + Fts3Cursor *pCur, /* The fulltest search cursor */ + Snippet *pSnippet, /* The Snippet object to be filled in */ + int iColumn, /* Index of fulltext table column */ + const char *zDoc, /* Text of the fulltext table column */ + int nDoc /* Length of zDoc in bytes */ +){ + const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ + sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ + sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ + Fts3Table *pVtab; /* The full text index */ + int nColumn; /* Number of columns in the index */ + int i, j; /* Loop counters */ + int rc; /* Return code */ + unsigned int match, prevMatch; /* Phrase search bitmasks */ + const char *zToken; /* Next token from the tokenizer */ + int nToken; /* Size of zToken */ + int iBegin, iEnd, iPos; /* Offsets of beginning and end */ + + /* The following variables keep a circular buffer of the last + ** few tokens */ + unsigned int iRotor = 0; /* Index of current token */ + int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ + int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ + + pVtab = (Fts3Table *)pCur->base.pVtab; + nColumn = pVtab->nColumn; + pTokenizer = pVtab->pTokenizer; + pTModule = pTokenizer->pModule; + rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); + if( rc ) return rc; + pTCursor->pTokenizer = pTokenizer; + + prevMatch = 0; + while( (rc = pTModule->xNext(pTCursor, &zToken, &nToken, + &iBegin, &iEnd, &iPos))==SQLITE_OK ){ + Fts3Expr *pIter = pCur->pExpr; + int iIter = -1; + iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; + iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; + match = 0; + for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ + int nPhrase; /* Number of tokens in current phrase */ + struct PhraseToken *pToken; /* Current token */ + int iCol; /* Column index */ + + if( fts3ExprBeneathNot(pIter) ) continue; + nPhrase = pIter->pPhrase->nToken; + pToken = &pIter->pPhrase->aToken[iIter]; + iCol = pIter->pPhrase->iColumn; + if( iCol>=0 && iColn>nToken ) continue; + if( !pToken->isPrefix && pToken->nn<=nToken ); + if( memcmp(pToken->z, zToken, pToken->n) ) continue; + if( iIter>0 && (prevMatch & (1<=0; j--){ + int k = (iRotor-j) & FTS3_ROTOR_MASK; + rc = snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, + iRotorBegin[k], iRotorLen[k]); + if( rc ) goto end_offsets_of_column; + } + } + } + prevMatch = match<<1; + iRotor++; + } +end_offsets_of_column: + pTModule->xClose(pTCursor); + return rc==SQLITE_DONE ? SQLITE_OK : rc; +} + +/* +** Remove entries from the pSnippet structure to account for the NEAR +** operator. When this is called, pSnippet contains the list of token +** offsets produced by treating all NEAR operators as AND operators. +** This function removes any entries that should not be present after +** accounting for the NEAR restriction. For example, if the queried +** document is: +** +** "A B C D E A" +** +** and the query is: +** +** A NEAR/0 E +** +** then when this function is called the Snippet contains token offsets +** 0, 4 and 5. This function removes the "0" entry (because the first A +** is not near enough to an E). +** +** When this function is called, the value pointed to by parameter piLeft is +** the integer id of the left-most token in the expression tree headed by +** pExpr. This function increments *piLeft by the total number of tokens +** in the expression tree headed by pExpr. +** +** Return 1 if any trimming occurs. Return 0 if no trimming is required. +*/ +static int trimSnippetOffsets( + Fts3Expr *pExpr, /* The search expression */ + Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ + int *piLeft /* Index of left-most token in pExpr */ +){ + if( pExpr ){ + if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ + return 1; + } + + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: + *piLeft += pExpr->pPhrase->nToken; + break; + case FTSQUERY_NEAR: { + /* The right-hand-side of a NEAR operator is always a phrase. The + ** left-hand-side is either a phrase or an expression tree that is + ** itself headed by a NEAR operator. The following initializations + ** set local variable iLeft to the token number of the left-most + ** token in the right-hand phrase, and iRight to the right most + ** token in the same phrase. For example, if we had: + ** + ** MATCH '"abc def" NEAR/2 "ghi jkl"' + ** + ** then iLeft will be set to 2 (token number of ghi) and nToken will + ** be set to 4. + */ + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + int iLeft = *piLeft; + int nNear = pExpr->nNear; + int nToken = pRight->pPhrase->nToken; + int jj, ii; + if( pLeft->eType==FTSQUERY_NEAR ){ + pLeft = pLeft->pRight; + } + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + nToken += pLeft->pPhrase->nToken; + + for(ii=0; iinMatch; ii++){ + struct snippetMatch *p = &pSnippet->aMatch[ii]; + if( p->iTerm==iLeft ){ + int isOk = 0; + /* Snippet ii is an occurence of query term iLeft in the document. + ** It occurs at position (p->iToken) of the document. We now + ** search for an instance of token (iLeft-1) somewhere in the + ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within + ** the set of snippetMatch structures. If one is found, proceed. + ** If one cannot be found, then remove snippets ii..(ii+N-1) + ** from the matching snippets, where N is the number of tokens + ** in phrase pRight->pPhrase. + */ + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==(iLeft-1) ){ + if( p2->iToken>=(p->iToken-nNear-1) + && p2->iToken<(p->iToken+nNear+nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[kk+ii].iTerm = -2; + } + return 1; + } + } + if( p->iTerm==(iLeft-1) ){ + int isOk = 0; + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==iLeft ){ + if( p2->iToken<=(p->iToken+nNear+1) + && p2->iToken>(p->iToken-nNear-nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[ii-kk].iTerm = -2; + } + return 1; + } + } + } + break; + } + } + + if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ + return 1; + } + } + return 0; +} + +/* +** Compute all offsets for the current row of the query. +** If the offsets have already been computed, this routine is a no-op. +*/ +static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; /* The FTS3 virtual table */ + int nColumn; /* Number of columns. Docid does count */ + int iColumn; /* Index of of a column */ + int i; /* Loop index */ + int iFirst; /* First column to search */ + int iLast; /* Last coumn to search */ + int iTerm = 0; + Snippet *pSnippet; + int rc = SQLITE_OK; + + if( pCsr->pExpr==0 ){ + return SQLITE_OK; + } + + pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet)); + *ppSnippet = pSnippet; + if( !pSnippet ){ + return SQLITE_NOMEM; + } + memset(pSnippet, 0, sizeof(Snippet)); + + nColumn = p->nColumn; + iColumn = (pCsr->eSearch - 2); + if( iColumn<0 || iColumn>=nColumn ){ + /* Look for matches over all columns of the full-text index */ + iFirst = 0; + iLast = nColumn-1; + }else{ + /* Look for matches in the iColumn-th column of the index only */ + iFirst = iColumn; + iLast = iColumn; + } + for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){ + const char *zDoc; + int nDoc; + zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1); + if( zDoc==0 && sqlite3_column_type(pCsr->pStmt, i+1)!=SQLITE_NULL ){ + rc = SQLITE_NOMEM; + }else{ + rc = snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc); + } + } + + while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){ + iTerm = 0; + } + + return rc; +} + +/* +** Convert the information in the aMatch[] array of the snippet +** into the string zOffset[0..nOffset-1]. This string is used as +** the return of the SQL offsets() function. +*/ +static void snippetOffsetText(Snippet *p){ + int i; + int cnt = 0; + StringBuffer sb; + char zBuf[200]; + if( p->zOffset ) return; + fts3SnippetSbInit(&sb); + for(i=0; inMatch; i++){ + struct snippetMatch *pMatch = &p->aMatch[i]; + if( pMatch->iTerm>=0 ){ + /* If snippetMatch.iTerm is less than 0, then the match was + ** discarded as part of processing the NEAR operator (see the + ** trimSnippetOffsetsForNear() function for details). Ignore + ** it in this case + */ + zBuf[0] = ' '; + sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", + pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); + fts3SnippetAppend(&sb, zBuf, -1); + cnt++; + } + } + p->zOffset = sb.z; + p->nOffset = sb.z ? sb.nUsed : 0; +} + +/* +** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set +** of matching words some of which might be in zDoc. zDoc is column +** number iCol. +** +** iBreak is suggested spot in zDoc where we could begin or end an +** excerpt. Return a value similar to iBreak but possibly adjusted +** to be a little left or right so that the break point is better. +*/ +static int wordBoundary( + int iBreak, /* The suggested break point */ + const char *zDoc, /* Document text */ + int nDoc, /* Number of bytes in zDoc[] */ + struct snippetMatch *aMatch, /* Matching words */ + int nMatch, /* Number of entries in aMatch[] */ + int iCol /* The column number for zDoc[] */ +){ + int i; + if( iBreak<=10 ){ + return 0; + } + if( iBreak>=nDoc-10 ){ + return nDoc; + } + for(i=0; ALWAYS(i0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ + return aMatch[i-1].iStart; + } + } + for(i=1; i<=10; i++){ + if( fts3snippetIsspace(zDoc[iBreak-i]) ){ + return iBreak - i + 1; + } + if( fts3snippetIsspace(zDoc[iBreak+i]) ){ + return iBreak + i + 1; + } + } + return iBreak; +} + + + +/* +** Allowed values for Snippet.aMatch[].snStatus +*/ +#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ +#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ + +/* +** Generate the text of a snippet. +*/ +static void snippetText( + Fts3Cursor *pCursor, /* The cursor we need the snippet for */ + Snippet *pSnippet, + const char *zStartMark, /* Markup to appear before each match */ + const char *zEndMark, /* Markup to appear after each match */ + const char *zEllipsis /* Ellipsis mark */ +){ + int i, j; + struct snippetMatch *aMatch; + int nMatch; + int nDesired; + StringBuffer sb; + int tailCol; + int tailOffset; + int iCol; + int nDoc; + const char *zDoc; + int iStart, iEnd; + int tailEllipsis = 0; + int iMatch; + + + sqlite3_free(pSnippet->zSnippet); + pSnippet->zSnippet = 0; + aMatch = pSnippet->aMatch; + nMatch = pSnippet->nMatch; + fts3SnippetSbInit(&sb); + + for(i=0; i0; i++){ + if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; + nDesired--; + iCol = aMatch[i].iCol; + zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); + iStart = aMatch[i].iStart - 40; + iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); + if( iStart<=10 ){ + iStart = 0; + } + if( iCol==tailCol && iStart<=tailOffset+20 ){ + iStart = tailOffset; + } + if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ + fts3SnippetTrimWhiteSpace(&sb); + fts3SnippetAppendWhiteSpace(&sb); + fts3SnippetAppend(&sb, zEllipsis, -1); + fts3SnippetAppendWhiteSpace(&sb); + } + iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; + iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); + if( iEnd>=nDoc-10 ){ + iEnd = nDoc; + tailEllipsis = 0; + }else{ + tailEllipsis = 1; + } + while( iMatchzSnippet = sb.z; + pSnippet->nSnippet = sb.z ? sb.nUsed : 0; +} + +void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Snippet *p; /* Snippet structure */ + int rc = snippetAllOffsets(pCsr, &p); + if( rc==SQLITE_OK ){ + snippetOffsetText(p); + if( p->zOffset ){ + sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_nomem(pCtx); + } + }else{ + sqlite3_result_error_nomem(pCtx); + } + fts3SnippetFree(p); +} + +void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis /* Snippet ellipsis text - "..." */ +){ + Snippet *p; /* Snippet structure */ + int rc = snippetAllOffsets(pCsr, &p); + if( rc==SQLITE_OK ){ + snippetText(pCsr, p, zStart, zEnd, zEllipsis); + if( p->zSnippet ){ + sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_nomem(pCtx); + } + }else{ + sqlite3_result_error_nomem(pCtx); + } + fts3SnippetFree(p); +} + +#endif diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index ef19995..a316fcf 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -30,9 +30,10 @@ SQLITE_EXTENSION_INIT1 #endif -#include "fts3_hash.h" -#include "fts3_tokenizer.h" +#include "fts3Int.h" #include +#include +#include /* ** Implementation of the SQL scalar function for accessing the underlying @@ -59,14 +60,14 @@ static void scalarFunc( int argc, sqlite3_value **argv ){ - fts3Hash *pHash; + Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; int nName; assert( argc==1 || argc==2 ); - pHash = (fts3Hash *)sqlite3_user_data(context); + pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; @@ -97,6 +98,127 @@ static void scalarFunc( sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } +static int fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + switch( *z1 ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = &z1[1]; + while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++; + if( *z2 ) z2++; + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Possible tokenizer specification */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + if( !z ){ + zCopy = sqlite3_mprintf("simple"); + }else{ + if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ + return SQLITE_OK; + } + zCopy = sqlite3_mprintf("%s", &z[8]); + *pzTokenizer = zArg; + } + if( !zCopy ){ + return SQLITE_NOMEM; + } + + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1); + if( !m ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + #ifdef SQLITE_TEST #include @@ -133,7 +255,7 @@ static void testFunc( int argc, sqlite3_value **argv ){ - fts3Hash *pHash; + Fts3Hash *pHash; sqlite3_tokenizer_module *p; sqlite3_tokenizer *pTokenizer = 0; sqlite3_tokenizer_cursor *pCsr = 0; @@ -166,7 +288,7 @@ static void testFunc( zArg = (const char *)sqlite3_value_text(argv[1]); } - pHash = (fts3Hash *)sqlite3_user_data(context); + pHash = (Fts3Hash *)sqlite3_user_data(context); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ @@ -257,7 +379,7 @@ int queryTokenizer( sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } @@ -294,6 +416,9 @@ static void intTestFunc( const sqlite3_tokenizer_module *p2; sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + /* Test the query function */ sqlite3Fts3SimpleTokenizerModule(&p1); rc = queryTokenizer(db, "simple", &p2); @@ -335,7 +460,7 @@ static void intTestFunc( */ int sqlite3Fts3InitHashTable( sqlite3 *db, - fts3Hash *pHash, + Fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; @@ -353,13 +478,13 @@ int sqlite3Fts3InitHashTable( } #endif - if( rc!=SQLITE_OK - || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) - || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) + if( SQLITE_OK!=rc + || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) + || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) #ifdef SQLITE_TEST - || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) - || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) - || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) + || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) + || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) + || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) #endif ); diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index da255d9..654e55d 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -24,6 +24,7 @@ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#include "fts3Int.h" #include #include @@ -49,9 +50,6 @@ typedef struct simple_tokenizer_cursor { } simple_tokenizer_cursor; -/* Forward declaration */ -static const sqlite3_tokenizer_module simpleTokenizerModule; - static int simpleDelim(simple_tokenizer *t, unsigned char c){ return c<0x80 && t->delim[c]; } @@ -75,7 +73,7 @@ static int simpleCreate( ** information on the initial create. */ if( argc>1 ){ - int i, n = strlen(argv[1]); + int i, n = (int)strlen(argv[1]); for(i=0; idelim[i] = !isalnum(i); + t->delim[i] = !isalnum(i) ? -1 : 0; } } @@ -118,6 +116,8 @@ static int simpleOpen( ){ simple_tokenizer_cursor *c; + UNUSED_PARAMETER(pTokenizer); + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; @@ -191,7 +191,7 @@ static int simpleNext( ** case-insensitivity. */ unsigned char ch = p[iStartOffset+i]; - c->pToken[i] = ch<0x80 ? tolower(ch) : ch; + c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch); } *ppToken = c->pToken; *pnBytes = n; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c new file mode 100644 index 0000000..d240149 --- /dev/null +++ b/ext/fts3/fts3_write.c @@ -0,0 +1,2171 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "fts3Int.h" +#include +#include +#include + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** Data structure used while accumulating terms in the pending-terms hash +** table. The hash table entry maps from term (a string) to a malloced +** instance of this structure. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level */ + sqlite3_int64 iStartBlock; + sqlite3_int64 iEndBlock; + sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used to iterate through the current doclist */ + char *pOffsetList; + sqlite3_int64 iDocid; +}; + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_SELECT_CONTENT_BY_ROWID 5 +#define SQL_NEXT_SEGMENT_INDEX 6 +#define SQL_INSERT_SEGMENTS 7 +#define SQL_NEXT_SEGMENTS_ID 8 +#define SQL_INSERT_SEGDIR 9 +#define SQL_SELECT_LEVEL 10 +#define SQL_SELECT_ALL_LEVEL 11 +#define SQL_SELECT_LEVEL_COUNT 12 +#define SQL_SELECT_SEGDIR_COUNT_MAX 13 +#define SQL_DELETE_SEGDIR_BY_LEVEL 14 +#define SQL_DELETE_SEGMENTS_RANGE 15 +#define SQL_CONTENT_INSERT 16 +#define SQL_GET_BLOCK 17 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", +/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?", +/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'", +/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + +/* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", + +/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", +/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + int i; /* Iterator variable */ + char *zVarlist; /* The "?, ?, ..." string */ + zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2); + if( !zVarlist ){ + *pp = 0; + return SQLITE_NOMEM; + } + zVarlist[0] = '?'; + zVarlist[p->nColumn*2+1] = '\0'; + for(i=1; i<=p->nColumn; i++){ + zVarlist[i*2-1] = ','; + zVarlist[i*2] = '?'; + } + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ + int rc; + int iStart; + int iEnd; + int iPos; + + char const *zToken; + int nToken; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + pCsr->pTokenizer = pTokenizer; + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + PendingList *pList; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + } + + pModule->xClose(pCsr); + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + return SQLITE_OK; +} + +void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + Fts3HashElem *pElem; + for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(&p->pendingTerms); + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + if( zText ){ + int rc = fts3PendingTermsAdd(p, zText, i-2); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p){ + int rc; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the %_content, %_segments and %_segdir tables. */ + rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0); + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + } + return rc; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){ + int rc; + sqlite3_stmt *pSelect; + + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + for(i=1; i<=p->nColumn; i++){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, zText, -1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + return rc; + } + } + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + return rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pNextIdx, 1, iLevel); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=FTS3_MERGE_COUNT ){ + rc = fts3SegmentMerge(p, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext(Fts3SegReader *pReader){ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + int rc; + if( !pReader->pStmt ){ + pReader->aNode = 0; + return SQLITE_OK; + } + rc = sqlite3_step(pReader->pStmt); + if( rc!=SQLITE_ROW ){ + pReader->aNode = 0; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); + } + pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); + pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); + pNext = pReader->aNode; + } + + pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); + pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + + if( nPrefix+nSuffix>pReader->nTermAlloc ){ + int nNew = (nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); + assert( pNext<&pReader->aNode[pReader->nNode] ); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ + int n; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static void fts3SegReaderNextDocid( + Fts3SegReader *pReader, + char **ppOffsetList, + int *pnOffsetList +){ + char *p = pReader->pOffsetList; + char c = 0; + + /* Pointer p currently points at the first byte of an offset list. The + ** following two lines advance it to point one byte past the end of + ** the same offset list. + */ + while( *p | c ) c = *p++ & 0x80; + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* If there are no more entries in the doclist, set pOffsetList to + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and + ** Fts3SegReader.pOffsetList to point to the next offset list before + ** returning. + */ + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ + pReader->pOffsetList = 0; + }else{ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + pReader->iDocid += iDelta; + } +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ + if( pReader ){ + if( pReader->pStmt ){ + /* Move the leaf-range SELECT statement to the aLeavesStmt[] array, + ** so that it can be reused when required by another query. + */ + assert( p->nLeavesStmtnLeavesTotal ); + sqlite3_reset(pReader->pStmt); + p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt; + } + sqlite3_free(pReader->zTerm); + sqlite3_free(pReader); + } +} + +/* +** Allocate a new SegReader object. +*/ +int sqlite3Fts3SegReaderNew( + Fts3Table *p, /* Virtual table handle */ + int iAge, /* Segment "age". */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + if( iStartLeaf==0 ){ + nExtra = nRoot; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iStartBlock = iStartLeaf; + pReader->iIdx = iAge; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->nNode = nRoot; + memcpy(pReader->aNode, zRoot, nRoot); + }else{ + /* If the text of the SQL statement to iterate through a contiguous + ** set of entries in the %_segments table has not yet been composed, + ** compose it now. + */ + if( !p->zSelectLeaves ){ + p->zSelectLeaves = sqlite3_mprintf( + "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " + "ORDER BY blockid", p->zDb, p->zName + ); + if( !p->zSelectLeaves ){ + rc = SQLITE_NOMEM; + goto finished; + } + } + + /* If there are no free statements in the aLeavesStmt[] array, prepare + ** a new statement now. Otherwise, reuse a prepared statement from + ** aLeavesStmt[]. + */ + if( p->nLeavesStmt==0 ){ + if( p->nLeavesTotal==p->nLeavesAlloc ){ + int nNew = p->nLeavesAlloc + 16; + sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc( + p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *) + ); + if( !aNew ){ + rc = SQLITE_NOMEM; + goto finished; + } + p->nLeavesAlloc = nNew; + p->aLeavesStmt = aNew; + } + rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); + if( rc!=SQLITE_OK ){ + goto finished; + } + p->nLeavesTotal++; + }else{ + pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt]; + } + + /* Bind the start and end leaf blockids to the prepared SQL statement. */ + sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); + sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); + } + rc = fts3SegReaderNext(pReader); + + finished: + if( rc==SQLITE_OK ){ + *ppReader = pReader; + }else{ + sqlite3Fts3SegReaderFree(p, pReader); + } + return rc; +} + + +/* +** The second argument to this function is expected to be a statement of +** the form: +** +** SELECT +** idx, -- col 0 +** start_block, -- col 1 +** leaves_end_block, -- col 2 +** end_block, -- col 3 +** root -- col 4 +** FROM %_segdir ... +** +** This function allocates and initializes a Fts3SegReader structure to +** iterate through the terms stored in the segment identified by the +** current row that pStmt is pointing to. +** +** If successful, the Fts3SegReader is left pointing to the first term +** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned. +*/ +static int fts3SegReaderNew( + Fts3Table *p, /* Virtual table handle */ + sqlite3_stmt *pStmt, /* See above */ + int iAge, /* Segment "age". */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + return sqlite3Fts3SegReaderNew(p, iAge, + sqlite3_column_int64(pStmt, 1), + sqlite3_column_int64(pStmt, 2), + sqlite3_column_int64(pStmt, 3), + sqlite3_column_blob(pStmt, 4), + sqlite3_column_bytes(pStmt, 4), + ppReader + ); +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared with memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLevel, /* Value for "level" field */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + sqlite3_bind_int64(pStmt, 5, iEndBlock); + rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + int iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir( + p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir( + p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + } + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** apVal[0] would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Set *pnSegment to the number of segments of level iLevel in the database. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){ + sqlite3_stmt *pStmt; + int rc; + + assert( iLevel>=0 ); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iLevel); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnSegment = sqlite3_column_int(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** Set *pnSegment to the total number of segments in the database. Set +** *pnMax to the largest segment level in the database (segment levels +** are stored in the 'level' column of the %_segdir table). +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ + sqlite3_stmt *pStmt; + int rc; + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnSegment = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int(pStmt, 1); + } + return sqlite3_reset(pStmt); +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + for(i=0; rc==SQLITE_OK && iiStartBlock ){ + sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + + if( iLevel>=0 ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iLevel); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + }else{ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( pflags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); + + /* If there are zero segments, this function is a no-op. This scenario + ** comes about only when reading from an empty database. + */ + if( nSegment==0 ) goto finished; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + if( pFilter->zTerm ){ + int nTerm = pFilter->nTerm; + const char *zTerm = pFilter->zTerm; + for(i=0; iaNode ){ + int nTerm = apSegment[0]->nTerm; + char *zTerm = apSegment[0]->zTerm; + int nMerge = 1; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm ){ + if( nTermnTerm + || (!isPrefix && nTerm>pFilter->nTerm) + || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + goto finished; + } + } + + while( nMergeaNode + && apSegment[nMerge]->nTerm==nTerm + && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm) + ){ + nMerge++; + } + + if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){ + Fts3SegReader *p0 = apSegment[0]; + rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); + if( rc!=SQLITE_OK ) goto finished; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term added to the new segment. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList; + int nList; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>nAlloc ){ + char *aNew; + nAlloc = nDoclist+nByte*2; + aNew = sqlite3_realloc(aBuffer, nAlloc); + if( !aNew ){ + rc = SQLITE_NOMEM; + goto finished; + } + aBuffer = aNew; + } + nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&aBuffer[nDoclist], pList, nList); + nDoclist += nList; + aBuffer[nDoclist++] = '\0'; + } + } + + fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + } + + if( nDoclist>0 ){ + rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist); + if( rc!=SQLITE_OK ) goto finished; + } + } + + /* If there is a term specified to filter on, and this is not a prefix + ** search, return now. The callback that corresponds to the required + ** term (if such a term exists in the index) has already been made. + */ + if( pFilter->zTerm && !isPrefix ){ + goto finished; + } + + for(i=0; i0 ); + assert( iNewLevel>=0 ); + + /* Allocate space for an array of pointers to segment iterators. */ + apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); + if( !apSegment ){ + return SQLITE_NOMEM; + } + memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); + + /* Allocate a Fts3SegReader structure for each segment being merged. A + ** Fts3SegReader stores the state data required to iterate through all + ** entries on all leaves of a single segment. + */ + assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0); + if( rc!=SQLITE_OK ) goto finished; + sqlite3_bind_int(pStmt, 1, iLevel); + for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){ + rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]); + if( rc!=SQLITE_OK ){ + goto finished; + } + } + rc = sqlite3_reset(pStmt); + pStmt = 0; + if( rc!=SQLITE_OK ) goto finished; + + memset(&filter, 0, sizeof(Fts3SegFilter)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS; + filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, + &filter, fts3MergeCallback, (void *)&pWriter + ); + if( rc!=SQLITE_OK ) goto finished; + + rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment); + if( rc==SQLITE_OK ){ + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + } + + finished: + fts3SegWriterFree(pWriter); + if( apSegment ){ + for(i=0; ipendingTerms); + if( nElem==0 ){ + assert( p->nPendingData==0 ); + return SQLITE_OK; + } + + /* Determine the next index at level 0, merging as necessary. */ + rc = fts3AllocateSegdirIdx(p, 0, &idx); + if( rc!=SQLITE_OK ){ + return rc; + } + + apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *)); + if( !apElem ){ + return SQLITE_NOMEM; + } + + i = 0; + for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ + apElem[i++] = pElem; + } + assert( i==nElem ); + + /* TODO(shess) Should we allow user-defined collation sequences, + ** here? I think we only need that once we support prefix searches. + ** Also, should we be using qsort()? + */ + if( nElem>1 ){ + qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare); + } + + + /* Write the segment tree into the database. */ + for(i=0; rc==SQLITE_OK && iaData, pList->nData+1); + } + if( rc==SQLITE_OK ){ + rc = fts3SegWriterFlush(p, pWriter, 0, idx); + } + + /* Free all allocated resources before returning */ + fts3SegWriterFree(pWriter); + sqlite3_free(apElem); + sqlite3Fts3PendingTermsClear(p); + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. +*/ +int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + int isRemove = 0; /* True for an UPDATE or DELETE */ + sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + int isEmpty; + rc = fts3IsEmpty(p, apVal, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. + */ + rc = fts3DeleteAll(p); + }else{ + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); + rc = fts3PendingTermsDocid(p, iRemove); + if( rc==SQLITE_OK ){ + rc = fts3DeleteTerms(p, apVal); + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal); + } + } + } + } + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ + rc = fts3PendingTermsDocid(p, *pRowid); + } + if( rc==SQLITE_OK ){ + rc = fts3InsertTerms(p, apVal); + } + } + + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, -1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3 ; RELEASE fts3", 0, 0, 0); + } + } + return rc; +} + +#endif diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index f4d74bf..49c00a3 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains code for implementations of the r-tree and r*-tree ** algorithms packaged as an SQLite virtual table module. -** -** $Id: rtree.c,v 1.14 2009/08/06 18:36:47 danielk1977 Exp $ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 8c1675b..9d9a2b6 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -11,8 +11,6 @@ # # The focus of this file is testing the r-tree extension. # -# $Id: rtree1.test,v 1.7 2009/07/17 16:54:48 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] diff --git a/ext/rtree/rtree2.test b/ext/rtree/rtree2.test index 7e38c8f..307e7e0 100644 --- a/ext/rtree/rtree2.test +++ b/ext/rtree/rtree2.test @@ -11,8 +11,6 @@ # # The focus of this file is testing the r-tree extension. # -# $Id: rtree2.test,v 1.4 2008/07/14 15:37:01 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] diff --git a/ext/rtree/rtree3.test b/ext/rtree/rtree3.test index b83ceeb..1ce131c 100644 --- a/ext/rtree/rtree3.test +++ b/ext/rtree/rtree3.test @@ -12,8 +12,6 @@ # The focus of this file is testing that the r-tree correctly handles # out-of-memory conditions. # -# $Id: rtree3.test,v 1.2 2008/06/23 15:55:52 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] @@ -71,4 +69,3 @@ do_malloc_test rtree3-3 -sqlprep { } finish_test - diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test index d73e7a6..fa0d606 100644 --- a/ext/rtree/rtree4.test +++ b/ext/rtree/rtree4.test @@ -11,8 +11,6 @@ # # Randomized test cases for the rtree extension. # -# $Id: rtree4.test,v 1.3 2008/06/23 15:55:52 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 4fa007f..3620619 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -12,8 +12,6 @@ # The focus of this file is testing the r-tree extension when it is # configured to store values as 32 bit integers. # -# $Id: rtree5.test,v 1.1 2008/07/14 15:37:01 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index affa8fe..d23a74e 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -8,8 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# -# $Id: rtree6.test,v 1.1 2008/09/01 12:47:00 danielk1977 Exp $ +# # if {![info exists testdir]} { @@ -108,4 +107,3 @@ do_test rtree6.2.5 { ] finish_test - diff --git a/ext/rtree/rtree_perf.tcl b/ext/rtree/rtree_perf.tcl index fa3a4d3..e42e685 100644 --- a/ext/rtree/rtree_perf.tcl +++ b/ext/rtree/rtree_perf.tcl @@ -72,5 +72,3 @@ set rtree_select_time [time { } }] puts "$rtree_select_time" - - diff --git a/ext/rtree/rtree_util.tcl b/ext/rtree/rtree_util.tcl index 55482e4..50a1b58 100644 --- a/ext/rtree/rtree_util.tcl +++ b/ext/rtree/rtree_util.tcl @@ -13,8 +13,6 @@ # analyzing r-tree structures created with this module. It is # used by both test procedures and the r-tree viewer application. # -# $Id: rtree_util.tcl,v 1.1 2008/05/26 18:41:54 danielk1977 Exp $ -# #-------------------------------------------------------------------------- @@ -192,4 +190,3 @@ proc rtree_treedump {db zTab} { set d [rtree_depth $db $zTab] rtree_nodetreedump $db $zTab "" $d 1 } - diff --git a/ext/rtree/tkt3363.test b/ext/rtree/tkt3363.test index 11c5192..f072171 100644 --- a/ext/rtree/tkt3363.test +++ b/ext/rtree/tkt3363.test @@ -11,8 +11,6 @@ # # The focus of this file is testing that ticket #3363 is fixed. # -# $Id: tkt3363.test,v 1.1 2008/09/08 11:07:03 danielk1977 Exp $ -# if {![info exists testdir]} { set testdir [file join [file dirname $argv0] .. .. test] @@ -50,5 +48,3 @@ do_test tkt3363.1.4 { } {7} finish_test - - diff --git a/ext/rtree/viewrtree.tcl b/ext/rtree/viewrtree.tcl index 2b4dd1b..794677f 100644 --- a/ext/rtree/viewrtree.tcl +++ b/ext/rtree/viewrtree.tcl @@ -186,4 +186,3 @@ proc cell_report {db zTab iParent iCell} { view_node bind .c view_node - diff --git a/main.mk b/main.mk index 5e1d154..bcad3cc 100644 --- a/main.mk +++ b/main.mk @@ -54,7 +54,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ @@ -65,7 +65,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ table.o tokenize.o trigger.o \ update.o util.o vacuum.o \ - vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \ + vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbetrace.o \ walker.o where.o utf.o vtab.o @@ -154,6 +154,7 @@ SRC = \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbeblob.c \ $(TOP)/src/vdbemem.c \ + $(TOP)/src/vdbetrace.c \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vtab.c \ $(TOP)/src/walker.c \ @@ -182,15 +183,17 @@ SRC += \ SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ + $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_expr.h \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ + $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_tokenizer1.c + $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_write.c SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c @@ -231,6 +234,7 @@ TESTSRC = \ $(TOP)/src/test_func.c \ $(TOP)/src/test_hexio.c \ $(TOP)/src/test_init.c \ + $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ $(TOP)/src/test_mutex.c \ @@ -295,7 +299,7 @@ EXTHDR += \ $(TOP)/ext/fts2/fts2_tokenizer.h EXTHDR += \ $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3_expr.h \ + $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_tokenizer.h EXTHDR += \ @@ -434,6 +438,9 @@ fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c +fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c + fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c @@ -443,6 +450,9 @@ fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c + rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c diff --git a/manifest b/manifest index eb2afb9..d8e009b 100644 --- a/manifest +++ b/manifest @@ -1,14 +1,14 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Remove\s"const"\sfrom\sparameter\sof\ssqlite3BtreeFactory()\sto\savoid\sa\scompiler\nwarning. -D 2009-11-04T13:30:02 +C Version\s3.6.21\srelease\scandidate\s3. +D 2009-12-07T16:39:13 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 -F Makefile.in a77dfde96ad86aafd3f71651a4333a104debe86a +F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F Makefile.vxworks 10010ddbf52e2503c7c49c7c0b7c7a096f8638a6 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 1dc0be471f26258df462bc85387f8255daad5f26 +F VERSION aa30c4b131a981624ab576df19eb4c5c9cd582ac F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -22,13 +22,13 @@ F art/src_logo.gif 9341ef09f0e53cd44c0c9b6fc3c16f7f3d6c2ad9 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 868fdb48c028421a203470e15c69ada15b9ba673 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 71cfacd7732f55af4aecaa8e7bea518e11cecc5e +F configure beb6a7b293d15978ac274e6b43b37b356d3070b2 F configure.ac 14740970ddb674d92a9f5da89083dff1179014ff F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538 F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 F ext/async/README.txt 0c541f418b14b415212264cbaaf51c924ec62e5b -F ext/async/sqlite3async.c 3d5396cd69851f5633ef29c7491ca9249eac903a +F ext/async/sqlite3async.c 676066c2a111a8b3107aeb59bdbbbf335c348f4a F ext/async/sqlite3async.h a21e1252deb14a2c211f0e165c4b9122a8f1f344 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b @@ -59,42 +59,44 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 35bfa67d9cd659b799b8498895fe60b1e8bd3500 +F ext/fts3/fts3.c 684a55d603f11c8432323171082ff8a9437b4681 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3_expr.c 0bfdae44d0d8ea2cb3ccad32bb6d6843d78d1a2d -F ext/fts3/fts3_expr.h b5412dcf565c6d90d6a8c22090ceb9ed8c745634 -F ext/fts3/fts3_hash.c e15e84d18f8df149ab290029872d4559c4c7c15a -F ext/fts3/fts3_hash.h 004b759e1602ff16dfa02fea3ca1c77336ad6798 +F ext/fts3/fts3Int.h cc716c74afa7da8e0f8ef39404f33ea62a823eb3 +F ext/fts3/fts3_expr.c c18794a62c257d3456d3314c5a18e348ae0d84bd +F ext/fts3/fts3_hash.c 18feef38fca216992725e9eae775a0c7735e6724 +F ext/fts3/fts3_hash.h d410ff2c93c81a56b927fcf07b2099ccbfa7a479 F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 -F ext/fts3/fts3_porter.c 3063da945fb0a935781c135f7575f39166173eca -F ext/fts3/fts3_tokenizer.c fcc8fdb5c161df7d61c77285ec2991da131f0181 +F ext/fts3/fts3_porter.c a651e287e02b49b565a6ccf9441959d434489156 +F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496 +F ext/fts3/fts3_tokenizer.c 52112e7b50fbf24eb68e356081c5969917406cc6 F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c -F ext/fts3/fts3_tokenizer1.c 0a5bcc579f35de5d24a9345d7908dc25ae403ee7 +F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b +F ext/fts3/fts3_write.c ec51fb6886f910e78ae32158ec0301aa675f52d8 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 5275d8f851c366f4a01e8a0c63aa0af492567f28 +F ext/rtree/rtree.c 038d59b05783c2e6c927a7352bb118a76c31065a F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test 207041aba07fdcdd93aa797a745839d305181857 -F ext/rtree/rtree2.test 9ac9d28fa948779df66916c67a5dcf9704c3cb74 -F ext/rtree/rtree3.test 877a09c1a0c2b87af0f94f3a286e7dd3b65adf22 -F ext/rtree/rtree4.test 11724f766a74f48710998cdd7552cec140c55bf9 -F ext/rtree/rtree5.test 7d0643482829038f0263881ddf7e2d51bff1d60f -F ext/rtree/rtree6.test 119d991e8651fb1dbb1fa31af67ad280fbe4adf7 -F ext/rtree/rtree_perf.tcl 0fabb6d5c48cb8024e042ce5d4bb88998b6ec1cb -F ext/rtree/rtree_util.tcl ee0a0311eb12175319d78bfb37302320496cee6e -F ext/rtree/tkt3363.test 6662237ea75bb431cd5d262dfc9535e1023315fc -F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869 +F ext/rtree/rtree1.test f72885ed80a329d6bd7991043016d74b51edf2c5 +F ext/rtree/rtree2.test 7b665c44d25e51b3098068d983a39902b2e2d7a1 +F ext/rtree/rtree3.test dece988c363368af8c11862995c762071894918f +F ext/rtree/rtree4.test 94fdd570ab5bc47244d87d4590023be43ac786bd +F ext/rtree/rtree5.test 92508f5152a50110af6551fa5b769d1bbd7c4ef3 +F ext/rtree/rtree6.test 11aade5311789068ca659be24a47cc0d852b1971 +F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 +F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea +F ext/rtree/tkt3363.test 2bf324f7908084a5f463de3109db9c6e607feb1b +F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d588eab5d51b0bfe924a1cccdfdd2cbb4cbe40b4 +F main.mk a92b99b264c3b277f8cf74d50aadc22c55967ef0 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk 3fb9bf077053c968451f4dd03d11661ac373f9d1 -F mkopcodeh.awk 104fa333e4a7a689fac074437cbd51a31a803bd4 +F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F publish.sh c74b6c2b6b63435aa1b4b43b1396dfebfae84095 F publish_osx.sh 2ad2ee7d50632dff99949edc9c162dbb052f7534 @@ -102,122 +104,125 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c 9e4b52e6b1d3a26242cf2ce680c9fce801f00aa1 -F src/analyze.c 5a8b8aa3d170eac5e71af45458cec61f83c623ee -F src/attach.c 1f2ae6ca3de365c8e959f1d56beb6af589fef75b -F src/auth.c a5471a6951a18f79d783da34be22cd94dfbe603a -F src/backup.c 6f1c2d9862c8a3feb7739dfcca02c1f5352e37f3 -F src/bitvec.c ed215b95734045e58358c3b3e16448f8fe6a235a -F src/btmutex.c 0f43a75bb5b8147b386e8e1c3e71ba734e3863b7 -F src/btree.c 3b29423f08987bddeb115562d6a699b002f653ae -F src/btree.h 577448a890c2ab9b21e6ab74f073526184bceebe -F src/btreeInt.h 1c863e543bb55772fa8db95535c370bb386c578c -F src/build.c 3c5762687d0554ebe8844dfaddb828fcc15fe16d -F src/callback.c 10d237171472865f58fb07d515737238c9e06688 -F src/complete.c 5ad5c6cd4548211867c204c41a126d73a9fbcea0 -F src/date.c 657ff12ca0f1195b531561afacbb38b772d16638 -F src/delete.c 308e300d599d2d11b838687e2cf7309d42f29a1a -F src/expr.c 11c163003b79f38a5b5100228052aca57b454807 -F src/fault.c dc88c821842157460750d2d61a8a8b4197d047ff -F src/fkey.c 41219cba186bcf0a053e42327dfa23aaba4f834a -F src/func.c e536218d193b8d326aab91120bc4c6f28aa2b606 -F src/global.c 271952d199a8cc59d4ce840b3bbbfd2f30c8ba32 -F src/hash.c ebcaa921ffd9d86f7ea5ae16a0a29d1c871130a7 -F src/hash.h 35b216c13343d0b4f87d9f21969ac55ad72174e1 -F src/hwtime.h 4a1d45f4cae1f402ea19686acf24acf4f0cb53cb -F src/insert.c 2fe2ef7bd03d6e0120e4525727c4ae7de5a2d571 -F src/journal.c e00df0c0da8413ab6e1bb7d7cab5665d4a9000d0 -F src/legacy.c 303b4ffcf1ae652fcf5ef635846c563c254564f6 +F src/alter.c 92ba938565d7cc6bfe92aad6cc90c00800ff21d3 +F src/analyze.c 55155f05ee9ab4ce33b7a4d19c449053f8935200 +F src/attach.c 0ba38b38252a34bb9721de35514a1d14058a8e49 +F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 +F src/backup.c 744e98359dfc79fed43e8dec911e33e108b06aae +F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 +F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff +F src/btree.c efdef3953c49e28f8b8fa9cc0ac5754cc1a7489a +F src/btree.h 7944a9dac59eb3e541aad45fd2747f1051e7c63d +F src/btreeInt.h 54f4245decd0409ea52cf9aee422d3d761d7ac10 +F src/build.c a48e74d24897100017d39ceba5de255e53ec9488 +F src/callback.c 908f3e0172c3d4058f4ca0acd42c637c52e9669f +F src/complete.c 417df1ef5ea798532bb6290b0cc4265fef82980a +F src/date.c a79c0a8f219370b972e320741f995a3bef9df33f +F src/delete.c 8b8afb9cd7783d573eae55a3f4208bc0637a2bb8 +F src/expr.c 50385ed51f1cd7f1ab289629cd0f87d5b2fcca52 +F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb +F src/fkey.c e2116672a6bd610dc888e27df292ebc7999c9bb0 +F src/func.c bf54e1202cbfb28bf4b1fd9b58899009ae76716f +F src/global.c 75946a4a2ab41c6ae58f10ca0ed31b3449694b26 +F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af +F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 +F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 +F src/insert.c f9c6098988675ac258b2f98ea5f7e370fc9990fa +F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c +F src/legacy.c 9304428e71b1d622b764913e1432e69156814755 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e -F src/loadext.c 0e88a335665db0b2fb4cece3e49dcb65d832635a +F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 F src/main.c aae32d5af35b88faff0664e0f937ee7133d77c8d -F src/malloc.c 685561d2f1602042e349aea5d7a88c3f10a63454 -F src/mem0.c f2f84062d1f35814d6535c9f9e33de3bfb3b132c -F src/mem1.c e6d5c23941288df8191b8a98c28e3f57771e2270 -F src/mem2.c d02bd6a5b34f2d59012a852615621939d9c09548 -F src/mem3.c 805ab642adfafa171781a5d8ab112119dfaef118 +F src/malloc.c 5fa175797f982b178eaf38afba9c588a866be729 +F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 +F src/mem1.c 552f7e11486272f27948d2de9c012884d1f52908 +F src/mem2.c 3f196f6fd3f4320035eb4acbe4530686da2f14b1 +F src/mem3.c 9b237d911ba9904142a804be727cc6664873f8a3 F src/mem5.c 4837b795ebdecc0cfe1522cd0c8b2c5d84ea490d -F src/memjournal.c e68cb5f7e828b84d5bf2ea16c5d87f1ed7e9fe7f -F src/mutex.c 828a40bc7bf79f01ef95ccbf8db8b02857739449 -F src/mutex.h 32ddef38560ce0128d7e7b3eb063f5c6eff889a3 -F src/mutex_noop.c f5a07671f25a1a9bd7c10ad7107bc2585446200f -F src/mutex_os2.c 6b5a74f812082a8483c3df05b47bbaac2424b9a0 -F src/mutex_unix.c aff26c77f698b552becfedfa677ad1036c42d790 -F src/mutex_w32.c b2c1481ee93b0ac7a8fa5346570fd173b6763fdb -F src/notify.c 0127121816d8a861deb0dfd111b495346bf233db -F src/os.c 8d62d8d98ad7909cb0dd294c1e5f3835c887ccb6 -F src/os.h 00a1334a4eecee7f7bef79ac606b88d325119f21 -F src/os_common.h 8c61457df58f1a4bd5f5adc3e90e01b37bf7afbc -F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5 -F src/os_unix.c a4b4ea928ce31ed34cb8f90ed36a35df19312fad +F src/memjournal.c 5bfc2f33c914946e2f77ed3f882aff14dfc9355d +F src/mutex.c 581a272e09098040ca3ef543cb5f3d643eff7d50 +F src/mutex.h 6fde601e55fa6c3fae768783c439797ab84c87c6 +F src/mutex_noop.c 5f58eaa31f2d742cb8957a747f7887ae98f16053 +F src/mutex_os2.c 20477db50cf3817c2f1cd3eb61e5c177e50231db +F src/mutex_unix.c 04a25238abce7e3d06b358dcf706e26624270809 +F src/mutex_w32.c 9ec75bcef0ca722821be7968c320fd725abfb984 +F src/notify.c f799bbda67ab6619b36b0a24153b49518874a203 +F src/os.c 4500ff276e277730776fe9b6c6c5930383ec4000 +F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c +F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 +F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f +F src/os_unix.c fe85acfeded5cc13c2340ab73c9baf841de4e6d9 F src/os_win.c 5ffab20249a61e0625f869efe157fa009747039b -F src/pager.c 729f73feeb33355ae1f0982a74f112ce190c74aa -F src/pager.h 11852d044c86cf5a9d6e34171fb0c4fcf1f6265f -F src/parse.y 0204f0dfe8974dc2a0d46eb9ab98a433a1f963d6 -F src/pcache.c c92ffd4f3e1279b3766854c6d18b5bf4aac0d1fa -F src/pcache.h 435ef324197f79391f9c92b71d7f92b548ad7a36 -F src/pcache1.c 211295a9ff6a5b30f1ca50516731a5cf3e9bf82c -F src/pragma.c c25d0d15dd0bbc5ec34e9760629353358705a447 -F src/prepare.c 665d52303135833c53b9be03e68533e249e1de54 -F src/printf.c 508a1c59433353552b6553cba175eaa7331f8fc1 -F src/random.c 676b9d7ac820fe81e6fb2394ac8c10cff7f38628 -F src/resolve.c f263d685bf21d0707b595455e0a0c95a3f5398f6 -F src/rowset.c c64dafba1f9fd876836c8db8682966b9d197eb1f -F src/select.c cbe366a0ce114856e66f5daf0f848d7c48a88298 -F src/shell.c f66531a57fff927f95c98d99c28237d88e400c86 -F src/sqlite.h.in 9106176cf206c36f01f8b761ba62671818bbe6ff -F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17 -F src/sqliteInt.h 67a9ae42527204726d8eb7a58cdb238d659a82fa -F src/sqliteLimit.h 38b2fffcd01faeaeaadea71b2b47695a81580c8b -F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76 -F src/table.c cc86ad3d6ad54df7c63a3e807b5783c90411a08d -F src/tclsqlite.c c2c4047177213baf485d4401c7dbb30a3c2ba322 -F src/test1.c 2232a39540a6b72f3be8f84b34d1ca0714f92aee -F src/test2.c 0de743ec8890ca4f09e0bce5d6d5a681f5957fec -F src/test3.c 2445c2beb5e7a0c91fd8136dc1339ec369a24898 -F src/test4.c b5fd530f02a6a0dbffb23be202168a690985dedd -F src/test5.c 162a1cea2105a2c460a3f39fa6919617b562a288 -F src/test6.c 1a0a7a1f179469044b065b4a88aab9faee114101 -F src/test7.c b94e68c2236de76889d82b8d7d8e00ad6a4d80b1 -F src/test8.c 34719910286a0a6ca233f10ba66558be938494dd -F src/test9.c 963d380922f25c1c323712d05db01b19197ee6f7 -F src/test_async.c 731d23f953ece5bf40ce87810cfb7607218953c5 -F src/test_autoext.c f53b0cdf7bf5f08100009572a5d65cdb540bd0ad -F src/test_backup.c 1384a18985a5a2d275c2662e48473bf1542ebd08 -F src/test_btree.c 5adbba9b138988a3cf4d3b5424dbc7c85651da02 -F src/test_config.c 4ac1e6257dcf926a71b7934410b71c5c326e68f2 -F src/test_devsym.c 9f4bc2551e267ce7aeda195f3897d0f30c5228f4 -F src/test_func.c c6e9d7cfbd7bb0bd7c392a10d76adab4b48e813b -F src/test_hexio.c 2f1122aa3f012fa0142ee3c36ce5c902a70cd12f -F src/test_init.c f6a5dfaf2fb52d697eec1c825a641e5893c339d2 -F src/test_journal.c dab49b7c47b53242f039c9563b18cafb67ebfe03 -F src/test_loadext.c 97dc8800e46a46ed002c2968572656f37e9c0dd9 -F src/test_malloc.c d054506b095d711e4e5575558dd576a2cbf035a2 -F src/test_mutex.c 482d9d987c1c678199691efc23c8cd3464e01ff5 -F src/test_onefile.c d2c3126633592aeef14e8d268fc40c74449b69d8 -F src/test_osinst.c 9a70a61e127f9e72bcfca000b20368b1c5367873 -F src/test_pcache.c d770f933888b2afe856c1abcefc64eda5941ffef -F src/test_schema.c 4b4bf7bb329326458c491b0e6facd4c8c4c5b479 -F src/test_server.c f0a403b5f699c09bd2b1236b6f69830fd6221f6b -F src/test_tclvar.c 9e42fa59d3d2f064b7ab8628e7ab2dc8a9fe93d4 -F src/test_thread.c b8a1ab7ca1a632f18e8a361880d5d65eeea08eac -F src/test_wsd.c 3ae5101de6cbfda2720152ab659ea84079719241 -F src/tokenize.c af8a56e6a50c5042fc305bfa796275e9bf26ff2b -F src/trigger.c 2053afa9952f69cf451bc0e6ea88072701f2925e -F src/update.c 8e8535f66c32d946199cb1caad19646a97ead3a7 -F src/utf.c 3586a6a2457c5c88b6816f6cda58a54e291883f8 -F src/util.c 59d4e9456bf1fe581f415a783fa0cee6115c8f35 -F src/vacuum.c 48e1282bbd5eac4b461587c51658378658c00770 -F src/vdbe.c a435ffcf6bfc7f14eb40998062ccbd7dfa482319 -F src/vdbe.h 449323a21c02226790acb6189dae78af17b92b78 -F src/vdbeInt.h 53b430ad3ff91bc5c963d5573299801c54cb7cba -F src/vdbeapi.c 44b5f387459d5faa158aa8d3a26967f0c8596efd -F src/vdbeaux.c 0fac44db16b5dc116eb1cacbe619033cb09569c3 -F src/vdbeblob.c 9bfaeab22e261a6a7b6df04e7faaf7d6dfdbef5a -F src/vdbemem.c 7055a2941a7802094f4704cedc7a28cc88a23749 -F src/vtab.c 3e54fe39374e5feb8b174de32a90e7a21966025d -F src/walker.c 1edca756275f158b80f20eb6f104c8d3fcc96a04 -F src/where.c fac37876e09e005e52cdfcf896090b6f5ebbf25b +F src/pager.c a0ed14b86de9d012a962b83389ca01003b6acccb +F src/pager.h 1b32faf2e578ac3e7bcf9c9d11217128261c5c54 +F src/parse.y b172fba9a954855502271556497c440506b6daf4 +F src/pcache.c 3b079306376e0e04c0d3df40c0a4b750a1839310 +F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 +F src/pcache1.c 2bb2261190b42a348038f5b1c285c8cef415fcc8 +F src/pragma.c 6936d7df5e04b9f996f8f320d15e65b6944b2caa +F src/prepare.c ad90970bba3aead154266d8bb6faf9fbb5233b94 +F src/printf.c 644bc7d59df3dc56d6d8b9a510914bfc6b51bc69 +F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 +F src/resolve.c d052e5c44bab34f83b3c1741aaa07478d18b5dd5 +F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 +F src/select.c 2f9ed7482e7a25b0b127fc41693bbdbe1caf5647 +F src/shell.c f4948cb6d30665d755a6b5e0ec313d1094aab828 +F src/sqlite.h.in 2d34605565e021851255e0bbcb15f8c1930d1f6f +F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 +F src/sqliteInt.h e946a6a3f2df015cdbc7668e9626987e8badbb5f +F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 +F src/status.c e651be6b30d397d86384c6867bc016e4913bcac7 +F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e +F src/tclsqlite.c bad6570a005b234ea670b9f7b48256da19a032d3 +F src/test1.c db4d8fd2849ab9aca0f27fd3773b8d68d078cf86 +F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb +F src/test3.c f17eeaf8114205844d76f4e69bab27ea341087af +F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d +F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1 +F src/test6.c a8ece4284d0e34477f349ac05655db73c48e0926 +F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e +F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea +F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 +F src/test_async.c c1656facbaf43cb2e71b62621e5b9eb080e2621c +F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e +F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de +F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 +F src/test_config.c 220a67047af393756f55760fdf442d935d0d88f3 +F src/test_devsym.c de3c9af2bb9a8b1e44525c449e4ec3f88e3d4110 +F src/test_func.c 1c94388a23d4a9e7cd62ec79d612d1bae2451fa2 +F src/test_hexio.c 415adbdb88cd9388831ce10edff76cb9e73d8a0b +F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c +F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 +F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 +F src/test_journal.c adc0ce3840ed19b49feb1d583b2212f560ef7866 +F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e +F src/test_malloc.c f777d15df756bea0e98271932464ac5d882e66fe +F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 +F src/test_onefile.c 06da7e085dce42924cf062b91763dd4bb84c6101 +F src/test_osinst.c 90fb03d396f39956897dfb4bd0e62c6711db1cca +F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 +F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 +F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 +F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa +F src/test_thread.c 00fed80690ae7f1525483a35861511c48bc579f2 +F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 +F src/tokenize.c 9f7d39da1a1346fa0cf106ea0bf10bb6b8b61ddf +F src/trigger.c d46f9389e3bf3dd1cc1d288aba2f289c96b34200 +F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 +F src/utf.c dad16adcc0c35ef2437dca125a4b07419d361052 +F src/util.c ad4f03079ba0fe83590d1cc9197e8e4844e38592 +F src/vacuum.c 03309a08d549f9389cc3a3589afd4fadbdaf0679 +F src/vdbe.c 5ed06318aac5d57849170a8bf39e807c22c5fedd +F src/vdbe.h bea1f0cd530775bdb58a340265f3cf3ee920e9b2 +F src/vdbeInt.h d7ea821ac7813c9bea0fe87558c35e07b2c7c44d +F src/vdbeapi.c bb128b819b9ef1a2ce211a36a6cb70a1643fa239 +F src/vdbeaux.c 0981dcb5b933b74ae7bc9bfa7770df5e4da849b3 +F src/vdbeblob.c 84f924700a7a889152aeebef77ca5f4e3875ffb4 +F src/vdbemem.c 1e16e3a16e55f4c3452834f0e041726021aa66e0 +F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 +F src/vtab.c 456fc226614569f0e46f216e33265bea268bd917 +F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f +F src/where.c 11b5b00c49d53e767a7eb855bc60790edeca6185 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45 @@ -228,13 +233,13 @@ F test/alter4.test 9386ffd1e9c7245f43eca412b2058d747509cc1f F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/analyze.test ad5329098fe4de4a96852231d53e3e9e6283ad4b F test/analyze2.test a2ad7b0a4e13801ee3968fe70f22aff52326569c -F test/analyze3.test 851bcc0738b87e1c1990d8dbad6ab53cdfcc19d6 +F test/analyze3.test ae06e0f8b3eaae0dd644ac9ac9d617058b5ac131 F test/async.test 8c75d31b8330f8b70cf2571b014d4476a063efdb F test/async2.test bf5e2ca2c96763b4cba3d016249ad7259a5603b6 F test/async3.test 93edaa9122f498e56ea98c36c72abc407f4fb11e F test/async4.test aafa6328c559d3e4bb587de770cbdecfca06f0da F test/async5.test f3592d79c84d6e83a5f50d3fd500445f7d97dfdf -F test/attach.test 1d1be27b9e4c654f9bb14d011a4a87753c0b197a +F test/attach.test 826f7676c41c12b035181d257299b8c8a17d64f3 F test/attach2.test a295d2d7061adcee5884ef4a93c7c96a82765437 F test/attach3.test bd9830bc3a0d22ed1310c9bff6896927937017dc F test/attachmalloc.test cf8cf17d183de357b1147a9baacbdfc85b940b61 @@ -274,6 +279,7 @@ F test/capi3c.test d9d293ce8fd4dc2944ce2dae5718fc7a6184a567 F test/capi3d.test 57d83b690d7364bde02cddbf8339a4b50d80ce23 F test/cast.test 166951664a0b0a2e0f8fb5997a152490c6363932 F test/check.test b897cd3cc839b34b31cdd073e9882ccd03da977b +F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c @@ -287,7 +293,7 @@ F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/colmeta.test 087c42997754b8c648819832241daf724f813322 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 0ed68b11f22721052d880ee80bd528a0e0828236 -F test/corrupt.test f47f220959b40f79f30c4271eefcec0587e984e3 +F test/corrupt.test 85c3fececa01bc6d24ff5d7bf1373844840c0b98 F test/corrupt2.test a571e30ea4e82318f319a24b6cc55935ce862079 F test/corrupt3.test 263e8bb04e2728df832fddf6973cf54c91db0c32 F test/corrupt4.test acdb01afaedf529004b70e55de1a6f5a05ae7fff @@ -322,7 +328,8 @@ F test/descidx2.test 1310ed1326cdfed4ea2c55169631579f082d174f F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_fkey.test af2750eddb48280cfd5ce1d8b2f7948bbd8779ad +F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17 +F test/e_fts3.test 8907e25b2c7d6bda9f7077356f64bc5e26c251a7 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041 @@ -372,31 +379,35 @@ F test/fts2p.test 4b48c35c91e6a7dbf5ac8d1e5691823cc999aafb F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a -F test/fts3.test efb41507c90f47e8af2a9101d7460cddeb84656b -F test/fts3aa.test 432d1d5c41939bb5405d4d6c80a9ec759b363393 -F test/fts3ab.test 7f6cf260ae80dda064023df8e8e503e9a412b91f +F test/fts3.test ae0433b09b12def08105640e57693726c4949338 +F test/fts3_common.tcl 31935839b1b601a5955572cb4e8060513c96bde0 +F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0 +F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69 -F test/fts3ad.test 32a114c6f214081f244f642bde9fd5517938788e -F test/fts3ae.test 31d8137fc7c14b5b991e3c4fa041ad2ac1255c7b +F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49 +F test/fts3ae.test ce32a13b34b0260928e4213b4481acf801533bda F test/fts3af.test d394978c534eabf22dd0837e718b913fd66b499c -F test/fts3ag.test 1c316bedb40a7c962e38998df854ea3ae26a3daa +F test/fts3ag.test 38d9c7dd4b607929498e8e0b32299af5665da1ab F test/fts3ah.test ba181d6a3dee0c929f0d69df67cac9c47cda6bff F test/fts3ai.test d29cee6ed653e30de478066881cec8aa766531b2 F test/fts3aj.test 584facbc9ac4381a7ec624bfde677340ffc2a5a4 F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d F test/fts3al.test 6d19619402d2133773262652fc3f185cdf6be667 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 -F test/fts3an.test 4b4fdab5abe2f308bdc47f6e822df2bcae30361c +F test/fts3an.test 931fa21bd80641ca594bfa32e105250a8a07918b F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa -F test/fts3b.test b3a25180a633873d37d86e1ccd00ed690d37237a -F test/fts3c.test 4c7ef29b37aca3e8ebb6a39b57910caa6506034e -F test/fts3d.test d92a47fe8ed59c9e53d2d8e6d2685bb380aadadc +F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 +F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 +F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a +F test/fts3malloc.test d02ee86b21edd2b43044e0d6dfdcd26cb6efddcb F test/fts3near.test dc196dd17b4606f440c580d45b3d23aa975fd077 +F test/fts3rnd.test ec82795eb358b7a4d6ce79e764d8d55556197584 F test/func.test af106ed834001738246d276659406823e35cde7b +F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/fuzz.test a4174c3009a3e2c2e14b31b364ebf7ddb49de2c9 F test/fuzz2.test ea38692ce2da99ad79fe0be5eb1a452c1c4d37bb F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 @@ -424,6 +435,7 @@ F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 6e382eaf7295a4463e6f29ea20fcd8e63d097eeb F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 +F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 F test/intpkey.test 537669fd535f62632ca64828e435b9e54e8d677f F test/io.test e7bd58edb4e2131a8ecd81b4b00af3ee5c79d464 @@ -511,10 +523,10 @@ F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 F test/permutations.test 1ce2874df8fec876d0b963c7a3ef61c4e9df8827 F test/pragma.test 5aeb48a442dba3c3e8e38773b121371814ab3b17 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 -F test/printf.test 47e9e5bbec8509023479d54ceb71c9d05a95308a +F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/quick.test 12fdc7656b4d20a537a686fb223eb99b5fe54483 +F test/quick.test d6591e74f3ac19da7fd076845f06dca48fd43cff F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a @@ -578,7 +590,7 @@ F test/tclsqlite.test bf4227eb236a4c097aa7974a2bf7d3225acf34be F test/tempdb.test 1bf52da28a9c24e29717362a87722dff08feb72b F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl 2caf7980d7dbb99dab9507ae0646802bc4d12c79 +F test/tester.tcl 02f671e71d1646440d226bed2dde8433f0a7bfa9 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -593,7 +605,7 @@ F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-3fe897352e.test 8084dad39807eac10b10720c84193bd1a5980973 F test/tkt-4a03edc4c8.test 2865e4edbc075b954daa82f8da7cc973033ec76e F test/tkt-5ee23731f.test 3581260f2a71e51db94e1506ba6b0f7311d002a9 -F test/tkt-94c04eaadb.test 40e6b1fce420fbecf8c2379d3ec3cb6889e49091 +F test/tkt-94c04eaadb.test be5ea61cb04dfdc047d19b5c5a9e75fa3da67a7f F test/tkt-d82e3f3721.txt cbed12b1a1e4740382de43ef1bb45c6bc0f8f473 F test/tkt-f777251dc7a.test 6f24c053bc5cdb7e1e19be9a72c8887cf41d5e87 F test/tkt1435.test f8c52c41de6e5ca02f1845f3a46e18e25cadac00 @@ -681,7 +693,7 @@ F test/tkt3992.test f3e7d548ac26f763b47bc0f750da3d03c81071da F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 -F test/trace.test 19ffbc09885c3321d56358a5738feae8587fb377 +F test/trace.test 2739f8dcc6739a9235523819c4cd30e78c44985c F test/trans.test d887cb07630dc39879a322d958ad8b006137485c F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 @@ -696,7 +708,7 @@ F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 F test/triggerA.test 0718ad2d9bfef27c7af00e636df79bee6b988da7 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe -F test/triggerC.test 0acc1d22d9c3d6cf0018bf795d544732a25657dc +F test/triggerC.test 4083c64d80854d271bad211268a08985f3d61cbd F test/types.test 9a825ec8eea4e965d7113b74c76a78bb5240f2ac F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test a0f66bf12f80fad89493535474f7a6d16fa58150 @@ -722,6 +734,7 @@ F test/vtabA.test 0dcd4c81ffb56649f47d1b5fb9c5ae807ccf41f7 F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796 F test/vtabC.test 1cf7896ab6859bfe3074244b2b0e12de5cbdd766 F test/vtabD.test 74167b1578e5886fe4c886d6bef2fd1406444c42 +F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d @@ -732,7 +745,7 @@ F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 42c4373595f4409d9c6a9987b4a60000ad664faf F test/where7.test fdd58ab9dec9f97679e65d4414bf5e07d725d79f -F test/where8.test 8d3704d04a683e792d373005f2e4e13bfd7e2dd5 +F test/where8.test 434f08974964b10378d67867773a2c3aedaf1d4b F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test be19e1a92f80985c1a121b4678bf7d2123eaa623 F test/whereA.test 1d1674254614147c866ab9b59af6582f454a858c @@ -748,11 +761,13 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c 9216336085e7a7c226a35c0bd780239968f8304f F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl a7e87ce780cbf30782bca3fd1197e86f92ae6f24 +F tool/mksqlite3c.tcl 1da28229695fdebdfe8a7d19902ef6c76d6c1c2d F tool/mksqlite3h.tcl eb100dce83f24b501b325b340f8b5eb8e5106b3b F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 27d6f6e3b1e95aeb26a1c140e6eb57771c6d794a F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c +F tool/shell1.test f85b976bef45f435fa39bf3690501cd5c0bdbb86 +F tool/shell2.test c985140ca75d96dad5bef8a5f49ed51b9c002dfb F tool/showdb.c 8ab8b3b53884312aafb7ef60982e255a6c31d238 F tool/showjournal.c ec3b171be148656827c4949fbfb8ab4370822f87 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -764,14 +779,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 24a4d520d540d92b611abc4eb57dc6da9be4eac6 -R a2178242a921307d19fbc8b7d2eec4f0 +P cd50acf37fd1e3b388f98fb2df7ed03cff454b24 +R 5d0b6a47a50c8a9cdc60d3d4d61a8cf2 U drh -Z a8d4aa9309aed4b69860cd88128a4708 +Z 8b76b0c0726e570aef93558fa0c3fdc7 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFK8YHdoxKgR168RlERAs0DAJ447A8mNmnCD0zy/gACLgqIHsmF8QCfYMHu -do95dSeuDXCMLnSZQtGGeQE= -=FnZg +iD8DBQFLHS+0oxKgR168RlERArAKAJ97NTyJvzQHmWC2GXt2Xt7D52m04QCeLeQt +HYPTNDWgvT/KRZ19RtHNjDQ= +=tE2+ -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 1bf4c6c..8b33d06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb7a544fe49d1626bacecfe53ddc03fe082e3243 +1ed88e9d01e9eda5cbc622e7614277f29bcc551c diff --git a/mkopcodeh.awk b/mkopcodeh.awk index 572331b..f6b90c1 100644 --- a/mkopcodeh.awk +++ b/mkopcodeh.awk @@ -49,6 +49,7 @@ in1[name] = 0 in2[name] = 0 in3[name] = 0 + out2[name] = 0 out3[name] = 0 for(i=3; ipDbPage) ); assert( pPage->pBt ); @@ -1152,7 +1151,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - assert( nBytepBt->usableSize-8 ); + usableSize = pPage->pBt->usableSize; + assert( nByte < usableSize-8 ); nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); @@ -1175,7 +1175,11 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ int pc, addr; for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size = get2byte(&data[pc+2]); /* Size of free slot */ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ int x = size - nByte; testcase( x==4 ); @@ -1185,6 +1189,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** fragmented bytes within the page. */ memcpy(&data[addr], &data[pc], 2); data[hdr+7] = (u8)(nFrag + x); + }else if( size+pc > usableSize ){ + return SQLITE_CORRUPT_BKPT; }else{ /* The slot remains on the free-list. Reduce its size to account ** for the portion used by the new allocation. */ @@ -1477,7 +1483,9 @@ static void zeroPage(MemPage *pPage, int flags){ assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pBt->mutex) ); - /*memset(&data[hdr], 0, pBt->usableSize - hdr);*/ +#ifdef SQLITE_SECURE_DELETE + memset(&data[hdr], 0, pBt->usableSize - hdr); +#endif data[hdr] = (char)flags; first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); memset(&data[hdr+1], 0, 4); @@ -1606,7 +1614,6 @@ static int getAndInitPage( */ static void releasePage(MemPage *pPage){ if( pPage ){ - assert( pPage->nOverflow==0 || sqlite3PagerPageRefcount(pPage->pDbPage)>1 ); assert( pPage->aData ); assert( pPage->pBt ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); @@ -2346,11 +2353,8 @@ static int newDatabase(BtShared *pBt){ int nPage; assert( sqlite3_mutex_held(pBt->mutex) ); - /* The database size has already been measured and cached, so failure - ** is impossible here. If the original size measurement failed, then - ** processing aborts before entering this routine. */ rc = sqlite3PagerPagecount(pBt->pPager, &nPage); - if( NEVER(rc!=SQLITE_OK) || nPage>0 ){ + if( rc!=SQLITE_OK || nPage>0 ){ return rc; } pP1 = pBt->pPage1; @@ -3280,8 +3284,8 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** root page of a b-tree. If it is not, then the cursor acquired ** will not work correctly. ** -** It is assumed that the sqlite3BtreeCursorSize() bytes of memory -** pointed to by pCur have been zeroed by the caller. +** It is assumed that the sqlite3BtreeCursorZero() has been called +** on pCur to initialize the memory space prior to invoking this routine. */ static int btreeCursor( Btree *p, /* The btree */ @@ -3354,7 +3358,19 @@ int sqlite3BtreeCursor( ** this routine. */ int sqlite3BtreeCursorSize(void){ - return sizeof(BtCursor); + return ROUND8(sizeof(BtCursor)); +} + +/* +** Initialize memory that will be converted into a BtCursor object. +** +** The simple approach here would be to memset() the entire object +** to zero. But it turns out that the apPage[] and aiIdx[] arrays +** do not need to be zeroed and they are large, so we can save a lot +** of run-time by skipping the initialization of those elements. +*/ +void sqlite3BtreeCursorZero(BtCursor *p){ + memset(p, 0, offsetof(BtCursor, iPage)); } /* @@ -5286,8 +5302,13 @@ static void insertCell( assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); - assert( sz==cellSizePtr(pPage, pCell) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + /* The cell should normally be sized correctly. However, when moving a + ** malformed cell from a leaf page to an interior page, if the cell size + ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size + ** might be less than 8 (leaf-size + pointer) on the interior node. Hence + ** the term after the || in the following assert(). */ + assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); @@ -5566,7 +5587,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ u8 * const aTo = pTo->aData; int const iFromHdr = pFrom->hdrOffset; int const iToHdr = ((pTo->pgno==1) ? 100 : 0); - TESTONLY(int rc;) + int rc; int iData; @@ -5580,11 +5601,16 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); /* Reinitialize page pTo so that the contents of the MemPage structure - ** match the new data. The initialization of pTo "cannot" fail, as the - ** data copied from pFrom is known to be valid. */ + ** match the new data. The initialization of pTo can actually fail under + ** fairly obscure circumstances, even though it is a copy of initialized + ** page pFrom. + */ pTo->isInit = 0; - TESTONLY(rc = ) btreeInitPage(pTo); - assert( rc==SQLITE_OK ); + rc = btreeInitPage(pTo); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. @@ -6845,9 +6871,9 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ */ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ - Pgno pgno, /* Page number to clear */ - int freePageFlag, /* Deallocate page if true */ - int *pnChange + Pgno pgno, /* Page number to clear */ + int freePageFlag, /* Deallocate page if true */ + int *pnChange /* Add number of Cells freed to this counter */ ){ MemPage *pPage; int rc; diff --git a/src/btree.h b/src/btree.h index ce44756..7852d4a 100644 --- a/src/btree.h +++ b/src/btree.h @@ -12,8 +12,6 @@ ** This header file defines the interface that the sqlite B-Tree file ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. -** -** @(#) $Id: btree.h,v 1.120 2009/07/22 00:35:24 drh Exp $ */ #ifndef _BTREE_H_ #define _BTREE_H_ @@ -150,6 +148,7 @@ int sqlite3BtreeCursor( BtCursor *pCursor /* Space to write cursor structure */ ); int sqlite3BtreeCursorSize(void); +void sqlite3BtreeCursorZero(BtCursor*); int sqlite3BtreeCloseCursor(BtCursor*); int sqlite3BtreeMovetoUnpacked( diff --git a/src/btreeInt.h b/src/btreeInt.h index c71d9a6..c9fca97 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -9,8 +9,6 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btreeInt.h,v 1.52 2009/07/15 17:25:46 drh Exp $ -** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to ** diff --git a/src/build.c b/src/build.c index 7b0ea04..d8097a7 100644 --- a/src/build.c +++ b/src/build.c @@ -21,8 +21,6 @@ ** BEGIN TRANSACTION ** COMMIT ** ROLLBACK -** -** $Id: build.c,v 1.557 2009/07/24 17:58:53 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/callback.c b/src/callback.c index 80dc693..e6c51bc 100644 --- a/src/callback.c +++ b/src/callback.c @@ -12,8 +12,6 @@ ** ** This file contains functions used to access the internal hash tables ** of user defined functions and collation sequences. -** -** $Id: callback.c,v 1.42 2009/06/17 00:35:31 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/complete.c b/src/complete.c index 8993cfd..0f7b187 100644 --- a/src/complete.c +++ b/src/complete.c @@ -15,8 +15,6 @@ ** This code used to be part of the tokenizer.c source file. But by ** separating it out, the code will be automatically omitted from ** static links that do not use it. -** -** $Id: complete.c,v 1.8 2009/04/28 04:46:42 drh Exp $ */ #include "sqliteInt.h" #ifndef SQLITE_OMIT_COMPLETE @@ -26,8 +24,7 @@ */ #ifndef SQLITE_AMALGAMATION #ifdef SQLITE_ASCII -extern const char sqlite3IsAsciiIdChar[]; -#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20])) +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) #endif #ifdef SQLITE_EBCDIC extern const char sqlite3IsEbcdicIdChar[]; @@ -184,7 +181,9 @@ int sqlite3_complete(const char *zSql){ break; } default: { - int c; +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif if( IdChar((u8)*zSql) ){ /* Keywords and unquoted identifiers */ int nId; diff --git a/src/date.c b/src/date.c index abea60c..a171591 100644 --- a/src/date.c +++ b/src/date.c @@ -16,8 +16,6 @@ ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** $Id: date.c,v 1.107 2009/05/03 20:23:53 drh Exp $ -** ** SQLite processes all times and dates as Julian Day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian diff --git a/src/delete.c b/src/delete.c index 9904d98..5b30888 100644 --- a/src/delete.c +++ b/src/delete.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains C code routines that are called by the parser ** in order to generate code for DELETE FROM statements. -** -** $Id: delete.c,v 1.207 2009/08/08 18:01:08 drh Exp $ */ #include "sqliteInt.h" @@ -498,7 +496,9 @@ void sqlite3GenerateRowDelete( /* TODO: Could use temporary registers here. Also could attempt to ** avoid copying the contents of the rowid register. */ - mask = sqlite3TriggerOldmask(pParse, pTrigger, 0, pTab, onconf); + mask = sqlite3TriggerColmask( + pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + ); mask |= sqlite3FkOldmask(pParse, pTab); iOld = pParse->nMem+1; pParse->nMem += (1 + pTab->nCol); @@ -635,4 +635,3 @@ int sqlite3GenerateIndexKey( sqlite3ReleaseTempRange(pParse, regBase, nCol+1); return regBase; } - diff --git a/src/expr.c b/src/expr.c index 0cc3093..aee2b74 100644 --- a/src/expr.c +++ b/src/expr.c @@ -627,11 +627,10 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ } /* -** Clear an expression structure without deleting the structure itself. -** Substructure is deleted. +** Recursively delete an expression tree. */ -void sqlite3ExprClear(sqlite3 *db, Expr *p){ - assert( p!=0 ); +void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p==0 ) return; if( !ExprHasAnyProperty(p, EP_TokenOnly) ){ sqlite3ExprDelete(db, p->pLeft); sqlite3ExprDelete(db, p->pRight); @@ -644,14 +643,6 @@ void sqlite3ExprClear(sqlite3 *db, Expr *p){ sqlite3ExprListDelete(db, p->x.pList); } } -} - -/* -** Recursively delete an expression tree. -*/ -void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p==0 ) return; - sqlite3ExprClear(db, p); if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFree(db, p); } @@ -1258,6 +1249,94 @@ int sqlite3ExprIsInteger(Expr *p, int *pValue){ return rc; } +/* +** Return FALSE if there is no chance that the expression can be NULL. +** +** If the expression might be NULL or if the expression is too complex +** to tell return TRUE. +** +** This routine is used as an optimization, to skip OP_IsNull opcodes +** when we know that a value cannot be NULL. Hence, a false positive +** (returning TRUE when in fact the expression can never be NULL) might +** be a small performance hit but is otherwise harmless. On the other +** hand, a false negative (returning FALSE when the result could be NULL) +** will likely result in an incorrect answer. So when in doubt, return +** TRUE. +*/ +int sqlite3ExprCanBeNull(const Expr *p){ + u8 op; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + return 0; + default: + return 1; + } +} + +/* +** Generate an OP_IsNull instruction that tests register iReg and jumps +** to location iDest if the value in iReg is NULL. The value in iReg +** was computed by pExpr. If we can look at pExpr at compile-time and +** determine that it can never generate a NULL, then the OP_IsNull operation +** can be omitted. +*/ +void sqlite3ExprCodeIsNullJump( + Vdbe *v, /* The VDBE under construction */ + const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */ + int iReg, /* Test the value in this register for NULL */ + int iDest /* Jump here if the value is null */ +){ + if( sqlite3ExprCanBeNull(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest); + } +} + +/* +** Return TRUE if the given expression is a constant which would be +** unchanged by OP_Affinity with the affinity given in the second +** argument. +** +** This routine is used to determine if the OP_Affinity operation +** can be omitted. When in doubt return FALSE. A false negative +** is harmless. A false positive, however, can result in the wrong +** answer. +*/ +int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ + u8 op; + if( aff==SQLITE_AFF_NONE ) return 1; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: { + return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + } + case TK_FLOAT: { + return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + } + case TK_STRING: { + return aff==SQLITE_AFF_TEXT; + } + case TK_BLOB: { + return 1; + } + case TK_COLUMN: { + assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ + return p->iColumn<0 + && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + } + default: { + return 0; + } + } +} + /* ** Return TRUE if the given string is a row-id column name. */ @@ -1345,16 +1424,16 @@ static int isCandidateForInOpt(Select *p){ ** When the b-tree is being used for membership tests, the calling function ** needs to know whether or not the structure contains an SQL NULL ** value in order to correctly evaluate expressions like "X IN (Y, Z)". -** If there is a chance that the b-tree might contain a NULL value at +** If there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written -** to *prNotFound. If there is no chance that the b-tree contains a +** to *prNotFound. If there is no chance that the (...) contains a ** NULL value, then *prNotFound is left unchanged. ** ** If a register is allocated and its location stored in *prNotFound, then -** its initial value is NULL. If the b-tree does not remain constant -** for the duration of the query (i.e. the SELECT that generates the b-tree +** its initial value is NULL. If the (...) does not remain constant +** for the duration of the query (i.e. the SELECT within the (...) ** is a correlated subquery) then the value of the allocated register is -** reset to NULL each time the b-tree is repopulated. This allows the +** reset to NULL each time the subquery is rerun. This allows the ** caller to use vdbe code equivalent to the following: ** ** if( register==NULL ){ @@ -1546,7 +1625,7 @@ int sqlite3CodeSubselect( affinity = sqlite3ExprAffinity(pLeft); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' - ** expression it is handled the same way. A virtual table is + ** expression it is handled the same way. An ephemeral table is ** filled with single-field index keys representing the results ** from the SELECT or the . ** @@ -1694,6 +1773,128 @@ int sqlite3CodeSubselect( } #endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for an IN expression. +** +** x IN (SELECT ...) +** x IN (value, value, ...) +** +** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) +** is an array of zero or more values. The expression is true if the LHS is +** contained within the RHS. The value of the expression is unknown (NULL) +** if the LHS is NULL or if the LHS is not contained within the RHS and the +** RHS contains one or more NULL values. +** +** This routine generates code will jump to destIfFalse if the LHS is not +** contained within the RHS. If due to NULLs we cannot determine if the LHS +** is contained in the RHS then jump to destIfNull. If the LHS is contained +** within the RHS then fall through. +*/ +static void sqlite3ExprCodeIN( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The IN expression */ + int destIfFalse, /* Jump here if LHS is not contained in the RHS */ + int destIfNull /* Jump here if the results are unknown due to NULLs */ +){ + int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + char affinity; /* Comparison affinity to use */ + int eType; /* Type of the RHS */ + int r1; /* Temporary use register */ + Vdbe *v; /* Statement under construction */ + + /* Compute the RHS. After this step, the table with cursor + ** pExpr->iTable will contains the values that make up the RHS. + */ + v = pParse->pVdbe; + assert( v!=0 ); /* OOM detected prior to this routine */ + VdbeNoopComment((v, "begin IN expr")); + eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); + + /* Figure out the affinity to use to create a key from the results + ** of the expression. affinityStr stores a static string suitable for + ** P4 of OP_MakeRecord. + */ + affinity = comparisonAffinity(pExpr); + + /* Code the LHS, the from " IN (...)". + */ + sqlite3ExprCachePush(pParse); + r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCode(pParse, pExpr->pLeft, r1); + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); + + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree + */ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + }else{ + /* In this case, the RHS is an index b-tree. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + if( rRhsHasNull==0 || destIfFalse==destIfNull ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1, j2, j3; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the presence of NULLs in the RHS does not matter, so jump + ** over all of the code that follows. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + + /* Here we begin generating code that runs if the LHS is not + ** contained within the RHS. Generate additional code that + ** tests the RHS for NULLs. If the RHS contains a NULL then + ** jump to destIfNull. If there are no NULLs in the RHS then + ** jump to destIfFalse. + */ + j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull); + j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); + sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull); + sqlite3VdbeJumpHere(v, j3); + sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1); + sqlite3VdbeJumpHere(v, j2); + + /* Jump to the appropriate target depending on whether or not + ** the RHS contains a NULL + */ + sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + + /* The OP_Found at the top of this branch jumps here when true, + ** causing the overall IN expression evaluation to fall through. + */ + sqlite3VdbeJumpHere(v, j1); + } + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ExprCachePop(pParse, 1); + VdbeComment((v, "end IN expr")); +} +#endif /* SQLITE_OMIT_SUBQUERY */ + /* ** Duplicate an 8-byte value */ @@ -2394,6 +2595,27 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } + + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evalation of + ** arguments past the first non-NULL argument. + */ + if( pDef->flags & SQLITE_FUNC_COALESCE ){ + int endCoalesce = sqlite3VdbeMakeLabel(v); + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + sqlite3ExprCachePop(pParse, 1); + } + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + + if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ @@ -2451,95 +2673,19 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } case TK_IN: { - int rNotFound = 0; - int rMayHaveNull = 0; - int j2, j3, j4, j5; - char affinity; - int eType; - - VdbeNoopComment((v, "begin IN expr r%d", target)); - eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull); - if( rMayHaveNull ){ - rNotFound = ++pParse->nMem; - } - - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); - - - /* Code the from " IN (...)". The temporary table - ** pExpr->iTable contains the values that make up the (...) set. - */ - sqlite3ExprCachePush(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, target); - j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target); - if( eType==IN_INDEX_ROWID ){ - j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target); - j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target); - sqlite3VdbeAddOp2(v, OP_Integer, 1, target); - j5 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, j3); - sqlite3VdbeJumpHere(v, j4); - sqlite3VdbeAddOp2(v, OP_Integer, 0, target); - }else{ - r2 = regFree2 = sqlite3GetTempReg(pParse); - - /* Create a record and test for set membership. If the set contains - ** the value, then jump to the end of the test code. The target - ** register still contains the true (1) value written to it earlier. - */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1); - sqlite3VdbeAddOp2(v, OP_Integer, 1, target); - j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - if( rNotFound==0 ){ - /* This branch runs if it is known at compile time (now) that - ** the set contains no NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. No need - ** to test the data structure at runtime in this case. - */ - sqlite3VdbeAddOp2(v, OP_Integer, 0, target); - }else{ - /* This block populates the rNotFound register with either NULL - ** or 0 (an integer value). If the data structure contains one - ** or more NULLs, then set rNotFound to NULL. Otherwise, set it - ** to 0. If register rMayHaveNull is already set to some value - ** other than NULL, then the test has already been run and - ** rNotFound is already populated. - */ - static const char nullRecord[] = { 0x02, 0x00 }; - j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull); - sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound); - sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0, - nullRecord, P4_STATIC); - j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull); - sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound); - sqlite3VdbeJumpHere(v, j4); - sqlite3VdbeJumpHere(v, j3); - - /* Copy the value of register rNotFound (which is either NULL or 0) - ** into the target register. This will be the result of the - ** expression. - */ - sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target); - } - } - sqlite3VdbeJumpHere(v, j2); - sqlite3VdbeJumpHere(v, j5); - sqlite3ExprCachePop(pParse, 1); - VdbeComment((v, "end IN expr r%d", target)); + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + sqlite3VdbeResolveLabel(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); + sqlite3VdbeResolveLabel(v, destIfNull); break; } -#endif +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* ** x BETWEEN y AND z ** @@ -2970,6 +3116,62 @@ int sqlite3ExprCodeExprList( return n; } +/* +** Generate code for a BETWEEN operator. +** +** x BETWEEN y AND z +** +** The above is equivalent to +** +** x>=y AND x<=z +** +** Code it as such, taking care to do the common subexpression +** elementation of x. +*/ +static void exprCodeBetween( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The BETWEEN expression */ + int dest, /* Jump here if the jump is taken */ + int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +){ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr compLeft; /* The x>=y term */ + Expr compRight; /* The x<=z term */ + Expr exprX; /* The x subexpression */ + int regFree1 = 0; /* Temporary use register */ + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + exprX = *pExpr->pLeft; + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = &exprX; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = &exprX; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); + exprX.op = TK_REGISTER; + if( jumpIfTrue ){ + sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + }else{ + sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + } + sqlite3ReleaseTempReg(pParse, regFree1); + + /* Ensure adequate test coverage */ + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); + testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); +} + /* ** Generate code for a boolean expression such that a jump is made ** to the label "dest" if the expression is true but execution @@ -3069,36 +3271,16 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_BETWEEN: { - /* x BETWEEN y AND z - ** - ** Is equivalent to - ** - ** x>=y AND x<=z - ** - ** Code it as such, taking care to do the common subexpression - ** elementation of x. - */ - Expr exprAnd; - Expr compLeft; - Expr compRight; - Expr exprX; - - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprX = *pExpr->pLeft; - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); - testcase( regFree1==0 ); - exprX.op = TK_REGISTER; testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + break; + } + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfNull = jumpIfNull ? dest : destIfFalse; + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeResolveLabel(v, destIfFalse); break; } default: { @@ -3182,6 +3364,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_NOT: { + testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } @@ -3229,36 +3412,18 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_BETWEEN: { - /* x BETWEEN y AND z - ** - ** Is equivalent to - ** - ** x>=y AND x<=z - ** - ** Code it as such, taking care to do the common subexpression - ** elementation of x. - */ - Expr exprAnd; - Expr compLeft; - Expr compRight; - Expr exprX; - - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprX = *pExpr->pLeft; - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); - testcase( regFree1==0 ); - exprX.op = TK_REGISTER; testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + break; + } + case TK_IN: { + if( jumpIfNull ){ + sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + }else{ + int destIfNull = sqlite3VdbeMakeLabel(v); + sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); + sqlite3VdbeResolveLabel(v, destIfNull); + } break; } default: { diff --git a/src/fault.c b/src/fault.c index b36dd36..c3028c4 100644 --- a/src/fault.c +++ b/src/fault.c @@ -10,10 +10,6 @@ ** ************************************************************************* ** -** $Id: fault.c,v 1.11 2008/09/02 00:52:52 drh Exp $ -*/ - -/* ** This file contains code to support the concept of "benign" ** malloc failures (when the xMalloc() or xRealloc() method of the ** sqlite3_mem_methods structure fails to allocate a block of memory diff --git a/src/fkey.c b/src/fkey.c index df0a5b4..e6f4bfc 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -399,7 +399,7 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec); sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0); - sqlite3VdbeAddOp3(v, OP_Found, iCur, iOk, regRec); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempRange(pParse, regTemp, nCol); diff --git a/src/func.c b/src/func.c index 2766fa3..8cb7a03 100644 --- a/src/func.c +++ b/src/func.c @@ -157,6 +157,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ ** If x is a blob, then we count bytes. ** ** If p1 is negative, then we begin abs(p1) from the end of x[]. +** +** If p2 is negative, return the p2 characters preceeding p1. */ static void substrFunc( sqlite3_context *context, @@ -177,6 +179,7 @@ static void substrFunc( return; } p0type = sqlite3_value_type(argv[0]); + p1 = sqlite3_value_int(argv[1]); if( p0type==SQLITE_BLOB ){ len = sqlite3_value_bytes(argv[0]); z = sqlite3_value_blob(argv[0]); @@ -186,11 +189,12 @@ static void substrFunc( z = sqlite3_value_text(argv[0]); if( z==0 ) return; len = 0; - for(z2=z; *z2; len++){ - SQLITE_SKIP_UTF8(z2); + if( p1<0 ){ + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } } } - p1 = sqlite3_value_int(argv[1]); if( argc==3 ){ p2 = sqlite3_value_int(argv[2]); if( p2<0 ){ @@ -220,10 +224,6 @@ static void substrFunc( } } assert( p1>=0 && p2>=0 ); - if( p1+p2>len ){ - p2 = len-p1; - if( p2<0 ) p2 = 0; - } if( p0type!=SQLITE_BLOB ){ while( *z && p1 ){ SQLITE_SKIP_UTF8(z); @@ -234,6 +234,10 @@ static void substrFunc( } sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); }else{ + if( p1+p2>len ){ + p2 = len-p1; + if( p2<0 ) p2 = 0; + } sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); } } @@ -335,6 +339,14 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } } + +#if 0 /* This function is never used. */ +/* +** The COALESCE() and IFNULL() functions used to be implemented as shown +** here. But now they are implemented as VDBE code so that unused arguments +** do not have to be computed. This legacy implementation is retained as +** comment. +*/ /* ** Implementation of the IFNULL(), NVL(), and COALESCE() functions. ** All three do the same thing. They return the first non-NULL @@ -353,6 +365,8 @@ static void ifnullFunc( } } } +#endif /* NOT USED */ +#define ifnullFunc versionFunc /* Substitute function - never called */ /* ** Implementation of random(). Return a random integer. @@ -1437,10 +1451,12 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), FUNCTION(coalesce, 0, 0, 0, 0 ), +/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ + {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0}, FUNCTION(hex, 1, 0, 0, hexFunc ), - FUNCTION(ifnull, 2, 0, 1, ifnullFunc ), +/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ + {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0}, FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/global.c b/src/global.c index 07b133d..bdfd1ff 100644 --- a/src/global.c +++ b/src/global.c @@ -14,7 +14,6 @@ */ #include "sqliteInt.h" - /* An array to map all upper-case characters into their corresponding ** lower-case character. ** @@ -70,6 +69,7 @@ const unsigned char sqlite3UpperToLower[] = { ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 +** SQLite identifier character 0x40 ** ** Bit 0x20 is set if the mapped character requires translation to upper ** case. i.e. if the character is a lower-case ASCII character. @@ -81,6 +81,11 @@ const unsigned char sqlite3UpperToLower[] = { ** Standard function tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** +** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +** ** SQLite's versions are identical to the standard versions assuming a ** locale of "C". They are implemented as macros in sqliteInt.h. */ @@ -90,7 +95,7 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ + 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ @@ -98,29 +103,29 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ - 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58..5f XYZ[\]^_ */ + 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80..87 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88..8f ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90..97 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98..9f ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a0..a7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a8..af ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0..b7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b8..bf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0..c7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c8..cf ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d0..d7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d8..df ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e0..e7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e8..ef ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* f0..f7 ........ */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* f8..ff ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ }; #endif @@ -188,3 +193,12 @@ SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; ** and dileterious behavior. */ int sqlite3PendingByte = 0x40000000; + +#include "opcodes.h" +/* +** Properties of opcodes. The OPFLG_INITIALIZER macro is +** created by mkopcodeh.awk during compilation. Data is obtained +** from the comments following the "case OP_xxxx:" statements in +** the vdbe.c file. +*/ +const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; diff --git a/src/hash.c b/src/hash.c index 9f1587a..d4daf92 100644 --- a/src/hash.c +++ b/src/hash.c @@ -11,8 +11,6 @@ ************************************************************************* ** This is the implementation of generic hash-tables ** used in SQLite. -** -** $Id: hash.c,v 1.38 2009/05/09 23:29:12 drh Exp $ */ #include "sqliteInt.h" #include diff --git a/src/hash.h b/src/hash.h index 959f509..990a2d6 100644 --- a/src/hash.h +++ b/src/hash.h @@ -11,8 +11,6 @@ ************************************************************************* ** This is the header file for the generic hash-table implemenation ** used in SQLite. -** -** $Id: hash.h,v 1.15 2009/05/02 13:29:38 drh Exp $ */ #ifndef _SQLITE_HASH_H_ #define _SQLITE_HASH_H_ diff --git a/src/hwtime.h b/src/hwtime.h index 896041e..b8bc5a2 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -12,8 +12,6 @@ ** ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. -** -** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $ */ #ifndef _HWTIME_H_ #define _HWTIME_H_ diff --git a/src/insert.c b/src/insert.c index 94b741a..5266af3 100644 --- a/src/insert.c +++ b/src/insert.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. -** -** $Id: insert.c,v 1.270 2009/07/24 17:58:53 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/journal.c b/src/journal.c index 9466e69..2a806e3 100644 --- a/src/journal.c +++ b/src/journal.c @@ -10,12 +10,6 @@ ** ************************************************************************* ** -** @(#) $Id: journal.c,v 1.9 2009/01/20 17:06:27 danielk1977 Exp $ -*/ - -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - -/* ** This file implements a special kind of sqlite3_file object used ** by SQLite to create journal files if the atomic-write optimization ** is enabled. @@ -30,7 +24,7 @@ ** buffer, or ** 2) The sqlite3JournalCreate() function is called. */ - +#ifdef SQLITE_ENABLE_ATOMIC_WRITE #include "sqliteInt.h" diff --git a/src/legacy.c b/src/legacy.c index 7342103..d432d29 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -13,8 +13,6 @@ ** implement the programmer interface to the library. Routines in ** other files are for internal use by SQLite and should not be ** accessed by users of the library. -** -** $Id: legacy.c,v 1.35 2009/08/07 16:56:00 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/loadext.c b/src/loadext.c index 1fa6aa3..8fe35d9 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains code used to dynamically load extensions into ** the SQLite library. -** -** $Id: loadext.c,v 1.60 2009/06/03 01:24:54 drh Exp $ */ #ifndef SQLITE_CORE diff --git a/src/malloc.c b/src/malloc.c index c0c695d..6b25187 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -11,8 +11,6 @@ ************************************************************************* ** ** Memory allocation functions used throughout sqlite. -** -** $Id: malloc.c,v 1.66 2009/07/17 11:44:07 drh Exp $ */ #include "sqliteInt.h" #include diff --git a/src/mem0.c b/src/mem0.c index 0cac07b..0d0b666 100644 --- a/src/mem0.c +++ b/src/mem0.c @@ -15,8 +15,6 @@ ** here always fail. SQLite will not operate with these drivers. These ** are merely placeholders. Real drivers must be substituted using ** sqlite3_config() before SQLite will operate. -** -** $Id: mem0.c,v 1.1 2008/10/28 18:58:20 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/mem1.c b/src/mem1.c index e3b8863..fbfa35e 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -16,8 +16,6 @@ ** ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. -** -** $Id: mem1.c,v 1.30 2009/03/23 04:33:33 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/mem2.c b/src/mem2.c index 8c498a2..965f304 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -18,8 +18,6 @@ ** ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. -** -** $Id: mem2.c,v 1.45 2009/03/23 04:33:33 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/mem3.c b/src/mem3.c index a12a337..e2d6681 100644 --- a/src/mem3.c +++ b/src/mem3.c @@ -22,8 +22,6 @@ ** ** This version of the memory allocation subsystem is included ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined. -** -** $Id: mem3.c,v 1.25 2008/11/19 16:52:44 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/memjournal.c b/src/memjournal.c index df770b4..0d81ecf 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -13,8 +13,6 @@ ** This file contains code use to implement an in-memory rollback journal. ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. -** -** @(#) $Id: memjournal.c,v 1.12 2009/05/04 11:42:30 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/mutex.c b/src/mutex.c index 79637b9..11d498c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -12,9 +12,6 @@ ** This file contains the C functions that implement mutexes. ** ** This file contains code that is common across all mutex implementations. - -** -** $Id: mutex.c,v 1.31 2009/07/16 18:21:18 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/mutex.h b/src/mutex.h index 55575d6..037d487 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -18,8 +18,6 @@ ** NOTE: source files should *not* #include this header file directly. ** Source files should #include the sqliteInt.h file and let that file ** include this one indirectly. -** -** $Id: mutex.h,v 1.9 2008/10/07 15:25:48 drh Exp $ */ diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 7c13244..ee74da1 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -24,8 +24,6 @@ ** If compiled with SQLITE_DEBUG, then additional logic is inserted ** that does error checking on mutexes to make sure they are being ** called correctly. -** -** $Id: mutex_noop.c,v 1.3 2008/12/05 17:17:08 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/mutex_os2.c b/src/mutex_os2.c index 15ffd2c..f16dc21 100644 --- a/src/mutex_os2.c +++ b/src/mutex_os2.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file contains the C functions that implement mutexes for OS/2 -** -** $Id: mutex_os2.c,v 1.11 2008/11/22 19:50:54 pweilbacher Exp $ */ #include "sqliteInt.h" @@ -160,6 +158,39 @@ static void os2MutexFree(sqlite3_mutex *p){ sqlite3_free( p ); } +#ifdef SQLITE_DEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int os2MutexHeld(sqlite3_mutex *p){ + TID tid; + PID pid; + ULONG ulCount; + PTIB ptib; + if( p!=0 ) { + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + } else { + DosGetInfoBlocks(&ptib, NULL); + tid = ptib->tib_ptib2->tib2_ultid; + } + return p==0 || (p->nRef!=0 && p->owner==tid); +} +static int os2MutexNotheld(sqlite3_mutex *p){ + TID tid; + PID pid; + ULONG ulCount; + PTIB ptib; + if( p!= 0 ) { + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + } else { + DosGetInfoBlocks(&ptib, NULL); + tid = ptib->tib_ptib2->tib2_ultid; + } + return p==0 || p->nRef==0 || p->owner!=tid; +} +#endif + /* ** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. If another thread is already within the mutex, @@ -220,39 +251,6 @@ static void os2MutexLeave(sqlite3_mutex *p){ DosReleaseMutexSem(p->mutex); } -#ifdef SQLITE_DEBUG -/* -** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are -** intended for use inside assert() statements. -*/ -static int os2MutexHeld(sqlite3_mutex *p){ - TID tid; - PID pid; - ULONG ulCount; - PTIB ptib; - if( p!=0 ) { - DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); - } else { - DosGetInfoBlocks(&ptib, NULL); - tid = ptib->tib_ptib2->tib2_ultid; - } - return p==0 || (p->nRef!=0 && p->owner==tid); -} -static int os2MutexNotheld(sqlite3_mutex *p){ - TID tid; - PID pid; - ULONG ulCount; - PTIB ptib; - if( p!= 0 ) { - DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); - } else { - DosGetInfoBlocks(&ptib, NULL); - tid = ptib->tib_ptib2->tib2_ultid; - } - return p==0 || p->nRef==0 || p->owner!=tid; -} -#endif - sqlite3_mutex_methods *sqlite3DefaultMutex(void){ static sqlite3_mutex_methods sMutex = { os2MutexInit, diff --git a/src/mutex_unix.c b/src/mutex_unix.c index dc57988..402757f 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file contains the C functions that implement mutexes for pthreads -** -** $Id: mutex_unix.c,v 1.16 2008/12/08 18:19:18 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/mutex_w32.c b/src/mutex_w32.c index d82913b..181f821 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file contains the C functions that implement mutexes for win32 -** -** $Id: mutex_w32.c,v 1.18 2009/08/10 03:23:21 shane Exp $ */ #include "sqliteInt.h" diff --git a/src/notify.c b/src/notify.c index afa7626..6362678 100644 --- a/src/notify.c +++ b/src/notify.c @@ -12,8 +12,6 @@ ** ** This file contains the implementation of the sqlite3_unlock_notify() ** API method and its associated functionality. -** -** $Id: notify.c,v 1.4 2009/04/07 22:06:57 drh Exp $ */ #include "sqliteInt.h" #include "btreeInt.h" diff --git a/src/os.c b/src/os.c index 9420721..598383d 100644 --- a/src/os.c +++ b/src/os.c @@ -12,8 +12,6 @@ ** ** This file contains OS interface code that is common to all ** architectures. -** -** $Id: os.c,v 1.127 2009/07/27 11:41:21 danielk1977 Exp $ */ #define _SQLITE_OS_C_ 1 #include "sqliteInt.h" @@ -140,6 +138,7 @@ int sqlite3OsFullPathname( int nPathOut, char *zPathOut ){ + zPathOut[0] = 0; return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); } #ifndef SQLITE_OMIT_LOAD_EXTENSION diff --git a/src/os.h b/src/os.h index 7bc848f..089901e 100644 --- a/src/os.h +++ b/src/os.h @@ -16,8 +16,6 @@ ** ** This header file is #include-ed by sqliteInt.h and thus ends up ** being included by every source file. -** -** $Id: os.h,v 1.108 2009/02/05 16:31:46 drh Exp $ */ #ifndef _SQLITE_OS_H_ #define _SQLITE_OS_H_ diff --git a/src/os_common.h b/src/os_common.h index 03d1d74..6a2e2d9 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -16,8 +16,6 @@ ** ** This file should be #included by the os_*.c files only. It is not a ** general purpose header file. -** -** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $ */ #ifndef _OS_COMMON_H_ #define _OS_COMMON_H_ diff --git a/src/os_os2.c b/src/os_os2.c index 07e9e6d..572b6a3 100644 --- a/src/os_os2.c +++ b/src/os_os2.c @@ -11,8 +11,6 @@ ****************************************************************************** ** ** This file contains code that is specific to OS/2. -** -** $Id: os_os2.c,v 1.63 2008/12/10 19:26:24 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/os_unix.c b/src/os_unix.c index b31fbab..d7a9d8e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1159,7 +1159,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ #endif unixLeaveMutex(); - OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + OSTRACE4("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved); *pResOut = reserved; return rc; @@ -1292,7 +1292,7 @@ static int unixLock(sqlite3_file *id, int locktype){ int tErrno; assert( pFile ); - OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d\n", pFile->h, + OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, locktypeName(locktype), locktypeName(pFile->locktype), locktypeName(pLock->locktype), pLock->cnt , getpid()); @@ -1301,7 +1301,7 @@ static int unixLock(sqlite3_file *id, int locktype){ ** unixEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ - OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h, + OSTRACE3("LOCK %d %s ok (already held) (unix)\n", pFile->h, locktypeName(locktype)); return SQLITE_OK; } @@ -1471,7 +1471,7 @@ static int unixLock(sqlite3_file *id, int locktype){ end_lock: unixLeaveMutex(); - OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), + OSTRACE4("LOCK %d %s %s (unix)\n", pFile->h, locktypeName(locktype), rc==SQLITE_OK ? "ok" : "failed"); return rc; } @@ -1535,7 +1535,7 @@ static int unixUnlock(sqlite3_file *id, int locktype){ int tErrno; /* Error code from system call errors */ assert( pFile ); - OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d\n", pFile->h, locktype, + OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, locktype, pFile->locktype, pFile->pLock->locktype, pFile->pLock->cnt, getpid()); assert( locktype<=SHARED_LOCK ); @@ -1816,7 +1816,7 @@ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { const char *zLockFile = (const char*)pFile->lockingContext; reserved = access(zLockFile, 0)==0; } - OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + OSTRACE4("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved); *pResOut = reserved; return rc; } @@ -1906,7 +1906,7 @@ static int dotlockUnlock(sqlite3_file *id, int locktype) { char *zLockFile = (char *)pFile->lockingContext; assert( pFile ); - OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + OSTRACE5("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, locktype, pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); @@ -2020,7 +2020,7 @@ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ } } } - OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + OSTRACE4("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ @@ -2087,7 +2087,7 @@ static int flockLock(sqlite3_file *id, int locktype) { /* got it, set the type and return ok */ pFile->locktype = locktype; } - OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), + OSTRACE4("LOCK %d %s %s (flock)\n", pFile->h, locktypeName(locktype), rc==SQLITE_OK ? "ok" : "failed"); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ @@ -2109,7 +2109,7 @@ static int flockUnlock(sqlite3_file *id, int locktype) { unixFile *pFile = (unixFile*)id; assert( pFile ); - OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + OSTRACE5("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, locktype, pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); @@ -2211,7 +2211,7 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { sem_post(pSem); } } - OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + OSTRACE4("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved); *pResOut = reserved; return rc; @@ -2286,7 +2286,7 @@ static int semUnlock(sqlite3_file *id, int locktype) { assert( pFile ); assert( pSem ); - OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + OSTRACE5("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, locktype, pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); @@ -2456,7 +2456,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ } } - OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + OSTRACE4("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved); *pResOut = reserved; return rc; @@ -2492,7 +2492,7 @@ static int afpLock(sqlite3_file *id, int locktype){ afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; assert( pFile ); - OSTRACE5("LOCK %d %s was %s pid=%d\n", pFile->h, + OSTRACE5("LOCK %d %s was %s pid=%d (afp)\n", pFile->h, locktypeName(locktype), locktypeName(pFile->locktype), getpid()); /* If there is already a lock of this type or more restrictive on the @@ -2500,7 +2500,7 @@ static int afpLock(sqlite3_file *id, int locktype){ ** unixEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ - OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h, + OSTRACE3("LOCK %d %s ok (already held) (afp)\n", pFile->h, locktypeName(locktype)); return SQLITE_OK; } @@ -2619,7 +2619,7 @@ static int afpLock(sqlite3_file *id, int locktype){ afp_end_lock: unixLeaveMutex(); - OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), + OSTRACE4("LOCK %d %s %s (afp)\n", pFile->h, locktypeName(locktype), rc==SQLITE_OK ? "ok" : "failed"); return rc; } @@ -2637,7 +2637,7 @@ static int afpUnlock(sqlite3_file *id, int locktype) { afpLockingContext *pCtx = (afpLockingContext *) pFile->lockingContext; assert( pFile ); - OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + OSTRACE5("UNLOCK %d %d was %d pid=%d (afp)\n", pFile->h, locktype, pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); @@ -3113,6 +3113,19 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ((unixFile*)id)->lastErrno = errno; return SQLITE_IOERR_TRUNCATE; }else{ +#ifndef NDEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) and we truncate the file to zero length, + ** that effectively updates the change counter. This might happen + ** when restoring a database using the backup API from a zero-length + ** source. + */ + if( ((unixFile*)id)->inNormalWrite && nByte==0 ){ + ((unixFile*)id)->transCntrChng = 1; + } +#endif + return SQLITE_OK; } } diff --git a/src/pager.c b/src/pager.c index 9ab6262..508119e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -17,8 +17,6 @@ ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. -** -** @(#) $Id: pager.c,v 1.629 2009/08/10 17:48:57 drh Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -913,10 +911,10 @@ static int readJournalHdr( /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power - ** of two greater than or equal to 512, and not greater than their + ** of two greater than or equal to 512 or 32, and not greater than their ** respective compile time maximum limits. */ - if( iPageSize<512 || iSectorSize<512 + if( iPageSize<512 || iSectorSize<32 || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 ){ @@ -1149,6 +1147,7 @@ static void pager_unlock(Pager *pPager){ pPager->changeCountDone = 0; pPager->state = PAGER_UNLOCK; + pPager->dbModified = 0; } } @@ -1423,7 +1422,7 @@ static int pager_playback_one_page( PgHdr *pPg; /* An existing page in the cache */ Pgno pgno; /* The page number of a page in journal */ u32 cksum; /* Checksum used for sanity checking */ - u8 *aData; /* Temporary storage for the page */ + char *aData; /* Temporary storage for the page */ sqlite3_file *jfd; /* The file descriptor for the journal file */ assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ @@ -1431,7 +1430,7 @@ static int pager_playback_one_page( assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ - aData = (u8*)pPager->pTmpSpace; + aData = pPager->pTmpSpace; assert( aData ); /* Temp storage must have already been allocated */ /* Read the page number and page data from the journal or sub-journal @@ -1440,7 +1439,7 @@ static int pager_playback_one_page( jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; rc = read32bits(jfd, *pOffset, &pgno); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4); + rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); if( rc!=SQLITE_OK ) return rc; *pOffset += pPager->pageSize + 4 + isMainJrnl*4; @@ -1459,7 +1458,7 @@ static int pager_playback_one_page( if( isMainJrnl ){ rc = read32bits(jfd, (*pOffset)-4, &cksum); if( rc ) return rc; - if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){ + if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ return SQLITE_DONE; } } @@ -1505,8 +1504,8 @@ static int pager_playback_one_page( pPg = pager_lookup(pPager, pgno); assert( pPg || !MEMDB ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", - PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData), - (isMainJrnl?"main-journal":"sub-journal") + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), + (isMainJrnl?"main-journal":"sub-journal") )); if( (pPager->state>=PAGER_EXCLUSIVE) && (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC)) @@ -1514,14 +1513,14 @@ static int pager_playback_one_page( && !isUnsync ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } if( pPager->pBackup ){ CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); - sqlite3BackupUpdate(pPager->pBackup, pgno, aData); - CODEC1(pPager, aData, pgno, 0, rc=SQLITE_NOMEM); + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to @@ -1556,7 +1555,7 @@ static int pager_playback_one_page( */ void *pData; pData = pPg->pData; - memcpy(pData, aData, pPager->pageSize); + memcpy(pData, (u8*)aData, pPager->pageSize); pPager->xReiniter(pPg); if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ /* If the contents of this page were just restored from the main @@ -1781,8 +1780,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ ** For temporary files the effective sector size is always 512 bytes. ** ** Otherwise, for non-temporary files, the effective sector size is -** the value returned by the xSectorSize() method rounded up to 512 if -** it is less than 512, or rounded down to MAX_SECTOR_SIZE if it +** the value returned by the xSectorSize() method rounded up to 32 if +** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it ** is greater than MAX_SECTOR_SIZE. */ static void setSectorSize(Pager *pPager){ @@ -1795,7 +1794,7 @@ static void setSectorSize(Pager *pPager){ */ pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); } - if( pPager->sectorSize<512 ){ + if( pPager->sectorSize<32 ){ pPager->sectorSize = 512; } if( pPager->sectorSize>MAX_SECTOR_SIZE ){ @@ -2525,8 +2524,11 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ assert( PAGER_RESERVED==RESERVED_LOCK ); assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - /* If the file is currently unlocked then the size must be unknown */ + /* If the file is currently unlocked then the size must be unknown. It + ** must not have been modified at this point. + */ assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); + assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 ); /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler @@ -2873,7 +2875,9 @@ static int pager_write_pagelist(PgHdr *pList){ ** any such pages to the file. ** ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag - ** set (set by sqlite3PagerDontWrite()). + ** set (set by sqlite3PagerDontWrite()). Note that if compiled with + ** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so + ** the second test is always true. */ if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ @@ -3838,7 +3842,7 @@ int sqlite3PagerAcquire( goto pager_acquire_err; } - if( nMax<(int)pgno || MEMDB || noContent ){ + if( MEMDB || nMax<(int)pgno || noContent ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -4382,6 +4386,7 @@ int sqlite3PagerIswriteable(DbPage *pPg){ } #endif +#ifndef SQLITE_SECURE_DELETE /* ** A call to this routine tells the pager that it is not necessary to ** write the information on page pPg back to the disk, even though @@ -4407,6 +4412,7 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ #endif } } +#endif /* !defined(SQLITE_SECURE_DELETE) */ /* ** This routine is called to increment the value of the database file @@ -4446,7 +4452,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ #endif assert( pPager->state>=PAGER_RESERVED ); - if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ u32 change_counter; /* Initial value of change-counter field */ @@ -5054,7 +5060,7 @@ static void sqlite3PagerSetCodec( void *pCodec ){ if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); - pPager->xCodec = xCodec; + pPager->xCodec = pPager->memDb ? 0 : xCodec; pPager->xCodecSizeChng = xCodecSizeChng; pPager->xCodecFree = xCodecFree; pPager->pCodec = pCodec; @@ -5099,6 +5105,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( pPg->nRef>0 ); + /* In order to be able to rollback, an in-memory database must journal + ** the page we are moving from. + */ + if( MEMDB ){ + rc = sqlite3PagerWrite(pPg); + if( rc ) return rc; + } + /* If the page being moved is dirty and has not been saved by the latest ** savepoint, then save the current contents of the page into the ** sub-journal now. This is required to handle the following scenario: @@ -5117,7 +5131,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** one or more savepoint bitvecs. This is the reason this function ** may return SQLITE_NOMEM. */ - if( pPg->flags&PGHDR_DIRTY + if( pPg->flags&PGHDR_DIRTY && subjRequiresPage(pPg) && SQLITE_OK!=(rc = subjournalPage(pPg)) ){ @@ -5152,7 +5166,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - sqlite3PcacheDrop(pPgOld); + if( MEMDB ){ + /* Do not discard pages from an in-memory database since we might + ** need to rollback later. Just move the page out of the way. */ + assert( pPager->dbSizeValid ); + sqlite3PcacheMove(pPgOld, pPager->dbSize+1); + }else{ + sqlite3PcacheDrop(pPgOld); + } } origPgno = pPg->pgno; @@ -5197,18 +5218,12 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* ** For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. We allocate - ** the page now, instead of at rollback, because we can better deal - ** with an out-of-memory error now. Ticket #3761. + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. */ if( MEMDB ){ - DbPage *pNew; - rc = sqlite3PagerAcquire(pPager, origPgno, &pNew, 1); - if( rc!=SQLITE_OK ){ - sqlite3PcacheMove(pPg, origPgno); - return rc; - } - sqlite3PagerUnref(pNew); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); } return SQLITE_OK; diff --git a/src/pager.h b/src/pager.h index b01ec4f..0fe1917 100644 --- a/src/pager.h +++ b/src/pager.h @@ -12,8 +12,6 @@ ** This header file defines the interface that the sqlite page cache ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. -** -** @(#) $Id: pager.h,v 1.104 2009/07/24 19:01:19 drh Exp $ */ #ifndef _PAGER_H_ diff --git a/src/parse.y b/src/parse.y index 41d3948..f1c4216 100644 --- a/src/parse.y +++ b/src/parse.y @@ -13,8 +13,6 @@ ** using the lemon parser generator to generate C code that runs ** the parser. Lemon will also generate a header file containing ** numeric codes for all of the tokens. -** -** @(#) $Id: parse.y,v 1.286 2009/08/10 03:57:58 shane Exp $ */ // All token codes are small integers with #defines that begin with "TK_" @@ -886,6 +884,19 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) escape(E). [LIKE_KW] { expr(A) ::= expr(X) ISNULL|NOTNULL(E). {spanUnaryPostfix(&A,pParse,@E,&X,&E);} expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} +%include { + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( db->mallocFailed==0 && pY->op==TK_NULL ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } +} + // expr1 IS expr2 // expr1 IS NOT expr2 // @@ -894,15 +905,11 @@ expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} // expr(A) ::= expr(X) IS expr(Y). { spanBinaryExpr(&A,pParse,TK_IS,&X,&Y); - if( pParse->db->mallocFailed==0 && Y.pExpr->op==TK_NULL ){ - A.pExpr->op = TK_ISNULL; - } + binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_ISNULL); } expr(A) ::= expr(X) IS NOT expr(Y). { spanBinaryExpr(&A,pParse,TK_ISNOT,&X,&Y); - if( pParse->db->mallocFailed==0 && Y.pExpr->op==TK_NULL ){ - A.pExpr->op = TK_NOTNULL; - } + binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_NOTNULL); } %include { diff --git a/src/parse.y.ex1 b/src/parse.y.ex1 deleted file mode 100644 index c413dbc..0000000 --- a/src/parse.y.ex1 +++ /dev/null @@ -1,1346 +0,0 @@ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains SQLite's grammar for SQL. Process this file -** using the lemon parser generator to generate C code that runs -** the parser. Lemon will also generate a header file containing -** numeric codes for all of the tokens. -** -** @(#) $Id: parse.y,v 1.286 2009/08/10 03:57:58 shane Exp $ -*/ - -// All token codes are small integers with #defines that begin with "TK_" -%token_prefix TK_ - -// The type of the data attached to each token is Token. This is also the -// default type for non-terminals. -// -%token_type {Token} -%default_type {Token} - -// The generated parser function takes a 4th argument as follows: -%extra_argument {Parse *pParse} - -// This code runs whenever there is a syntax error -// -%syntax_error { - UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ - assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - pParse->parseError = 1; -} -%stack_overflow { - UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ - sqlite3ErrorMsg(pParse, "parser stack overflow"); - pParse->parseError = 1; -} - -// The name of the generated procedure that implements the parser -// is as follows: -%name sqlite3Parser - -// The following text is included near the beginning of the C source -// code file that implements the parser. -// -%include { -#include "sqliteInt.h" - -/* -** Disable all error recovery processing in the parser push-down -** automaton. -*/ -#define YYNOERRORRECOVERY 1 - -/* -** Make yytestcase() the same as testcase() -*/ -#define yytestcase(X) testcase(X) - -/* -** An instance of this structure holds information about the -** LIMIT clause of a SELECT statement. -*/ -struct LimitVal { - Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ - Expr *pOffset; /* The OFFSET expression. NULL if there is none */ -}; - -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ -}; - -/* -** An instance of the following structure describes the event of a -** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, -** TK_DELETE, or TK_INSTEAD. If the event is of the form -** -** UPDATE ON (a,b,c) -** -** Then the "b" IdList records the list "a,b,c". -*/ -struct TrigEvent { int a; IdList * b; }; - -/* -** An instance of this structure holds the ATTACH key and the key type. -*/ -struct AttachKey { int type; Token key; }; - -} // end %include - -// Input is a single SQL command -input ::= cmdlist. -cmdlist ::= cmdlist ecmd. -cmdlist ::= ecmd. -ecmd ::= SEMI. -ecmd ::= explain cmdx SEMI. -explain ::= . { sqlite3BeginParse(pParse, 0); } -%ifndef SQLITE_OMIT_EXPLAIN -explain ::= EXPLAIN. { sqlite3BeginParse(pParse, 1); } -explain ::= EXPLAIN QUERY PLAN. { sqlite3BeginParse(pParse, 2); } -%endif SQLITE_OMIT_EXPLAIN -cmdx ::= cmd. { sqlite3FinishCoding(pParse); } - -///////////////////// Begin and end transactions. //////////////////////////// -// - -cmd ::= BEGIN transtype(Y) trans_opt. {sqlite3BeginTransaction(pParse, Y);} -trans_opt ::= . -trans_opt ::= TRANSACTION. -trans_opt ::= TRANSACTION nm. -%type transtype {int} -transtype(A) ::= . {A = TK_DEFERRED;} -transtype(A) ::= DEFERRED(X). {A = @X;} -transtype(A) ::= IMMEDIATE(X). {A = @X;} -transtype(A) ::= EXCLUSIVE(X). {A = @X;} -cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} -cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} -cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} - -savepoint_opt ::= SAVEPOINT. -savepoint_opt ::= . -cmd ::= SAVEPOINT nm(X). { - sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &X); -} -cmd ::= RELEASE savepoint_opt nm(X). { - sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &X); -} -cmd ::= ROLLBACK trans_opt TO savepoint_opt nm(X). { - sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &X); -} - -///////////////////// The CREATE TABLE statement //////////////////////////// -// -cmd ::= create_table create_table_args. -create_table ::= createkw temp(T) TABLE ifnotexists(E) nm(Y) dbnm(Z). { - sqlite3StartTable(pParse,&Y,&Z,T,0,0,E); -} -createkw(A) ::= CREATE(X). { - pParse->db->lookaside.bEnabled = 0; - A = X; -} -%type ifnotexists {int} -ifnotexists(A) ::= . {A = 0;} -ifnotexists(A) ::= IF NOT EXISTS. {A = 1;} -%type temp {int} -%ifndef SQLITE_OMIT_TEMPDB -temp(A) ::= TEMP. {A = 1;} -%endif SQLITE_OMIT_TEMPDB -temp(A) ::= . {A = 0;} -create_table_args ::= LP columnlist conslist_opt(X) RP(Y). { - sqlite3EndTable(pParse,&X,&Y,0); -} -create_table_args ::= AS select(S). { - sqlite3EndTable(pParse,0,0,S); - sqlite3SelectDelete(pParse->db, S); -} -columnlist ::= columnlist COMMA column. -columnlist ::= column. - -// A "column" is a complete description of a single column in a -// CREATE TABLE statement. This includes the column name, its -// datatype, and other keywords such as PRIMARY KEY, UNIQUE, REFERENCES, -// NOT NULL and so forth. -// -column(A) ::= columnid(X) type carglist. { - A.z = X.z; - A.n = (int)(pParse->sLastToken.z-X.z) + pParse->sLastToken.n; -} -columnid(A) ::= nm(X). { - sqlite3AddColumn(pParse,&X); - A = X; -} - - -// An IDENTIFIER can be a generic identifier, or one of several -// keywords. Any non-standard keyword can also be an identifier. -// -%type id {Token} -id(A) ::= ID(X). {A = X;} -id(A) ::= INDEXED(X). {A = X;} - -// The following directive causes tokens ABORT, AFTER, ASC, etc. to -// fallback to ID if they will not parse as their original value. -// This obviates the need for the "id" nonterminal. -// -%fallback ID - ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW - CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR - IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK - SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL -%ifdef SQLITE_OMIT_COMPOUND_SELECT - EXCEPT INTERSECT UNION -%endif SQLITE_OMIT_COMPOUND_SELECT - REINDEX RENAME CTIME_KW IF - . - -// Define operator precedence early so that this is the first occurance -// of the operator tokens in the grammer. Keeping the operators together -// causes them to be assigned integer values that are close together, -// which keeps parser tables smaller. -// -// The token values assigned to these symbols is determined by the order -// in which lemon first sees them. It must be the case that ISNULL/NOTNULL, -// NE/EQ, GT/LE, and GE/LT are separated by only a single value. See -// the sqlite3ExprIfFalse() routine for additional information on this -// constraint. -// -%left OR. -%left AND. -%right NOT. -%left IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. -%left GT LE LT GE. -%right ESCAPE. -%left BITAND BITOR LSHIFT RSHIFT. -%left PLUS MINUS. -%left STAR SLASH REM. -%left CONCAT. -%left COLLATE. -%right BITNOT. - -// And "ids" is an identifer-or-string. -// -%type ids {Token} -ids(A) ::= ID|STRING(X). {A = X;} - -// The name of a column or table can be any of the following: -// -%type nm {Token} -nm(A) ::= id(X). {A = X;} -nm(A) ::= STRING(X). {A = X;} -nm(A) ::= JOIN_KW(X). {A = X;} - -// A typetoken is really one or more tokens that form a type name such -// as can be found after the column name in a CREATE TABLE statement. -// Multiple tokens are concatenated to form the value of the typetoken. -// -%type typetoken {Token} -type ::= . -type ::= typetoken(X). {sqlite3AddColumnType(pParse,&X);} -typetoken(A) ::= typename(X). {A = X;} -typetoken(A) ::= typename(X) LP signed RP(Y). { - A.z = X.z; - A.n = (int)(&Y.z[Y.n] - X.z); -} -typetoken(A) ::= typename(X) LP signed COMMA signed RP(Y). { - A.z = X.z; - A.n = (int)(&Y.z[Y.n] - X.z); -} -%type typename {Token} -typename(A) ::= ids(X). {A = X;} -typename(A) ::= typename(X) ids(Y). {A.z=X.z; A.n=Y.n+(int)(Y.z-X.z);} -signed ::= plus_num. -signed ::= minus_num. - -// "carglist" is a list of additional constraints that come after the -// column name and column type in a CREATE TABLE statement. -// -carglist ::= carglist carg. -carglist ::= . -carg ::= CONSTRAINT nm ccons. -carg ::= ccons. -ccons ::= DEFAULT term(X). {sqlite3AddDefaultValue(pParse,&X);} -ccons ::= DEFAULT LP expr(X) RP. {sqlite3AddDefaultValue(pParse,&X);} -ccons ::= DEFAULT PLUS term(X). {sqlite3AddDefaultValue(pParse,&X);} -ccons ::= DEFAULT MINUS(A) term(X). { - ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, X.pExpr, 0, 0); - v.zStart = A.z; - v.zEnd = X.zEnd; - sqlite3AddDefaultValue(pParse,&v); -} -ccons ::= DEFAULT id(X). { - ExprSpan v; - spanExpr(&v, pParse, TK_STRING, &X); - sqlite3AddDefaultValue(pParse,&v); -} - -// In addition to the type name, we also care about the primary key and -// UNIQUE constraints. -// -ccons ::= NULL onconf. -ccons ::= NOT NULL onconf(R). {sqlite3AddNotNull(pParse, R);} -ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I). - {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} -ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);} -ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X.pExpr);} -ccons ::= REFERENCES nm(T) idxlist_opt(TA) refargs(R). - {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} -ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} -ccons ::= COLLATE ids(C). {sqlite3AddCollateType(pParse, &C);} - -// The optional AUTOINCREMENT keyword -%type autoinc {int} -autoinc(X) ::= . {X = 0;} -autoinc(X) ::= AUTOINCR. {X = 1;} - -// The next group of rules parses the arguments to a REFERENCES clause -// that determine if the referential integrity checking is deferred or -// or immediate and which determine what action to take if a ref-integ -// check fails. -// -%type refargs {int} -refargs(A) ::= . { A = OE_None*0x0101; /* EV: R-19803-45884 */} -refargs(A) ::= refargs(X) refarg(Y). { A = (X & ~Y.mask) | Y.value; } -%type refarg {struct {int value; int mask;}} -refarg(A) ::= MATCH nm. { A.value = 0; A.mask = 0x000000; } -refarg(A) ::= ON DELETE refact(X). { A.value = X; A.mask = 0x0000ff; } -refarg(A) ::= ON UPDATE refact(X). { A.value = X<<8; A.mask = 0x00ff00; } -%type refact {int} -refact(A) ::= SET NULL. { A = OE_SetNull; /* EV: R-33326-45252 */} -refact(A) ::= SET DEFAULT. { A = OE_SetDflt; /* EV: R-33326-45252 */} -refact(A) ::= CASCADE. { A = OE_Cascade; /* EV: R-33326-45252 */} -refact(A) ::= RESTRICT. { A = OE_Restrict; /* EV: R-33326-45252 */} -refact(A) ::= NO ACTION. { A = OE_None; /* EV: R-33326-45252 */} -%type defer_subclause {int} -defer_subclause(A) ::= NOT DEFERRABLE init_deferred_pred_opt. {A = 0;} -defer_subclause(A) ::= DEFERRABLE init_deferred_pred_opt(X). {A = X;} -%type init_deferred_pred_opt {int} -init_deferred_pred_opt(A) ::= . {A = 0;} -init_deferred_pred_opt(A) ::= INITIALLY DEFERRED. {A = 1;} -init_deferred_pred_opt(A) ::= INITIALLY IMMEDIATE. {A = 0;} - -// For the time being, the only constraint we care about is the primary -// key and UNIQUE. Both create indices. -// -conslist_opt(A) ::= . {A.n = 0; A.z = 0;} -conslist_opt(A) ::= COMMA(X) conslist. {A = X;} -conslist ::= conslist COMMA tcons. -conslist ::= conslist tcons. -conslist ::= tcons. -tcons ::= CONSTRAINT nm. -tcons ::= PRIMARY KEY LP idxlist(X) autoinc(I) RP onconf(R). - {sqlite3AddPrimaryKey(pParse,X,R,I,0);} -tcons ::= UNIQUE LP idxlist(X) RP onconf(R). - {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);} -tcons ::= CHECK LP expr(E) RP onconf. - {sqlite3AddCheckConstraint(pParse,E.pExpr);} -tcons ::= FOREIGN KEY LP idxlist(FA) RP - REFERENCES nm(T) idxlist_opt(TA) refargs(R) defer_subclause_opt(D). { - sqlite3CreateForeignKey(pParse, FA, &T, TA, R); - sqlite3DeferForeignKey(pParse, D); -} -%type defer_subclause_opt {int} -defer_subclause_opt(A) ::= . {A = 0;} -defer_subclause_opt(A) ::= defer_subclause(X). {A = X;} - -// The following is a non-standard extension that allows us to declare the -// default behavior when there is a constraint conflict. -// -%type onconf {int} -%type orconf {u8} -%type resolvetype {int} -onconf(A) ::= . {A = OE_Default;} -onconf(A) ::= ON CONFLICT resolvetype(X). {A = X;} -orconf(A) ::= . {A = OE_Default;} -orconf(A) ::= OR resolvetype(X). {A = (u8)X;} -resolvetype(A) ::= raisetype(X). {A = X;} -resolvetype(A) ::= IGNORE. {A = OE_Ignore;} -resolvetype(A) ::= REPLACE. {A = OE_Replace;} - -////////////////////////// The DROP TABLE ///////////////////////////////////// -// -cmd ::= DROP TABLE ifexists(E) fullname(X). { - sqlite3DropTable(pParse, X, 0, E); -} -%type ifexists {int} -ifexists(A) ::= IF EXISTS. {A = 1;} -ifexists(A) ::= . {A = 0;} - -///////////////////// The CREATE VIEW statement ///////////////////////////// -// -%ifndef SQLITE_OMIT_VIEW -cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) AS select(S). { - sqlite3CreateView(pParse, &X, &Y, &Z, S, T, E); -} -cmd ::= DROP VIEW ifexists(E) fullname(X). { - sqlite3DropTable(pParse, X, 1, E); -} -%endif SQLITE_OMIT_VIEW - -//////////////////////// The SELECT statement ///////////////////////////////// -// -cmd ::= select(X). { - SelectDest dest = {SRT_Output, 0, 0, 0, 0}; - sqlite3Select(pParse, X, &dest); - sqlite3SelectDelete(pParse->db, X); -} - -%type select {Select*} -%destructor select {sqlite3SelectDelete(pParse->db, $$);} -%type oneselect {Select*} -%destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} - -select(A) ::= oneselect(X). {A = X;} -%ifndef SQLITE_OMIT_COMPOUND_SELECT -select(A) ::= select(X) multiselect_op(Y) oneselect(Z). { - if( Z ){ - Z->op = (u8)Y; - Z->pPrior = X; - }else{ - sqlite3SelectDelete(pParse->db, X); - } - A = Z; -} -%type multiselect_op {int} -multiselect_op(A) ::= UNION(OP). {A = @OP;} -multiselect_op(A) ::= UNION ALL. {A = TK_ALL;} -multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP;} -%endif SQLITE_OMIT_COMPOUND_SELECT -oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) - groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { - A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); -} - -// The "distinct" nonterminal is true (1) if the DISTINCT keyword is -// present and false (0) if it is not. -// -%type distinct {int} -distinct(A) ::= DISTINCT. {A = 1;} -distinct(A) ::= ALL. {A = 0;} -distinct(A) ::= . {A = 0;} - -// selcollist is a list of expressions that are to become the return -// values of the SELECT statement. The "*" in statements like -// "SELECT * FROM ..." is encoded as a special expression with an -// opcode of TK_ALL. -// -%type selcollist {ExprList*} -%destructor selcollist {sqlite3ExprListDelete(pParse->db, $$);} -%type sclp {ExprList*} -%destructor sclp {sqlite3ExprListDelete(pParse->db, $$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} -sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = sqlite3ExprListAppend(pParse, P, X.pExpr); - if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1); - sqlite3ExprListSetSpan(pParse,A,&X); -} -selcollist(A) ::= sclp(P) STAR. { - Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - A = sqlite3ExprListAppend(pParse, P, p); -} -selcollist(A) ::= sclp(P) nm(X) DOT STAR(Y). { - Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &Y); - Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - A = sqlite3ExprListAppend(pParse,P, pDot); -} - -// An option "AS " phrase that can follow one of the expressions that -// define the result set, or one of the tables in the FROM clause. -// -%type as {Token} -as(X) ::= AS nm(Y). {X = Y;} -as(X) ::= ids(Y). {X = Y;} -as(X) ::= . {X.n = 0;} - - -%type seltablist {SrcList*} -%destructor seltablist {sqlite3SrcListDelete(pParse->db, $$);} -%type stl_prefix {SrcList*} -%destructor stl_prefix {sqlite3SrcListDelete(pParse->db, $$);} -%type from {SrcList*} -%destructor from {sqlite3SrcListDelete(pParse->db, $$);} - -// A complete FROM clause. -// -from(A) ::= . {A = sqlite3DbMallocZero(pParse->db, sizeof(*A));} -from(A) ::= FROM seltablist(X). { - A = X; - sqlite3SrcListShiftJoinType(A); -} - -// "seltablist" is a "Select Table List" - the content of the FROM clause -// in a SELECT statement. "stl_prefix" is a prefix of this list. -// -stl_prefix(A) ::= seltablist(X) joinop(Y). { - A = X; - if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].jointype = (u8)Y; -} -stl_prefix(A) ::= . {A = 0;} -seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) using_opt(U). { - A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); - sqlite3SrcListIndexedBy(pParse, A, &I); -} -%ifndef SQLITE_OMIT_SUBQUERY - seltablist(A) ::= stl_prefix(X) LP select(S) RP - as(Z) on_opt(N) using_opt(U). { - A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,S,N,U); - } - seltablist(A) ::= stl_prefix(X) LP seltablist(F) RP - as(Z) on_opt(N) using_opt(U). { - if( X==0 && Z.n==0 && N==0 && U==0 ){ - A = F; - }else{ - Select *pSubquery; - sqlite3SrcListShiftJoinType(F); - pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); - A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,pSubquery,N,U); - } - } - - // A seltablist_paren nonterminal represents anything in a FROM that - // is contained inside parentheses. This can be either a subquery or - // a grouping of table and subqueries. - // -// %type seltablist_paren {Select*} -// %destructor seltablist_paren {sqlite3SelectDelete(pParse->db, $$);} -// seltablist_paren(A) ::= select(S). {A = S;} -// seltablist_paren(A) ::= seltablist(F). { -// sqlite3SrcListShiftJoinType(F); -// A = sqlite3SelectNew(pParse,0,F,0,0,0,0,0,0,0); -// } -%endif SQLITE_OMIT_SUBQUERY - -%type dbnm {Token} -dbnm(A) ::= . {A.z=0; A.n=0;} -dbnm(A) ::= DOT nm(X). {A = X;} - -%type fullname {SrcList*} -%destructor fullname {sqlite3SrcListDelete(pParse->db, $$);} -fullname(A) ::= nm(X) dbnm(Y). {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y);} - -%type joinop {int} -%type joinop2 {int} -joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } -joinop(X) ::= JOIN_KW(A) JOIN. { X = sqlite3JoinType(pParse,&A,0,0); } -joinop(X) ::= JOIN_KW(A) nm(B) JOIN. { X = sqlite3JoinType(pParse,&A,&B,0); } -joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN. - { X = sqlite3JoinType(pParse,&A,&B,&C); } - -%type on_opt {Expr*} -%destructor on_opt {sqlite3ExprDelete(pParse->db, $$);} -on_opt(N) ::= ON expr(E). {N = E.pExpr;} -on_opt(N) ::= . {N = 0;} - -// Note that this block abuses the Token type just a little. If there is -// no "INDEXED BY" clause, the returned token is empty (z==0 && n==0). If -// there is an INDEXED BY clause, then the token is populated as per normal, -// with z pointing to the token data and n containing the number of bytes -// in the token. -// -// If there is a "NOT INDEXED" clause, then (z==0 && n==1), which is -// normally illegal. The sqlite3SrcListIndexedBy() function -// recognizes and interprets this as a special case. -// -%type indexed_opt {Token} -indexed_opt(A) ::= . {A.z=0; A.n=0;} -indexed_opt(A) ::= INDEXED BY nm(X). {A = X;} -indexed_opt(A) ::= NOT INDEXED. {A.z=0; A.n=1;} - -%type using_opt {IdList*} -%destructor using_opt {sqlite3IdListDelete(pParse->db, $$);} -using_opt(U) ::= USING LP inscollist(L) RP. {U = L;} -using_opt(U) ::= . {U = 0;} - - -%type orderby_opt {ExprList*} -%destructor orderby_opt {sqlite3ExprListDelete(pParse->db, $$);} -%type sortlist {ExprList*} -%destructor sortlist {sqlite3ExprListDelete(pParse->db, $$);} -%type sortitem {Expr*} -%destructor sortitem {sqlite3ExprDelete(pParse->db, $$);} - -orderby_opt(A) ::= . {A = 0;} -orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} -sortlist(A) ::= sortlist(X) COMMA sortitem(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,X,Y); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; -} -sortlist(A) ::= sortitem(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,0,Y); - if( A && ALWAYS(A->a) ) A->a[0].sortOrder = (u8)Z; -} -sortitem(A) ::= expr(X). {A = X.pExpr;} - -%type sortorder {int} - -sortorder(A) ::= ASC. {A = SQLITE_SO_ASC;} -sortorder(A) ::= DESC. {A = SQLITE_SO_DESC;} -sortorder(A) ::= . {A = SQLITE_SO_ASC;} - -%type groupby_opt {ExprList*} -%destructor groupby_opt {sqlite3ExprListDelete(pParse->db, $$);} -groupby_opt(A) ::= . {A = 0;} -groupby_opt(A) ::= GROUP BY nexprlist(X). {A = X;} - -%type having_opt {Expr*} -%destructor having_opt {sqlite3ExprDelete(pParse->db, $$);} -having_opt(A) ::= . {A = 0;} -having_opt(A) ::= HAVING expr(X). {A = X.pExpr;} - -%type limit_opt {struct LimitVal} - -// The destructor for limit_opt will never fire in the current grammar. -// The limit_opt non-terminal only occurs at the end of a single production -// rule for SELECT statements. As soon as the rule that create the -// limit_opt non-terminal reduces, the SELECT statement rule will also -// reduce. So there is never a limit_opt non-terminal on the stack -// except as a transient. So there is never anything to destroy. -// -//%destructor limit_opt { -// sqlite3ExprDelete(pParse->db, $$.pLimit); -// sqlite3ExprDelete(pParse->db, $$.pOffset); -//} -limit_opt(A) ::= . {A.pLimit = 0; A.pOffset = 0;} -limit_opt(A) ::= LIMIT expr(X). {A.pLimit = X.pExpr; A.pOffset = 0;} -limit_opt(A) ::= LIMIT expr(X) OFFSET expr(Y). - {A.pLimit = X.pExpr; A.pOffset = Y.pExpr;} -limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). - {A.pOffset = X.pExpr; A.pLimit = Y.pExpr;} - -/////////////////////////// The DELETE statement ///////////////////////////// -// -%ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) - orderby_opt(O) limit_opt(L). { - sqlite3SrcListIndexedBy(pParse, X, &I); - W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); - sqlite3DeleteFrom(pParse,X,W); -} -%endif -%ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { - sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3DeleteFrom(pParse,X,W); -} -%endif - -%type where_opt {Expr*} -%destructor where_opt {sqlite3ExprDelete(pParse->db, $$);} - -where_opt(A) ::= . {A = 0;} -where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} - -////////////////////////// The UPDATE command //////////////////////////////// -// -%ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) orderby_opt(O) limit_opt(L). { - sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3ExprListCheckLength(pParse,Y,"set list"); - W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); - sqlite3Update(pParse,X,Y,W,R); -} -%endif -%ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { - sqlite3SrcListIndexedBy(pParse, X, &I); - sqlite3ExprListCheckLength(pParse,Y,"set list"); - sqlite3Update(pParse,X,Y,W,R); -} -%endif - -%type setlist {ExprList*} -%destructor setlist {sqlite3ExprListDelete(pParse->db, $$);} - -setlist(A) ::= setlist(Z) COMMA nm(X) EQ expr(Y). { - A = sqlite3ExprListAppend(pParse, Z, Y.pExpr); - sqlite3ExprListSetName(pParse, A, &X, 1); -} -setlist(A) ::= nm(X) EQ expr(Y). { - A = sqlite3ExprListAppend(pParse, 0, Y.pExpr); - sqlite3ExprListSetName(pParse, A, &X, 1); -} - -////////////////////////// The INSERT command ///////////////////////////////// -// -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) - VALUES LP itemlist(Y) RP. - {sqlite3Insert(pParse, X, Y, 0, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). - {sqlite3Insert(pParse, X, 0, S, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. - {sqlite3Insert(pParse, X, 0, 0, F, R);} - -%type insert_cmd {u8} -insert_cmd(A) ::= INSERT orconf(R). {A = R;} -insert_cmd(A) ::= REPLACE. {A = OE_Replace;} - - -%type itemlist {ExprList*} -%destructor itemlist {sqlite3ExprListDelete(pParse->db, $$);} - -itemlist(A) ::= itemlist(X) COMMA expr(Y). - {A = sqlite3ExprListAppend(pParse,X,Y.pExpr);} -itemlist(A) ::= expr(X). - {A = sqlite3ExprListAppend(pParse,0,X.pExpr);} - -%type inscollist_opt {IdList*} -%destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);} -%type inscollist {IdList*} -%destructor inscollist {sqlite3IdListDelete(pParse->db, $$);} - -inscollist_opt(A) ::= . {A = 0;} -inscollist_opt(A) ::= LP inscollist(X) RP. {A = X;} -inscollist(A) ::= inscollist(X) COMMA nm(Y). - {A = sqlite3IdListAppend(pParse->db,X,&Y);} -inscollist(A) ::= nm(Y). - {A = sqlite3IdListAppend(pParse->db,0,&Y);} - -/////////////////////////// Expression Processing ///////////////////////////// -// - -%type expr {ExprSpan} -%destructor expr {sqlite3ExprDelete(pParse->db, $$.pExpr);} -%type term {ExprSpan} -%destructor term {sqlite3ExprDelete(pParse->db, $$.pExpr);} - -%include { - /* This is a utility routine used to set the ExprSpan.zStart and - ** ExprSpan.zEnd values of pOut so that the span covers the complete - ** range of text beginning with pStart and going to the end of pEnd. - */ - static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ - pOut->zStart = pStart->z; - pOut->zEnd = &pEnd->z[pEnd->n]; - } - - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. - */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); - pOut->zStart = pValue->z; - pOut->zEnd = &pValue->z[pValue->n]; - } -} - -expr(A) ::= term(X). {A = X;} -expr(A) ::= LP(B) expr(X) RP(E). {A.pExpr = X.pExpr; spanSet(&A,&B,&E);} -term(A) ::= NULL(X). {spanExpr(&A, pParse, @X, &X);} -expr(A) ::= id(X). {spanExpr(&A, pParse, TK_ID, &X);} -expr(A) ::= JOIN_KW(X). {spanExpr(&A, pParse, TK_ID, &X);} -expr(A) ::= nm(X) DOT nm(Y). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); - A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&A,&X,&Y); -} -expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Z); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&A,&X,&Z); -} -term(A) ::= INTEGER|FLOAT|BLOB(X). {spanExpr(&A, pParse, @X, &X);} -term(A) ::= STRING(X). {spanExpr(&A, pParse, @X, &X);} -expr(A) ::= REGISTER(X). { - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); - A.pExpr = 0; - }else{ - A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); - if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); - } - spanSet(&A, &X, &X); -} -expr(A) ::= VARIABLE(X). { - spanExpr(&A, pParse, TK_VARIABLE, &X); - sqlite3ExprAssignVarNumber(pParse, A.pExpr); - spanSet(&A, &X, &X); -} -expr(A) ::= expr(E) COLLATE ids(C). { - A.pExpr = sqlite3ExprSetColl(pParse, E.pExpr, &C); - A.zStart = E.zStart; - A.zEnd = &C.z[C.n]; -} -%ifndef SQLITE_OMIT_CAST -expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). { - A.pExpr = sqlite3PExpr(pParse, TK_CAST, E.pExpr, 0, &T); - spanSet(&A,&X,&Y); -} -%endif SQLITE_OMIT_CAST -expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). { - if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X); - } - A.pExpr = sqlite3ExprFunction(pParse, Y, &X); - spanSet(&A,&X,&E); - if( D && A.pExpr ){ - A.pExpr->flags |= EP_Distinct; - } -} -expr(A) ::= ID(X) LP STAR RP(E). { - A.pExpr = sqlite3ExprFunction(pParse, 0, &X); - spanSet(&A,&X,&E); -} -term(A) ::= CTIME_KW(OP). { - /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are - ** treated as functions that return constants */ - A.pExpr = sqlite3ExprFunction(pParse, 0,&OP); - if( A.pExpr ){ - A.pExpr->op = TK_CONST_FUNC; - } - spanSet(&A, &OP, &OP); -} - -%include { - /* This routine constructs a binary expression node out of two ExprSpan - ** objects and uses the result to populate a new ExprSpan object. - */ - static void spanBinaryExpr( - ExprSpan *pOut, /* Write the result here */ - Parse *pParse, /* The parsing context. Errors accumulate here */ - int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand */ - ExprSpan *pRight /* The right operand */ - ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pOut->zStart = pLeft->zStart; - pOut->zEnd = pRight->zEnd; - } -} - -expr(A) ::= expr(X) AND(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) OR(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) LT|GT|GE|LE(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) EQ|NE(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) PLUS|MINUS(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -%type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} -%type escape {ExprSpan} -%destructor escape {sqlite3ExprDelete(pParse->db, $$.pExpr);} -escape(X) ::= ESCAPE expr(A). [ESCAPE] {X = A;} -escape(X) ::= . [ESCAPE] {memset(&X,0,sizeof(X));} -expr(A) ::= expr(X) likeop(OP) expr(Y) escape(E). [LIKE_KW] { - ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); - if( E.pExpr ){ - pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); - } - A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); - A.zStart = X.zStart; - A.zEnd = Y.zEnd; - if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; -} - -%include { - /* Construct an expression node for a unary postfix operator - */ - static void spanUnaryPostfix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPostOp /* The operand token for setting the span */ - ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pOperand->zStart; - pOut->zEnd = &pPostOp->z[pPostOp->n]; - } -} - -expr(A) ::= expr(X) ISNULL|NOTNULL(E). {spanUnaryPostfix(&A,pParse,@E,&X,&E);} -expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} - -// expr1 IS expr2 -// expr1 IS NOT expr2 -// -// If expr2 is NULL then code as TK_ISNULL or TK_NOTNULL. If expr2 -// is any other expression, code as TK_IS or TK_ISNOT. -// -expr(A) ::= expr(X) IS expr(Y). { - spanBinaryExpr(&A,pParse,TK_IS,&X,&Y); - if( pParse->db->mallocFailed==0 && Y.pExpr->op==TK_NULL ){ - A.pExpr->op = TK_ISNULL; - } -} -expr(A) ::= expr(X) IS NOT expr(Y). { - spanBinaryExpr(&A,pParse,TK_ISNOT,&X,&Y); - if( pParse->db->mallocFailed==0 && Y.pExpr->op==TK_NULL ){ - A.pExpr->op = TK_NOTNULL; - } -} - -%include { - /* Construct an expression node for a unary prefix operator - */ - static void spanUnaryPrefix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPreOp /* The operand token for setting the span */ - ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pPreOp->z; - pOut->zEnd = pOperand->zEnd; - } -} - - - -expr(A) ::= NOT(B) expr(X). {spanUnaryPrefix(&A,pParse,@B,&X,&B);} -expr(A) ::= BITNOT(B) expr(X). {spanUnaryPrefix(&A,pParse,@B,&X,&B);} -expr(A) ::= MINUS(B) expr(X). [BITNOT] - {spanUnaryPrefix(&A,pParse,TK_UMINUS,&X,&B);} -expr(A) ::= PLUS(B) expr(X). [BITNOT] - {spanUnaryPrefix(&A,pParse,TK_UPLUS,&X,&B);} - -%type between_op {int} -between_op(A) ::= BETWEEN. {A = 0;} -between_op(A) ::= NOT BETWEEN. {A = 1;} -expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { - ExprList *pList = sqlite3ExprListAppend(pParse,0, X.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, Y.pExpr); - A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, W.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pList = pList; - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); - A.zStart = W.zStart; - A.zEnd = Y.zEnd; -} -%ifndef SQLITE_OMIT_SUBQUERY - %type in_op {int} - in_op(A) ::= IN. {A = 0;} - in_op(A) ::= NOT IN. {A = 1;} - expr(A) ::= expr(X) in_op(N) LP exprlist(Y) RP(E). [IN] { - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pList = Y; - sqlite3ExprSetHeight(pParse, A.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, Y); - } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); - A.zStart = X.zStart; - A.zEnd = &E.z[E.n]; - } - expr(A) ::= LP(B) select(X) RP(E). { - A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = X; - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, X); - } - A.zStart = B.z; - A.zEnd = &E.z[E.n]; - } - expr(A) ::= expr(X) in_op(N) LP select(Y) RP(E). [IN] { - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = Y; - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); - A.zStart = X.zStart; - A.zEnd = &E.z[E.n]; - } - expr(A) ::= expr(X) in_op(N) nm(Y) dbnm(Z). [IN] { - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z); - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); - }else{ - sqlite3SrcListDelete(pParse->db, pSrc); - } - if( N ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); - A.zStart = X.zStart; - A.zEnd = Z.z ? &Z.z[Z.n] : &Y.z[Y.n]; - } - expr(A) ::= EXISTS(B) LP select(Y) RP(E). { - Expr *p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - if( p ){ - p->x.pSelect = Y; - ExprSetProperty(p, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, p); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } - A.zStart = B.z; - A.zEnd = &E.z[E.n]; - } -%endif SQLITE_OMIT_SUBQUERY - -/* CASE expressions */ -expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { - A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, Z, 0); - if( A.pExpr ){ - A.pExpr->x.pList = Y; - sqlite3ExprSetHeight(pParse, A.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, Y); - } - A.zStart = C.z; - A.zEnd = &E.z[E.n]; -} -%type case_exprlist {ExprList*} -%destructor case_exprlist {sqlite3ExprListDelete(pParse->db, $$);} -case_exprlist(A) ::= case_exprlist(X) WHEN expr(Y) THEN expr(Z). { - A = sqlite3ExprListAppend(pParse,X, Y.pExpr); - A = sqlite3ExprListAppend(pParse,A, Z.pExpr); -} -case_exprlist(A) ::= WHEN expr(Y) THEN expr(Z). { - A = sqlite3ExprListAppend(pParse,0, Y.pExpr); - A = sqlite3ExprListAppend(pParse,A, Z.pExpr); -} -%type case_else {Expr*} -%destructor case_else {sqlite3ExprDelete(pParse->db, $$);} -case_else(A) ::= ELSE expr(X). {A = X.pExpr;} -case_else(A) ::= . {A = 0;} -%type case_operand {Expr*} -%destructor case_operand {sqlite3ExprDelete(pParse->db, $$);} -case_operand(A) ::= expr(X). {A = X.pExpr;} -case_operand(A) ::= . {A = 0;} - -%type exprlist {ExprList*} -%destructor exprlist {sqlite3ExprListDelete(pParse->db, $$);} -%type nexprlist {ExprList*} -%destructor nexprlist {sqlite3ExprListDelete(pParse->db, $$);} - -exprlist(A) ::= nexprlist(X). {A = X;} -exprlist(A) ::= . {A = 0;} -nexprlist(A) ::= nexprlist(X) COMMA expr(Y). - {A = sqlite3ExprListAppend(pParse,X,Y.pExpr);} -nexprlist(A) ::= expr(Y). - {A = sqlite3ExprListAppend(pParse,0,Y.pExpr);} - - -///////////////////////////// The CREATE INDEX command /////////////////////// -// -cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) - ON nm(Y) LP idxlist(Z) RP(E). { - sqlite3CreateIndex(pParse, &X, &D, - sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U, - &S, &E, SQLITE_SO_ASC, NE); -} - -%type uniqueflag {int} -uniqueflag(A) ::= UNIQUE. {A = OE_Abort;} -uniqueflag(A) ::= . {A = OE_None;} - -%type idxlist {ExprList*} -%destructor idxlist {sqlite3ExprListDelete(pParse->db, $$);} -%type idxlist_opt {ExprList*} -%destructor idxlist_opt {sqlite3ExprListDelete(pParse->db, $$);} - -idxlist_opt(A) ::= . {A = 0;} -idxlist_opt(A) ::= LP idxlist(X) RP. {A = X;} -idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetColl(pParse, p, &C); - } - A = sqlite3ExprListAppend(pParse,X, p); - sqlite3ExprListSetName(pParse,A,&Y,1); - sqlite3ExprListCheckLength(pParse, A, "index"); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; -} -idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { - Expr *p = 0; - if( C.n>0 ){ - p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetColl(pParse, p, &C); - } - A = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, A, &Y, 1); - sqlite3ExprListCheckLength(pParse, A, "index"); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; -} - -%type collate {Token} -collate(C) ::= . {C.z = 0; C.n = 0;} -collate(C) ::= COLLATE ids(X). {C = X;} - - -///////////////////////////// The DROP INDEX command ///////////////////////// -// -cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} - -///////////////////////////// The VACUUM command ///////////////////////////// -// -%ifndef SQLITE_OMIT_VACUUM -%ifndef SQLITE_OMIT_ATTACH -cmd ::= VACUUM. {sqlite3Vacuum(pParse);} -cmd ::= VACUUM nm. {sqlite3Vacuum(pParse);} -%endif SQLITE_OMIT_ATTACH -%endif SQLITE_OMIT_VACUUM - -///////////////////////////// The PRAGMA command ///////////////////////////// -// -%ifndef SQLITE_OMIT_PRAGMA -cmd ::= PRAGMA nm(X) dbnm(Z). {sqlite3Pragma(pParse,&X,&Z,0,0);} -cmd ::= PRAGMA nm(X) dbnm(Z) EQ nmnum(Y). {sqlite3Pragma(pParse,&X,&Z,&Y,0);} -cmd ::= PRAGMA nm(X) dbnm(Z) LP nmnum(Y) RP. {sqlite3Pragma(pParse,&X,&Z,&Y,0);} -cmd ::= PRAGMA nm(X) dbnm(Z) EQ minus_num(Y). - {sqlite3Pragma(pParse,&X,&Z,&Y,1);} -cmd ::= PRAGMA nm(X) dbnm(Z) LP minus_num(Y) RP. - {sqlite3Pragma(pParse,&X,&Z,&Y,1);} - -nmnum(A) ::= plus_num(X). {A = X;} -nmnum(A) ::= nm(X). {A = X;} -nmnum(A) ::= ON(X). {A = X;} -nmnum(A) ::= DELETE(X). {A = X;} -nmnum(A) ::= DEFAULT(X). {A = X;} -%endif SQLITE_OMIT_PRAGMA -plus_num(A) ::= plus_opt number(X). {A = X;} -minus_num(A) ::= MINUS number(X). {A = X;} -number(A) ::= INTEGER|FLOAT(X). {A = X;} -plus_opt ::= PLUS. -plus_opt ::= . - -//////////////////////////// The CREATE TRIGGER command ///////////////////// - -%ifndef SQLITE_OMIT_TRIGGER - -cmd ::= createkw trigger_decl(A) BEGIN trigger_cmd_list(S) END(Z). { - Token all; - all.z = A.z; - all.n = (int)(Z.z - A.z) + Z.n; - sqlite3FinishTrigger(pParse, S, &all); -} - -trigger_decl(A) ::= temp(T) TRIGGER ifnotexists(NOERR) nm(B) dbnm(Z) - trigger_time(C) trigger_event(D) - ON fullname(E) foreach_clause when_clause(G). { - sqlite3BeginTrigger(pParse, &B, &Z, C, D.a, D.b, E, G, T, NOERR); - A = (Z.n==0?B:Z); -} - -%type trigger_time {int} -trigger_time(A) ::= BEFORE. { A = TK_BEFORE; } -trigger_time(A) ::= AFTER. { A = TK_AFTER; } -trigger_time(A) ::= INSTEAD OF. { A = TK_INSTEAD;} -trigger_time(A) ::= . { A = TK_BEFORE; } - -%type trigger_event {struct TrigEvent} -%destructor trigger_event {sqlite3IdListDelete(pParse->db, $$.b);} -trigger_event(A) ::= DELETE|INSERT(OP). {A.a = @OP; A.b = 0;} -trigger_event(A) ::= UPDATE(OP). {A.a = @OP; A.b = 0;} -trigger_event(A) ::= UPDATE OF inscollist(X). {A.a = TK_UPDATE; A.b = X;} - -foreach_clause ::= . -foreach_clause ::= FOR EACH ROW. - -%type when_clause {Expr*} -%destructor when_clause {sqlite3ExprDelete(pParse->db, $$);} -when_clause(A) ::= . { A = 0; } -when_clause(A) ::= WHEN expr(X). { A = X.pExpr; } - -%type trigger_cmd_list {TriggerStep*} -%destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);} -trigger_cmd_list(A) ::= trigger_cmd_list(Y) trigger_cmd(X) SEMI. { - assert( Y!=0 ); - Y->pLast->pNext = X; - Y->pLast = X; - A = Y; -} -trigger_cmd_list(A) ::= trigger_cmd(X) SEMI. { - assert( X!=0 ); - X->pLast = X; - A = X; -} - -// Disallow qualified table names on INSERT, UPDATE, and DELETE statements -// within a trigger. The table to INSERT, UPDATE, or DELETE is always in -// the same database as the table that the trigger fires on. -// -%type trnm {Token} -trnm(A) ::= nm(X). {A = X;} -trnm(A) ::= nm DOT nm(X). { - A = X; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - -// Disallow the INDEX BY and NOT INDEXED clauses on UPDATE and DELETE -// statements within triggers. We make a specific error message for this -// since it is an exception to the default grammar rules. -// -tridxby ::= . -tridxby ::= INDEXED BY nm. { - sqlite3ErrorMsg(pParse, - "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} -tridxby ::= NOT INDEXED. { - sqlite3ErrorMsg(pParse, - "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - - - -%type trigger_cmd {TriggerStep*} -%destructor trigger_cmd {sqlite3DeleteTriggerStep(pParse->db, $$);} -// UPDATE -trigger_cmd(A) ::= - UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z). - { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); } - -// INSERT -trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) VALUES LP itemlist(Y) RP. - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y, 0, R);} - -trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, 0, S, R);} - -// DELETE -trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y). - {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y);} - -// SELECT -trigger_cmd(A) ::= select(X). {A = sqlite3TriggerSelectStep(pParse->db, X); } - -// The special RAISE expression that may occur in trigger programs -expr(A) ::= RAISE(X) LP IGNORE RP(Y). { - A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( A.pExpr ){ - A.pExpr->affinity = OE_Ignore; - } - A.zStart = X.z; - A.zEnd = &Y.z[Y.n]; -} -expr(A) ::= RAISE(X) LP raisetype(T) COMMA nm(Z) RP(Y). { - A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &Z); - if( A.pExpr ) { - A.pExpr->affinity = (char)T; - } - A.zStart = X.z; - A.zEnd = &Y.z[Y.n]; -} -%endif !SQLITE_OMIT_TRIGGER - -%type raisetype {int} -raisetype(A) ::= ROLLBACK. {A = OE_Rollback;} -raisetype(A) ::= ABORT. {A = OE_Abort;} -raisetype(A) ::= FAIL. {A = OE_Fail;} - - -//////////////////////// DROP TRIGGER statement ////////////////////////////// -%ifndef SQLITE_OMIT_TRIGGER -cmd ::= DROP TRIGGER ifexists(NOERR) fullname(X). { - sqlite3DropTrigger(pParse,X,NOERR); -} -%endif !SQLITE_OMIT_TRIGGER - -//////////////////////// ATTACH DATABASE file AS name ///////////////////////// -%ifndef SQLITE_OMIT_ATTACH -cmd ::= ATTACH database_kw_opt expr(F) AS expr(D) key_opt(K). { - sqlite3Attach(pParse, F.pExpr, D.pExpr, K); -} -cmd ::= DETACH database_kw_opt expr(D). { - sqlite3Detach(pParse, D.pExpr); -} - -%type key_opt {Expr*} -%destructor key_opt {sqlite3ExprDelete(pParse->db, $$);} -key_opt(A) ::= . { A = 0; } -key_opt(A) ::= KEY expr(X). { A = X.pExpr; } - -database_kw_opt ::= DATABASE. -database_kw_opt ::= . -%endif SQLITE_OMIT_ATTACH - -////////////////////////// REINDEX collation ////////////////////////////////// -%ifndef SQLITE_OMIT_REINDEX -cmd ::= REINDEX. {sqlite3Reindex(pParse, 0, 0);} -cmd ::= REINDEX nm(X) dbnm(Y). {sqlite3Reindex(pParse, &X, &Y);} -%endif SQLITE_OMIT_REINDEX - -/////////////////////////////////// ANALYZE /////////////////////////////////// -%ifndef SQLITE_OMIT_ANALYZE -cmd ::= ANALYZE. {sqlite3Analyze(pParse, 0, 0);} -cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} -%endif - -//////////////////////// ALTER TABLE table ... //////////////////////////////// -%ifndef SQLITE_OMIT_ALTERTABLE -cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). { - sqlite3AlterRenameTable(pParse,X,&Z); -} -cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column(Y). { - sqlite3AlterFinishAddColumn(pParse, &Y); -} -add_column_fullname ::= fullname(X). { - pParse->db->lookaside.bEnabled = 0; - sqlite3AlterBeginAddColumn(pParse, X); -} -kwcolumn_opt ::= . -kwcolumn_opt ::= COLUMNKW. -%endif SQLITE_OMIT_ALTERTABLE - -//////////////////////// CREATE VIRTUAL TABLE ... ///////////////////////////// -%wildcard ANY. -%ifndef SQLITE_OMIT_VIRTUALTABLE -cmd ::= create_vtab. {sqlite3VtabFinishParse(pParse,0);} -cmd ::= create_vtab LP vtabarglist RP(X). {sqlite3VtabFinishParse(pParse,&X);} -create_vtab ::= createkw VIRTUAL TABLE nm(X) dbnm(Y) USING nm(Z). { - sqlite3VtabBeginParse(pParse, &X, &Y, &Z); -} -vtabarglist ::= vtabarg. -vtabarglist ::= vtabarglist COMMA vtabarg. -vtabarg ::= . {sqlite3VtabArgInit(pParse);} -vtabarg ::= vtabarg vtabargtoken. -vtabargtoken ::= ANY(X). {sqlite3VtabArgExtend(pParse,&X);} -vtabargtoken ::= lp anylist RP(X). {sqlite3VtabArgExtend(pParse,&X);} -lp ::= LP(X). {sqlite3VtabArgExtend(pParse,&X);} -anylist ::= . -anylist ::= anylist LP anylist RP. -anylist ::= anylist ANY. -%endif SQLITE_OMIT_VIRTUALTABLE diff --git a/src/pcache.c b/src/pcache.c index ea5d8a9..bd46e54 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file implements that page cache. -** -** @(#) $Id: pcache.c,v 1.47 2009/07/25 11:46:49 danielk1977 Exp $ */ #include "sqliteInt.h" diff --git a/src/pcache.h b/src/pcache.h index 0894590..33735d2 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -11,8 +11,6 @@ ************************************************************************* ** This header file defines the interface that the sqlite page cache ** subsystem. -** -** @(#) $Id: pcache.h,v 1.20 2009/07/25 11:46:49 danielk1977 Exp $ */ #ifndef _PCACHE_H_ diff --git a/src/pcache1.c b/src/pcache1.c index 036af54..a3bd0fc 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -15,8 +15,6 @@ ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overriden, then neither of ** these two features are available. -** -** @(#) $Id: pcache1.c,v 1.19 2009/07/17 11:44:07 drh Exp $ */ #include "sqliteInt.h" @@ -667,15 +665,7 @@ static void pcache1Rekey( pPage->iKey = iNew; pPage->pNext = pCache->apHash[h]; pCache->apHash[h] = pPage; - - /* The xRekey() interface is only used to move pages earlier in the - ** database file (in order to move all free pages to the end of the - ** file where they can be truncated off.) Hence, it is not possible - ** for the new page number to be greater than the largest previously - ** fetched page. But we retain the following test in case xRekey() - ** begins to be used in different ways in the future. - */ - if( NEVER(iNew>pCache->iMaxKey) ){ + if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } diff --git a/src/pragma.c b/src/pragma.c index 3d779a7..0afce97 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file contains code used to implement the PRAGMA command. -** -** $Id: pragma.c,v 1.214 2009/07/02 07:47:33 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -1130,6 +1128,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2; + int r1; static const VdbeOpList idxErr[] = { { OP_AddImm, 1, -1, 0}, { OP_String8, 0, 3, 0}, /* 1 */ @@ -1143,8 +1142,8 @@ void sqlite3Pragma( { OP_IfPos, 1, 0, 0}, /* 9 */ { OP_Halt, 0, 0, 0}, }; - sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1); - jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0); + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1); addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr); sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC); sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC); diff --git a/src/prepare.c b/src/prepare.c index 5970b6f..0789ca5 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -12,8 +12,6 @@ ** This file contains the implementation of the sqlite3_prepare() ** interface, and routines that contribute to loading the database schema ** from disk. -** -** $Id: prepare.c,v 1.131 2009/08/06 17:43:31 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/printf.c b/src/printf.c index d8d8386..17edb09 100644 --- a/src/printf.c +++ b/src/printf.c @@ -5,8 +5,6 @@ ** an historical reference. Most of the "enhancements" have been backed ** out so that the functionality is now the same as standard printf(). ** -** $Id: printf.c,v 1.104 2009/06/03 01:24:54 drh Exp $ -** ************************************************************************** ** ** The following modules is an enhanced replacement for the "printf" subroutines @@ -647,14 +645,15 @@ void sqlite3VXPrintf( case etSQLESCAPE: case etSQLESCAPE2: case etSQLESCAPE3: { - int i, j, n, isnull; + int i, j, k, n, isnull; int needQuote; char ch; char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg = va_arg(ap,char*); isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); - for(i=n=0; (ch=escarg[i])!=0; i++){ + k = precision; + for(i=n=0; (ch=escarg[i])!=0 && k!=0; i++, k--){ if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; @@ -670,15 +669,17 @@ void sqlite3VXPrintf( } j = 0; if( needQuote ) bufpt[j++] = q; - for(i=0; (ch=escarg[i])!=0; i++){ - bufpt[j++] = ch; + k = i; + for(i=0; i=0 && precision=0 && precision @@ -89,7 +87,13 @@ static void resolveAlias( pDup->pColl = pExpr->pColl; pDup->flags |= EP_ExpCollate; } - sqlite3ExprClear(db, pExpr); + + /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This + ** prevents ExprDelete() from deleting the Expr structure itself, + ** allowing it to be repopulated by the memcpy() on the following line. + */ + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(db, pExpr); memcpy(pExpr, pDup, sizeof(*pExpr)); sqlite3DbFree(db, pDup); } @@ -260,6 +264,10 @@ static int lookupName( testcase( iCol==31 ); testcase( iCol==32 ); pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iColumn = (i16)iCol; pExpr->pTab = pTab; diff --git a/src/rowset.c b/src/rowset.c index 8d5bd5c..d84bb93 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -59,8 +59,6 @@ ** ** There is an added cost of O(N) when switching between TEST and ** SMALLEST primitives. -** -** $Id: rowset.c,v 1.7 2009/05/22 01:00:13 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/select.c b/src/select.c index 0f6a2b2..7555a39 100644 --- a/src/select.c +++ b/src/select.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. -** -** $Id: select.c,v 1.526 2009/08/01 15:09:58 drh Exp $ */ #include "sqliteInt.h" @@ -441,8 +439,8 @@ static void codeDistinct( v = pParse->pVdbe; r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); - sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); sqlite3ReleaseTempReg(pParse, r1); } @@ -683,8 +681,7 @@ static void selectInnerLoop( if( p->iLimit ){ assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to ** pushOntoSorter() would have cleared p->iLimit */ - sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); - sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak); + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } } @@ -1310,7 +1307,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1; + int addr1, n; if( p->iLimit ) return; /* @@ -1325,10 +1322,18 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ - sqlite3ExprCode(pParse, p->pLimit, iLimit); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); - VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); + if( sqlite3ExprIsInteger(p->pLimit, &n) ){ + sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); + VdbeComment((v, "LIMIT counter")); + if( n==0 ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + } + }else{ + sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); + } if( p->pOffset ){ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ @@ -1664,7 +1669,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); - sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, 0, -1, &dest, iCont, iBreak); @@ -1883,8 +1888,7 @@ static int generateOutputSubroutine( /* Jump to the end of the loop if the LIMIT is reached. */ if( p->iLimit ){ - sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); - sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak); + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } /* Generate the subroutine return diff --git a/src/shell.c b/src/shell.c index b454581..973fc5a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains code to implement the "sqlite" command line ** utility for accessing SQLite databases. -** -** $Id: shell.c,v 1.210 2009/05/31 17:16:10 drh Exp $ */ #if defined(_WIN32) || defined(WIN32) /* This needs to come before any includes for MSVC compiler */ @@ -2140,7 +2138,8 @@ static char zHelp[] = " LIKE pattern TABLE.\n" ".echo ON|OFF Turn command echo on or off\n" ".exit Exit this program\n" - ".explain ON|OFF Turn output mode suitable for EXPLAIN on or off.\n" + ".explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off.\n" + " With no args, it turns EXPLAIN on.\n" #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_SUBQUERY) ".genfkey ?OPTIONS? Options are:\n" " --no-drop: Do not drop old fkey triggers.\n" @@ -2186,7 +2185,7 @@ static char zHelp[] = " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" ".timeout MS Try opening locked tables for MS milliseconds\n" - ".width NUM NUM ... Set column widths for \"column\" mode\n" + ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" ; static char zTimerHelp[] = @@ -2313,7 +2312,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; - if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 && nArg>1 ){ + if( c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0 && nArg>1 && nArg<4){ const char *zDestFile; const char *zDb; sqlite3 *pDest; @@ -2349,11 +2348,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ sqlite3_close(pDest); }else - if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 && nArg>1 ){ + if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 && nArg>1 && nArg<3 ){ bail_on_error = booleanValue(azArg[1]); }else - if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 && nArg==1 ){ struct callback_data data; char *zErrMsg = 0; open_db(p); @@ -2372,7 +2371,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else - if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + if( c=='d' && strncmp(azArg[0], "dump", n)==0 && nArg<3 ){ char *zErrMsg = 0; open_db(p); /* When playing back a "dump", the content might appear in an order @@ -2425,15 +2424,15 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else - if( c=='e' && strncmp(azArg[0], "echo", n)==0 && nArg>1 ){ + if( c=='e' && strncmp(azArg[0], "echo", n)==0 && nArg>1 && nArg<3 ){ p->echoOn = booleanValue(azArg[1]); }else - if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + if( c=='e' && strncmp(azArg[0], "exit", n)==0 && nArg==1 ){ rc = 2; }else - if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + if( c=='e' && strncmp(azArg[0], "explain", n)==0 && nArg<3 ){ int val = nArg>=2 ? booleanValue(azArg[1]) : 1; if(val == 1) { if(!p->explainPrev.valid) { @@ -2480,7 +2479,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ #endif if( c=='h' && (strncmp(azArg[0], "header", n)==0 || - strncmp(azArg[0], "headers", n)==0 )&& nArg>1 ){ + strncmp(azArg[0], "headers", n)==0) && nArg>1 && nArg<3 ){ p->showHeader = booleanValue(azArg[1]); }else @@ -2491,7 +2490,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else - if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg>=3 ){ + if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg==3 ){ char *zTable = azArg[2]; /* Insert data into this table */ char *zFile = azArg[1]; /* The file from which to extract data */ sqlite3_stmt *pStmt = NULL; /* A statement */ @@ -2608,7 +2607,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ sqlite3_exec(p->db, zCommit, 0, 0, 0); }else - if( c=='i' && strncmp(azArg[0], "indices", n)==0 ){ + if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg<3 ){ struct callback_data data; char *zErrMsg = 0; open_db(p); @@ -2687,35 +2686,31 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else #endif - if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg>=2 ){ + if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==2 ){ int n2 = strlen30(azArg[1]); - if( strncmp(azArg[1],"line",n2)==0 + if( (n2==4 && strncmp(azArg[1],"line",n2)==0) || - strncmp(azArg[1],"lines",n2)==0 ){ + (n2==5 && strncmp(azArg[1],"lines",n2)==0) ){ p->mode = MODE_Line; - }else if( strncmp(azArg[1],"column",n2)==0 + }else if( (n2==6 && strncmp(azArg[1],"column",n2)==0) || - strncmp(azArg[1],"columns",n2)==0 ){ + (n2==7 && strncmp(azArg[1],"columns",n2)==0) ){ p->mode = MODE_Column; - }else if( strncmp(azArg[1],"list",n2)==0 ){ + }else if( n2==4 && strncmp(azArg[1],"list",n2)==0 ){ p->mode = MODE_List; - }else if( strncmp(azArg[1],"html",n2)==0 ){ + }else if( n2==4 && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; - }else if( strncmp(azArg[1],"tcl",n2)==0 ){ + }else if( n2==3 && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; - }else if( strncmp(azArg[1],"csv",n2)==0 ){ + }else if( n2==3 && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->separator), p->separator, ","); - }else if( strncmp(azArg[1],"tabs",n2)==0 ){ + }else if( n2==4 && strncmp(azArg[1],"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->separator), p->separator, "\t"); - }else if( strncmp(azArg[1],"insert",n2)==0 ){ + }else if( n2==6 && strncmp(azArg[1],"insert",n2)==0 ){ p->mode = MODE_Insert; - if( nArg>=3 ){ - set_table_name(p, azArg[2]); - }else{ - set_table_name(p, "table"); - } + set_table_name(p, "table"); }else { fprintf(stderr,"Error: mode should be one of: " "column csv html insert line list tabs tcl\n"); @@ -2723,6 +2718,18 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==3 ){ + int n2 = strlen30(azArg[1]); + if( n2==6 && strncmp(azArg[1],"insert",n2)==0 ){ + p->mode = MODE_Insert; + set_table_name(p, azArg[2]); + }else { + fprintf(stderr, "Error: invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[2]); + rc = 1; + } + }else + if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 && nArg==2 ) { sqlite3_snprintf(sizeof(p->nullvalue), p->nullvalue, "%.*s", (int)ArraySize(p->nullvalue)-1, azArg[1]); @@ -2756,7 +2763,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else - if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ + if( c=='q' && strncmp(azArg[0], "quit", n)==0 && nArg==1 ){ rc = 2; }else @@ -2771,7 +2778,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else - if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 && nArg>1 ){ + if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 && nArg>1 && nArg<4){ const char *zSrcFile; const char *zDb; sqlite3 *pSrc; @@ -2818,7 +2825,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ sqlite3_close(pSrc); }else - if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + if( c=='s' && strncmp(azArg[0], "schema", n)==0 && nArg<3 ){ struct callback_data data; char *zErrMsg = 0; open_db(p); @@ -2896,7 +2903,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ "%.*s", (int)sizeof(p->separator)-1, azArg[1]); }else - if( c=='s' && strncmp(azArg[0], "show", n)==0){ + if( c=='s' && strncmp(azArg[0], "show", n)==0 && nArg==1 ){ int i; fprintf(p->out,"%9.9s: %s\n","echo", p->echoOn ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","explain", p->explainPrev.valid ? "on" :"off"); @@ -2917,7 +2924,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ fprintf(p->out,"\n"); }else - if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 ){ + if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 && nArg<3 ){ char **azResult; int nRow; char *zErrMsg; @@ -2975,12 +2982,16 @@ static int do_meta_command(char *zLine, struct callback_data *p){ sqlite3_free_table(azResult); }else - if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 && nArg>=2 ){ + if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 && nArg==2 ){ open_db(p); sqlite3_busy_timeout(p->db, atoi(azArg[1])); - }else if( HAS_TIMER && c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 && nArg>1 ){ + }else + + if( HAS_TIMER && c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 && nArg==2 ){ enableTimer = booleanValue(azArg[1]); - }else if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + }else + + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); for(j=1; jcolWidth); j++){ @@ -3283,6 +3294,7 @@ static int process_sqliterc( ** Show available command line options */ static const char zOptions[] = + " -help show this message\n" " -init filename read/process named file\n" " -echo print commands before execution\n" " -[no]header turn headers on or off\n" @@ -3361,7 +3373,6 @@ int main(int argc, char **argv){ ** we do the actual processing of arguments later in a second pass. */ }else if( strcmp(argv[i],"-batch")==0 ){ - i++; stdin_is_interactive = 0; } } @@ -3381,6 +3392,11 @@ int main(int argc, char **argv){ if( i=argc){ + fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"Use -help for a list of options.\n"); + return 1; + } sqlite3_snprintf(sizeof(data.separator), data.separator, "%.*s",(int)sizeof(data.separator)-1,argv[i]); }else if( strcmp(z,"-nullvalue")==0 ){ i++; + if(i>=argc){ + fprintf(stderr,"%s: Error: missing argument for option: %s\n", Argv0, z); + fprintf(stderr,"Use -help for a list of options.\n"); + return 1; + } sqlite3_snprintf(sizeof(data.nullvalue), data.nullvalue, "%.*s",(int)sizeof(data.nullvalue)-1,argv[i]); }else if( strcmp(z,"-header")==0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 967986e..2f21c1e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -820,7 +820,7 @@ struct sqlite3_vfs { ** The sqlite3_initialize() routine initializes the ** SQLite library. The sqlite3_shutdown() routine ** deallocates any resources that were allocated by sqlite3_initialize(). -** This routines are designed to aid in process initialization and +** These routines are designed to aid in process initialization and ** shutdown on embedded systems. Workstation applications using ** SQLite normally do not need to invoke either of these routines. ** @@ -1290,6 +1290,9 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** For the purposes of this routine, an [INSERT] is considered to ** be successful even if it is subsequently rolled back. ** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** ** Requirements: ** [H12221] [H12223] ** @@ -1347,8 +1350,8 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); ** However, the number returned does not include changes ** caused by subtriggers since those have their own context. ** -** See also the [sqlite3_total_changes()] interface and the -** [count_changes pragma]. +** See also the [sqlite3_total_changes()] interface, the +** [count_changes pragma], and the [changes() SQL function]. ** ** Requirements: ** [H12241] [H12243] @@ -1375,8 +1378,8 @@ int sqlite3_changes(sqlite3*); ** completed (when the statement handle is passed to [sqlite3_reset()] or ** [sqlite3_finalize()]). ** -** See also the [sqlite3_changes()] interface and the -** [count_changes pragma]. +** See also the [sqlite3_changes()] interface, the +** [count_changes pragma], and the [total_changes() SQL function]. ** ** Requirements: ** [H12261] [H12263] @@ -2390,7 +2393,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** In the "v2" interfaces, the prepared statement ** that is returned (the [sqlite3_stmt] object) contains a copy of the ** original SQL text. This causes the [sqlite3_step()] interface to -** behave a differently in three ways: +** behave differently in three ways: ** **
    **
  1. @@ -4137,6 +4140,8 @@ int sqlite3_table_column_metadata( ** {H12606} Extension loading must be enabled using ** [sqlite3_enable_load_extension()] prior to calling this API, ** otherwise an error will be returned. +** +** See also the [load_extension() SQL function]. */ int sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 5526646..0d37bbe 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -14,8 +14,6 @@ ** an SQLite instance. Shared libraries that intend to be loaded ** as extensions by SQLite should #include this file instead of ** sqlite3.h. -** -** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $ */ #ifndef _SQLITE3EXT_H_ #define _SQLITE3EXT_H_ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b47eba4..1202971 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -484,9 +484,19 @@ extern const int sqlite3one; #define ROUNDDOWN8(x) ((x)&~7) /* -** Assert that the pointer X is aligned to an 8-byte boundary. +** Assert that the pointer X is aligned to an 8-byte boundary. This +** macro is used only within assert() to verify that the code gets +** all alignment restrictions correct. +** +** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the +** underlying malloc() implemention might return us 4-byte aligned +** pointers. In that case, only verify 4-byte alignment. */ -#define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +#endif /* @@ -959,6 +969,7 @@ struct FuncDef { #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ #define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */ #define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1508,7 +1519,7 @@ struct AggInfo { ** the option is available (at compile-time). */ #if SQLITE_MAX_VARIABLE_NUMBER<=32767 -typedef i64 ynVar; +typedef i16 ynVar; #else typedef int ynVar; #endif @@ -2058,15 +2069,16 @@ struct AutoincInfo { ** The Parse.pTriggerPrg list never contains two entries with the same ** values for both pTrigger and orconf. ** -** The TriggerPrg.oldmask variable is set to a mask of old.* columns +** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns ** accessed (or set to 0 for triggers fired as a result of INSERT -** statements). +** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to +** a mask of new.* columns used by the program. */ struct TriggerPrg { Trigger *pTrigger; /* Trigger this program was coded from */ int orconf; /* Default ON CONFLICT policy */ SubProgram *pProgram; /* Program implementing pTrigger/orconf */ - u32 oldmask; /* Mask of old.* columns accessed */ + u32 aColmask[2]; /* Masks of old.*, new.* columns accessed */ TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ }; @@ -2138,6 +2150,7 @@ struct Parse { Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ u8 disableTriggers; /* True to disable triggers */ @@ -2516,6 +2529,9 @@ void sqlite3StatusSet(int, int); int sqlite3IsNaN(double); void sqlite3VXPrintf(StrAccum*, int, const char*, va_list); +#ifndef SQLITE_OMIT_TRACE +void sqlite3XPrintf(StrAccum*, const char*, ...); +#endif char *sqlite3MPrintf(sqlite3*,const char*, ...); char *sqlite3VMPrintf(sqlite3*,const char*, va_list); char *sqlite3MAppendf(sqlite3*,char*,const char*,...); @@ -2543,7 +2559,6 @@ Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*); Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); void sqlite3ExprAssignVarNumber(Parse*, Expr*); -void sqlite3ExprClear(sqlite3*, Expr*); void sqlite3ExprDelete(sqlite3*, Expr*); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); @@ -2673,6 +2688,9 @@ int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*); int sqlite3ExprIsInteger(Expr*, int*); +int sqlite3ExprCanBeNull(const Expr*); +void sqlite3ExprCodeIsNullJump(Vdbe*, const Expr*, int, int); +int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); void sqlite3GenerateRowDelete(Parse*, Table*, int, int, int, Trigger *, int); void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int*); @@ -2730,7 +2748,7 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, int); TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); - u32 sqlite3TriggerOldmask(Parse*,Trigger*,ExprList*,Table*,int); + u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 @@ -2741,7 +2759,7 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, int); # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p -# define sqlite3TriggerOldmask(A,B,C,D,E) 0 +# define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); @@ -2844,6 +2862,7 @@ char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *); int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION +extern const unsigned char sqlite3OpcodeProperty[]; extern const unsigned char sqlite3UpperToLower[]; extern const unsigned char sqlite3CtypeMap[]; extern SQLITE_WSD struct Sqlite3Config sqlite3Config; @@ -2954,6 +2973,7 @@ int sqlite3VtabCallDestroy(sqlite3*, int, const char *); int sqlite3VtabBegin(sqlite3 *, VTable *); FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**); +int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 77d7858..faf80d4 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -11,8 +11,6 @@ ************************************************************************* ** ** This file defines various limits of what SQLite can process. -** -** @(#) $Id: sqliteLimit.h,v 1.10 2009/01/10 16:15:09 danielk1977 Exp $ */ /* @@ -197,9 +195,5 @@ ** may be executed. */ #ifndef SQLITE_MAX_TRIGGER_DEPTH -#if defined(SQLITE_SMALL_STACK) -# define SQLITE_MAX_TRIGGER_DEPTH 10 -#else # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif -#endif diff --git a/src/status.c b/src/status.c index f1a54df..58a7e68 100644 --- a/src/status.c +++ b/src/status.c @@ -12,8 +12,6 @@ ** ** This module implements the sqlite3_status() interface and related ** functionality. -** -** $Id: status.c,v 1.9 2008/09/02 00:52:52 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/table.c b/src/table.c index cf0a3ee..26bbfb4 100644 --- a/src/table.c +++ b/src/table.c @@ -15,8 +15,6 @@ ** ** These routines are in a separate files so that they will not be linked ** if they are not used. -** -** $Id: table.c,v 1.40 2009/04/10 14:28:00 drh Exp $ */ #include "sqliteInt.h" #include diff --git a/src/tclsqlite.c b/src/tclsqlite.c index ca433ac..7048e4e 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -33,10 +33,11 @@ ** appended to the amalgamation. */ #ifndef SQLITE_AMALGAMATION -# include "sqliteInt.h" +# include "sqlite3.h" # include # include # include + typedef unsigned char u8; #endif #include @@ -3447,16 +3448,6 @@ static char zMainloop[] = ; #endif -/* -** If the macro TCLSH is two, then get the main loop code out of -** the separate file "spaceanal_tcl.h". -*/ -#if TCLSH==2 -static char zMainloop[] = -#include "spaceanal_tcl.h" -; -#endif - #define TCLSH_MAIN main /* Needed to fake out mktclapp */ int TCLSH_MAIN(int argc, char **argv){ Tcl_Interp *interp; @@ -3498,6 +3489,7 @@ int TCLSH_MAIN(int argc, char **argv){ extern int SqlitetestOnefile_Init(); extern int SqlitetestOsinst_Init(Tcl_Interp*); extern int Sqlitetestbackup_Init(Tcl_Interp*); + extern int Sqlitetestintarray_Init(Tcl_Interp*); Sqliteconfig_Init(interp); Sqlitetest1_Init(interp); @@ -3522,13 +3514,14 @@ int TCLSH_MAIN(int argc, char **argv){ SqlitetestOnefile_Init(interp); SqlitetestOsinst_Init(interp); Sqlitetestbackup_Init(interp); + Sqlitetestintarray_Init(interp); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); #endif } #endif - if( argc>=2 || TCLSH==2 ){ + if( argc>=2 ){ int i; char zArgc[32]; sqlite3_snprintf(sizeof(zArgc), zArgc, "%d", argc-(3-TCLSH)); @@ -3539,14 +3532,14 @@ int TCLSH_MAIN(int argc, char **argv){ Tcl_SetVar(interp, "argv", argv[i], TCL_GLOBAL_ONLY | TCL_LIST_ELEMENT | TCL_APPEND_VALUE); } - if( TCLSH==1 && Tcl_EvalFile(interp, argv[1])!=TCL_OK ){ + if( Tcl_EvalFile(interp, argv[1])!=TCL_OK ){ const char *zInfo = Tcl_GetVar(interp, "errorInfo", TCL_GLOBAL_ONLY); if( zInfo==0 ) zInfo = Tcl_GetStringResult(interp); fprintf(stderr,"%s: %s\n", *argv, zInfo); return 1; } } - if( argc<=1 || TCLSH==2 ){ + if( argc<=1 ){ Tcl_GlobalEval(interp, zMainloop); } return 0; diff --git a/src/test1.c b/src/test1.c index 6ea48e6..5a2505a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -12,8 +12,6 @@ ** Code for testing all sorts of SQLite interfaces. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test1.c,v 1.354 2009/08/10 04:37:50 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test2.c b/src/test2.c index 9aaa442..13d36e7 100644 --- a/src/test2.c +++ b/src/test2.c @@ -12,8 +12,6 @@ ** Code for testing the pager.c module in SQLite. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test2.c,v 1.74 2009/07/24 19:01:20 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test3.c b/src/test3.c index f630275..06222f5 100644 --- a/src/test3.c +++ b/src/test3.c @@ -12,8 +12,6 @@ ** Code for testing the btree.c module in SQLite. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test3.c,v 1.111 2009/07/09 05:07:38 danielk1977 Exp $ */ #include "sqliteInt.h" #include "btreeInt.h" diff --git a/src/test4.c b/src/test4.c index 654b572..cb5087b 100644 --- a/src/test4.c +++ b/src/test4.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** Code for testing the the SQLite library in a multithreaded environment. -** -** $Id: test4.c,v 1.24 2008/10/12 00:27:54 shane Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test5.c b/src/test5.c index 369580b..dc7dafe 100644 --- a/src/test5.c +++ b/src/test5.c @@ -14,8 +14,6 @@ ** testing of the SQLite library. Specifically, the code in this file ** is used for testing the SQLite routines for converting between ** the various supported unicode encodings. -** -** $Id: test5.c,v 1.22 2008/08/12 15:04:59 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" diff --git a/src/test6.c b/src/test6.c index ab3654d..b6fa3aa 100644 --- a/src/test6.c +++ b/src/test6.c @@ -13,8 +13,6 @@ ** This file contains code that modified the OS layer in order to simulate ** the effect on the database file of an OS crash or power failure. This ** is used to test the ability of SQLite to recover from those situations. -** -** $Id: test6.c,v 1.43 2009/02/11 14:27:04 danielk1977 Exp $ */ #if SQLITE_TEST /* This file is used for testing only */ #include "sqliteInt.h" diff --git a/src/test7.c b/src/test7.c index 6eebc7c..77b3661 100644 --- a/src/test7.c +++ b/src/test7.c @@ -11,8 +11,6 @@ ************************************************************************* ** Code for testing the client/server version of the SQLite library. ** Derived from test4.c. -** -** $Id: test7.c,v 1.13 2008/10/12 00:27:54 shane Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test8.c b/src/test8.c index a313c26..110aa65 100644 --- a/src/test8.c +++ b/src/test8.c @@ -12,8 +12,6 @@ ** Code for testing the virtual table interfaces. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test8.c,v 1.78 2009/04/29 11:50:54 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test9.c b/src/test9.c index 222bdc3..e5993e8 100644 --- a/src/test9.c +++ b/src/test9.c @@ -13,8 +13,6 @@ ** This file contains obscure tests of the C-interface required ** for completeness. Test code is written in C for these cases ** as there is not much point in binding to Tcl. -** -** $Id: test9.c,v 1.7 2009/04/02 18:32:27 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test_async.c b/src/test_async.c index 52ed94b..c0c0cd1 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** -** $Id: test_async.c,v 1.62 2009/04/28 13:01:09 drh Exp $ -** ** This file contains a binding of the asynchronous IO extension interface ** (defined in ext/async/sqlite3async.h) to Tcl. */ diff --git a/src/test_autoext.c b/src/test_autoext.c index 11f5413..6b1e297 100644 --- a/src/test_autoext.c +++ b/src/test_autoext.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** Test extension for testing the sqlite3_auto_extension() function. -** -** $Id: test_autoext.c,v 1.5 2008/07/08 02:12:37 drh Exp $ */ #include "tcl.h" #include "sqlite3ext.h" diff --git a/src/test_backup.c b/src/test_backup.c index 7c561af..2727137 100644 --- a/src/test_backup.c +++ b/src/test_backup.c @@ -9,8 +9,8 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* +** This file contains test logic for the sqlite3_backup() interface. ** -** $Id: test_backup.c,v 1.3 2009/03/30 12:56:52 drh Exp $ */ #include "tcl.h" diff --git a/src/test_btree.c b/src/test_btree.c index 2767f5c..0048397 100644 --- a/src/test_btree.c +++ b/src/test_btree.c @@ -12,8 +12,6 @@ ** Code for testing the btree.c module in SQLite. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test_btree.c,v 1.9 2009/07/09 02:48:24 shane Exp $ */ #include "btreeInt.h" #include diff --git a/src/test_config.c b/src/test_config.c index 6340b5e..1cb33e8 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -15,8 +15,6 @@ ** ** The focus of this file is providing the TCL testing layer ** access to compile-time constants. -** -** $Id: test_config.c,v 1.50 2009/06/19 14:06:03 drh Exp $ */ #include "sqliteLimit.h" diff --git a/src/test_devsym.c b/src/test_devsym.c index d428e4d..836a450 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -13,8 +13,6 @@ ** This file contains code that modified the OS layer in order to simulate ** different device types (by overriding the return values of the ** xDeviceCharacteristics() and xSectorSize() methods). -** -** $Id: test_devsym.c,v 1.9 2008/12/09 01:32:03 drh Exp $ */ #if SQLITE_TEST /* This file is used for testing only */ diff --git a/src/test_func.c b/src/test_func.c index 3557674..00c45d0 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -11,8 +11,6 @@ ************************************************************************* ** Code for testing all sorts of SQLite interfaces. This code ** implements new SQL functions used by the test scripts. -** -** $Id: test_func.c,v 1.16 2009/07/22 07:27:57 danielk1977 Exp $ */ #include "sqlite3.h" #include "tcl.h" diff --git a/src/test_hexio.c b/src/test_hexio.c index 9e1e8de..db1aa44 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -16,8 +16,6 @@ ** command of TCL to do a lot of this, but there are some issues ** with historical versions of the "binary" command. So it seems ** easier and safer to build our own mechanism. -** -** $Id: test_hexio.c,v 1.7 2008/05/12 16:17:42 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -318,6 +316,48 @@ static int utf8_to_utf8( return TCL_OK; } +static int getFts3Varint(const char *p, sqlite_int64 *v){ + const unsigned char *q = (const unsigned char *) p; + sqlite_uint64 x = 0, y = 1; + while( (*q & 0x80) == 0x80 ){ + x += y * (*q++ & 0x7f); + y <<= 7; + } + x += y * (*q++); + *v = (sqlite_int64) x; + return (int) (q - (unsigned char *)p); +} + + +/* +** USAGE: read_varint BLOB VARNAME +** +** Read a varint from the start of BLOB. Set variable VARNAME to contain +** the interpreted value. Return the number of bytes of BLOB consumed. +*/ +static int read_varint( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int nBlob; + unsigned char *zBlob; + sqlite3_int64 iVal; + int nVal; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BLOB VARNAME"); + return TCL_ERROR; + } + zBlob = Tcl_GetByteArrayFromObj(objv[1], &nBlob); + + nVal = getFts3Varint((char*)zBlob, (sqlite3_int64 *)(&iVal)); + Tcl_ObjSetVar2(interp, objv[2], 0, Tcl_NewWideIntObj(iVal), 0); + Tcl_SetObjResult(interp, Tcl_NewIntObj(nVal)); + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. @@ -333,6 +373,7 @@ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, + { "read_varint", read_varint }, }; int i; for(i=0; i +#include + + +/* +** Definition of the sqlite3_intarray object. +** +** The internal representation of an intarray object is subject +** to change, is not externally visible, and should be used by +** the implementation of intarray only. This object is opaque +** to users. +*/ +struct sqlite3_intarray { + int n; /* Number of elements in the array */ + sqlite3_int64 *a; /* Contents of the array */ + void (*xFree)(void*); /* Function used to free a[] */ +}; + +/* Objects used internally by the virtual table implementation */ +typedef struct intarray_vtab intarray_vtab; +typedef struct intarray_cursor intarray_cursor; + +/* A intarray table object */ +struct intarray_vtab { + sqlite3_vtab base; /* Base class */ + sqlite3_intarray *pContent; /* Content of the integer array */ +}; + +/* A intarray cursor object */ +struct intarray_cursor { + sqlite3_vtab_cursor base; /* Base class */ + int i; /* Current cursor position */ +}; + +/* +** None of this works unless we have virtual tables. +*/ +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Free an sqlite3_intarray object. +*/ +static void intarrayFree(sqlite3_intarray *p){ + if( p->xFree ){ + p->xFree(p->a); + } + sqlite3_free(p); +} + +/* +** Table destructor for the intarray module. +*/ +static int intarrayDestroy(sqlite3_vtab *p){ + intarray_vtab *pVtab = (intarray_vtab*)p; + sqlite3_free(pVtab); + return 0; +} + +/* +** Table constructor for the intarray module. +*/ +static int intarrayCreate( + sqlite3 *db, /* Database where module is created */ + void *pAux, /* clientdata for the module */ + int argc, /* Number of arguments */ + const char *const*argv, /* Value for all arguments */ + sqlite3_vtab **ppVtab, /* Write the new virtual table object here */ + char **pzErr /* Put error message text here */ +){ + int rc = SQLITE_NOMEM; + intarray_vtab *pVtab = sqlite3_malloc(sizeof(intarray_vtab)); + + if( pVtab ){ + memset(pVtab, 0, sizeof(intarray_vtab)); + pVtab->pContent = (sqlite3_intarray*)pAux; + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value INTEGER PRIMARY KEY)"); + } + *ppVtab = (sqlite3_vtab *)pVtab; + return rc; +} + +/* +** Open a new cursor on the intarray table. +*/ +static int intarrayOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + intarray_cursor *pCur; + pCur = sqlite3_malloc(sizeof(intarray_cursor)); + if( pCur ){ + memset(pCur, 0, sizeof(intarray_cursor)); + *ppCursor = (sqlite3_vtab_cursor *)pCur; + rc = SQLITE_OK; + } + return rc; +} + +/* +** Close a intarray table cursor. +*/ +static int intarrayClose(sqlite3_vtab_cursor *cur){ + intarray_cursor *pCur = (intarray_cursor *)cur; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Retrieve a column of data. +*/ +static int intarrayColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + intarray_cursor *pCur = (intarray_cursor*)cur; + intarray_vtab *pVtab = (intarray_vtab*)cur->pVtab; + if( pCur->i>=0 && pCur->ipContent->n ){ + sqlite3_result_int64(ctx, pVtab->pContent->a[pCur->i]); + } + return SQLITE_OK; +} + +/* +** Retrieve the current rowid. +*/ +static int intarrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + intarray_cursor *pCur = (intarray_cursor *)cur; + *pRowid = pCur->i; + return SQLITE_OK; +} + +static int intarrayEof(sqlite3_vtab_cursor *cur){ + intarray_cursor *pCur = (intarray_cursor *)cur; + intarray_vtab *pVtab = (intarray_vtab *)cur->pVtab; + return pCur->i>=pVtab->pContent->n; +} + +/* +** Advance the cursor to the next row. +*/ +static int intarrayNext(sqlite3_vtab_cursor *cur){ + intarray_cursor *pCur = (intarray_cursor *)cur; + pCur->i++; + return SQLITE_OK; +} + +/* +** Reset a intarray table cursor. +*/ +static int intarrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + intarray_cursor *pCur = (intarray_cursor *)pVtabCursor; + pCur->i = 0; + return SQLITE_OK; +} + +/* +** Analyse the WHERE condition. +*/ +static int intarrayBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + return SQLITE_OK; +} + +/* +** A virtual table module that merely echos method calls into TCL +** variables. +*/ +static sqlite3_module intarrayModule = { + 0, /* iVersion */ + intarrayCreate, /* xCreate - create a new virtual table */ + intarrayCreate, /* xConnect - connect to an existing vtab */ + intarrayBestIndex, /* xBestIndex - find the best query index */ + intarrayDestroy, /* xDisconnect - disconnect a vtab */ + intarrayDestroy, /* xDestroy - destroy a vtab */ + intarrayOpen, /* xOpen - open a cursor */ + intarrayClose, /* xClose - close a cursor */ + intarrayFilter, /* xFilter - configure scan constraints */ + intarrayNext, /* xNext - advance a cursor */ + intarrayEof, /* xEof */ + intarrayColumn, /* xColumn - read data */ + intarrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + +/* +** Invoke this routine to create a specific instance of an intarray object. +** The new intarray object is returned by the 3rd parameter. +** +** Each intarray object corresponds to a virtual table in the TEMP table +** with a name of zName. +** +** Destroy the intarray object by dropping the virtual table. If not done +** explicitly by the application, the virtual table will be dropped implicitly +** by the system when the database connection is closed. +*/ +int sqlite3_intarray_create( + sqlite3 *db, + const char *zName, + sqlite3_intarray **ppReturn +){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_intarray *p; + + *ppReturn = p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(*p)); + rc = sqlite3_create_module_v2(db, zName, &intarrayModule, p, + (void(*)(void*))intarrayFree); + if( rc==SQLITE_OK ){ + char *zSql; + zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE temp.%Q USING %Q", + zName, zName); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +#endif + return rc; +} + +/* +** Bind a new array array of integers to a specific intarray object. +** +** The array of integers bound must be unchanged for the duration of +** any query against the corresponding virtual table. If the integer +** array does change or is deallocated undefined behavior will result. +*/ +int sqlite3_intarray_bind( + sqlite3_intarray *pIntArray, /* The intarray object to bind to */ + int nElements, /* Number of elements in the intarray */ + sqlite3_int64 *aElements, /* Content of the intarray */ + void (*xFree)(void*) /* How to dispose of the intarray when done */ +){ + if( pIntArray->xFree ){ + pIntArray->xFree(pIntArray->a); + } + pIntArray->n = nElements; + pIntArray->a = aElements; + pIntArray->xFree = xFree; + return SQLITE_OK; +} + + +/***************************************************************************** +** Everything below is interface for testing this module. +*/ +#ifdef SQLITE_TEST +#include + +/* +** Routines to encode and decode pointers +*/ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern void *sqlite3TestTextToPtr(const char*); +extern int sqlite3TestMakePointerStr(Tcl_Interp*, char *zPtr, void*); +extern const char *sqlite3TestErrorName(int); + +/* +** sqlite3_intarray_create DB NAME +** +** Invoke the sqlite3_intarray_create interface. A string that becomes +** the first parameter to sqlite3_intarray_bind. +*/ +static int test_intarray_create( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zName; + sqlite3_intarray *pArray; + int rc = SQLITE_OK; + char zPtr[100]; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_intarray_create(db, zName, &pArray); +#endif + if( rc!=SQLITE_OK ){ + assert( pArray==0 ); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + return TCL_ERROR; + } + sqlite3TestMakePointerStr(interp, zPtr, pArray); + Tcl_AppendResult(interp, zPtr, (char*)0); + return TCL_OK; +} + +/* +** sqlite3_intarray_bind INTARRAY ?VALUE ...? +** +** Invoke the sqlite3_intarray_bind interface on the given array of integers. +*/ +static int test_intarray_bind( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_intarray *pArray; + int rc = SQLITE_OK; + int i, n; + sqlite3_int64 *a; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "INTARRAY"); + return TCL_ERROR; + } + pArray = (sqlite3_intarray*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); + n = objc - 2; +#ifndef SQLITE_OMIT_VIRTUALTABLE + a = sqlite3_malloc( sizeof(a[0])*n ); + if( a==0 ){ + Tcl_AppendResult(interp, "SQLITE_NOMEM", (char*)0); + return TCL_ERROR; + } + for(i=0; i #include "sqlite3ext.h" diff --git a/src/test_malloc.c b/src/test_malloc.c index ff2ced3..5556cc1 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -12,8 +12,6 @@ ** ** This file contains code used to implement test interfaces to the ** memory allocation subsystem. -** -** $Id: test_malloc.c,v 1.55 2009/07/01 18:09:02 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" diff --git a/src/test_mutex.c b/src/test_mutex.c index 69030ae..ff388f3 100644 --- a/src/test_mutex.c +++ b/src/test_mutex.c @@ -9,8 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** -** $Id: test_mutex.c,v 1.15 2009/03/20 13:15:30 drh Exp $ +** This file contains test logic for the sqlite3_mutex interfaces. */ #include "tcl.h" diff --git a/src/test_onefile.c b/src/test_onefile.c index 9d176be..818bd17 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** -** $Id: test_onefile.c,v 1.12 2009/04/07 11:21:29 danielk1977 Exp $ -** ** OVERVIEW: ** ** This file contains some example code demonstrating how the SQLite diff --git a/src/test_osinst.c b/src/test_osinst.c index b60664a..283afe6 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -13,8 +13,6 @@ ** This file contains the implementation of an SQLite vfs wrapper that ** adds instrumentation to all vfs and file methods. C and Tcl interfaces ** are provided to control the instrumentation. -** -** $Id: test_osinst.c,v 1.19 2009/01/08 17:57:32 danielk1977 Exp $ */ #ifdef SQLITE_ENABLE_INSTVFS diff --git a/src/test_pcache.c b/src/test_pcache.c index dfef141..98aa136 100644 --- a/src/test_pcache.c +++ b/src/test_pcache.c @@ -20,8 +20,6 @@ ** ** This pagecache implementation is designed for simplicity ** not speed. -** -** $Id: test_pcache.c,v 1.3 2009/04/11 11:38:54 drh Exp $ */ #include "sqlite3.h" #include diff --git a/src/test_schema.c b/src/test_schema.c index 51099f5..1264446 100644 --- a/src/test_schema.c +++ b/src/test_schema.c @@ -12,8 +12,6 @@ ** Code for testing the virtual table interfaces. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. -** -** $Id: test_schema.c,v 1.15 2008/07/07 14:50:14 drh Exp $ */ /* The code in this file defines a sqlite3 virtual-table module that diff --git a/src/test_server.c b/src/test_server.c index 6862d7c..3eb7544 100644 --- a/src/test_server.c +++ b/src/test_server.c @@ -10,8 +10,6 @@ ** ****************************************************************************** ** -** $Id: test_server.c,v 1.8 2008/06/26 10:41:19 danielk1977 Exp $ -** ** This file contains demonstration code. Nothing in this file gets compiled ** or linked into the SQLite library unless you use a non-standard option: ** diff --git a/src/test_tclvar.c b/src/test_tclvar.c index acabd21..1219190 100644 --- a/src/test_tclvar.c +++ b/src/test_tclvar.c @@ -15,8 +15,6 @@ ** ** The emphasis of this file is a virtual table that provides ** access to TCL variables. -** -** $Id: test_tclvar.c,v 1.17 2008/08/12 14:48:41 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -167,6 +165,15 @@ static int tclvarFilter( Tcl_ListObjAppendElement(0, p, pArg); } Tcl_EvalObjEx(interp, p, TCL_EVAL_GLOBAL); + if( pCur->pList1 ){ + Tcl_DecrRefCount(pCur->pList1); + } + if( pCur->pList2 ){ + Tcl_DecrRefCount(pCur->pList2); + pCur->pList2 = 0; + } + pCur->i1 = 0; + pCur->i2 = 0; pCur->pList1 = Tcl_GetObjResult(interp); Tcl_IncrRefCount(pCur->pList1); assert( pCur->i1==0 && pCur->i2==0 && pCur->pList2==0 ); diff --git a/src/test_thread.c b/src/test_thread.c index fc372a2..6a319fa 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -13,8 +13,6 @@ ** This file contains the implementation of some Tcl commands used to ** test that sqlite3 database handles may be concurrently accessed by ** multiple threads. Right now this only works on unix. -** -** $Id: test_thread.c,v 1.15 2009/03/27 12:32:56 drh Exp $ */ #include "sqliteInt.h" diff --git a/src/test_wsd.c b/src/test_wsd.c index a5d10a7..99e4a05 100644 --- a/src/test_wsd.c +++ b/src/test_wsd.c @@ -13,8 +13,6 @@ ** The code in this file contains sample implementations of the ** sqlite3_wsd_init() and sqlite3_wsd_find() functions required if the ** SQLITE_OMIT_WSD symbol is defined at build time. -** -** $Id: test_wsd.c,v 1.4 2009/03/23 04:33:33 danielk1977 Exp $ */ #if defined(SQLITE_OMIT_WSD) && defined(SQLITE_TEST) diff --git a/src/tokenize.c b/src/tokenize.c index 97c4b7b..782d60e 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -14,8 +14,6 @@ ** This file contains C code that splits an SQL input string up into ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. -** -** $Id: tokenize.c,v 1.163 2009/07/03 22:54:37 drh Exp $ */ #include "sqliteInt.h" #include @@ -84,16 +82,7 @@ const unsigned char ebcdicToAscii[] = { ** But the feature is undocumented. */ #ifdef SQLITE_ASCII -const char sqlite3IsAsciiIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ -}; -#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20])) +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) #endif #ifdef SQLITE_EBCDIC const char sqlite3IsEbcdicIdChar[] = { diff --git a/src/trigger.c b/src/trigger.c index 82acfbf..51969ce 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -8,9 +8,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** -** -** $Id: trigger.c,v 1.143 2009/08/10 03:57:58 shane Exp $ +** This file contains the implementation for TRIGGERs */ #include "sqliteInt.h" @@ -813,7 +811,8 @@ static TriggerPrg *codeRowTrigger( pProgram->nRef = 1; pPrg->pTrigger = pTrigger; pPrg->orconf = orconf; - pPrg->oldmask = 0xffffffff; + pPrg->aColmask[0] = 0xffffffff; + pPrg->aColmask[1] = 0xffffffff; /* Allocate and populate a new Parse context to use for coding the ** trigger sub-program. */ @@ -874,7 +873,8 @@ static TriggerPrg *codeRowTrigger( pProgram->nMem = pSubParse->nMem; pProgram->nCsr = pSubParse->nTab; pProgram->token = (void *)pTrigger; - pPrg->oldmask = pSubParse->oldmask; + pPrg->aColmask[0] = pSubParse->oldmask; + pPrg->aColmask[1] = pSubParse->newmask; sqlite3VdbeDelete(v); } @@ -1034,28 +1034,36 @@ void sqlite3CodeRowTrigger( } /* -** Triggers fired by UPDATE or DELETE statements may access values stored -** in the old.* pseudo-table. This function returns a 32-bit bitmask -** indicating which columns of the old.* table actually are used by -** triggers. This information may be used by the caller to avoid having -** to load the entire old.* record into memory when executing an UPDATE -** or DELETE command. +** Triggers may access values stored in the old.* or new.* pseudo-table. +** This function returns a 32-bit bitmask indicating which columns of the +** old.* or new.* tables actually are used by triggers. This information +** may be used by the caller, for example, to avoid having to load the entire +** old.* record into memory when executing an UPDATE or DELETE command. ** ** Bit 0 of the returned mask is set if the left-most column of the -** table may be accessed using an old. reference. Bit 1 is set if +** table may be accessed using an [old|new]. reference. Bit 1 is set if ** the second leftmost column value is required, and so on. If there ** are more than 32 columns in the table, and at least one of the columns ** with an index greater than 32 may be accessed, 0xffffffff is returned. ** -** It is not possible to determine if the old.rowid column is accessed -** by triggers. The caller must always assume that it is. +** It is not possible to determine if the old.rowid or new.rowid column is +** accessed by triggers. The caller must always assume that it is. ** -** There is no equivalent function for new.* references. +** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned +** applies to the old.* table. If 1, the new.* table. +** +** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE +** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only +** included in the returned mask if the TRIGGER_BEFORE bit is set in the +** tr_tm parameter. Similarly, values accessed by AFTER triggers are only +** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm. */ -u32 sqlite3TriggerOldmask( +u32 sqlite3TriggerColmask( Parse *pParse, /* Parse context */ Trigger *pTrigger, /* List of triggers on table pTab */ ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */ + int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ Table *pTab, /* The table to code triggers from */ int orconf /* Default ON CONFLICT policy for trigger steps */ ){ @@ -1063,12 +1071,15 @@ u32 sqlite3TriggerOldmask( u32 mask = 0; Trigger *p; + assert( isNew==1 || isNew==0 ); for(p=pTrigger; p; p=p->pNext){ - if( p->op==op && checkColumnOverlap(p->pColumns,pChanges) ){ + if( p->op==op && (tr_tm&p->tr_tm) + && checkColumnOverlap(p->pColumns,pChanges) + ){ TriggerPrg *pPrg; pPrg = getRowTrigger(pParse, p, pTab, orconf); if( pPrg ){ - mask |= pPrg->oldmask; + mask |= pPrg->aColmask[isNew]; } } } diff --git a/src/update.c b/src/update.c index 3f5e15f..66bf8ca 100644 --- a/src/update.c +++ b/src/update.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. -** -** $Id: update.c,v 1.207 2009/08/08 18:01:08 drh Exp $ */ #include "sqliteInt.h" @@ -113,14 +111,15 @@ void sqlite3Update( AuthContext sContext; /* The authorization context */ NameContext sNC; /* The name-context to resolve expressions in */ int iDb; /* Database containing the table being updated */ - int j1; /* Addresses of jump instructions */ int okOnePass; /* True for one-pass algorithm without the FIFO */ int hasFK; /* True if foreign key processing is required */ #ifndef SQLITE_OMIT_TRIGGER - int isView; /* Trying to update a view */ - Trigger *pTrigger; /* List of triggers on pTab, if required */ + int isView; /* True when updating a view (INSTEAD OF trigger) */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ #endif + int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -148,11 +147,13 @@ void sqlite3Update( ** updated is a view. */ #ifndef SQLITE_OMIT_TRIGGER - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, 0); + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); isView = pTab->pSelect!=0; + assert( pTrigger || tmask==0 ); #else # define pTrigger 0 # define isView 0 +# define tmask 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -162,7 +163,7 @@ void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ goto update_cleanup; } aXRef = sqlite3DbMallocRaw(db, sizeof(int) * pTab->nCol ); @@ -390,7 +391,9 @@ void sqlite3Update( ** with the required old.* column data. */ if( hasFK || pTrigger ){ u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); - oldmask |= sqlite3TriggerOldmask(pParse, pTrigger, pChanges, pTab, onError); + oldmask |= sqlite3TriggerColmask(pParse, + pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError + ); for(i=0; inCol; i++){ if( aXRef[i]<0 || oldmask==0xffffffff || (oldmask & (1<nCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); }else{ j = aXRef[i]; - if( j<0 ){ + if( j>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask&(1<a[j].pExpr, regNew+i); } } } /* Fire any BEFORE UPDATE triggers. This happens before constraints are - ** verified. One could argue that this is wrong. */ - if( pTrigger ){ + ** verified. One could argue that this is wrong. + */ + if( tmask&TRIGGER_BEFORE ){ sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol); sqlite3TableAffinityStr(v, pTab); sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, @@ -434,11 +457,25 @@ void sqlite3Update( ** case, jump to the next row. No updates or AFTER triggers are ** required. This behaviour - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the - ** documentation. */ + ** documentation. + */ sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, regOldRowid); + + /* If it did not delete it, the row-trigger may still have modified + ** some of the columns of the row being updated. Load the values for + ** all columns not modified by the update statement into their + ** registers in case this has happened. + */ + for(i=0; inCol; i++){ + if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3VdbeAddOp3(v, OP_Column, iCur, i, regNew+i); + sqlite3ColumnDefault(v, pTab, i, regNew+i); + } + } } if( !isView ){ + int j1; /* Address of jump instruction */ /* Do constraint checks. */ sqlite3GenerateConstraintChecks(pParse, pTab, iCur, regNewRowid, diff --git a/src/utf.c b/src/utf.c index 30db9a9..4ea63ee 100644 --- a/src/utf.c +++ b/src/utf.c @@ -12,8 +12,6 @@ ** This file contains routines used to translate between UTF-8, ** UTF-16, UTF-16BE, and UTF-16LE. ** -** $Id: utf.c,v 1.73 2009/04/01 18:40:32 drh Exp $ -** ** Notes on UTF-8: ** ** Byte-0 Byte-1 Byte-2 Byte-3 Value diff --git a/src/util.c b/src/util.c index d12bcd5..81e42b4 100644 --- a/src/util.c +++ b/src/util.c @@ -1,1094 +1,1094 @@ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Utility functions used throughout sqlite. -** -** This file contains functions for allocating memory, comparing -** strings, and stuff like that. -** -*/ -#include "sqliteInt.h" -#include -#ifdef SQLITE_HAVE_ISNAN -# include -#endif - -/* -** Routine needed to support the testcase() macro. -*/ -#ifdef SQLITE_COVERAGE_TEST -void sqlite3Coverage(int x){ - static int dummy = 0; - dummy += x; -} -#endif - -/* -** Return true if the floating point value is Not a Number (NaN). -** -** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. -** Otherwise, we have our own implementation that works on most systems. -*/ -int sqlite3IsNaN(double x){ - int rc; /* The value return */ -#if !defined(SQLITE_HAVE_ISNAN) - /* - ** Systems that support the isnan() library function should probably - ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have - ** found that many systems do not have a working isnan() function so - ** this implementation is provided as an alternative. - ** - ** This NaN test sometimes fails if compiled on GCC with -ffast-math. - ** On the other hand, the use of -ffast-math comes with the following - ** warning: - ** - ** This option [-ffast-math] should never be turned on by any - ** -O option since it can result in incorrect output for programs - ** which depend on an exact implementation of IEEE or ISO - ** rules/specifications for math functions. - ** - ** Under MSVC, this NaN test may fail if compiled with a floating- - ** point precision mode other than /fp:precise. From the MSDN - ** documentation: - ** - ** The compiler [with /fp:precise] will properly handle comparisons - ** involving NaN. For example, x != x evaluates to true if x is NaN - ** ... - */ -#ifdef __FAST_MATH__ -# error SQLite will not work correctly with the -ffast-math option of GCC. -#endif - volatile double y = x; - volatile double z = y; - rc = (y!=z); -#else /* if defined(SQLITE_HAVE_ISNAN) */ - rc = isnan(x); -#endif /* SQLITE_HAVE_ISNAN */ - testcase( rc ); - return rc; -} - -/* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -** -** The value returned will never be negative. Nor will it ever be greater -** than the actual length of the string. For very long strings (greater -** than 1GiB) the value returned might be less than the true string length. -*/ -int sqlite3Strlen30(const char *z){ - const char *z2 = z; - if( z==0 ) return 0; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); -} - -/* -** Set the most recent error code and error string for the sqlite -** handle "db". The error code is set to "err_code". -** -** If it is not NULL, string zFormat specifies the format of the -** error string in the style of the printf functions: The following -** format characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList -** -** zFormat and any string tokens that follow it are assumed to be -** encoded in UTF-8. -** -** To clear the most recent error for sqlite handle "db", sqlite3Error -** should be called with err_code set to SQLITE_OK and zFormat set -** to NULL. -*/ -void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ - if( db && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ - db->errCode = err_code; - if( zFormat ){ - char *z; - va_list ap; - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); - }else{ - sqlite3ValueSetStr(db->pErr, 0, 0, SQLITE_UTF8, SQLITE_STATIC); - } - } -} - -/* -** Add an error message to pParse->zErrMsg and increment pParse->nErr. -** The following formatting characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList -** -** This function should be used to report any error that occurs whilst -** compiling an SQL statement (i.e. within sqlite3_prepare()). The -** last thing the sqlite3_prepare() function does is copy the error -** stored by this function into the database handle using sqlite3Error(). -** Function sqlite3Error() should be used during statement execution -** (sqlite3_step() etc.). -*/ -void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ - va_list ap; - sqlite3 *db = pParse->db; - pParse->nErr++; - sqlite3DbFree(db, pParse->zErrMsg); - va_start(ap, zFormat); - pParse->zErrMsg = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - pParse->rc = SQLITE_ERROR; -} - -/* -** Clear the error message in pParse, if any -*/ -void sqlite3ErrorClear(Parse *pParse){ - sqlite3DbFree(pParse->db, pParse->zErrMsg); - pParse->zErrMsg = 0; - pParse->nErr = 0; -} - -/* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. -** -** The input string must be zero-terminated. A new zero-terminator -** is added to the dequoted string. -** -** The return value is -1 if no dequoting occurs or the length of the -** dequoted string, exclusive of the zero terminator, if dequoting does -** occur. -** -** 2002-Feb-14: This routine is extended to remove MS-Access style -** brackets from around identifers. For example: "[a-b-c]" becomes -** "a-b-c". -*/ -int sqlite3Dequote(char *z){ - char quote; - int i, j; - if( z==0 ) return -1; - quote = z[0]; - switch( quote ){ - case '\'': break; - case '"': break; - case '`': break; /* For MySQL compatibility */ - case '[': quote = ']'; break; /* For MS SqlServer compatibility */ - default: return -1; - } - for(i=1, j=0; ALWAYS(z[i]); i++){ - if( z[i]==quote ){ - if( z[i+1]==quote ){ - z[j++] = quote; - i++; - }else{ - break; - } - }else{ - z[j++] = z[i]; - } - } - z[j] = 0; - return j; -} - -/* Convenient short-hand */ -#define UpperToLower sqlite3UpperToLower - -/* -** Some systems have stricmp(). Others have strcasecmp(). Because -** there is no consistency, we will define our own. -*/ -int sqlite3StrICmp(const char *zLeft, const char *zRight){ - register unsigned char *a, *b; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return UpperToLower[*a] - UpperToLower[*b]; -} -int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ - register unsigned char *a, *b; - a = (unsigned char *)zLeft; - b = (unsigned char *)zRight; - while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } - return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; -} - -/* -** Return TRUE if z is a pure numeric string. Return FALSE and leave -** *realnum unchanged if the string contains any character which is not -** part of a number. -** -** If the string is pure numeric, set *realnum to TRUE if the string -** contains the '.' character or an "E+000" style exponentiation suffix. -** Otherwise set *realnum to FALSE. Note that just becaue *realnum is -** false does not mean that the number can be successfully converted into -** an integer - it might be too big. -** -** An empty string is considered non-numeric. -*/ -int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ - int incr = (enc==SQLITE_UTF8?1:2); - if( enc==SQLITE_UTF16BE ) z++; - if( *z=='-' || *z=='+' ) z += incr; - if( !sqlite3Isdigit(*z) ){ - return 0; - } - z += incr; - *realnum = 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - if( *z=='.' ){ - z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z += incr; - if( *z=='+' || *z=='-' ) z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } - return *z==0; -} - -/* -** The string z[] is an ASCII representation of a real number. -** Convert this string to a double. -** -** This routine assumes that z[] really is a valid number. If it -** is not, the result is undefined. -** -** This routine is used instead of the library atof() function because -** the library atof() might want to use "," as the decimal point instead -** of "." depending on how locale is set. But that would cause problems -** for SQL. So this routine always uses "." regardless of locale. -*/ -int sqlite3AtoF(const char *z, double *pResult){ -#ifndef SQLITE_OMIT_FLOATING_POINT - const char *zBegin = z; - /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ - double result; - int nDigits = 0; - - /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; - /* get sign of significand */ - if( *z=='-' ){ - sign = -1; - z++; - }else if( *z=='+' ){ - z++; - } - /* skip leading zeroes */ - while( z[0]=='0' ) z++, nDigits++; - - /* copy max significant digits to significand */ - while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ - s = s*10 + (*z - '0'); - z++, nDigits++; - } - /* skip non-significant significand digits - ** (increase exponent by d to shift decimal left) */ - while( sqlite3Isdigit(*z) ) z++, nDigits++, d++; - - /* if decimal point is present */ - if( *z=='.' ){ - z++; - /* copy digits from after decimal to significand - ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ - s = s*10 + (*z - '0'); - z++, nDigits++, d--; - } - /* skip non-significant digits */ - while( sqlite3Isdigit(*z) ) z++, nDigits++; - } - - /* if exponent is present */ - if( *z=='e' || *z=='E' ){ - z++; - /* get sign of exponent */ - if( *z=='-' ){ - esign = -1; - z++; - }else if( *z=='+' ){ - z++; - } - /* copy digits to exponent */ - while( sqlite3Isdigit(*z) ){ - e = e*10 + (*z - '0'); - z++; - } - } - - /* adjust exponent by d, and update sign */ - e = (e*esign) + d; - if( e<0 ) { - esign = -1; - e *= -1; - } else { - esign = 1; - } - - /* if 0 significand */ - if( !s ) { - /* In the IEEE 754 standard, zero is signed. - ** Add the sign if we've seen at least one digit */ - result = (sign<0 && nDigits) ? -(double)0 : (double)0; - } else { - /* attempt to reduce exponent */ - if( esign>0 ){ - while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; - }else{ - while( !(s%10) && e>0 ) e--,s/=10; - } - - /* adjust the sign of significand */ - s = sign<0 ? -s : s; - - /* if exponent, scale significand as appropriate - ** and store in result. */ - if( e ){ - double scale = 1.0; - /* attempt to handle extremely small/large numbers better */ - if( e>307 && e<342 ){ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else{ - /* 1.0e+22 is the largest power of 10 than can be - ** represented exactly. */ - while( e%22 ) { scale *= 1.0e+1; e -= 1; } - while( e>0 ) { scale *= 1.0e+22; e -= 22; } - if( esign<0 ){ - result = s / scale; - }else{ - result = s * scale; - } - } - } else { - result = (double)s; - } - } - - /* store the result */ - *pResult = result; - - /* return number of characters used */ - return (int)(z - zBegin); -#else - return sqlite3Atoi64(z, pResult); -#endif /* SQLITE_OMIT_FLOATING_POINT */ -} - -/* -** Compare the 19-character string zNum against the text representation -** value 2^63: 9223372036854775808. Return negative, zero, or positive -** if zNum is less than, equal to, or greater than the string. -** -** Unlike memcmp() this routine is guaranteed to return the difference -** in the values of the last digit if the only difference is in the -** last digit. So, for example, -** -** compare2pow63("9223372036854775800") -** -** will return -8. -*/ -static int compare2pow63(const char *zNum){ - int c; - c = memcmp(zNum,"922337203685477580",18)*10; - if( c==0 ){ - c = zNum[18] - '8'; - } - return c; -} - - -/* -** Return TRUE if zNum is a 64-bit signed integer and write -** the value of the integer into *pNum. If zNum is not an integer -** or is an integer that is too large to be expressed with 64 bits, -** then return false. -** -** When this routine was originally written it dealt with only -** 32-bit numbers. At that time, it was much faster than the -** atoi() library routine in RedHat 7.2. -*/ -int sqlite3Atoi64(const char *zNum, i64 *pNum){ - i64 v = 0; - int neg; - int i, c; - const char *zStart; - while( sqlite3Isspace(*zNum) ) zNum++; - if( *zNum=='-' ){ - neg = 1; - zNum++; - }else if( *zNum=='+' ){ - neg = 0; - zNum++; - }else{ - neg = 0; - } - zStart = zNum; - while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */ - for(i=0; (c=zNum[i])>='0' && c<='9'; i++){ - v = v*10 + c - '0'; - } - *pNum = neg ? -v : v; - if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ - /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranting that it is too large) */ - return 0; - }else if( i<19 ){ - /* Less than 19 digits, so we know that it fits in 64 bits */ - return 1; - }else{ - /* 19-digit numbers must be no larger than 9223372036854775807 if positive - ** or 9223372036854775808 if negative. Note that 9223372036854665808 - ** is 2^63. */ - return compare2pow63(zNum)='0' && zNum[0]<='9' ); /* zNum is an unsigned number */ - - if( negFlag ) neg = 1-neg; - while( *zNum=='0' ){ - zNum++; /* Skip leading zeros. Ticket #2454 */ - } - for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } - if( i<19 ){ - /* Guaranteed to fit if less than 19 digits */ - return 1; - }else if( i>19 ){ - /* Guaranteed to be too big if greater than 19 digits */ - return 0; - }else{ - /* Compare against 2^63. */ - return compare2pow63(zNum)=0 && c<=9; i++){ - v = v*10 + c; - } - - /* The longest decimal representation of a 32 bit integer is 10 digits: - ** - ** 1234567890 - ** 2^31 -> 2147483648 - */ - if( i>10 ){ - return 0; - } - if( v-neg>2147483647 ){ - return 0; - } - if( neg ){ - v = -v; - } - *pValue = (int)v; - return 1; -} - -/* -** The variable-length integer encoding is as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** C = xxxxxxxx 8 bits of data -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** 28 bits - BBBA -** 35 bits - BBBBA -** 42 bits - BBBBBA -** 49 bits - BBBBBBA -** 56 bits - BBBBBBBA -** 64 bits - BBBBBBBBC -*/ - -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data write will be between 1 and 9 bytes. The number -** of bytes written is returned. -** -** A variable-length integer consists of the lower 7 bits of each byte -** for all bytes that have the 8th bit set and one byte with the 8th -** bit clear. Except, if we get to the 9th byte, it stores the full -** 8 bits and is the last byte. -*/ -int sqlite3PutVarint(unsigned char *p, u64 v){ - int i, j, n; - u8 buf[10]; - if( v & (((u64)0xff000000)<<32) ){ - p[8] = (u8)v; - v >>= 8; - for(i=7; i>=0; i--){ - p[i] = (u8)((v & 0x7f) | 0x80); - v >>= 7; - } - return 9; - } - n = 0; - do{ - buf[n++] = (u8)((v & 0x7f) | 0x80); - v >>= 7; - }while( v!=0 ); - buf[0] &= 0x7f; - assert( n<=9 ); - for(i=0, j=n-1; j>=0; j--, i++){ - p[i] = buf[j]; - } - return n; -} - -/* -** This routine is a faster version of sqlite3PutVarint() that only -** works for 32-bit positive integers and which is optimized for -** the common case of small integers. A MACRO version, putVarint32, -** is provided which inlines the single-byte case. All code should use -** the MACRO version as this function assumes the single-byte case has -** already been handled. -*/ -int sqlite3PutVarint32(unsigned char *p, u32 v){ -#ifndef putVarint32 - if( (v & ~0x7f)==0 ){ - p[0] = v; - return 1; - } -#endif - if( (v & ~0x3fff)==0 ){ - p[0] = (u8)((v>>7) | 0x80); - p[1] = (u8)(v & 0x7f); - return 2; - } - return sqlite3PutVarint(p, v); -} - -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read. The value is stored in *v. -*/ -u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ - u32 a,b,s; - - a = *p; - /* a: p0 (unmasked) */ - if (!(a&0x80)) - { - *v = a; - return 1; - } - - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - a &= 0x7f; - a = a<<7; - a |= b; - *v = a; - return 2; - } - - p++; - a = a<<14; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - a &= (0x7f<<14)|(0x7f); - b &= 0x7f; - b = b<<7; - a |= b; - *v = a; - return 3; - } - - /* CSE1 from below */ - a &= (0x7f<<14)|(0x7f); - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - b &= (0x7f<<14)|(0x7f); - /* moved CSE1 up */ - /* a &= (0x7f<<14)|(0x7f); */ - a = a<<7; - a |= b; - *v = a; - return 4; - } - - /* a: p0<<14 | p2 (masked) */ - /* b: p1<<14 | p3 (unmasked) */ - /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - /* moved CSE1 up */ - /* a &= (0x7f<<14)|(0x7f); */ - b &= (0x7f<<14)|(0x7f); - s = a; - /* s: p0<<14 | p2 (masked) */ - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* we can skip these cause they were (effectively) done above in calc'ing s */ - /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ - /* b &= (0x7f<<14)|(0x7f); */ - b = b<<7; - a |= b; - s = s>>18; - *v = ((u64)s)<<32 | a; - return 5; - } - - /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - s = s<<7; - s |= b; - /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - - p++; - b = b<<14; - b |= *p; - /* b: p1<<28 | p3<<14 | p5 (unmasked) */ - if (!(b&0x80)) - { - /* we can skip this cause it was (effectively) done above in calc'ing s */ - /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ - a &= (0x7f<<14)|(0x7f); - a = a<<7; - a |= b; - s = s>>18; - *v = ((u64)s)<<32 | a; - return 6; - } - - p++; - a = a<<14; - a |= *p; - /* a: p2<<28 | p4<<14 | p6 (unmasked) */ - if (!(a&0x80)) - { - a &= (0x1f<<28)|(0x7f<<14)|(0x7f); - b &= (0x7f<<14)|(0x7f); - b = b<<7; - a |= b; - s = s>>11; - *v = ((u64)s)<<32 | a; - return 7; - } - - /* CSE2 from below */ - a &= (0x7f<<14)|(0x7f); - p++; - b = b<<14; - b |= *p; - /* b: p3<<28 | p5<<14 | p7 (unmasked) */ - if (!(b&0x80)) - { - b &= (0x1f<<28)|(0x7f<<14)|(0x7f); - /* moved CSE2 up */ - /* a &= (0x7f<<14)|(0x7f); */ - a = a<<7; - a |= b; - s = s>>4; - *v = ((u64)s)<<32 | a; - return 8; - } - - p++; - a = a<<15; - a |= *p; - /* a: p4<<29 | p6<<15 | p8 (unmasked) */ - - /* moved CSE2 up */ - /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ - b &= (0x7f<<14)|(0x7f); - b = b<<8; - a |= b; - - s = s<<4; - b = p[-4]; - b &= 0x7f; - b = b>>3; - s |= b; - - *v = ((u64)s)<<32 | a; - - return 9; -} - -/* -** Read a 32-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read. The value is stored in *v. -** -** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned -** integer, then set *v to 0xffffffff. -** -** A MACRO version, getVarint32, is provided which inlines the -** single-byte case. All code should use the MACRO version as -** this function assumes the single-byte case has already been handled. -*/ -u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ - u32 a,b; - - /* The 1-byte case. Overwhelmingly the most common. Handled inline - ** by the getVarin32() macro */ - a = *p; - /* a: p0 (unmasked) */ -#ifndef getVarint32 - if (!(a&0x80)) - { - /* Values between 0 and 127 */ - *v = a; - return 1; - } -#endif - - /* The 2-byte case */ - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 128 and 16383 */ - a &= 0x7f; - a = a<<7; - *v = a | b; - return 2; - } - - /* The 3-byte case */ - p++; - a = a<<14; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 16384 and 2097151 */ - a &= (0x7f<<14)|(0x7f); - b &= 0x7f; - b = b<<7; - *v = a | b; - return 3; - } - - /* A 32-bit varint is used to store size information in btrees. - ** Objects are rarely larger than 2MiB limit of a 3-byte varint. - ** A 3-byte varint is sufficient, for example, to record the size - ** of a 1048569-byte BLOB or string. - ** - ** We only unroll the first 1-, 2-, and 3- byte cases. The very - ** rare larger cases can be handled by the slower 64-bit varint - ** routine. - */ -#if 1 - { - u64 v64; - u8 n; - - p -= 2; - n = sqlite3GetVarint(p, &v64); - assert( n>3 && n<=9 ); - if( (v64 & SQLITE_MAX_U32)!=v64 ){ - *v = 0xffffffff; - }else{ - *v = (u32)v64; - } - return n; - } - -#else - /* For following code (kept for historical record only) shows an - ** unrolling for the 3- and 4-byte varint cases. This code is - ** slightly faster, but it is also larger and much harder to test. - */ - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 2097152 and 268435455 */ - b &= (0x7f<<14)|(0x7f); - a &= (0x7f<<14)|(0x7f); - a = a<<7; - *v = a | b; - return 4; - } - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* Walues between 268435456 and 34359738367 */ - a &= (0x1f<<28)|(0x7f<<14)|(0x7f); - b &= (0x1f<<28)|(0x7f<<14)|(0x7f); - b = b<<7; - *v = a | b; - return 5; - } - - /* We can only reach this point when reading a corrupt database - ** file. In that case we are not in any hurry. Use the (relatively - ** slow) general-purpose sqlite3GetVarint() routine to extract the - ** value. */ - { - u64 v64; - u8 n; - - p -= 4; - n = sqlite3GetVarint(p, &v64); - assert( n>5 && n<=9 ); - *v = (u32)v64; - return n; - } -#endif -} - -/* -** Return the number of bytes that will be needed to store the given -** 64-bit integer. -*/ -int sqlite3VarintLen(u64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 && ALWAYS(i<9) ); - return i; -} - - -/* -** Read or write a four-byte big-endian integer value. -*/ -u32 sqlite3Get4byte(const u8 *p){ - return (p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; -} -void sqlite3Put4byte(unsigned char *p, u32 v){ - p[0] = (u8)(v>>24); - p[1] = (u8)(v>>16); - p[2] = (u8)(v>>8); - p[3] = (u8)v; -} - - - -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) -/* -** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F -*/ -static u8 hexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); -#ifdef SQLITE_ASCII - h += 9*(1&(h>>6)); -#endif -#ifdef SQLITE_EBCDIC - h += 9*(1&~(h>>4)); -#endif - return (u8)(h & 0xf); -} -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ - -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) -/* -** Convert a BLOB literal of the form "x'hhhhhh'" into its binary -** value. Return a pointer to its binary value. Space to hold the -** binary value has been obtained from malloc and must be freed by -** the calling routine. -*/ -void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ - char *zBlob; - int i; - - zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); - n--; - if( zBlob ){ - for(i=0; imagic is not a valid open value, take care not -** to modify the db structure at all. It could be that db is a stale -** pointer. In other words, it could be that there has been a prior -** call to sqlite3_close(db) and db has been deallocated. And we do -** not want to write into deallocated memory. -*/ -#ifdef SQLITE_DEBUG -int sqlite3SafetyOn(sqlite3 *db){ - if( db->magic==SQLITE_MAGIC_OPEN ){ - db->magic = SQLITE_MAGIC_BUSY; - assert( sqlite3_mutex_held(db->mutex) ); - return 0; - }else if( db->magic==SQLITE_MAGIC_BUSY ){ - db->magic = SQLITE_MAGIC_ERROR; - db->u1.isInterrupted = 1; - } - return 1; -} -#endif - -/* -** Change the magic from SQLITE_MAGIC_BUSY to SQLITE_MAGIC_OPEN. -** Return an error (non-zero) if the magic was not SQLITE_MAGIC_BUSY -** when this routine is called. -*/ -#ifdef SQLITE_DEBUG -int sqlite3SafetyOff(sqlite3 *db){ - if( db->magic==SQLITE_MAGIC_BUSY ){ - db->magic = SQLITE_MAGIC_OPEN; - assert( sqlite3_mutex_held(db->mutex) ); - return 0; - }else{ - db->magic = SQLITE_MAGIC_ERROR; - db->u1.isInterrupted = 1; - return 1; - } -} -#endif - -/* -** Check to make sure we have a valid db pointer. This test is not -** foolproof but it does provide some measure of protection against -** misuse of the interface such as passing in db pointers that are -** NULL or which have been previously closed. If this routine returns -** 1 it means that the db pointer is valid and 0 if it should not be -** dereferenced for any reason. The calling function should invoke -** SQLITE_MISUSE immediately. -** -** sqlite3SafetyCheckOk() requires that the db pointer be valid for -** use. sqlite3SafetyCheckSickOrOk() allows a db pointer that failed to -** open properly and is not fit for general use but which can be -** used as an argument to sqlite3_errmsg() or sqlite3_close(). -*/ -int sqlite3SafetyCheckOk(sqlite3 *db){ - u32 magic; - if( db==0 ) return 0; - magic = db->magic; - if( magic!=SQLITE_MAGIC_OPEN -#ifdef SQLITE_DEBUG - && magic!=SQLITE_MAGIC_BUSY -#endif - ){ - return 0; - }else{ - return 1; - } -} -int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ - u32 magic; - magic = db->magic; - if( magic!=SQLITE_MAGIC_SICK && - magic!=SQLITE_MAGIC_OPEN && - magic!=SQLITE_MAGIC_BUSY ) return 0; - return 1; -} +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Utility functions used throughout sqlite. +** +** This file contains functions for allocating memory, comparing +** strings, and stuff like that. +** +*/ +#include "sqliteInt.h" +#include +#ifdef SQLITE_HAVE_ISNAN +# include +#endif + +/* +** Routine needed to support the testcase() macro. +*/ +#ifdef SQLITE_COVERAGE_TEST +void sqlite3Coverage(int x){ + static int dummy = 0; + dummy += x; +} +#endif + +/* +** Return true if the floating point value is Not a Number (NaN). +** +** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. +** Otherwise, we have our own implementation that works on most systems. +*/ +int sqlite3IsNaN(double x){ + int rc; /* The value return */ +#if !defined(SQLITE_HAVE_ISNAN) + /* + ** Systems that support the isnan() library function should probably + ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have + ** found that many systems do not have a working isnan() function so + ** this implementation is provided as an alternative. + ** + ** This NaN test sometimes fails if compiled on GCC with -ffast-math. + ** On the other hand, the use of -ffast-math comes with the following + ** warning: + ** + ** This option [-ffast-math] should never be turned on by any + ** -O option since it can result in incorrect output for programs + ** which depend on an exact implementation of IEEE or ISO + ** rules/specifications for math functions. + ** + ** Under MSVC, this NaN test may fail if compiled with a floating- + ** point precision mode other than /fp:precise. From the MSDN + ** documentation: + ** + ** The compiler [with /fp:precise] will properly handle comparisons + ** involving NaN. For example, x != x evaluates to true if x is NaN + ** ... + */ +#ifdef __FAST_MATH__ +# error SQLite will not work correctly with the -ffast-math option of GCC. +#endif + volatile double y = x; + volatile double z = y; + rc = (y!=z); +#else /* if defined(SQLITE_HAVE_ISNAN) */ + rc = isnan(x); +#endif /* SQLITE_HAVE_ISNAN */ + testcase( rc ); + return rc; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +** +** The value returned will never be negative. Nor will it ever be greater +** than the actual length of the string. For very long strings (greater +** than 1GiB) the value returned might be less than the true string length. +*/ +int sqlite3Strlen30(const char *z){ + const char *z2 = z; + if( z==0 ) return 0; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Set the most recent error code and error string for the sqlite +** handle "db". The error code is set to "err_code". +** +** If it is not NULL, string zFormat specifies the format of the +** error string in the style of the printf functions: The following +** format characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** zFormat and any string tokens that follow it are assumed to be +** encoded in UTF-8. +** +** To clear the most recent error for sqlite handle "db", sqlite3Error +** should be called with err_code set to SQLITE_OK and zFormat set +** to NULL. +*/ +void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ + if( db && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ + db->errCode = err_code; + if( zFormat ){ + char *z; + va_list ap; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); + }else{ + sqlite3ValueSetStr(db->pErr, 0, 0, SQLITE_UTF8, SQLITE_STATIC); + } + } +} + +/* +** Add an error message to pParse->zErrMsg and increment pParse->nErr. +** The following formatting characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** This function should be used to report any error that occurs whilst +** compiling an SQL statement (i.e. within sqlite3_prepare()). The +** last thing the sqlite3_prepare() function does is copy the error +** stored by this function into the database handle using sqlite3Error(). +** Function sqlite3Error() should be used during statement execution +** (sqlite3_step() etc.). +*/ +void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ + va_list ap; + sqlite3 *db = pParse->db; + pParse->nErr++; + sqlite3DbFree(db, pParse->zErrMsg); + va_start(ap, zFormat); + pParse->zErrMsg = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + pParse->rc = SQLITE_ERROR; +} + +/* +** Clear the error message in pParse, if any +*/ +void sqlite3ErrorClear(Parse *pParse){ + sqlite3DbFree(pParse->db, pParse->zErrMsg); + pParse->zErrMsg = 0; + pParse->nErr = 0; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** The input string must be zero-terminated. A new zero-terminator +** is added to the dequoted string. +** +** The return value is -1 if no dequoting occurs or the length of the +** dequoted string, exclusive of the zero terminator, if dequoting does +** occur. +** +** 2002-Feb-14: This routine is extended to remove MS-Access style +** brackets from around identifers. For example: "[a-b-c]" becomes +** "a-b-c". +*/ +int sqlite3Dequote(char *z){ + char quote; + int i, j; + if( z==0 ) return -1; + quote = z[0]; + switch( quote ){ + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + case '[': quote = ']'; break; /* For MS SqlServer compatibility */ + default: return -1; + } + for(i=1, j=0; ALWAYS(z[i]); i++){ + if( z[i]==quote ){ + if( z[i+1]==quote ){ + z[j++] = quote; + i++; + }else{ + break; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; + return j; +} + +/* Convenient short-hand */ +#define UpperToLower sqlite3UpperToLower + +/* +** Some systems have stricmp(). Others have strcasecmp(). Because +** there is no consistency, we will define our own. +*/ +int sqlite3StrICmp(const char *zLeft, const char *zRight){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return UpperToLower[*a] - UpperToLower[*b]; +} +int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; +} + +/* +** Return TRUE if z is a pure numeric string. Return FALSE and leave +** *realnum unchanged if the string contains any character which is not +** part of a number. +** +** If the string is pure numeric, set *realnum to TRUE if the string +** contains the '.' character or an "E+000" style exponentiation suffix. +** Otherwise set *realnum to FALSE. Note that just becaue *realnum is +** false does not mean that the number can be successfully converted into +** an integer - it might be too big. +** +** An empty string is considered non-numeric. +*/ +int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ + int incr = (enc==SQLITE_UTF8?1:2); + if( enc==SQLITE_UTF16BE ) z++; + if( *z=='-' || *z=='+' ) z += incr; + if( !sqlite3Isdigit(*z) ){ + return 0; + } + z += incr; + *realnum = 0; + while( sqlite3Isdigit(*z) ){ z += incr; } + if( *z=='.' ){ + z += incr; + if( !sqlite3Isdigit(*z) ) return 0; + while( sqlite3Isdigit(*z) ){ z += incr; } + *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z += incr; + if( *z=='+' || *z=='-' ) z += incr; + if( !sqlite3Isdigit(*z) ) return 0; + while( sqlite3Isdigit(*z) ){ z += incr; } + *realnum = 1; + } + return *z==0; +} + +/* +** The string z[] is an ASCII representation of a real number. +** Convert this string to a double. +** +** This routine assumes that z[] really is a valid number. If it +** is not, the result is undefined. +** +** This routine is used instead of the library atof() function because +** the library atof() might want to use "," as the decimal point instead +** of "." depending on how locale is set. But that would cause problems +** for SQL. So this routine always uses "." regardless of locale. +*/ +int sqlite3AtoF(const char *z, double *pResult){ +#ifndef SQLITE_OMIT_FLOATING_POINT + const char *zBegin = z; + /* sign * significand * (10 ^ (esign * exponent)) */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + double result; + int nDigits = 0; + + /* skip leading spaces */ + while( sqlite3Isspace(*z) ) z++; + /* get sign of significand */ + if( *z=='-' ){ + sign = -1; + z++; + }else if( *z=='+' ){ + z++; + } + /* skip leading zeroes */ + while( z[0]=='0' ) z++, nDigits++; + + /* copy max significant digits to significand */ + while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + s = s*10 + (*z - '0'); + z++, nDigits++; + } + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( sqlite3Isdigit(*z) ) z++, nDigits++, d++; + + /* if decimal point is present */ + if( *z=='.' ){ + z++; + /* copy digits from after decimal to significand + ** (decrease exponent by d to shift decimal right) */ + while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + s = s*10 + (*z - '0'); + z++, nDigits++, d--; + } + /* skip non-significant digits */ + while( sqlite3Isdigit(*z) ) z++, nDigits++; + } + + /* if exponent is present */ + if( *z=='e' || *z=='E' ){ + z++; + /* get sign of exponent */ + if( *z=='-' ){ + esign = -1; + z++; + }else if( *z=='+' ){ + z++; + } + /* copy digits to exponent */ + while( sqlite3Isdigit(*z) ){ + e = e*10 + (*z - '0'); + z++; + } + } + + /* adjust exponent by d, and update sign */ + e = (e*esign) + d; + if( e<0 ) { + esign = -1; + e *= -1; + } else { + esign = 1; + } + + /* if 0 significand */ + if( !s ) { + /* In the IEEE 754 standard, zero is signed. + ** Add the sign if we've seen at least one digit */ + result = (sign<0 && nDigits) ? -(double)0 : (double)0; + } else { + /* attempt to reduce exponent */ + if( esign>0 ){ + while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; + }else{ + while( !(s%10) && e>0 ) e--,s/=10; + } + + /* adjust the sign of significand */ + s = sign<0 ? -s : s; + + /* if exponent, scale significand as appropriate + ** and store in result. */ + if( e ){ + double scale = 1.0; + /* attempt to handle extremely small/large numbers better */ + if( e>307 && e<342 ){ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else{ + /* 1.0e+22 is the largest power of 10 than can be + ** represented exactly. */ + while( e%22 ) { scale *= 1.0e+1; e -= 1; } + while( e>0 ) { scale *= 1.0e+22; e -= 22; } + if( esign<0 ){ + result = s / scale; + }else{ + result = s * scale; + } + } + } else { + result = (double)s; + } + } + + /* store the result */ + *pResult = result; + + /* return number of characters used */ + return (int)(z - zBegin); +#else + return sqlite3Atoi64(z, pResult); +#endif /* SQLITE_OMIT_FLOATING_POINT */ +} + +/* +** Compare the 19-character string zNum against the text representation +** value 2^63: 9223372036854775808. Return negative, zero, or positive +** if zNum is less than, equal to, or greater than the string. +** +** Unlike memcmp() this routine is guaranteed to return the difference +** in the values of the last digit if the only difference is in the +** last digit. So, for example, +** +** compare2pow63("9223372036854775800") +** +** will return -8. +*/ +static int compare2pow63(const char *zNum){ + int c; + c = memcmp(zNum,"922337203685477580",18)*10; + if( c==0 ){ + c = zNum[18] - '8'; + } + return c; +} + + +/* +** Return TRUE if zNum is a 64-bit signed integer and write +** the value of the integer into *pNum. If zNum is not an integer +** or is an integer that is too large to be expressed with 64 bits, +** then return false. +** +** When this routine was originally written it dealt with only +** 32-bit numbers. At that time, it was much faster than the +** atoi() library routine in RedHat 7.2. +*/ +int sqlite3Atoi64(const char *zNum, i64 *pNum){ + i64 v = 0; + int neg; + int i, c; + const char *zStart; + while( sqlite3Isspace(*zNum) ) zNum++; + if( *zNum=='-' ){ + neg = 1; + zNum++; + }else if( *zNum=='+' ){ + neg = 0; + zNum++; + }else{ + neg = 0; + } + zStart = zNum; + while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */ + for(i=0; (c=zNum[i])>='0' && c<='9'; i++){ + v = v*10 + c - '0'; + } + *pNum = neg ? -v : v; + if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ + /* zNum is empty or contains non-numeric text or is longer + ** than 19 digits (thus guaranting that it is too large) */ + return 0; + }else if( i<19 ){ + /* Less than 19 digits, so we know that it fits in 64 bits */ + return 1; + }else{ + /* 19-digit numbers must be no larger than 9223372036854775807 if positive + ** or 9223372036854775808 if negative. Note that 9223372036854665808 + ** is 2^63. */ + return compare2pow63(zNum)='0' && zNum[0]<='9' ); /* zNum is an unsigned number */ + + if( negFlag ) neg = 1-neg; + while( *zNum=='0' ){ + zNum++; /* Skip leading zeros. Ticket #2454 */ + } + for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } + if( i<19 ){ + /* Guaranteed to fit if less than 19 digits */ + return 1; + }else if( i>19 ){ + /* Guaranteed to be too big if greater than 19 digits */ + return 0; + }else{ + /* Compare against 2^63. */ + return compare2pow63(zNum)=0 && c<=9; i++){ + v = v*10 + c; + } + + /* The longest decimal representation of a 32 bit integer is 10 digits: + ** + ** 1234567890 + ** 2^31 -> 2147483648 + */ + if( i>10 ){ + return 0; + } + if( v-neg>2147483647 ){ + return 0; + } + if( neg ){ + v = -v; + } + *pValue = (int)v; + return 1; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +int sqlite3PutVarint(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + +/* +** This routine is a faster version of sqlite3PutVarint() that only +** works for 32-bit positive integers and which is optimized for +** the common case of small integers. A MACRO version, putVarint32, +** is provided which inlines the single-byte case. All code should use +** the MACRO version as this function assumes the single-byte case has +** already been handled. +*/ +int sqlite3PutVarint32(unsigned char *p, u32 v){ +#ifndef putVarint32 + if( (v & ~0x7f)==0 ){ + p[0] = v; + return 1; + } +#endif + if( (v & ~0x3fff)==0 ){ + p[0] = (u8)((v>>7) | 0x80); + p[1] = (u8)(v & 0x7f); + return 2; + } + return sqlite3PutVarint(p, v); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + a = *p; + /* a: p0 (unmasked) */ + if (!(a&0x80)) + { + *v = a; + return 1; + } + + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + a &= 0x7f; + a = a<<7; + a |= b; + *v = a; + return 2; + } + + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= (0x7f<<14)|(0x7f); + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= (0x7f<<14)|(0x7f); + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= (0x7f<<14)|(0x7f); + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= (0x7f<<14)|(0x7f); + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above in calc'ing s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= (0x7f<<14)|(0x7f); + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= (0x1f<<28)|(0x7f<<14)|(0x7f); + b &= (0x7f<<14)|(0x7f); + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= (0x7f<<14)|(0x7f); + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= (0x1f<<28)|(0x7f<<14)|(0x7f); + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= (0x7f<<14)|(0x7f); + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** Read a 32-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +** +** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned +** integer, then set *v to 0xffffffff. +** +** A MACRO version, getVarint32, is provided which inlines the +** single-byte case. All code should use the MACRO version as +** this function assumes the single-byte case has already been handled. +*/ +u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ + u32 a,b; + + /* The 1-byte case. Overwhelmingly the most common. Handled inline + ** by the getVarin32() macro */ + a = *p; + /* a: p0 (unmasked) */ +#ifndef getVarint32 + if (!(a&0x80)) + { + /* Values between 0 and 127 */ + *v = a; + return 1; + } +#endif + + /* The 2-byte case */ + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 128 and 16383 */ + a &= 0x7f; + a = a<<7; + *v = a | b; + return 2; + } + + /* The 3-byte case */ + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 16384 and 2097151 */ + a &= (0x7f<<14)|(0x7f); + b &= 0x7f; + b = b<<7; + *v = a | b; + return 3; + } + + /* A 32-bit varint is used to store size information in btrees. + ** Objects are rarely larger than 2MiB limit of a 3-byte varint. + ** A 3-byte varint is sufficient, for example, to record the size + ** of a 1048569-byte BLOB or string. + ** + ** We only unroll the first 1-, 2-, and 3- byte cases. The very + ** rare larger cases can be handled by the slower 64-bit varint + ** routine. + */ +#if 1 + { + u64 v64; + u8 n; + + p -= 2; + n = sqlite3GetVarint(p, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ + *v = (u32)v64; + } + return n; + } + +#else + /* For following code (kept for historical record only) shows an + ** unrolling for the 3- and 4-byte varint cases. This code is + ** slightly faster, but it is also larger and much harder to test. + */ + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 2097152 and 268435455 */ + b &= (0x7f<<14)|(0x7f); + a &= (0x7f<<14)|(0x7f); + a = a<<7; + *v = a | b; + return 4; + } + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* Walues between 268435456 and 34359738367 */ + a &= (0x1f<<28)|(0x7f<<14)|(0x7f); + b &= (0x1f<<28)|(0x7f<<14)|(0x7f); + b = b<<7; + *v = a | b; + return 5; + } + + /* We can only reach this point when reading a corrupt database + ** file. In that case we are not in any hurry. Use the (relatively + ** slow) general-purpose sqlite3GetVarint() routine to extract the + ** value. */ + { + u64 v64; + u8 n; + + p -= 4; + n = sqlite3GetVarint(p, &v64); + assert( n>5 && n<=9 ); + *v = (u32)v64; + return n; + } +#endif +} + +/* +** Return the number of bytes that will be needed to store the given +** 64-bit integer. +*/ +int sqlite3VarintLen(u64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 && ALWAYS(i<9) ); + return i; +} + + +/* +** Read or write a four-byte big-endian integer value. +*/ +u32 sqlite3Get4byte(const u8 *p){ + return (p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +} +void sqlite3Put4byte(unsigned char *p, u32 v){ + p[0] = (u8)(v>>24); + p[1] = (u8)(v>>16); + p[2] = (u8)(v>>8); + p[3] = (u8)v; +} + + + +#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +static u8 hexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} +#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ + +#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) +/* +** Convert a BLOB literal of the form "x'hhhhhh'" into its binary +** value. Return a pointer to its binary value. Space to hold the +** binary value has been obtained from malloc and must be freed by +** the calling routine. +*/ +void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ + char *zBlob; + int i; + + zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); + n--; + if( zBlob ){ + for(i=0; imagic is not a valid open value, take care not +** to modify the db structure at all. It could be that db is a stale +** pointer. In other words, it could be that there has been a prior +** call to sqlite3_close(db) and db has been deallocated. And we do +** not want to write into deallocated memory. +*/ +#ifdef SQLITE_DEBUG +int sqlite3SafetyOn(sqlite3 *db){ + if( db->magic==SQLITE_MAGIC_OPEN ){ + db->magic = SQLITE_MAGIC_BUSY; + assert( sqlite3_mutex_held(db->mutex) ); + return 0; + }else if( db->magic==SQLITE_MAGIC_BUSY ){ + db->magic = SQLITE_MAGIC_ERROR; + db->u1.isInterrupted = 1; + } + return 1; +} +#endif + +/* +** Change the magic from SQLITE_MAGIC_BUSY to SQLITE_MAGIC_OPEN. +** Return an error (non-zero) if the magic was not SQLITE_MAGIC_BUSY +** when this routine is called. +*/ +#ifdef SQLITE_DEBUG +int sqlite3SafetyOff(sqlite3 *db){ + if( db->magic==SQLITE_MAGIC_BUSY ){ + db->magic = SQLITE_MAGIC_OPEN; + assert( sqlite3_mutex_held(db->mutex) ); + return 0; + }else{ + db->magic = SQLITE_MAGIC_ERROR; + db->u1.isInterrupted = 1; + return 1; + } +} +#endif + +/* +** Check to make sure we have a valid db pointer. This test is not +** foolproof but it does provide some measure of protection against +** misuse of the interface such as passing in db pointers that are +** NULL or which have been previously closed. If this routine returns +** 1 it means that the db pointer is valid and 0 if it should not be +** dereferenced for any reason. The calling function should invoke +** SQLITE_MISUSE immediately. +** +** sqlite3SafetyCheckOk() requires that the db pointer be valid for +** use. sqlite3SafetyCheckSickOrOk() allows a db pointer that failed to +** open properly and is not fit for general use but which can be +** used as an argument to sqlite3_errmsg() or sqlite3_close(). +*/ +int sqlite3SafetyCheckOk(sqlite3 *db){ + u32 magic; + if( db==0 ) return 0; + magic = db->magic; + if( magic!=SQLITE_MAGIC_OPEN +#ifdef SQLITE_DEBUG + && magic!=SQLITE_MAGIC_BUSY +#endif + ){ + return 0; + }else{ + return 1; + } +} +int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ + u32 magic; + magic = db->magic; + if( magic!=SQLITE_MAGIC_SICK && + magic!=SQLITE_MAGIC_OPEN && + magic!=SQLITE_MAGIC_BUSY ) return 0; + return 1; +} diff --git a/src/vacuum.c b/src/vacuum.c index 8373b38..43545f3 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -13,8 +13,6 @@ ** ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. -** -** $Id: vacuum.c,v 1.91 2009/07/02 07:47:33 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" diff --git a/src/vdbe.c b/src/vdbe.c index 3e316fc..10401a4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -42,8 +42,6 @@ ** of the code in this file is, therefore, important. See other comments ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. -** -** $Id: vdbe.c,v 1.874 2009/07/24 17:58:53 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -172,23 +170,6 @@ void sqlite3VdbeMemStoreType(Mem *pMem){ } } -/* -** Properties of opcodes. The OPFLG_INITIALIZER macro is -** created by mkopcodeh.awk during compilation. Data is obtained -** from the comments following the "case OP_xxxx:" statements in -** this file. -*/ -static const unsigned char opcodeProperty[] = OPFLG_INITIALIZER; - -/* -** Return true if an opcode has any of the OPFLG_xxx properties -** specified by mask. -*/ -int sqlite3VdbeOpcodeHasProperty(int opcode, int mask){ - assert( opcode>0 && opcode<(int)sizeof(opcodeProperty) ); - return (opcodeProperty[opcode]&mask)!=0; -} - /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -223,7 +204,7 @@ static VdbeCursor *allocateCursor( int nByte; VdbeCursor *pCx = 0; nByte = - sizeof(VdbeCursor) + + ROUND8(sizeof(VdbeCursor)) + (isBtreeCursor?sqlite3BtreeCursorSize():0) + 2*nField*sizeof(u32); @@ -234,15 +215,16 @@ static VdbeCursor *allocateCursor( } if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; - memset(pMem->z, 0, nByte); + memset(pCx, 0, sizeof(VdbeCursor)); pCx->iDb = iDb; pCx->nField = nField; if( nField ){ - pCx->aType = (u32 *)&pMem->z[sizeof(VdbeCursor)]; + pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))]; } if( isBtreeCursor ){ pCx->pCursor = (BtCursor*) - &pMem->z[sizeof(VdbeCursor)+2*nField*sizeof(u32)]; + &pMem->z[ROUND8(sizeof(VdbeCursor))+2*nField*sizeof(u32)]; + sqlite3BtreeCursorZero(pCx->pCursor); } } return pCx; @@ -557,23 +539,26 @@ int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ int pc; /* The program counter */ + Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp; /* Current operation */ int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ + u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */ u8 encoding = ENC(db); /* The database encoding */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int checkProgress; /* True if progress callbacks are enabled */ + int nProgressOps = 0; /* Opcodes executed since progress callback. */ +#endif + Mem *aMem = p->aMem; /* Copy of p->aMem */ Mem *pIn1 = 0; /* 1st input operand */ Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ - u8 opProperty; int iCompare = 0; /* Result of last OP_Compare operation */ int *aPermute = 0; /* Permutation of columns for OP_Compare */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ int origPc; /* Program counter at start of opcode */ -#endif -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - int nProgressOps = 0; /* Opcodes executed since progress callback. */ #endif /*** INSERT STACK UNION HERE ***/ @@ -592,6 +577,9 @@ int sqlite3VdbeExec( db->busyHandler.nBusy = 0; CHECK_FOR_INTERRUPT; sqlite3VdbeIOTraceSql(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + checkProgress = db->xProgress!=0; +#endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); if( p->pc==0 @@ -601,7 +589,7 @@ int sqlite3VdbeExec( printf("VDBE Program Listing:\n"); sqlite3VdbePrintSql(p); for(i=0; inOp; i++){ - sqlite3VdbePrintOp(stdout, i, &p->aOp[i]); + sqlite3VdbePrintOp(stdout, i, &aOp[i]); } } if( fileExists(db, "vdbe_trace") ){ @@ -616,7 +604,7 @@ int sqlite3VdbeExec( origPc = pc; start = sqlite3Hwtime(); #endif - pOp = &p->aOp[pc]; + pOp = &aOp[pc]; /* Only allow tracing if SQLITE_DEBUG is defined. */ @@ -657,7 +645,7 @@ int sqlite3VdbeExec( ** If the progress callback returns non-zero, exit the virtual machine with ** a return code SQLITE_ABORT. */ - if( db->xProgress ){ + if( checkProgress ){ if( db->nProgressOps==nProgressOps ){ int prc; if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; @@ -673,66 +661,47 @@ int sqlite3VdbeExec( } #endif - /* Do common setup processing for any opcode that is marked - ** with the "out2-prerelease" tag. Such opcodes have a single - ** output which is specified by the P2 parameter. The P2 register - ** is initialized to a NULL. + /* On any opcode with the "out2-prerelase" tag, free any + ** external allocations out of mem[p2] and set mem[p2] to be + ** an undefined integer. Opcodes will either fill in the integer + ** value or convert mem[p2] to a different type. */ - opProperty = opcodeProperty[pOp->opcode]; - if( (opProperty & OPFLG_OUT2_PRERELEASE)!=0 ){ + assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); + if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); - pOut = &p->aMem[pOp->p2]; + pOut = &aMem[pOp->p2]; sqlite3VdbeMemReleaseExternal(pOut); - pOut->flags = MEM_Null; - pOut->n = 0; - }else - - /* Do common setup for opcodes marked with one of the following - ** combinations of properties. - ** - ** in1 - ** in1 in2 - ** in1 in2 out3 - ** in1 in3 - ** - ** Variables pIn1, pIn2, and pIn3 are made to point to appropriate - ** registers for inputs. Variable pOut points to the output register. - */ - if( (opProperty & OPFLG_IN1)!=0 ){ - assert( pOp->p1>0 ); - assert( pOp->p1<=p->nMem ); - pIn1 = &p->aMem[pOp->p1]; - REGISTER_TRACE(pOp->p1, pIn1); - if( (opProperty & OPFLG_IN2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - pIn2 = &p->aMem[pOp->p2]; - REGISTER_TRACE(pOp->p2, pIn2); - /* As currently implemented, in2 implies out3. There is no reason - ** why this has to be, it just worked out that way. */ - assert( (opProperty & OPFLG_OUT3)!=0 ); - assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); - pOut = &p->aMem[pOp->p3]; - }else if( (opProperty & OPFLG_IN3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); - pIn3 = &p->aMem[pOp->p3]; - REGISTER_TRACE(pOp->p3, pIn3); - } - }else if( (opProperty & OPFLG_IN2)!=0 ){ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - pIn2 = &p->aMem[pOp->p2]; - REGISTER_TRACE(pOp->p2, pIn2); - }else if( (opProperty & OPFLG_IN3)!=0 ){ - assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); - pIn3 = &p->aMem[pOp->p3]; - REGISTER_TRACE(pOp->p3, pIn3); + pOut->flags = MEM_Int; } + /* Sanity checking on other operands */ +#ifdef SQLITE_DEBUG + if( (pOp->opflags & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=p->nMem ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (pOp->opflags & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=p->nMem ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (pOp->opflags & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=p->nMem ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (pOp->opflags & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=p->nMem ); + } + if( (pOp->opflags & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=p->nMem ); + } +#endif + switch( pOp->opcode ){ /***************************************************************************** @@ -788,10 +757,8 @@ case OP_Goto: { /* jump */ ** Write the current address onto register P1 ** and then jump to address P2. */ -case OP_Gosub: { /* jump */ - assert( pOp->p1>0 ); - assert( pOp->p1<=p->nMem ); - pIn1 = &p->aMem[pOp->p1]; +case OP_Gosub: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); pIn1->flags = MEM_Int; pIn1->u.i = pc; @@ -805,6 +772,7 @@ case OP_Gosub: { /* jump */ ** Jump to the next instruction after the address in register P1. */ case OP_Return: { /* in1 */ + pIn1 = &aMem[pOp->p1]; assert( pIn1->flags & MEM_Int ); pc = (int)pIn1->u.i; break; @@ -816,6 +784,7 @@ case OP_Return: { /* in1 */ */ case OP_Yield: { /* in1 */ int pcDest; + pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); pIn1->flags = MEM_Int; pcDest = (int)pIn1->u.i; @@ -832,6 +801,7 @@ case OP_Yield: { /* in1 */ ** value in register P3 is not NULL, then this routine is a no-op. */ case OP_HaltIfNull: { /* in3 */ + pIn3 = &aMem[pOp->p3]; if( (pIn3->flags & MEM_Null)==0 ) break; /* Fall through into OP_Halt */ } @@ -871,6 +841,8 @@ case OP_Halt: { ** as the p2 of the calling OP_Program. */ pc = p->aOp[pc].p2-1; } + aOp = p->aOp; + aMem = p->aMem; break; } @@ -897,7 +869,6 @@ case OP_Halt: { ** The 32-bit integer value P1 is written into register P2. */ case OP_Integer: { /* out2-prerelease */ - pOut->flags = MEM_Int; pOut->u.i = pOp->p1; break; } @@ -909,7 +880,6 @@ case OP_Integer: { /* out2-prerelease */ */ case OP_Int64: { /* out2-prerelease */ assert( pOp->p4.pI64!=0 ); - pOut->flags = MEM_Int; pOut->u.i = *pOp->p4.pI64; break; } @@ -979,6 +949,7 @@ case OP_String: { /* out2-prerelease */ ** Write a NULL into register P2. */ case OP_Null: { /* out2-prerelease */ + pOut->flags = MEM_Null; break; } @@ -1026,7 +997,7 @@ case OP_Variable: { if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } - pOut = &p->aMem[p2++]; + pOut = &aMem[p2++]; sqlite3VdbeMemReleaseExternal(pOut); pOut->flags = MEM_Null; sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); @@ -1054,11 +1025,11 @@ case OP_Move: { assert( n>0 && p1>0 && p2>0 ); assert( p1+n<=p2 || p2+n<=p1 ); - pIn1 = &p->aMem[p1]; - pOut = &p->aMem[p2]; + pIn1 = &aMem[p1]; + pOut = &aMem[p2]; while( n-- ){ - assert( pOut<=&p->aMem[p->nMem] ); - assert( pIn1<=&p->aMem[p->nMem] ); + assert( pOut<=&aMem[p->nMem] ); + assert( pIn1<=&aMem[p->nMem] ); zMalloc = pOut->zMalloc; pOut->zMalloc = 0; sqlite3VdbeMemMove(pOut, pIn1); @@ -1077,10 +1048,9 @@ case OP_Move: { ** This instruction makes a deep copy of the value. A duplicate ** is made of any string or blob constant. See also OP_SCopy. */ -case OP_Copy: { /* in1 */ - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - pOut = &p->aMem[pOp->p2]; +case OP_Copy: { /* in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); Deephemeralize(pOut); @@ -1100,11 +1070,9 @@ case OP_Copy: { /* in1 */ ** during the lifetime of the copy. Use OP_Copy to make a complete ** copy. */ -case OP_SCopy: { /* in1 */ - REGISTER_TRACE(pOp->p1, pIn1); - assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); - pOut = &p->aMem[pOp->p2]; +case OP_SCopy: { /* in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); REGISTER_TRACE(pOp->p2, pOut); @@ -1163,7 +1131,7 @@ case OP_ResultRow: { ** and have an assigned type. The results are de-ephemeralized as ** as side effect. */ - pMem = p->pResultSet = &p->aMem[pOp->p1]; + pMem = p->pResultSet = &aMem[pOp->p1]; for(i=0; ip2; i++){ sqlite3VdbeMemNulTerminate(&pMem[i]); sqlite3VdbeMemStoreType(&pMem[i]); @@ -1193,6 +1161,9 @@ case OP_ResultRow: { case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ i64 nByte; + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; assert( pIn1!=pOut ); if( (pIn1->flags | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); @@ -1266,8 +1237,11 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rA; /* Real value of left operand */ double rB; /* Real value of right operand */ + pIn1 = &aMem[pOp->p1]; applyNumericAffinity(pIn1); + pIn2 = &aMem[pOp->p2]; applyNumericAffinity(pIn2); + pOut = &aMem[pOp->p3]; flags = pIn1->flags | pIn2->flags; if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ @@ -1382,7 +1356,7 @@ case OP_Function: { assert( n==0 || (pOp->p2>0 && pOp->p2+n<=p->nMem+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pArg = &p->aMem[pOp->p2]; + pArg = &aMem[pOp->p2]; for(i=0; ip3>0 && pOp->p3<=p->nMem ); - pOut = &p->aMem[pOp->p3]; + pOut = &aMem[pOp->p3]; ctx.s.flags = MEM_Null; ctx.s.db = db; ctx.s.xDel = 0; @@ -1414,7 +1388,7 @@ case OP_Function: { ctx.isError = 0; if( ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); + assert( pOp>aOp ); assert( pOp[-1].p4type==P4_COLLSEQ ); assert( pOp[-1].opcode==OP_CollSeq ); ctx.pColl = pOp[-1].p4.pColl; @@ -1498,6 +1472,9 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ i64 a; i64 b; + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; if( (pIn1->flags | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; @@ -1524,6 +1501,7 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ ** To force any register to be an integer, just add 0. */ case OP_AddImm: { /* in1 */ + pIn1 = &aMem[pOp->p1]; sqlite3VdbeMemIntegerify(pIn1); pIn1->u.i += pOp->p2; break; @@ -1537,6 +1515,7 @@ case OP_AddImm: { /* in1 */ ** raise an SQLITE_MISMATCH exception. */ case OP_MustBeInt: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); if( (pIn1->flags & MEM_Int)==0 ){ if( pOp->p2==0 ){ @@ -1561,6 +1540,7 @@ case OP_MustBeInt: { /* jump, in1 */ ** to have only a real value. */ case OP_RealAffinity: { /* in1 */ + pIn1 = &aMem[pOp->p1]; if( pIn1->flags & MEM_Int ){ sqlite3VdbeMemRealify(pIn1); } @@ -1578,6 +1558,7 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToText: { /* same as TK_TO_TEXT, in1 */ + pIn1 = &aMem[pOp->p1]; if( pIn1->flags & MEM_Null ) break; assert( MEM_Str==(MEM_Blob>>3) ); pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; @@ -1599,6 +1580,7 @@ case OP_ToText: { /* same as TK_TO_TEXT, in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ + pIn1 = &aMem[pOp->p1]; if( pIn1->flags & MEM_Null ) break; if( (pIn1->flags & MEM_Blob)==0 ){ applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); @@ -1622,6 +1604,7 @@ case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ + pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){ sqlite3VdbeMemNumerify(pIn1); } @@ -1639,6 +1622,7 @@ case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToInt: { /* same as TK_TO_INT, in1 */ + pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Null)==0 ){ sqlite3VdbeMemIntegerify(pIn1); } @@ -1656,6 +1640,7 @@ case OP_ToInt: { /* same as TK_TO_INT, in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ + pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Null)==0 ){ sqlite3VdbeMemRealify(pIn1); } @@ -1744,6 +1729,8 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ int res; /* Result of the comparison of pIn1 against pIn3 */ char affinity; /* Affinity to use for comparison */ + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; if( (pIn1->flags | pIn3->flags)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ @@ -1759,7 +1746,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. */ if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &p->aMem[pOp->p2]; + pOut = &aMem[pOp->p2]; MemSetTypeFlag(pOut, MEM_Null); REGISTER_TRACE(pOp->p2, pOut); }else if( pOp->p5 & SQLITE_JUMPIFNULL ){ @@ -1791,7 +1778,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &p->aMem[pOp->p2]; + pOut = &aMem[pOp->p2]; MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = res; REGISTER_TRACE(pOp->p2, pOut); @@ -1860,12 +1847,12 @@ case OP_Compare: { #endif /* SQLITE_DEBUG */ for(i=0; iaMem[p1+idx]); - REGISTER_TRACE(p2+idx, &p->aMem[p2+idx]); + REGISTER_TRACE(p1+idx, &aMem[p1+idx]); + REGISTER_TRACE(p2+idx, &aMem[p2+idx]); assert( inField ); pColl = pKeyInfo->aColl[i]; bRev = pKeyInfo->aSortOrder[i]; - iCompare = sqlite3MemCompare(&p->aMem[p1+idx], &p->aMem[p2+idx], pColl); + iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); if( iCompare ){ if( bRev ) iCompare = -iCompare; break; @@ -1915,11 +1902,13 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + pIn1 = &aMem[pOp->p1]; if( pIn1->flags & MEM_Null ){ v1 = 2; }else{ v1 = sqlite3VdbeIntValue(pIn1)!=0; } + pIn2 = &aMem[pOp->p2]; if( pIn2->flags & MEM_Null ){ v2 = 2; }else{ @@ -1932,6 +1921,7 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 }; v1 = or_logic[v1*3+v2]; } + pOut = &aMem[pOp->p3]; if( v1==2 ){ MemSetTypeFlag(pOut, MEM_Null); }else{ @@ -1947,8 +1937,9 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ ** boolean complement in register P2. If the value in register P1 is ** NULL, then a NULL is stored in P2. */ -case OP_Not: { /* same as TK_NOT, in1 */ - pOut = &p->aMem[pOp->p2]; +case OP_Not: { /* same as TK_NOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; if( pIn1->flags & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); }else{ @@ -1963,8 +1954,9 @@ case OP_Not: { /* same as TK_NOT, in1 */ ** ones-complement of the P1 value into register P2. If P1 holds ** a NULL then store a NULL in P2. */ -case OP_BitNot: { /* same as TK_BITNOT, in1 */ - pOut = &p->aMem[pOp->p2]; +case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; if( pIn1->flags & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); }else{ @@ -1988,6 +1980,7 @@ case OP_BitNot: { /* same as TK_BITNOT, in1 */ case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ int c; + pIn1 = &aMem[pOp->p1]; if( pIn1->flags & MEM_Null ){ c = pOp->p3; }else{ @@ -2009,6 +2002,7 @@ case OP_IfNot: { /* jump, in1 */ ** Jump to P2 if the value in register P1 is NULL. */ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Null)!=0 ){ pc = pOp->p2 - 1; } @@ -2020,6 +2014,7 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ ** Jump to P2 if the value in register P1 is not NULL. */ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Null)==0 ){ pc = pOp->p2 - 1; } @@ -2076,7 +2071,7 @@ case OP_Column: { memset(&sMem, 0, sizeof(sMem)); assert( p1nCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pDest = &p->aMem[pOp->p3]; + pDest = &aMem[pOp->p3]; MemSetTypeFlag(pDest, MEM_Null); zRec = 0; @@ -2122,7 +2117,7 @@ case OP_Column: { assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } }else if( pC->pseudoTableReg>0 ){ - pReg = &p->aMem[pC->pseudoTableReg]; + pReg = &aMem[pC->pseudoTableReg]; assert( pReg->flags & MEM_Blob ); payloadSize = pReg->n; zRec = pReg->z; @@ -2333,17 +2328,18 @@ op_column_out: ** memory cell in the range. */ case OP_Affinity: { - char *zAffinity; /* The affinity to be applied */ - Mem *pData0; /* First register to which to apply affinity */ - Mem *pLast; /* Last register to which to apply affinity */ - Mem *pRec; /* Current register */ + const char *zAffinity; /* The affinity to be applied */ + char cAff; /* A single character of affinity */ zAffinity = pOp->p4.z; - pData0 = &p->aMem[pOp->p1]; - pLast = &pData0[pOp->p2-1]; - for(pRec=pData0; pRec<=pLast; pRec++){ - ExpandBlob(pRec); - applyAffinity(pRec, zAffinity[pRec-pData0], encoding); + assert( zAffinity!=0 ); + assert( zAffinity[pOp->p2]==0 ); + pIn1 = &aMem[pOp->p1]; + while( (cAff = *(zAffinity++))!=0 ){ + assert( pIn1 <= &p->aMem[p->nMem] ); + ExpandBlob(pIn1); + applyAffinity(pIn1, cAff, encoding); + pIn1++; } break; } @@ -2405,7 +2401,7 @@ case OP_MakeRecord: { nField = pOp->p1; zAffinity = pOp->p4.z; assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=p->nMem+1 ); - pData0 = &p->aMem[nField]; + pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; file_format = p->minWriteFileFormat; @@ -2449,7 +2445,7 @@ case OP_MakeRecord: { ** sqlite3VdbeMemGrow() could clobber the value before it is used). */ assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); - pOut = &p->aMem[pOp->p3]; + pOut = &aMem[pOp->p3]; if( sqlite3VdbeMemGrow(pOut, (int)nByte, 0) ){ goto no_mem; } @@ -2496,7 +2492,6 @@ case OP_Count: { /* out2-prerelease */ }else{ nEntry = 0; } - pOut->flags = MEM_Int; pOut->u.i = nEntry; break; } @@ -2817,7 +2812,6 @@ case OP_ReadCookie: { /* out2-prerelease */ sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); pOut->u.i = iMeta; - MemSetTypeFlag(pOut, MEM_Int); break; } @@ -2838,6 +2832,7 @@ case OP_SetCookie: { /* in3 */ assert( (p->btreeMask & (1<p1))!=0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); + pIn3 = &aMem[pOp->p3]; sqlite3VdbeMemIntegerify(pIn3); /* See note about index shifting on OP_ReadCookie */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); @@ -2996,7 +2991,7 @@ case OP_OpenWrite: { if( pOp->p5 ){ assert( p2>0 ); assert( p2<=p->nMem ); - pIn2 = &p->aMem[p2]; + pIn2 = &aMem[p2]; sqlite3VdbeMemIntegerify(pIn2); p2 = (int)pIn2->u.i; /* The p2 value always comes from a prior OP_CreateTable opcode and @@ -3212,6 +3207,9 @@ case OP_SeekGt: { /* jump, in3 */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->pseudoTableReg==0 ); + assert( OP_SeekLe == OP_SeekLt+1 ); + assert( OP_SeekGe == OP_SeekLt+2 ); + assert( OP_SeekGt == OP_SeekLt+3 ); if( pC->pCursor!=0 ){ oc = pOp->opcode; pC->nullRow = 0; @@ -3219,6 +3217,7 @@ case OP_SeekGt: { /* jump, in3 */ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so covert it. */ + pIn3 = &aMem[pOp->p3]; applyNumericAffinity(pIn3); iKey = sqlite3VdbeIntValue(pIn3); pC->rowidIsValid = 0; @@ -3241,12 +3240,12 @@ case OP_SeekGt: { /* jump, in3 */ ** integer. */ res = 1; if( pIn3->r<0 ){ - if( oc==OP_SeekGt || oc==OP_SeekGe ){ + if( oc>=OP_SeekGe ){ assert( oc==OP_SeekGe || oc==OP_SeekGt ); rc = sqlite3BtreeFirst(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ - if( oc==OP_SeekLt || oc==OP_SeekLe ){ + if( oc<=OP_SeekLe ){ assert( oc==OP_SeekLt || oc==OP_SeekLe ); rc = sqlite3BtreeLast(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; } @@ -3278,12 +3277,22 @@ case OP_SeekGt: { /* jump, in3 */ assert( nField>0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)nField; - if( oc==OP_SeekGt || oc==OP_SeekLe ){ - r.flags = UNPACKED_INCRKEY; - }else{ - r.flags = 0; - } - r.aMem = &p->aMem[pOp->p3]; + + /* The next line of code computes as follows, only faster: + ** if( oc==OP_SeekGt || oc==OP_SeekLe ){ + ** r.flags = UNPACKED_INCRKEY; + ** }else{ + ** r.flags = 0; + ** } + */ + r.flags = (u16)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); + assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); + assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); + assert( oc!=OP_SeekGe || r.flags==0 ); + assert( oc!=OP_SeekLt || r.flags==0 ); + + r.aMem = &aMem[pOp->p3]; + ExpandBlob(r.aMem); rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, &r, 0, 0, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; @@ -3295,7 +3304,7 @@ case OP_SeekGt: { /* jump, in3 */ #ifdef SQLITE_TEST sqlite3_search_count++; #endif - if( oc==OP_SeekGe || oc==OP_SeekGt ){ + if( oc>=OP_SeekGe ){ assert( oc==OP_SeekGe || oc==OP_SeekGt ); if( res<0 || (res==0 && oc==OP_SeekGt) ){ rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; @@ -3348,6 +3357,7 @@ case OP_Seek: { /* in2 */ if( ALWAYS(pC->pCursor!=0) ){ assert( pC->isTable ); pC->nullRow = 0; + pIn2 = &aMem[pOp->p2]; pC->movetoTarget = sqlite3VdbeIntValue(pIn2); pC->rowidIsValid = 0; pC->deferredMoveto = 1; @@ -3356,33 +3366,27 @@ case OP_Seek: { /* in2 */ } -/* Opcode: Found P1 P2 P3 * * +/* Opcode: Found P1 P2 P3 P4 * ** -** Register P3 holds a blob constructed by MakeRecord. P1 is an index. -** If an entry that matches the value in register p3 exists in P1 then -** jump to P2. If the P3 value does not match any entry in P1 -** then fall thru. The P1 cursor is left pointing at the matching entry -** if it exists. +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. ** -** This instruction is used to implement the IN operator where the -** left-hand side is a SELECT statement. P1 may be a true index, or it -** may be a temporary index that holds the results of the SELECT -** statement. This instruction is also used to implement the -** DISTINCT keyword in SELECT statements. -** -** This instruction checks if index P1 contains a record for which -** the first N serialized values exactly match the N serialized values -** in the record in register P3, where N is the total number of values in -** the P3 record (the P3 record is a prefix of the P1 record). -** -** See also: NotFound, IsUnique, NotExists +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is a prefix of any entry in P1 then a jump is made to P2 and +** P1 is left pointing at the matching entry. */ -/* Opcode: NotFound P1 P2 P3 * * +/* Opcode: NotFound P1 P2 P3 P4 * ** -** Register P3 holds a blob constructed by MakeRecord. P1 is -** an index. If no entry exists in P1 that matches the blob then jump -** to P2. If an entry does existing, fall through. The cursor is left -** pointing to the entry that matches. +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is not the prefix of any entry in P1 then a jump is made to P2. If P1 +** does contain an entry whose prefix matches the P3/P4 record then control +** falls through to the next instruction and P1 is left pointing at the +** matching entry. ** ** See also: Found, NotExists, IsUnique */ @@ -3392,6 +3396,7 @@ case OP_Found: { /* jump, in3 */ VdbeCursor *pC; int res; UnpackedRecord *pIdxKey; + UnpackedRecord r; char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; #ifdef SQLITE_TEST @@ -3400,23 +3405,33 @@ case OP_Found: { /* jump, in3 */ alreadyExists = 0; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + pIn3 = &aMem[pOp->p3]; if( ALWAYS(pC->pCursor!=0) ){ assert( pC->isTable==0 ); - assert( pIn3->flags & MEM_Blob ); - ExpandBlob(pIn3); - pIdxKey = sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, - aTempRec, sizeof(aTempRec)); - if( pIdxKey==0 ){ - goto no_mem; - } - if( pOp->opcode==OP_Found ){ + if( pOp->p4.i>0 ){ + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p4.i; + r.aMem = pIn3; + r.flags = UNPACKED_PREFIX_MATCH; + pIdxKey = &r; + }else{ + assert( pIn3->flags & MEM_Blob ); + ExpandBlob(pIn3); + pIdxKey = sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, + aTempRec, sizeof(aTempRec)); + if( pIdxKey==0 ){ + goto no_mem; + } pIdxKey->flags |= UNPACKED_PREFIX_MATCH; } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + if( pOp->p4.i==0 ){ + sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + } if( rc!=SQLITE_OK ){ break; } @@ -3434,9 +3449,10 @@ case OP_Found: { /* jump, in3 */ /* Opcode: IsUnique P1 P2 P3 P4 * ** -** Cursor P1 is open on an index. So it has no data and its key consists -** of a record generated by OP_MakeRecord where the last field is the -** rowid of the entry that the index refers to. +** Cursor P1 is open on an index b-tree - that is to say, a btree which +** no data and where the key are records generated by OP_MakeRecord with +** the list field being the integer ROWID of the entry that the index +** entry refers to. ** ** The P3 register contains an integer record number. Call this record ** number R. Register P4 is the first in a set of N contiguous registers @@ -3462,11 +3478,12 @@ case OP_IsUnique: { /* jump, in3 */ VdbeCursor *pCx; BtCursor *pCrsr; u16 nField; - Mem *aMem; + Mem *aMx; UnpackedRecord r; /* B-Tree index search key */ i64 R; /* Rowid stored in register P3 */ - aMem = &p->aMem[pOp->p4.i]; + pIn3 = &aMem[pOp->p3]; + aMx = &aMem[pOp->p4.i]; /* Assert that the values of parameters P1 and P4 are in range. */ assert( pOp->p4type==P4_INT32 ); assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem ); @@ -3482,20 +3499,20 @@ case OP_IsUnique: { /* jump, in3 */ /* If any of the values are NULL, take the jump. */ nField = pCx->pKeyInfo->nField; for(ii=0; iip2 - 1; pCrsr = 0; break; } } - assert( (aMem[nField].flags & MEM_Null)==0 ); + assert( (aMx[nField].flags & MEM_Null)==0 ); if( pCrsr!=0 ){ /* Populate the index search key. */ r.pKeyInfo = pCx->pKeyInfo; r.nField = nField + 1; r.flags = UNPACKED_PREFIX_SEARCH; - r.aMem = aMem; + r.aMem = aMx; /* Extract the value of R from register P3. */ sqlite3VdbeMemIntegerify(pIn3); @@ -3534,6 +3551,7 @@ case OP_NotExists: { /* jump, in3 */ int res; u64 iKey; + pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -3577,7 +3595,6 @@ case OP_Sequence: { /* out2-prerelease */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( p->apCsr[pOp->p1]!=0 ); pOut->u.i = p->apCsr[pOp->p1]->seqCount++; - MemSetTypeFlag(pOut, MEM_Int); break; } @@ -3671,7 +3688,7 @@ case OP_NewRowid: { /* out2-prerelease */ }else{ /* Assert that P3 is a valid memory cell. */ assert( pOp->p3<=p->nMem ); - pMem = &p->aMem[pOp->p3]; + pMem = &aMem[pOp->p3]; } REGISTER_TRACE(pOp->p3, pMem); @@ -3714,7 +3731,6 @@ case OP_NewRowid: { /* out2-prerelease */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } - MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = v; break; } @@ -3775,7 +3791,7 @@ case OP_InsertInt: { const char *zTbl; /* Table name - used by the opdate hook */ int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ - pData = &p->aMem[pOp->p2]; + pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -3785,7 +3801,7 @@ case OP_InsertInt: { REGISTER_TRACE(pOp->p2, pData); if( pOp->opcode==OP_Insert ){ - pKey = &p->aMem[pOp->p3]; + pKey = &aMem[pOp->p3]; assert( pKey->flags & MEM_Int ); REGISTER_TRACE(pOp->p3, pKey); iKey = pKey->u.i; @@ -3933,7 +3949,7 @@ case OP_RowData: { u32 n; i64 n64; - pOut = &p->aMem[pOp->p2]; + pOut = &aMem[pOp->p2]; /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -4006,7 +4022,7 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC!=0 ); assert( pC->pseudoTableReg==0 ); if( pC->nullRow ){ - /* Do nothing so that reg[P2] remains NULL */ + pOut->flags = MEM_Null; break; }else if( pC->deferredMoveto ){ v = pC->movetoTarget; @@ -4034,7 +4050,6 @@ case OP_Rowid: { /* out2-prerelease */ } } pOut->u.i = v; - MemSetTypeFlag(pOut, MEM_Int); break; } @@ -4219,6 +4234,7 @@ case OP_IdxInsert: { /* in2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); pCrsr = pC->pCursor; if( ALWAYS(pCrsr!=0) ){ @@ -4259,7 +4275,7 @@ case OP_IdxDelete: { r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.flags = 0; - r.aMem = &p->aMem[pOp->p2]; + r.aMem = &aMem[pOp->p2]; rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); if( rc==SQLITE_OK && res==0 ){ rc = sqlite3BtreeDelete(pCrsr); @@ -4287,6 +4303,7 @@ case OP_IdxRowid: { /* out2-prerelease */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->pCursor; + pOut->flags = MEM_Null; if( ALWAYS(pCrsr!=0) ){ rc = sqlite3VdbeCursorMoveto(pC); if( NEVER(rc) ) goto abort_due_to_error; @@ -4297,8 +4314,8 @@ case OP_IdxRowid: { /* out2-prerelease */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = rowid; + pOut->flags = MEM_Int; } } break; @@ -4330,8 +4347,8 @@ case OP_IdxRowid: { /* out2-prerelease */ ** If P5 is non-zero then the key value is increased by an epsilon prior ** to the comparison. This makes the opcode work like IdxLE. */ -case OP_IdxLT: /* jump, in3 */ -case OP_IdxGE: { /* jump, in3 */ +case OP_IdxLT: /* jump */ +case OP_IdxGE: { /* jump */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -4350,7 +4367,7 @@ case OP_IdxGE: { /* jump, in3 */ }else{ r.flags = UNPACKED_IGNORE_ROWID; } - r.aMem = &p->aMem[pOp->p3]; + r.aMem = &aMem[pOp->p3]; rc = sqlite3VdbeIdxKeyCompare(pC, &r, &res); if( pOp->opcode==OP_IdxLT ){ res = -res; @@ -4400,6 +4417,7 @@ case OP_Destroy: { /* out2-prerelease */ #else iCnt = db->activeVdbeCnt; #endif + pOut->flags = MEM_Null; if( iCnt>1 ){ rc = SQLITE_LOCKED; p->errorAction = OE_Abort; @@ -4408,11 +4426,12 @@ case OP_Destroy: { /* out2-prerelease */ assert( iCnt==1 ); assert( (p->btreeMask & (1<aDb[iDb].pBt, pOp->p1, &iMoved); - MemSetTypeFlag(pOut, MEM_Int); + pOut->flags = MEM_Int; pOut->u.i = iMoved; #ifndef SQLITE_OMIT_AUTOVACUUM if( rc==SQLITE_OK && iMoved!=0 ){ sqlite3RootPageMoved(&db->aDb[iDb], iMoved, pOp->p1); + resetSchemaOnFault = 1; } #endif } @@ -4448,7 +4467,7 @@ case OP_Clear: { if( pOp->p3 ){ p->nChange += nChange; if( pOp->p3>0 ){ - p->aMem[pOp->p3].u.i += nChange; + aMem[pOp->p3].u.i += nChange; } } break; @@ -4495,7 +4514,6 @@ case OP_CreateTable: { /* out2-prerelease */ } rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); pOut->u.i = pgno; - MemSetTypeFlag(pOut, MEM_Int); break; } @@ -4658,10 +4676,10 @@ case OP_IntegrityCk: { aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); if( aRoot==0 ) goto no_mem; assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pnErr = &p->aMem[pOp->p3]; + pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &p->aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1]; for(j=0; jp1>0 && pOp->p1<=p->nMem ); - pIdx = &p->aMem[pOp->p1]; - assert( pOp->p2>0 && pOp->p2<=p->nMem ); - pVal = &p->aMem[pOp->p2]; - assert( (pVal->flags & MEM_Int)!=0 ); - if( (pIdx->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIdx); - if( (pIdx->flags & MEM_RowSet)==0 ) goto no_mem; +case OP_RowSetAdd: { /* in1, in2 */ + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Int)!=0 ); + if( (pIn1->flags & MEM_RowSet)==0 ){ + sqlite3VdbeMemSetRowSet(pIn1); + if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; } - sqlite3RowSetInsert(pIdx->u.pRowSet, pVal->u.i); + sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); break; } @@ -4715,23 +4729,19 @@ case OP_RowSetAdd: { /* in2 */ ** register P3. Or, if boolean index P1 is initially empty, leave P3 ** unchanged and jump to instruction P2. */ -case OP_RowSetRead: { /* jump, out3 */ - Mem *pIdx; +case OP_RowSetRead: { /* jump, in1, out3 */ i64 val; - assert( pOp->p1>0 && pOp->p1<=p->nMem ); CHECK_FOR_INTERRUPT; - pIdx = &p->aMem[pOp->p1]; - pOut = &p->aMem[pOp->p3]; - if( (pIdx->flags & MEM_RowSet)==0 - || sqlite3RowSetNext(pIdx->u.pRowSet, &val)==0 + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_RowSet)==0 + || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0 ){ /* The boolean index is empty */ - sqlite3VdbeMemSetNull(pIdx); + sqlite3VdbeMemSetNull(pIn1); pc = pOp->p2 - 1; }else{ /* A value was pulled from the index */ - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - sqlite3VdbeMemSetInt64(pOut, val); + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); } break; } @@ -4763,6 +4773,8 @@ case OP_RowSetTest: { /* jump, in1, in3 */ int iSet; int exists; + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; iSet = pOp->p4.i; assert( pIn3->flags&MEM_Int ); @@ -4818,7 +4830,7 @@ case OP_Program: { /* jump */ void *t; /* Token identifying trigger */ pProgram = pOp->p4.pProgram; - pRt = &p->aMem[pOp->p3]; + pRt = &aMem[pOp->p3]; assert( pProgram->nOp>0 ); /* If the p5 flag is clear, then recursive invocation of triggers is @@ -4896,11 +4908,11 @@ case OP_Program: { /* jump */ pFrame->nChange = p->nChange; p->nChange = 0; p->pFrame = pFrame; - p->aMem = &VdbeFrameMem(pFrame)[-1]; + p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&p->aMem[p->nMem+1]; - p->aOp = pProgram->aOp; + p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; pc = -1; @@ -4986,9 +4998,10 @@ case OP_MemMax: { /* in2 */ for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); pIn1 = &pFrame->aMem[pOp->p1]; }else{ - pIn1 = &p->aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1]; } sqlite3VdbeMemIntegerify(pIn1); + pIn2 = &aMem[pOp->p2]; sqlite3VdbeMemIntegerify(pIn2); if( pIn1->u.iu.i){ pIn1->u.i = pIn2->u.i; @@ -5005,6 +5018,7 @@ case OP_MemMax: { /* in2 */ ** not contain an integer. An assertion fault will result if you try. */ case OP_IfPos: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); if( pIn1->u.i>0 ){ pc = pOp->p2 - 1; @@ -5020,6 +5034,7 @@ case OP_IfPos: { /* jump, in1 */ ** not contain an integer. An assertion fault will result if you try. */ case OP_IfNeg: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); if( pIn1->u.i<0 ){ pc = pOp->p2 - 1; @@ -5027,15 +5042,18 @@ case OP_IfNeg: { /* jump, in1 */ break; } -/* Opcode: IfZero P1 P2 * * * +/* Opcode: IfZero P1 P2 P3 * * ** -** If the value of register P1 is exactly 0, jump to P2. +** The register P1 must contain an integer. Add literal P3 to the +** value in register P1. If the result is exactly 0, jump to P2. ** ** It is illegal to use this instruction on a register that does ** not contain an integer. An assertion fault will result if you try. */ case OP_IfZero: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; if( pIn1->u.i==0 ){ pc = pOp->p2 - 1; } @@ -5062,7 +5080,7 @@ case OP_AggStep: { n = pOp->p5; assert( n>=0 ); - pRec = &p->aMem[pOp->p2]; + pRec = &aMem[pOp->p2]; apVal = p->apArg; assert( apVal || n==0 ); for(i=0; ip4.pFunc; assert( pOp->p3>0 && pOp->p3<=p->nMem ); - ctx.pMem = pMem = &p->aMem[pOp->p3]; + ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; ctx.s.flags = MEM_Null; ctx.s.z = 0; @@ -5110,7 +5128,7 @@ case OP_AggStep: { case OP_AggFinal: { Mem *pMem; assert( pOp->p1>0 && pOp->p1<=p->nMem ); - pMem = &p->aMem[pOp->p1]; + pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ @@ -5335,7 +5353,7 @@ case OP_VFilter: { /* jump */ int i; Mem **apArg; - pQuery = &p->aMem[pOp->p3]; + pQuery = &aMem[pOp->p3]; pArgc = &pQuery[1]; pCur = p->apCsr[pOp->p1]; REGISTER_TRACE(pOp->p3, pQuery); @@ -5396,7 +5414,7 @@ case OP_VColumn: { VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur->pVtabCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pDest = &p->aMem[pOp->p3]; + pDest = &aMem[pOp->p3]; if( pCur->nullRow ){ sqlite3VdbeMemSetNull(pDest); break; @@ -5428,8 +5446,8 @@ case OP_VColumn: { ** dynamic allocation in sContext.s (a Mem struct) is released. */ sqlite3VdbeChangeEncoding(&sContext.s, encoding); - REGISTER_TRACE(pOp->p3, pDest); sqlite3VdbeMemMove(pDest, &sContext.s); + REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); if( sqlite3SafetyOn(db) ){ @@ -5503,7 +5521,7 @@ case OP_VRename: { Mem *pName; pVtab = pOp->p4.pVtab->pVtab; - pName = &p->aMem[pOp->p1]; + pName = &aMem[pOp->p1]; assert( pVtab->pModule->xRename ); REGISTER_TRACE(pOp->p1, pName); assert( pName->flags & MEM_Str ); @@ -5557,7 +5575,7 @@ case OP_VUpdate: { assert( pOp->p4type==P4_VTAB ); if( ALWAYS(pModule->xUpdate) ){ apArg = p->apArg; - pX = &p->aMem[pOp->p3]; + pX = &aMem[pOp->p3]; for(i=0; iflags = MEM_Int; pOut->u.i = nPage; } break; @@ -5615,7 +5632,9 @@ case OP_Trace: { zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); if( zTrace ){ if( db->xTrace ){ - db->xTrace(db->pTraceArg, zTrace); + char *z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); } #ifdef SQLITE_DEBUG if( (db->flags & SQLITE_SqlTrace)!=0 ){ @@ -5658,7 +5677,7 @@ default: { /* This is really OP_Noop and OP_Explain */ pOp->cnt++; #if 0 fprintf(stdout, "%10llu ", elapsed); - sqlite3VdbePrintOp(stdout, origPc, &p->aOp[origPc]); + sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]); #endif } #endif @@ -5674,11 +5693,11 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef SQLITE_DEBUG if( p->trace ){ if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc); - if( opProperty & OPFLG_OUT2_PRERELEASE ){ - registerTrace(p->trace, pOp->p2, pOut); + if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ + registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]); } - if( opProperty & OPFLG_OUT3 ){ - registerTrace(p->trace, pOp->p3, pOut); + if( pOp->opflags & OPFLG_OUT3 ){ + registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]); } } #endif /* SQLITE_DEBUG */ @@ -5694,6 +5713,7 @@ vdbe_error_halt: sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; rc = SQLITE_ERROR; + if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0); /* This is the only way out of this procedure. We have to ** release the mutexes on btrees that were acquired at the diff --git a/src/vdbe.h b/src/vdbe.h index bdc1f24..f67e903 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -14,8 +14,6 @@ ** This header defines the interface to the virtual database engine ** or VDBE. The VDBE implements an abstract machine that runs a ** simple program to access and modify the underlying database. -** -** $Id: vdbe.h,v 1.142 2009/07/24 17:58:53 danielk1977 Exp $ */ #ifndef _SQLITE_VDBE_H_ #define _SQLITE_VDBE_H_ @@ -44,7 +42,7 @@ typedef struct SubProgram SubProgram; struct VdbeOp { u8 opcode; /* What operation to perform */ signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 opflags; /* Not currently used */ + u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */ u8 p5; /* Fifth parameter is an unsigned character */ int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ @@ -172,6 +170,7 @@ int sqlite3VdbeAddOp1(Vdbe*,int,int); int sqlite3VdbeAddOp2(Vdbe*,int,int,int); int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); @@ -204,6 +203,9 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); void sqlite3VdbeProgramDelete(sqlite3 *, SubProgram *, int); sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8); void sqlite3VdbeSetVarmask(Vdbe*, int); +#ifndef SQLITE_OMIT_TRACE + char *sqlite3VdbeExpandSql(Vdbe*, const char*); +#endif UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 964d55f..93550e9 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -14,8 +14,6 @@ ** source code file "vdbe.c". When that file became too big (over ** 6000 lines long) it was split up into several smaller files and ** this header information was factored out. -** -** $Id: vdbeInt.h,v 1.174 2009/06/23 14:15:04 drh Exp $ */ #ifndef _VDBEINT_H_ #define _VDBEINT_H_ @@ -381,7 +379,6 @@ void sqlite3VdbeMemRelease(Mem *p); void sqlite3VdbeMemReleaseExternal(Mem *p); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); -int sqlite3VdbeOpcodeHasProperty(int, int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); int sqlite3VdbeCloseStatement(Vdbe *, int); void sqlite3VdbeFrameDelete(VdbeFrame*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b9ee52b..a866708 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -12,8 +12,6 @@ ** ** This file contains code use to implement APIs that are part of the ** VDBE. -** -** $Id: vdbeapi.c,v 1.167 2009/06/25 01:47:12 drh Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -1122,8 +1120,7 @@ const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ ** with that name. If there is no variable with the given name, ** return 0. */ -int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ - Vdbe *p = (Vdbe*)pStmt; +int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ int i; if( p==0 ){ return 0; @@ -1132,13 +1129,16 @@ int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ if( zName ){ for(i=0; inVar; i++){ const char *z = p->azVar[i]; - if( z && strcmp(z,zName)==0 ){ + if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; } } } return 0; } +int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ + return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); +} /* ** Transfer all bindings from the first statement over to the second. diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b28ba96..0635086 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -13,8 +13,6 @@ ** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior ** to version 2.8.7, all this code was combined into the vdbe.c source file. ** But that file was getting too big so this subroutines were split out. -** -** $Id: vdbeaux.c,v 1.480 2009/08/08 18:01:08 drh Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -197,6 +195,22 @@ int sqlite3VdbeAddOp4( return addr; } +/* +** Add an opcode that includes the p4 value as an integer. +*/ +int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); + return addr; +} + /* ** Create a new symbolic label for an instruction that has yet to be ** coded. The symbolic label is really just a negative number. The @@ -372,6 +386,8 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument ** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by ** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** +** The Op.opflags field is set on all opcodes. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ int i; @@ -382,15 +398,14 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){ u8 opcode = pOp->opcode; + pOp->opflags = sqlite3OpcodeProperty[opcode]; if( opcode==OP_Function || opcode==OP_AggStep ){ if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; -#ifndef SQLITE_OMIT_VIRTUALTABLE - }else if( opcode==OP_VUpdate ){ - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; -#endif }else if( opcode==OP_Transaction && pOp->p2!=0 ){ p->readOnly = 0; #ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( opcode==OP_VUpdate ){ + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; }else if( opcode==OP_VFilter ){ int n; assert( p->nOp - i >= 3 ); @@ -400,7 +415,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ #endif } - if( sqlite3VdbeOpcodeHasProperty(opcode, OPFLG_JUMP) && pOp->p2<0 ){ + if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ assert( -1-pOp->p2nLabel ); pOp->p2 = aLabel[-1-pOp->p2]; } @@ -462,7 +477,7 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ VdbeOp *pOut = &p->aOp[i+addr]; pOut->opcode = pIn->opcode; pOut->p1 = pIn->p1; - if( p2<0 && sqlite3VdbeOpcodeHasProperty(pOut->opcode, OPFLG_JUMP) ){ + if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){ pOut->p2 = addr + ADDR(p2); }else{ pOut->p2 = p2; @@ -2772,9 +2787,17 @@ int sqlite3VdbeRecordCompare( pKeyInfo = pPKey2->pKeyInfo; mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; - mem1.flags = 0; - mem1.u.i = 0; /* not needed, here to silence compiler warning */ - mem1.zMalloc = 0; + /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + + /* Compilers may complain that mem1.u.i is potentially uninitialized. + ** We could initialize it, as shown here, to silence those complaints. + ** But in fact, mem1.u.i will never actually be used initialized, and doing + ** the unnecessary initialization has a measurable negative performance + ** impact, since this routine is a very high runner. And so, we choose + ** to ignore the compiler warnings and leave this variable uninitialized. + */ + /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; @@ -2798,47 +2821,52 @@ int sqlite3VdbeRecordCompare( rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], iaColl[i] : 0); if( rc!=0 ){ - break; + assert( mem1.zMalloc==0 ); /* See comment below */ + + /* Invert the result if we are using DESC sort order. */ + if( pKeyInfo->aSortOrder && iaSortOrder[i] ){ + rc = -rc; + } + + /* If the PREFIX_SEARCH flag is set and all fields except the final + ** rowid field were equal, then clear the PREFIX_SEARCH flag and set + ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1). + ** This is used by the OP_IsUnique opcode. + */ + if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){ + assert( idx1==szHdr1 && rc ); + assert( mem1.flags & MEM_Int ); + pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH; + pPKey2->rowid = mem1.u.i; + } + + return rc; } i++; } - /* No memory allocation is ever used on mem1. */ - if( NEVER(mem1.zMalloc) ) sqlite3VdbeMemRelease(&mem1); - - /* If the PREFIX_SEARCH flag is set and all fields except the final - ** rowid field were equal, then clear the PREFIX_SEARCH flag and set - ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1). - ** This is used by the OP_IsUnique opcode. + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){ - assert( idx1==szHdr1 && rc ); - assert( mem1.flags & MEM_Int ); - pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH; - pPKey2->rowid = mem1.u.i; - } + assert( mem1.zMalloc==0 ); - if( rc==0 ){ - /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. If the UNPACKED_INCRKEY - ** flag is set, then break the tie by treating key2 as larger. - ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes - ** are considered to be equal. Otherwise, the longer key is the - ** larger. As it happens, the pPKey2 will always be the longer - ** if there is a difference. - */ - if( pPKey2->flags & UNPACKED_INCRKEY ){ - rc = -1; - }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ - /* Leave rc==0 */ - }else if( idx1aSortOrder && inField - && pKeyInfo->aSortOrder[i] ){ - rc = -rc; + /* rc==0 here means that one of the keys ran out of fields and + ** all the fields up to that point were equal. If the UNPACKED_INCRKEY + ** flag is set, then break the tie by treating key2 as larger. + ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes + ** are considered to be equal. Otherwise, the longer key is the + ** larger. As it happens, the pPKey2 will always be the longer + ** if there is a difference. + */ + assert( rc==0 ); + if( pPKey2->flags & UNPACKED_INCRKEY ){ + rc = -1; + }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ + /* Leave rc==0 */ + }else if( idx1flags; f2 = pMem2->flags; combined_flags = f1|f2; @@ -1009,7 +1004,7 @@ int sqlite3ValueFromExpr( } op = pExpr->op; if( op==TK_REGISTER ){ - op = pExpr->op2; + op = pExpr->op2; /* This only happens with SQLITE_ENABLE_STAT2 */ } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ diff --git a/src/vdbetrace.c b/src/vdbetrace.c new file mode 100644 index 0000000..0d6b212 --- /dev/null +++ b/src/vdbetrace.c @@ -0,0 +1,142 @@ +/* +** 2009 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to insert the values of host parameters +** (aka "wildcards") into the SQL text output by sqlite3_trace(). +*/ +#include "sqliteInt.h" +#include "vdbeInt.h" + +#ifndef SQLITE_OMIT_TRACE + +/* +** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of +** bytes in this text up to but excluding the first character in +** a host parameter. If the text contains no host parameters, return +** the total number of bytes in the text. +*/ +static int findNextHostParameter(const char *zSql, int *pnToken){ + int tokenType; + int nTotal = 0; + int n; + + *pnToken = 0; + while( zSql[0] ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + assert( n>0 && tokenType!=TK_ILLEGAL ); + if( tokenType==TK_VARIABLE ){ + *pnToken = n; + break; + } + nTotal += n; + zSql += n; + } + return nTotal; +} + +/* +** Return a pointer to a string in memory obtained form sqlite3DbMalloc() which +** holds a copy of zRawSql but with host parameters expanded to their +** current bindings. +** +** The calling function is responsible for making sure the memory returned +** is eventually freed. +** +** ALGORITHM: Scan the input string looking for host parameters in any of +** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within +** string literals, quoted identifier names, and comments. For text forms, +** the host parameter index is found by scanning the perpared +** statement for the corresponding OP_Variable opcode. Once the host +** parameter index is known, locate the value in p->aVar[]. Then render +** the value as a literal in place of the host parameter name. +*/ +char *sqlite3VdbeExpandSql( + Vdbe *p, /* The prepared statement being evaluated */ + const char *zRawSql /* Raw text of the SQL statement */ +){ + sqlite3 *db; /* The database connection */ + int idx = 0; /* Index of a host parameter */ + int nextIndex = 1; /* Index of next ? host parameter */ + int n; /* Length of a token prefix */ + int nToken; /* Length of the parameter token */ + int i; /* Loop counter */ + Mem *pVar; /* Value of a host parameter */ + StrAccum out; /* Accumulate the output here */ + char zBase[100]; /* Initial working space */ + + db = p->db; + sqlite3StrAccumInit(&out, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + out.db = db; + while( zRawSql[0] ){ + n = findNextHostParameter(zRawSql, &nToken); + assert( n>0 ); + sqlite3StrAccumAppend(&out, zRawSql, n); + zRawSql += n; + assert( zRawSql[0] || nToken==0 ); + if( nToken==0 ) break; + if( zRawSql[0]=='?' ){ + if( nToken>1 ){ + assert( sqlite3Isdigit(zRawSql[1]) ); + sqlite3GetInt32(&zRawSql[1], &idx); + }else{ + idx = nextIndex; + } + }else{ + assert( zRawSql[0]==':' || zRawSql[0]=='$' || zRawSql[0]=='@' ); + testcase( zRawSql[0]==':' ); + testcase( zRawSql[0]=='$' ); + testcase( zRawSql[0]=='@' ); + idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); + assert( idx>0 ); + } + zRawSql += nToken; + nextIndex = idx + 1; + assert( idx>0 && idx<=p->nVar ); + pVar = &p->aVar[idx-1]; + if( pVar->flags & MEM_Null ){ + sqlite3StrAccumAppend(&out, "NULL", 4); + }else if( pVar->flags & MEM_Int ){ + sqlite3XPrintf(&out, "%lld", pVar->u.i); + }else if( pVar->flags & MEM_Real ){ + sqlite3XPrintf(&out, "%!.15g", pVar->r); + }else if( pVar->flags & MEM_Str ){ +#ifndef SQLITE_OMIT_UTF16 + u8 enc = ENC(db); + if( enc!=SQLITE_UTF8 ){ + Mem utf8; + memset(&utf8, 0, sizeof(utf8)); + utf8.db = db; + sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); + sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); + sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); + sqlite3VdbeMemRelease(&utf8); + }else +#endif + { + sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + } + }else if( pVar->flags & MEM_Zero ){ + sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); + }else{ + assert( pVar->flags & MEM_Blob ); + sqlite3StrAccumAppend(&out, "x'", 2); + for(i=0; in; i++){ + sqlite3XPrintf(&out, "%02x", pVar->z[i]&0xff); + } + sqlite3StrAccumAppend(&out, "'", 1); + } + } + return sqlite3StrAccumFinish(&out); +} + +#endif /* #ifndef SQLITE_OMIT_TRACE */ diff --git a/src/vtab.c b/src/vtab.c index 117f361..f5269e5 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -10,8 +10,6 @@ ** ************************************************************************* ** This file contains code used to help implement virtual tables. -** -** $Id: vtab.c,v 1.94 2009/08/08 18:01:08 drh Exp $ */ #ifndef SQLITE_OMIT_VIRTUALTABLE #include "sqliteInt.h" diff --git a/src/walker.c b/src/walker.c index 2700630..c95a9c1 100644 --- a/src/walker.c +++ b/src/walker.c @@ -11,8 +11,6 @@ ************************************************************************* ** This file contains routines used for walking the parser tree for ** an SQL statement. -** -** $Id: walker.c,v 1.7 2009/06/15 23:15:59 drh Exp $ */ #include "sqliteInt.h" #include diff --git a/src/where.c b/src/where.c index f17853d..067ec40 100644 --- a/src/where.c +++ b/src/where.c @@ -15,8 +15,6 @@ ** rows. Indices are selected and used to speed the search when doing ** so is applicable. Because this module is responsible for selecting ** indices, you might also think of this module as the "query optimizer". -** -** $Id: where.c,v 1.411 2009/07/31 06:14:52 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -2596,16 +2594,39 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ ** Code an OP_Affinity opcode to apply the column affinity string zAff ** to the n registers starting at base. ** -** Buffer zAff was allocated using sqlite3DbMalloc(). It is the -** responsibility of this function to arrange for it to be eventually -** freed using sqlite3DbFree(). +** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_NONE, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. */ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; + } assert( v!=0 ); - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, P4_DYNAMIC); - sqlite3ExprCacheAffinityChange(pParse, base, n); + + /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ + n--; + base++; + zAff++; + } + while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ + n--; + } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); + } } @@ -2676,7 +2697,7 @@ static int codeEqualityTerm( /* ** Generate code that will evaluate all == and IN constraints for an -** index. The values for all constraints are left on the stack. +** index. ** ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). ** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 @@ -2688,7 +2709,8 @@ static int codeEqualityTerm( ** ** In the example above nEq==2. But this subroutine works for any value ** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. ** ** This routine always allocates at least one memory cell and returns ** the index of that memory cell. The code that @@ -2766,11 +2788,15 @@ static int codeAllEqualityTerms( testcase( pTerm->eOperator & WO_ISNULL ); testcase( pTerm->eOperator & WO_IN ); if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); - if( zAff - && sqlite3CompareAffinity(pTerm->pExpr->pRight, zAff[j])==SQLITE_AFF_NONE - ){ - zAff[j] = SQLITE_AFF_NONE; + Expr *pRight = pTerm->pExpr->pRight; + sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk); + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ + zAff[j] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_NONE; + } } } } @@ -2850,6 +2876,7 @@ static Bitmask codeOneLoopStart( const struct sqlite3_index_constraint *aConstraint = pVtabIdx->aConstraint; + sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); for(j=1; j<=nConstraint; j++){ for(k=0; kp1 = iCur; pLevel->p2 = sqlite3VdbeCurrentAddr(v); sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse, 1); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -3096,15 +3124,18 @@ static Bitmask codeOneLoopStart( if( pRangeStart ){ Expr *pRight = pRangeStart->pExpr->pRight; sqlite3ExprCode(pParse, pRight, regBase+nEq); - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - if( zAff - && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE - ){ - /* Since the comparison is to be performed with no conversions applied - ** to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zAff[nConstraint] = SQLITE_AFF_NONE; - } + sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zAff[nConstraint] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){ + zAff[nConstraint] = SQLITE_AFF_NONE; + } + } nConstraint++; }else if( isMinQuery ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); @@ -3121,8 +3152,7 @@ static Bitmask codeOneLoopStart( testcase( op==OP_SeekGe ); testcase( op==OP_SeekLe ); testcase( op==OP_SeekLt ); - sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase, - SQLITE_INT_TO_PTR(nConstraint), P4_INT32); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); /* Load the value for the inequality constraint at the end of the ** range (if any). @@ -3132,19 +3162,22 @@ static Bitmask codeOneLoopStart( Expr *pRight = pRangeEnd->pExpr->pRight; sqlite3ExprCacheRemove(pParse, regBase+nEq); sqlite3ExprCode(pParse, pRight, regBase+nEq); - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - zAff = sqlite3DbStrDup(pParse->db, zAff); - if( zAff - && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE - ){ - /* Since the comparison is to be performed with no conversions applied - ** to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zAff[nConstraint] = SQLITE_AFF_NONE; - } + sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_NONE. */ + zAff[nConstraint] = SQLITE_AFF_NONE; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){ + zAff[nConstraint] = SQLITE_AFF_NONE; + } + } codeApplyAffinity(pParse, regBase, nEq+1, zAff); nConstraint++; } + sqlite3DbFree(pParse->db, zAff); /* Top of the loop body */ pLevel->p2 = sqlite3VdbeCurrentAddr(v); @@ -3155,8 +3188,7 @@ static Bitmask codeOneLoopStart( testcase( op==OP_IdxGE ); testcase( op==OP_IdxLT ); if( op!=OP_Noop ){ - sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase, - SQLITE_INT_TO_PTR(nConstraint), P4_INT32); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0); } @@ -3285,9 +3317,8 @@ static Bitmask codeOneLoopStart( int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, regRowid, 0); - sqlite3VdbeAddOp4(v, OP_RowSetTest, regRowset, - sqlite3VdbeCurrentAddr(v)+2, - r, SQLITE_INT_TO_PTR(iSet), P4_INT32); + sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, + sqlite3VdbeCurrentAddr(v)+2, r, iSet); } sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); @@ -3818,7 +3849,8 @@ WhereInfo *sqlite3WhereBegin( Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, SQLITE_INT_TO_PTR(n), P4_INT32); + sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, + SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } }else{ @@ -3946,7 +3978,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLevel->iLeftJoin ){ int addr; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); - sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); + if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + } if( pLevel->iIdxCur>=0 ){ sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } @@ -3997,7 +4033,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ int k, j, last; VdbeOp *pOp; Index *pIdx = pLevel->plan.u.pIdx; - int useIndexOnly = pLevel->plan.wsFlags & WHERE_IDX_ONLY; assert( pIdx!=0 ); pOp = sqlite3VdbeGetOp(v, pWInfo->iTop); @@ -4012,12 +4047,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ break; } } - assert(!useIndexOnly || jnColumn); + assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + || jnColumn ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; - }else if( pOp->opcode==OP_NullRow && useIndexOnly ){ - pOp->opcode = OP_Noop; } } } diff --git a/test/analyze3.test b/test/analyze3.test index 2aaa873..f623f98 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -598,4 +598,3 @@ do_test analyze3-5.1.3 { } {SQLITE_OK} finish_test - diff --git a/test/attach.test b/test/attach.test index 7ee6597..56d6445 100644 --- a/test/attach.test +++ b/test/attach.test @@ -787,5 +787,32 @@ do_test attach-8.4 { db2 close file delete -force test2.db +# Test that it is possible to attach the same database more than +# once when not in shared-cache mode. That this is not possible in +# shared-cache mode is tested in shared7.test. +do_test attach-9.1 { + file delete -force test4.db + execsql { + ATTACH 'test4.db' AS aux1; + CREATE TABLE aux1.t1(a, b); + INSERT INTO aux1.t1 VALUES(1, 2); + ATTACH 'test4.db' AS aux2; + SELECT * FROM aux2.t1; + } +} {1 2} +do_test attach-9.2 { + catchsql { + BEGIN; + INSERT INTO aux1.t1 VALUES(3, 4); + INSERT INTO aux2.t1 VALUES(5, 6); + } +} {1 {database is locked}} +do_test attach-9.3 { + execsql { + COMMIT; + SELECT * FROM aux2.t1; + } +} {1 2 3 4} + finish_test diff --git a/test/coalesce.test b/test/coalesce.test new file mode 100644 index 0000000..af4ea8d --- /dev/null +++ b/test/coalesce.test @@ -0,0 +1,84 @@ +# 2009 November 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Additional test cases for the COALESCE() and IFNULL() functions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + + +do_test coalesce-1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + INSERT INTO t1 VALUES(1, null, null, null); + INSERT INTO t1 VALUES(2, 2, 99, 99); + INSERT INTO t1 VALUES(3, null, 3, 99); + INSERT INTO t1 VALUES(4, null, null, 4); + INSERT INTO t1 VALUES(5, null, null, null); + INSERT INTO t1 VALUES(6, 22, 99, 99); + INSERT INTO t1 VALUES(7, null, 33, 99); + INSERT INTO t1 VALUES(8, null, null, 44); + + SELECT coalesce(b,c,d) FROM t1 ORDER BY a; + } +} {{} 2 3 4 {} 22 33 44} +do_test coalesce-1.1 { + db eval { + SELECT coalesce(d+c+b,d+c,d) FROM t1 ORDER BY a; + } +} {{} 200 102 4 {} 220 132 44} +do_test coalesce-1.2 { + db eval { + SELECT ifnull(d+c+b,ifnull(d+c,d)) FROM t1 ORDER BY a; + } +} {{} 200 102 4 {} 220 132 44} +do_test coalesce-1.3 { + db eval { + SELECT ifnull(ifnull(d+c+b,d+c),d) FROM t1 ORDER BY a; + } +} {{} 200 102 4 {} 220 132 44} +do_test coalesce-1.4 { + db eval { + SELECT ifnull(ifnull(b,c),d) FROM t1 ORDER BY a; + } +} {{} 2 3 4 {} 22 33 44} +do_test coalesce-1.5 { + db eval { + SELECT ifnull(b,ifnull(c,d)) FROM t1 ORDER BY a; + } +} {{} 2 3 4 {} 22 33 44} +do_test coalesce-1.6 { + db eval { + SELECT coalesce(b,NOT b,-b,abs(b),lower(b),length(b),min(b,5),b*123,c) + FROM t1 ORDER BY a; + } +} {{} 2 3 {} {} 22 33 {}} +do_test coalesce-1.7 { + db eval { + SELECT ifnull(nullif(a,4),99) + FROM t1 ORDER BY a; + } +} {1 2 3 99 5 6 7 8} +do_test coalesce-1.8 { + db eval { +pragma vdbe_listing=on; + SELECT coalesce( + CASE WHEN b=2 THEN 123 END, + CASE WHEN b=3 THEN 234 END, + CASE WHEN c=3 THEN 345 WHEN c=33 THEN 456 END, + d + ) + FROM t1 ORDER BY a; + } +} {{} 123 345 4 {} 99 456 44} + + +finish_test diff --git a/test/corrupt.test b/test/corrupt.test index 16f1a92..be6236e 100644 --- a/test/corrupt.test +++ b/test/corrupt.test @@ -257,4 +257,58 @@ do_test corrupt-6.1 { catchsql { INSERT INTO t1 VALUES( randomblob(10) ) } } {1 {database disk image is malformed}} +ifcapable oversize_cell_check { + db close + file delete -force test.db test.db-journal + sqlite3 db test.db + execsql { + PRAGMA page_size = 1024; CREATE TABLE t1(x); + } + + do_test corrupt-7.1 { + for {set i 0} {$i < 39} {incr i} { + execsql { + INSERT INTO t1 VALUES(X'000100020003000400050006000700080009000A'); + } + } + } {} + db close + + # Corrupt the root page of table t1 so that the first offset in the + # cell-offset array points to the data for the SQL blob associated with + # record (rowid=10). The root page still passes the checks in btreeInitPage(), + # because the start of said blob looks like the start of a legitimate + # page cell. + # + # Test case cc-2 overwrites the blob so that it no longer looks like a + # real cell. But, by the time it is overwritten, btreeInitPage() has already + # initialized the root page, so no corruption is detected. + # + # Test case cc-3 inserts an extra record into t1, forcing balance-deeper + # to run. After copying the contents of the root page to the new child, + # btreeInitPage() is called on the child. This time, it detects corruption + # (because the start of the blob associated with the (rowid=10) record + # no longer looks like a real cell). At one point the code assumed that + # detecting corruption was not possible at that point, and an assert() failed. + # + set fd [open test.db r+] + fconfigure $fd -translation binary -encoding binary + seek $fd [expr 1024+8] + puts -nonewline $fd "\x03\x14" + close $fd + + sqlite3 db test.db + do_test corrupt-7.2 { + execsql { + UPDATE t1 SET x = X'870400020003000400050006000700080009000A' + WHERE rowid = 10; + } + } {} + do_test corrupt-7.3 { + catchsql { + INSERT INTO t1 VALUES(X'000100020003000400050006000700080009000A'); + } + } {1 {database disk image is malformed}} +} + finish_test diff --git a/test/e_fkey.test b/test/e_fkey.test index 008e0e9..76095d1 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -37,7 +37,7 @@ proc eqp {sql {db db}} { uplevel execsql [list "EXPLAIN QUERY PLAN $sql"] $db } # foreign key functionality. # ifcapable trigger&&foreignkey { - do_test e_fkey-49 { + do_test e_fkey-1 { execsql { PRAGMA foreign_keys = ON; CREATE TABLE p(i PRIMARY KEY); @@ -63,7 +63,7 @@ ifcapable trigger&&foreignkey { # reset_db ifcapable !trigger&&foreignkey { - do_test e_fkey-51.1 { + do_test e_fkey-2.1 { execsql { PRAGMA foreign_keys = ON; CREATE TABLE p(i PRIMARY KEY); @@ -74,10 +74,10 @@ ifcapable !trigger&&foreignkey { SELECT * FROM c; } } {hello} - do_test e_fkey-51.2 { + do_test e_fkey-2.2 { execsql { PRAGMA foreign_key_list(c) } } {0 0 p j {} CASCADE {NO ACTION} NONE} - do_test e_fkey-51.3 { + do_test e_fkey-2.3 { execsql { PRAGMA foreign_keys } } {} } @@ -95,22 +95,22 @@ ifcapable !trigger&&foreignkey { # reset_db ifcapable !foreignkey { - do_test e_fkey-52.1 { + do_test e_fkey-3.1 { execsql { CREATE TABLE p(i PRIMARY KEY) } catchsql { CREATE TABLE c(j REFERENCES p ON UPDATE CASCADE) } } {1 {near "ON": syntax error}} - do_test e_fkey-52.2 { + do_test e_fkey-3.2 { # This is allowed, as in this build, "REFERENCES" is not a keyword. # The declared datatype of column j is "REFERENCES p". execsql { CREATE TABLE c(j REFERENCES p) } } {} - do_test e_fkey-52.3 { + do_test e_fkey-3.3 { execsql { PRAGMA table_info(c) } } {0 j {REFERENCES p} 0 {} 0} - do_test e_fkey-52.4 { + do_test e_fkey-3.4 { execsql { PRAGMA foreign_key_list(c) } } {} - do_test e_fkey-52.5 { + do_test e_fkey-3.5 { execsql { PRAGMA foreign_keys } } {} } @@ -130,7 +130,7 @@ reset_db # This also tests that foreign key constraints are disabled by default. # drop_all_tables -do_test e_fkey-53.1 { +do_test e_fkey-4.1 { execsql { CREATE TABLE p(i PRIMARY KEY); CREATE TABLE c(j REFERENCES p ON UPDATE CASCADE); @@ -140,7 +140,7 @@ do_test e_fkey-53.1 { SELECT * FROM c; } } {hello} -do_test e_fkey-53.2 { +do_test e_fkey-4.2 { execsql { DELETE FROM c; DELETE FROM p; @@ -161,16 +161,16 @@ do_test e_fkey-53.2 { # the example code in section 2 of foreignkeys.in. # reset_db -do_test e_fkey-54.1 { +do_test e_fkey-5.1 { execsql { PRAGMA foreign_keys } } {0} -do_test e_fkey-54.2 { +do_test e_fkey-5.2 { execsql { PRAGMA foreign_keys = ON; PRAGMA foreign_keys; } } {1} -do_test e_fkey-54.3 { +do_test e_fkey-5.3 { execsql { PRAGMA foreign_keys = OFF; PRAGMA foreign_keys; @@ -184,7 +184,7 @@ do_test e_fkey-54.3 { # while not in auto-commit mode. # reset_db -do_test e_fkey-55.1 { +do_test e_fkey-6.1 { execsql { PRAGMA foreign_keys = ON; CREATE TABLE t1(a UNIQUE, b); @@ -198,10 +198,10 @@ do_test e_fkey-55.1 { DELETE FROM t1 } } {1 {foreign key constraint failed}} -do_test e_fkey-55.2 { +do_test e_fkey-6.2 { execsql { PRAGMA foreign_keys } } {1} -do_test e_fkey-55.3 { +do_test e_fkey-6.3 { execsql { COMMIT; PRAGMA foreign_keys = OFF; @@ -211,7 +211,7 @@ do_test e_fkey-55.3 { PRAGMA foreign_keys; } } {0} -do_test e_fkey-55.4 { +do_test e_fkey-6.4 { execsql COMMIT } {} @@ -225,7 +225,7 @@ execsql "PRAGMA foreign_keys = ON" # # Verify that the syntax in the first example in section 1 is valid. # -do_test e_fkey-38.1 { +do_test e_fkey-7.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -246,14 +246,14 @@ do_test e_fkey-38.1 { # Attempting to insert a row into the 'track' table that corresponds # to no row in the 'artist' table fails. # -do_test e_fkey-39.1 { +do_test e_fkey-8.1 { catchsql { INSERT INTO track VALUES(1, 'track 1', 1) } } {1 {foreign key constraint failed}} -do_test e_fkey-39.2 { +do_test e_fkey-8.2 { execsql { INSERT INTO artist VALUES(2, 'artist 1') } catchsql { INSERT INTO track VALUES(1, 'track 1', 1) } } {1 {foreign key constraint failed}} -do_test e_fkey-39.2 { +do_test e_fkey-8.2 { execsql { INSERT INTO track VALUES(1, 'track 1', 2) } } {} @@ -263,10 +263,10 @@ do_test e_fkey-39.2 { # Attempting to delete a row from the 'artist' table while there are # dependent rows in the track table also fails. # -do_test e_fkey-40.1 { +do_test e_fkey-9.1 { catchsql { DELETE FROM artist WHERE artistid = 2 } } {1 {foreign key constraint failed}} -do_test e_fkey-40.2 { +do_test e_fkey-9.2 { execsql { DELETE FROM track WHERE trackartist = 2; DELETE FROM artist WHERE artistid = 2; @@ -279,27 +279,27 @@ do_test e_fkey-40.2 { # If the foreign key column (trackartist) in table 'track' is set to NULL, # there is no requirement for a matching row in the 'artist' table. # -do_test e_fkey-41.1 { +do_test e_fkey-10.1 { execsql { INSERT INTO track VALUES(1, 'track 1', NULL); INSERT INTO track VALUES(2, 'track 2', NULL); } } {} -do_test e_fkey-41.2 { +do_test e_fkey-10.2 { execsql { SELECT * FROM artist } } {} -do_test e_fkey-41.3 { +do_test e_fkey-10.3 { # Setting the trackid to a non-NULL value fails, of course. catchsql { UPDATE track SET trackartist = 5 WHERE trackid = 1 } } {1 {foreign key constraint failed}} -do_test e_fkey-41.4 { +do_test e_fkey-10.4 { execsql { INSERT INTO artist VALUES(5, 'artist 5'); UPDATE track SET trackartist = 5 WHERE trackid = 1; } catchsql { DELETE FROM artist WHERE artistid = 5} } {1 {foreign key constraint failed}} -do_test e_fkey-41.5 { +do_test e_fkey-10.5 { execsql { UPDATE track SET trackartist = NULL WHERE trackid = 1; DELETE FROM artist WHERE artistid = 5; @@ -329,7 +329,7 @@ proc test_r52486_21352 {tn sql} { error $res } - do_test e_fkey-42.$tn { + do_test e_fkey-11.$tn { execsql { SELECT count(*) FROM track WHERE NOT ( trackartist IS NULL OR @@ -368,7 +368,7 @@ for {set i 0} {$i < 500} {incr i} { # to prohibit NULL child keys from being inserted. # drop_all_tables -do_test e_fkey-48.1 { +do_test e_fkey-12.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -382,7 +382,7 @@ do_test e_fkey-48.1 { ); } } {} -do_test e_fkey-48.2 { +do_test e_fkey-12.2 { catchsql { INSERT INTO track VALUES(14, 'Mr. Bojangles', NULL) } } {1 {track.trackartist may not be NULL}} @@ -392,7 +392,7 @@ do_test e_fkey-48.2 { # Test an example from foreignkeys.html. # drop_all_tables -do_test e_fkey-43.1 { +do_test e_fkey-13.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -411,18 +411,18 @@ do_test e_fkey-43.1 { INSERT INTO track VALUES(13, 'My Way', 2); } } {} -do_test e_fkey-43.2 { +do_test e_fkey-13.2 { catchsql { INSERT INTO track VALUES(14, 'Mr. Bojangles', 3) } } {1 {foreign key constraint failed}} -do_test e_fkey-43.3 { +do_test e_fkey-13.3 { execsql { INSERT INTO track VALUES(14, 'Mr. Bojangles', NULL) } } {} -do_test e_fkey-43.4 { +do_test e_fkey-13.4 { catchsql { UPDATE track SET trackartist = 3 WHERE trackname = 'Mr. Bojangles'; } } {1 {foreign key constraint failed}} -do_test e_fkey-43.5 { +do_test e_fkey-13.5 { execsql { INSERT INTO artist VALUES(3, 'Sammy Davis Jr.'); UPDATE track SET trackartist = 3 WHERE trackname = 'Mr. Bojangles'; @@ -435,23 +435,23 @@ do_test e_fkey-43.5 { # # Test the second example from the first section of foreignkeys.html. # -do_test e_fkey-44.1 { +do_test e_fkey-14.1 { catchsql { DELETE FROM artist WHERE artistname = 'Frank Sinatra'; } } {1 {foreign key constraint failed}} -do_test e_fkey-44.2 { +do_test e_fkey-14.2 { execsql { DELETE FROM track WHERE trackname = 'My Way'; DELETE FROM artist WHERE artistname = 'Frank Sinatra'; } } {} -do_test e_fkey-44.3 { +do_test e_fkey-14.3 { catchsql { UPDATE artist SET artistid=4 WHERE artistname = 'Dean Martin'; } } {1 {foreign key constraint failed}} -do_test e_fkey-44.4 { +do_test e_fkey-14.4 { execsql { DELETE FROM track WHERE trackname IN('That''s Amore', 'Christmas Blues'); UPDATE artist SET artistid=4 WHERE artistname = 'Dean Martin'; @@ -473,7 +473,7 @@ do_test e_fkey-44.4 { # is a matching row in the parent table of a foreign key constraint. # drop_all_tables -do_test e_fkey-45.1 { +do_test e_fkey-15.1 { execsql { CREATE TABLE par(p PRIMARY KEY); CREATE TABLE chi(c REFERENCES par); @@ -486,11 +486,11 @@ do_test e_fkey-45.1 { } {integer text blob} proc test_efkey_45 {tn isError sql} { - do_test e_fkey-45.$tn.1 " + do_test e_fkey-15.$tn.1 " catchsql {$sql} " [lindex {{0 {}} {1 {foreign key constraint failed}}} $isError] - do_test e_fkey-45.$tn.2 { + do_test e_fkey-15.$tn.2 { execsql { SELECT * FROM chi WHERE c IS NOT NULL AND c NOT IN (SELECT p FROM par) } @@ -514,13 +514,13 @@ test_efkey_45 9 1 "INSERT INTO chi VALUES(X'32')" # default collation sequence of the parent key column is used. # drop_all_tables -do_test e_fkey-46.1 { +do_test e_fkey-16.1 { execsql { CREATE TABLE t1(a COLLATE nocase PRIMARY KEY); CREATE TABLE t2(b REFERENCES t1); } } {} -do_test e_fkey-46.2 { +do_test e_fkey-16.2 { execsql { INSERT INTO t1 VALUES('oNe'); INSERT INTO t2 VALUES('one'); @@ -529,10 +529,10 @@ do_test e_fkey-46.2 { UPDATE t1 SET a = 'ONE'; } } {} -do_test e_fkey-46.3 { +do_test e_fkey-16.3 { catchsql { UPDATE t2 SET b = 'two' WHERE rowid = 1 } } {1 {foreign key constraint failed}} -do_test e_fkey-46.4 { +do_test e_fkey-16.4 { catchsql { DELETE FROM t1 WHERE rowid = 1 } } {1 {foreign key constraint failed}} @@ -544,13 +544,13 @@ do_test e_fkey-46.4 { # before the comparison takes place. # drop_all_tables -do_test e_fkey-47.1 { +do_test e_fkey-17.1 { execsql { CREATE TABLE t1(a NUMERIC PRIMARY KEY); CREATE TABLE t2(b TEXT REFERENCES t1); } } {} -do_test e_fkey-47.2 { +do_test e_fkey-17.2 { execsql { INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2); @@ -559,10 +559,10 @@ do_test e_fkey-47.2 { SELECT b, typeof(b) FROM t2; } } {2.0 text} -do_test e_fkey-47.3 { +do_test e_fkey-17.3 { execsql { SELECT typeof(a) FROM t1 } } {integer integer text} -do_test e_fkey-47.4 { +do_test e_fkey-17.4 { catchsql { DELETE FROM t1 WHERE rowid = 2 } } {1 {foreign key constraint failed}} @@ -584,7 +584,7 @@ do_test e_fkey-47.4 { # columns. # drop_all_tables -do_test e_fkey-57.1 { +do_test e_fkey-18.1 { execsql { CREATE TABLE t2(a REFERENCES t1(x)); } @@ -592,7 +592,7 @@ do_test e_fkey-57.1 { proc test_efkey_57 {tn isError sql} { catchsql { DROP TABLE t1 } execsql $sql - do_test e_fkey-57.$tn { + do_test e_fkey-18.$tn { catchsql { INSERT INTO t2 VALUES(NULL) } } [lindex {{0 {}} {1 {foreign key mismatch}}} $isError] } @@ -633,7 +633,7 @@ test_efkey_57 9 1 { # Problem with FK on child6 and child7. # drop_all_tables -do_test e_fkey-56.1 { +do_test e_fkey-19.1 { execsql { CREATE TABLE parent(a PRIMARY KEY, b UNIQUE, c, d, e, f); CREATE UNIQUE INDEX i1 ON parent(c, d); @@ -649,7 +649,7 @@ do_test e_fkey-56.1 { CREATE TABLE child7(r REFERENCES parent(c)); -- Err } } {} -do_test e_fkey-56.2 { +do_test e_fkey-19.2 { execsql { INSERT INTO parent VALUES(1, 2, 3, 4, 5, 6); INSERT INTO child1 VALUES('xxx', 1); @@ -657,16 +657,16 @@ do_test e_fkey-56.2 { INSERT INTO child3 VALUES(3, 4); } } {} -do_test e_fkey-56.2 { +do_test e_fkey-19.2 { catchsql { INSERT INTO child4 VALUES('xxx', 5) } } {1 {foreign key mismatch}} -do_test e_fkey-56.3 { +do_test e_fkey-19.3 { catchsql { INSERT INTO child5 VALUES('xxx', 6) } } {1 {foreign key mismatch}} -do_test e_fkey-56.4 { +do_test e_fkey-19.4 { catchsql { INSERT INTO child6 VALUES(2, 3) } } {1 {foreign key mismatch}} -do_test e_fkey-56.5 { +do_test e_fkey-19.5 { catchsql { INSERT INTO child7 VALUES(3) } } {1 {foreign key mismatch}} @@ -680,7 +680,7 @@ do_test e_fkey-56.5 { # DML statements. The error text for these messages always matches # either "foreign key mismatch" or "no such table*" (using [string match]). # -do_test e_fkey-66.1 { +do_test e_fkey-20.1 { execsql { CREATE TABLE c1(c REFERENCES nosuchtable, d); @@ -715,24 +715,24 @@ foreach {tn tbl ptbl err} { 7 c6 p6 "foreign key mismatch" 8 c7 p7 "foreign key mismatch" } { - do_test e_fkey-66.$tn.1 { + do_test e_fkey-20.$tn.1 { catchsql "INSERT INTO $tbl VALUES('a', 'b')" } [list 1 $err] - do_test e_fkey-66.$tn.2 { + do_test e_fkey-20.$tn.2 { catchsql "UPDATE $tbl SET c = ?, d = ?" } [list 1 $err] - do_test e_fkey-66.$tn.3 { + do_test e_fkey-20.$tn.3 { catchsql "INSERT INTO $tbl SELECT ?, ?" } [list 1 $err] if {$ptbl ne ""} { - do_test e_fkey-66.$tn.4 { + do_test e_fkey-20.$tn.4 { catchsql "DELETE FROM $ptbl" } [list 1 $err] - do_test e_fkey-66.$tn.5 { + do_test e_fkey-20.$tn.5 { catchsql "UPDATE $ptbl SET a = ?, b = ?" } [list 1 $err] - do_test e_fkey-66.$tn.6 { + do_test e_fkey-20.$tn.6 { catchsql "INSERT INTO $ptbl SELECT ?, ?" } [list 1 $err] } @@ -746,7 +746,7 @@ foreach {tn tbl ptbl err} { # child key consists of a different number of columns to that primary key. # drop_all_tables -do_test e_fkey-58.1 { +do_test e_fkey-21.1 { execsql { CREATE TABLE parent2(a, b, PRIMARY KEY(a,b)); @@ -755,28 +755,28 @@ do_test e_fkey-58.1 { CREATE TABLE child10(x,y,z, FOREIGN KEY(x,y,z) REFERENCES parent2); -- Err } } {} -do_test e_fkey-58.2 { +do_test e_fkey-21.2 { execsql { INSERT INTO parent2 VALUES('I', 'II'); INSERT INTO child8 VALUES('I', 'II'); } } {} -do_test e_fkey-58.3 { +do_test e_fkey-21.3 { catchsql { INSERT INTO child9 VALUES('I') } } {1 {foreign key mismatch}} -do_test e_fkey-58.4 { +do_test e_fkey-21.4 { catchsql { INSERT INTO child9 VALUES('II') } } {1 {foreign key mismatch}} -do_test e_fkey-58.5 { +do_test e_fkey-21.5 { catchsql { INSERT INTO child9 VALUES(NULL) } } {1 {foreign key mismatch}} -do_test e_fkey-58.6 { +do_test e_fkey-21.6 { catchsql { INSERT INTO child10 VALUES('I', 'II', 'III') } } {1 {foreign key mismatch}} -do_test e_fkey-58.7 { +do_test e_fkey-21.7 { catchsql { INSERT INTO child10 VALUES(1, 2, 3) } } {1 {foreign key mismatch}} -do_test e_fkey-58.8 { +do_test e_fkey-21.8 { catchsql { INSERT INTO child10 VALUES(NULL, NULL, NULL) } } {1 {foreign key mismatch}} @@ -807,7 +807,7 @@ foreach fk [list OFF ON] { "CREATE TABLE child2(a, b, FOREIGN KEY(c, b) REFERENCES p(c, d))" {unknown column "c" in foreign key definition} } { - do_test e_fkey-59.$fk.[incr i] { + do_test e_fkey-22.$fk.[incr i] { catchsql $sql } [list 1 $error] } @@ -819,7 +819,7 @@ foreach fk [list OFF ON] { # Test that a REFERENCING clause that does not specify parent key columns # implicitly maps to the primary key of the parent table. # -do_test e_fkey-60.1 { +do_test e_fkey-23.1 { execsql { CREATE TABLE p1(a, b, PRIMARY KEY(a, b)); CREATE TABLE p2(a, b PRIMARY KEY); @@ -828,7 +828,7 @@ do_test e_fkey-60.1 { } } {} proc test_efkey_60 {tn isError sql} { - do_test e_fkey-60.$tn " + do_test e_fkey-23.$tn " catchsql {$sql} " [lindex {{0 {}} {1 {foreign key constraint failed}}} $isError] } @@ -852,7 +852,7 @@ test_efkey_60 7 0 "INSERT INTO c2 VALUES(239, 231)" # not make a difference whether or not it is a UNIQUE index. # drop_all_tables -do_test e_fkey-61.1 { +do_test e_fkey-24.1 { execsql { CREATE TABLE parent(x, y, UNIQUE(y, x)); CREATE TABLE c1(a, b, FOREIGN KEY(a, b) REFERENCES parent(x, y)); @@ -863,7 +863,7 @@ do_test e_fkey-61.1 { } } {} proc test_efkey_61 {tn isError sql} { - do_test e_fkey-61.$tn " + do_test e_fkey-24.$tn " catchsql {$sql} " [lindex {{0 {}} {1 {foreign key constraint failed}}} $isError] } @@ -888,7 +888,7 @@ foreach {tn c} [list 2 c1 3 c2 4 c3] { # Also test that if the SELECT above would return any rows, a foreign # key constraint is violated. # -do_test e_fkey-62.1 { +do_test e_fkey-25.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -902,20 +902,20 @@ do_test e_fkey-62.1 { ); } } {} -do_test e_fkey-62.2 { +do_test e_fkey-25.2 { execsql { PRAGMA foreign_keys = OFF; EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; EXPLAIN QUERY PLAN SELECT rowid FROM track WHERE trackartist = ?; } } {0 0 {TABLE artist} 0 0 {TABLE track}} -do_test e_fkey-62.3 { +do_test e_fkey-25.3 { execsql { PRAGMA foreign_keys = ON; EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; } } {0 0 {TABLE artist} 0 0 {TABLE track}} -do_test e_fkey-62.4 { +do_test e_fkey-25.4 { execsql { INSERT INTO artist VALUES(5, 'artist 5'); INSERT INTO artist VALUES(6, 'artist 6'); @@ -925,19 +925,19 @@ do_test e_fkey-62.4 { } } {} -do_test e_fkey-62.5 { +do_test e_fkey-25.5 { concat \ [execsql { SELECT rowid FROM track WHERE trackartist = 5 }] \ [catchsql { DELETE FROM artist WHERE artistid = 5 }] } {1 1 {foreign key constraint failed}} -do_test e_fkey-62.6 { +do_test e_fkey-25.6 { concat \ [execsql { SELECT rowid FROM track WHERE trackartist = 7 }] \ [catchsql { DELETE FROM artist WHERE artistid = 7 }] } {0 {}} -do_test e_fkey-62.7 { +do_test e_fkey-25.7 { concat \ [execsql { SELECT rowid FROM track WHERE trackartist = 6 }] \ [catchsql { DELETE FROM artist WHERE artistid = 6 }] @@ -963,7 +963,7 @@ do_test e_fkey-62.7 { # # drop_all_tables -do_test e_fkey-64.1 { +do_test e_fkey-26.1 { execsql { CREATE TABLE parent(x, y, UNIQUE(y, x)) } } {} foreach {tn sql} { @@ -993,8 +993,8 @@ foreach {tn sql} { ] execsql {PRAGMA foreign_keys = ON} - do_test e_fkey-64.$tn.1 { eqp "DELETE FROM parent WHERE 1" } $delete - do_test e_fkey-64.$tn.2 { eqp "UPDATE parent set x=?, y=?" } $update + do_test e_fkey-26.$tn.1 { eqp "DELETE FROM parent WHERE 1" } $delete + do_test e_fkey-26.$tn.2 { eqp "UPDATE parent set x=?, y=?" } $update execsql {DROP TABLE child} } @@ -1007,7 +1007,7 @@ foreach {tn sql} { # related operations on the parent table do not provoke linear scans. # drop_all_tables -do_test e_fkey-63.1 { +do_test e_fkey-27.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -1021,17 +1021,17 @@ do_test e_fkey-63.1 { CREATE INDEX trackindex ON track(trackartist); } } {} -do_test e_fkey-63.2 { +do_test e_fkey-27.2 { eqp { INSERT INTO artist VALUES(?, ?) } } {} -do_test e_fkey-63.3 { +do_test e_fkey-27.3 { eqp { UPDATE artist SET artistid = ?, artistname = ? } } [list \ 0 0 {TABLE artist} \ 0 0 {TABLE track WITH INDEX trackindex} \ 0 0 {TABLE track WITH INDEX trackindex} ] -do_test e_fkey-63.4 { +do_test e_fkey-27.4 { eqp { DELETE FROM artist } } [list \ 0 0 {TABLE artist} \ @@ -1070,9 +1070,9 @@ foreach {tn sql err} { {number of columns in foreign key does not match the number of columns in the referenced table} } { drop_all_tables - do_test e_fkey-65.$tn [list catchsql $sql] [list 1 $err] + do_test e_fkey-28.$tn [list catchsql $sql] [list 1 $err] } -do_test e_fkey-65.8 { +do_test e_fkey-28.8 { drop_all_tables execsql { CREATE TABLE p(x PRIMARY KEY); @@ -1080,7 +1080,7 @@ do_test e_fkey-65.8 { } catchsql {DELETE FROM p} } {1 {foreign key mismatch}} -do_test e_fkey-65.9 { +do_test e_fkey-28.9 { drop_all_tables execsql { CREATE TABLE p(x, y, PRIMARY KEY(x,y)); @@ -1096,7 +1096,7 @@ do_test e_fkey-65.9 { # Test the example schema in the "Composite Foreign Key Constraints" # section. # -do_test e_fkey-36.1 { +do_test e_fkey-29.1 { execsql { CREATE TABLE album( albumartist TEXT, @@ -1114,7 +1114,7 @@ do_test e_fkey-36.1 { } } {} -do_test e_fkey-36.2 { +do_test e_fkey-29.2 { execsql { INSERT INTO album VALUES('Elvis Presley', 'Elvis'' Christmas Album', NULL); INSERT INTO song VALUES( @@ -1122,7 +1122,7 @@ do_test e_fkey-36.2 { ); } } {} -do_test e_fkey-36.3 { +do_test e_fkey-29.3 { catchsql { INSERT INTO song VALUES(2, 'Elvis Presley', 'Elvis Is Back!', 'Fever'); } @@ -1135,7 +1135,7 @@ do_test e_fkey-36.3 { # Check that if any of the child key columns in the above schema are NULL, # there is no requirement for a corresponding parent key. # -do_test e_fkey-37.1 { +do_test e_fkey-30.1 { execsql { INSERT INTO song VALUES(2, 'Elvis Presley', NULL, 'Fever'); INSERT INTO song VALUES(3, NULL, 'Elvis Is Back', 'Soldier Boy'); @@ -1160,19 +1160,19 @@ do_test e_fkey-37.1 { # the statement rolled back. # drop_all_tables -do_test e_fkey-33.1 { +do_test e_fkey-31.1 { execsql { CREATE TABLE king(a, b, PRIMARY KEY(a)); CREATE TABLE prince(c REFERENCES king, d); } } {} -do_test e_fkey-33.2 { +do_test e_fkey-31.2 { # Execute a statement that violates the immediate FK constraint. catchsql { INSERT INTO prince VALUES(1, 2) } } {1 {foreign key constraint failed}} -do_test e_fkey-33.3 { +do_test e_fkey-31.3 { # This time, use a trigger to fix the constraint violation before the # statement has finished executing. Then execute the same statement as # in the previous test case. This time, no error. @@ -1188,7 +1188,7 @@ do_test e_fkey-33.3 { # Test that operating inside a transaction makes no difference to # immediate constraint violation handling. -do_test e_fkey-33.4 { +do_test e_fkey-31.4 { execsql { BEGIN; INSERT INTO prince VALUES(2, 3); @@ -1196,7 +1196,7 @@ do_test e_fkey-33.4 { } catchsql { INSERT INTO prince VALUES(3, 4) } } {1 {foreign key constraint failed}} -do_test e_fkey-33.5 { +do_test e_fkey-31.5 { execsql { COMMIT; SELECT * FROM king; @@ -1215,7 +1215,7 @@ do_test e_fkey-33.5 { # to COMMIT the transaction fail until the FK constraint is satisfied. # proc test_efkey_34 {tn isError sql} { - do_test e_fkey-34.$tn " + do_test e_fkey-32.$tn " catchsql {$sql} " [lindex {{0 {}} {1 {foreign key constraint failed}}} $isError] } @@ -1242,11 +1242,11 @@ test_efkey_34 9 0 "COMMIT" # drop_all_tables proc test_efkey_35 {tn isError sql} { - do_test e_fkey-35.$tn " + do_test e_fkey-33.$tn " catchsql {$sql} " [lindex {{0 {}} {1 {foreign key constraint failed}}} $isError] } -do_test e_fkey-35.1 { +do_test e_fkey-33.1 { execsql { CREATE TABLE parent(x, y); CREATE UNIQUE INDEX pi ON parent(x, y); @@ -1287,7 +1287,7 @@ test_efkey_35 4 0 "INSERT INTO child VALUES('x', 'y')" # /* EV: R-30323-21917 */ FKs are either IMMEDIATE or DEFERRED. # drop_all_tables -do_test e_fkey-29.1 { +do_test e_fkey-34.1 { execsql { CREATE TABLE parent(x, y, z, PRIMARY KEY(x,y,z)); CREATE TABLE c1(a, b, c, @@ -1331,7 +1331,7 @@ do_test e_fkey-29.1 { } {} proc test_efkey_29 {tn sql isError} { - do_test e_fkey-29.$tn "catchsql {$sql}" [ + do_test e_fkey-34.$tn "catchsql {$sql}" [ lindex {{0 {}} {1 {foreign key constraint failed}}} $isError ] } @@ -1386,7 +1386,7 @@ test_efkey_29 33 "ROLLBACK" 0 # Test an example from foreignkeys.html dealing with a deferred foreign # key constraint. # -do_test e_fkey-28.1 { +do_test e_fkey-35.1 { drop_all_tables execsql { CREATE TABLE artist( @@ -1400,14 +1400,14 @@ do_test e_fkey-28.1 { ); } } {} -do_test e_fkey-28.2 { +do_test e_fkey-35.2 { execsql { BEGIN; INSERT INTO track VALUES(1, 'White Christmas', 5); } catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-28.3 { +do_test e_fkey-35.3 { execsql { INSERT INTO artist VALUES(5, 'Bing Crosby'); COMMIT; @@ -1421,7 +1421,7 @@ do_test e_fkey-28.3 { # deferred foreign key constraints. # drop_all_tables -do_test e_fkey-30.1 { +do_test e_fkey-36.1 { execsql { CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t1 DEFERRABLE INITIALLY DEFERRED @@ -1431,7 +1431,7 @@ do_test e_fkey-30.1 { INSERT INTO t1 VALUES(3, 3); } } {} -do_test e_fkey-30.2 { +do_test e_fkey-36.2 { execsql { BEGIN; SAVEPOINT one; @@ -1439,10 +1439,10 @@ do_test e_fkey-30.2 { RELEASE one; } } {} -do_test e_fkey-30.3 { +do_test e_fkey-36.3 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-30.4 { +do_test e_fkey-36.4 { execsql { UPDATE t1 SET a = 5 WHERE a = 4; COMMIT; @@ -1457,7 +1457,7 @@ do_test e_fkey-30.4 { # the database was in auto-commit mode) cannot be released without # satisfying deferred foreign key constraints. It may be rolled back. # -do_test e_fkey-31.1 { +do_test e_fkey-37.1 { execsql { SAVEPOINT one; SAVEPOINT two; @@ -1465,16 +1465,16 @@ do_test e_fkey-31.1 { RELEASE two; } } {} -do_test e_fkey-31.2 { +do_test e_fkey-37.2 { catchsql {RELEASE one} } {1 {foreign key constraint failed}} -do_test e_fkey-31.3 { +do_test e_fkey-37.3 { execsql { UPDATE t1 SET a = 7 WHERE a = 6; RELEASE one; } } {} -do_test e_fkey-31.4 { +do_test e_fkey-37.4 { execsql { SAVEPOINT one; SAVEPOINT two; @@ -1482,10 +1482,10 @@ do_test e_fkey-31.4 { RELEASE two; } } {} -do_test e_fkey-31.5 { +do_test e_fkey-37.5 { catchsql {RELEASE one} } {1 {foreign key constraint failed}} -do_test e_fkey-31.6 { +do_test e_fkey-37.6 { execsql {ROLLBACK TO one ; RELEASE one} } {} @@ -1495,13 +1495,13 @@ do_test e_fkey-31.6 { # Test that if a COMMIT operation fails due to deferred foreign key # constraints, any nested savepoints remain open. # -do_test e_fkey-32.1 { +do_test e_fkey-38.1 { execsql { DELETE FROM t1 WHERE a>3; SELECT * FROM t1; } } {1 1 2 2 3 3} -do_test e_fkey-32.2 { +do_test e_fkey-38.2 { execsql { BEGIN; INSERT INTO t1 VALUES(4, 4); @@ -1510,10 +1510,10 @@ do_test e_fkey-32.2 { SELECT * FROM t1; } } {1 1 2 2 3 3 4 4 5 6} -do_test e_fkey-32.3 { +do_test e_fkey-38.3 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-32.4 { +do_test e_fkey-38.4 { execsql { ROLLBACK TO one; COMMIT; @@ -1521,7 +1521,7 @@ do_test e_fkey-32.4 { } } {1 1 2 2 3 3 4 4} -do_test e_fkey-32.5 { +do_test e_fkey-38.5 { execsql { SAVEPOINT a; INSERT INTO t1 VALUES(5, 5); @@ -1531,14 +1531,14 @@ do_test e_fkey-32.5 { INSERT INTO t1 VALUES(7, 8); } } {} -do_test e_fkey-32.6 { +do_test e_fkey-38.6 { catchsql {RELEASE a} } {1 {foreign key constraint failed}} -do_test e_fkey-32.7 { +do_test e_fkey-38.7 { execsql {ROLLBACK TO c} catchsql {RELEASE a} } {1 {foreign key constraint failed}} -do_test e_fkey-32.8 { +do_test e_fkey-38.8 { execsql { ROLLBACK TO b; RELEASE a; @@ -1561,7 +1561,7 @@ do_test e_fkey-32.8 { # Test that a single FK constraint may have different actions configured # for ON DELETE and ON UPDATE. # -do_test e_fkey-16.1 { +do_test e_fkey-39.1 { execsql { CREATE TABLE p(a, b PRIMARY KEY, c); CREATE TABLE c1(d, e, f DEFAULT 'k0' REFERENCES p @@ -1579,19 +1579,19 @@ do_test e_fkey-16.1 { INSERT INTO c1 VALUES(3, 'xx', 'k3'); } } {} -do_test e_fkey-16.2 { +do_test e_fkey-39.2 { execsql { UPDATE p SET b = 'k4' WHERE a = 1; SELECT * FROM c1; } } {1 xx k0 2 xx k2 3 xx k3} -do_test e_fkey-16.3 { +do_test e_fkey-39.3 { execsql { DELETE FROM p WHERE a = 2; SELECT * FROM c1; } } {1 xx k0 2 xx {} 3 xx k3} -do_test e_fkey-16.4 { +do_test e_fkey-39.4 { execsql { CREATE UNIQUE INDEX pi ON p(c); REPLACE INTO p VALUES(5, 'k5', 'III'); @@ -1610,7 +1610,7 @@ do_test e_fkey-16.4 { # If none is specified explicitly, "NO ACTION" is the default. # drop_all_tables -do_test e_fkey-17.1 { +do_test e_fkey-40.1 { execsql { CREATE TABLE parent(x PRIMARY KEY, y); CREATE TABLE child1(a, @@ -1644,7 +1644,7 @@ foreach {tn zTab lRes} { 8 child7 {0 0 parent b {} {NO ACTION} {NO ACTION} NONE} 9 child8 {0 0 parent b {} {NO ACTION} {NO ACTION} NONE} } { - do_test e_fkey-17.$tn { execsql "PRAGMA foreign_key_list($zTab)" } $lRes + do_test e_fkey-40.$tn { execsql "PRAGMA foreign_key_list($zTab)" } $lRes } #------------------------------------------------------------------------- @@ -1654,7 +1654,7 @@ foreach {tn zTab lRes} { # it's parent row is updated or deleted. # drop_all_tables -do_test e_fkey-18.1 { +do_test e_fkey-41.1 { execsql { CREATE TABLE parent(p1, p2, PRIMARY KEY(p1, p2)); CREATE TABLE child(c1, c2, @@ -1669,7 +1669,7 @@ do_test e_fkey-18.1 { INSERT INTO child VALUES('l', 'm'); } } {} -do_test e_fkey-18.2 { +do_test e_fkey-41.2 { execsql { BEGIN; UPDATE parent SET p1='k' WHERE p1='j'; @@ -1677,10 +1677,10 @@ do_test e_fkey-18.2 { SELECT * FROM child; } } {j k l m} -do_test e_fkey-18.3 { +do_test e_fkey-41.3 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-18.4 { +do_test e_fkey-41.4 { execsql ROLLBACK } {} @@ -1692,7 +1692,7 @@ do_test e_fkey-18.4 { # mapped to it. # drop_all_tables -do_test e_fkey-18.1 { +do_test e_fkey-41.1 { execsql { CREATE TABLE parent(p1, p2); CREATE UNIQUE INDEX parent_i ON parent(p1, p2); @@ -1704,7 +1704,7 @@ do_test e_fkey-18.1 { ); } } {} -do_test e_fkey-18.2 { +do_test e_fkey-41.2 { execsql { INSERT INTO parent VALUES('a', 'b'); INSERT INTO parent VALUES('c', 'd'); @@ -1712,10 +1712,10 @@ do_test e_fkey-18.2 { INSERT INTO child2 VALUES('d', 'c'); } } {} -do_test e_fkey-18.3 { +do_test e_fkey-41.3 { catchsql { DELETE FROM parent WHERE p1 = 'a' } } {1 {foreign key constraint failed}} -do_test e_fkey-18.4 { +do_test e_fkey-41.4 { catchsql { UPDATE parent SET p2 = 'e' WHERE p1 = 'c' } } {1 {foreign key constraint failed}} @@ -1727,7 +1727,7 @@ do_test e_fkey-18.4 { # statement. # drop_all_tables -do_test e_fkey-19.1 { +do_test e_fkey-42.1 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child1(c REFERENCES parent ON UPDATE RESTRICT); @@ -1744,10 +1744,10 @@ do_test e_fkey-19.1 { END; } } {} -do_test e_fkey-19.2 { +do_test e_fkey-42.2 { catchsql { UPDATE parent SET x = 'key one' WHERE x = 'key1' } } {1 {foreign key constraint failed}} -do_test e_fkey-19.3 { +do_test e_fkey-42.3 { execsql { UPDATE parent SET x = 'key two' WHERE x = 'key2'; SELECT * FROM child2; @@ -1755,7 +1755,7 @@ do_test e_fkey-19.3 { } {{key two}} drop_all_tables -do_test e_fkey-19.4 { +do_test e_fkey-42.4 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child1(c REFERENCES parent ON DELETE RESTRICT); @@ -1772,10 +1772,10 @@ do_test e_fkey-19.4 { END; } } {} -do_test e_fkey-19.5 { +do_test e_fkey-42.5 { catchsql { DELETE FROM parent WHERE x = 'key1' } } {1 {foreign key constraint failed}} -do_test e_fkey-19.6 { +do_test e_fkey-42.6 { execsql { DELETE FROM parent WHERE x = 'key2'; SELECT * FROM child2; @@ -1783,7 +1783,7 @@ do_test e_fkey-19.6 { } {{}} drop_all_tables -do_test e_fkey-19.7 { +do_test e_fkey-42.7 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child1(c REFERENCES parent ON DELETE RESTRICT); @@ -1795,10 +1795,10 @@ do_test e_fkey-19.7 { INSERT INTO child2 VALUES('key2'); } } {} -do_test e_fkey-19.8 { +do_test e_fkey-42.8 { catchsql { REPLACE INTO parent VALUES('key1') } } {1 {foreign key constraint failed}} -do_test e_fkey-19.9 { +do_test e_fkey-42.9 { execsql { REPLACE INTO parent VALUES('key2'); SELECT * FROM child2; @@ -1811,7 +1811,7 @@ do_test e_fkey-19.9 { # Test that RESTRICT is enforced immediately, even for a DEFERRED constraint. # drop_all_tables -do_test e_fkey-20.1 { +do_test e_fkey-43.1 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child1(c REFERENCES parent ON UPDATE RESTRICT @@ -1828,16 +1828,16 @@ do_test e_fkey-20.1 { BEGIN; } } {} -do_test e_fkey-20.2 { +do_test e_fkey-43.2 { catchsql { UPDATE parent SET x = 'key one' WHERE x = 'key1' } } {1 {foreign key constraint failed}} -do_test e_fkey-20.3 { +do_test e_fkey-43.3 { execsql { UPDATE parent SET x = 'key two' WHERE x = 'key2' } } {} -do_test e_fkey-20.4 { +do_test e_fkey-43.4 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-20.5 { +do_test e_fkey-43.5 { execsql { UPDATE child2 SET c = 'key two'; COMMIT; @@ -1845,7 +1845,7 @@ do_test e_fkey-20.5 { } {} drop_all_tables -do_test e_fkey-20.6 { +do_test e_fkey-43.6 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child1(c REFERENCES parent ON DELETE RESTRICT @@ -1862,16 +1862,16 @@ do_test e_fkey-20.6 { BEGIN; } } {} -do_test e_fkey-20.7 { +do_test e_fkey-43.7 { catchsql { DELETE FROM parent WHERE x = 'key1' } } {1 {foreign key constraint failed}} -do_test e_fkey-20.8 { +do_test e_fkey-43.8 { execsql { DELETE FROM parent WHERE x = 'key2' } } {} -do_test e_fkey-20.9 { +do_test e_fkey-43.9 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-20.10 { +do_test e_fkey-43.10 { execsql { UPDATE child2 SET c = NULL; COMMIT; @@ -1884,7 +1884,7 @@ do_test e_fkey-20.10 { # Test SET NULL actions. # drop_all_tables -do_test e_fkey-21.1 { +do_test e_fkey-44.1 { execsql { CREATE TABLE pA(x PRIMARY KEY); CREATE TABLE cA(c REFERENCES pA ON DELETE SET NULL); @@ -1896,24 +1896,24 @@ do_test e_fkey-21.1 { INSERT INTO cB VALUES(X'1234'); } } {} -do_test e_fkey-21.2 { +do_test e_fkey-44.2 { execsql { DELETE FROM pA WHERE rowid = 1; SELECT quote(x) FROM pA; } } {X'1234'} -do_test e_fkey-21.3 { +do_test e_fkey-44.3 { execsql { SELECT quote(c) FROM cA; } } {NULL} -do_test e_fkey-21.4 { +do_test e_fkey-44.4 { execsql { UPDATE pA SET x = X'8765' WHERE rowid = 2; SELECT quote(x) FROM pA; } } {X'8765'} -do_test e_fkey-21.5 { +do_test e_fkey-44.5 { execsql { SELECT quote(c) FROM cB } } {NULL} @@ -1923,7 +1923,7 @@ do_test e_fkey-21.5 { # Test SET DEFAULT actions. # drop_all_tables -do_test e_fkey-22.1 { +do_test e_fkey-45.1 { execsql { CREATE TABLE pA(x PRIMARY KEY); CREATE TABLE cA(c DEFAULT X'0000' REFERENCES pA ON DELETE SET DEFAULT); @@ -1938,22 +1938,22 @@ do_test e_fkey-22.1 { INSERT INTO cB VALUES(X'1234'); } } {} -do_test e_fkey-22.2 { +do_test e_fkey-45.2 { execsql { DELETE FROM pA WHERE rowid = 3; SELECT quote(x) FROM pA; } } {X'0000' X'9999' X'1234'} -do_test e_fkey-22.3 { +do_test e_fkey-45.3 { execsql { SELECT quote(c) FROM cA } } {X'0000'} -do_test e_fkey-22.4 { +do_test e_fkey-45.4 { execsql { UPDATE pA SET x = X'8765' WHERE rowid = 4; SELECT quote(x) FROM pA; } } {X'0000' X'9999' X'8765'} -do_test e_fkey-22.5 { +do_test e_fkey-45.5 { execsql { SELECT quote(c) FROM cB } } {X'9999'} @@ -1964,7 +1964,7 @@ do_test e_fkey-22.5 { # Test ON DELETE CASCADE actions. # drop_all_tables -do_test e_fkey-23.1 { +do_test e_fkey-46.1 { execsql { CREATE TABLE p1(a, b UNIQUE); CREATE TABLE c1(c REFERENCES p1(b) ON DELETE CASCADE, d); @@ -1977,19 +1977,19 @@ do_test e_fkey-23.1 { SELECT count(*) FROM c1; } } {3} -do_test e_fkey-23.2 { +do_test e_fkey-46.2 { execsql { DELETE FROM p1 WHERE a = 4; SELECT d, c FROM c1; } } {{} {} 5 5} -do_test e_fkey-23.3 { +do_test e_fkey-46.3 { execsql { DELETE FROM p1; SELECT d, c FROM c1; } } {{} {}} -do_test e_fkey-23.4 { +do_test e_fkey-46.4 { execsql { SELECT * FROM p1 } } {} @@ -2001,7 +2001,7 @@ do_test e_fkey-23.4 { # Test ON UPDATE CASCADE actions. # drop_all_tables -do_test e_fkey-24.1 { +do_test e_fkey-47.1 { execsql { CREATE TABLE p1(a, b UNIQUE); CREATE TABLE c1(c REFERENCES p1(b) ON UPDATE CASCADE, d); @@ -2014,25 +2014,25 @@ do_test e_fkey-24.1 { SELECT count(*) FROM c1; } } {3} -do_test e_fkey-24.2 { +do_test e_fkey-47.2 { execsql { UPDATE p1 SET b = 10 WHERE b = 5; SELECT d, c FROM c1; } } {{} {} 4 4 5 10} -do_test e_fkey-24.3 { +do_test e_fkey-47.3 { execsql { UPDATE p1 SET b = 11 WHERE b = 4; SELECT d, c FROM c1; } } {{} {} 4 11 5 10} -do_test e_fkey-24.4 { +do_test e_fkey-47.4 { execsql { UPDATE p1 SET b = 6 WHERE b IS NULL; SELECT d, c FROM c1; } } {{} {} 4 11 5 10} -do_test e_fkey-23.5 { +do_test e_fkey-46.5 { execsql { SELECT * FROM p1 } } {{} 6 4 11 5 10} @@ -2043,7 +2043,7 @@ do_test e_fkey-23.5 { # of foreignkeys.html. # drop_all_tables -do_test e_fkey-15.1 { +do_test e_fkey-48.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -2062,15 +2062,15 @@ do_test e_fkey-15.1 { INSERT INTO track VALUES(13, 'My Way', 2); } } {} -do_test e_fkey-15.2 { +do_test e_fkey-48.2 { execsql { UPDATE artist SET artistid = 100 WHERE artistname = 'Dean Martin'; } } {} -do_test e_fkey-15.3 { +do_test e_fkey-48.3 { execsql { SELECT * FROM artist } } {2 {Frank Sinatra} 100 {Dean Martin}} -do_test e_fkey-15.4 { +do_test e_fkey-48.4 { execsql { SELECT * FROM track } } {11 {That's Amore} 100 12 {Christmas Blues} 100 13 {My Way} 2} @@ -2082,7 +2082,7 @@ do_test e_fkey-15.4 { # requirement not to violate the foreign key constraint. # drop_all_tables -do_test e_fkey-25.1 { +do_test e_fkey-49.1 { execsql { CREATE TABLE parent(a COLLATE nocase, b, c, PRIMARY KEY(c, a)); CREATE TABLE child(d DEFAULT 'a', e, f DEFAULT 'c', @@ -2094,21 +2094,21 @@ do_test e_fkey-25.1 { INSERT INTO child VALUES('one', 'two', 'three'); } } {} -do_test e_fkey-25.2 { +do_test e_fkey-49.2 { execsql { BEGIN; UPDATE parent SET a = '' WHERE a = 'oNe'; SELECT * FROM child; } } {a two c} -do_test e_fkey-25.3 { +do_test e_fkey-49.3 { execsql { ROLLBACK; DELETE FROM parent WHERE a = 'A'; SELECT * FROM parent; } } {ONE two three} -do_test e_fkey-25.4 { +do_test e_fkey-49.4 { catchsql { UPDATE parent SET a = '' WHERE a = 'oNe' } } {1 {foreign key constraint failed}} @@ -2123,7 +2123,7 @@ do_test e_fkey-25.4 { # (R-28220-46694). # drop_all_tables -do_test e_fkey-14.1 { +do_test e_fkey-50.1 { execsql { CREATE TABLE artist( artistid INTEGER PRIMARY KEY, @@ -2138,19 +2138,19 @@ do_test e_fkey-14.1 { INSERT INTO track VALUES(14, 'Mr. Bojangles', 3); } } {} -do_test e_fkey-14.2 { +do_test e_fkey-50.2 { catchsql { DELETE FROM artist WHERE artistname = 'Sammy Davis Jr.' } } {1 {foreign key constraint failed}} -do_test e_fkey-14.3 { +do_test e_fkey-50.3 { execsql { INSERT INTO artist VALUES(0, 'Unknown Artist'); DELETE FROM artist WHERE artistname = 'Sammy Davis Jr.'; } } {} -do_test e_fkey-14.4 { +do_test e_fkey-50.4 { execsql { SELECT * FROM artist } } {0 {Unknown Artist}} -do_test e_fkey-14.5 { +do_test e_fkey-50.5 { execsql { SELECT * FROM track } } {14 {Mr. Bojangles} 0} @@ -2167,7 +2167,7 @@ do_test e_fkey-14.5 { # 5. Execute applicable AFTER trigger programs. # drop_all_tables -do_test e_fkey-27.1 { +do_test e_fkey-51.1 { proc maxparent {args} { db one {SELECT max(x) FROM parent} } db func maxparent maxparent @@ -2188,13 +2188,13 @@ do_test e_fkey-27.1 { INSERT INTO child VALUES(1); } } {} -do_test e_fkey-27.2 { +do_test e_fkey-51.2 { execsql { UPDATE parent SET x = 22; SELECT * FROM parent UNION ALL SELECT 'xxx' UNION ALL SELECT a FROM child; } } {22 21 23 xxx 22} -do_test e_fkey-27.3 { +do_test e_fkey-51.3 { execsql { DELETE FROM child; DELETE FROM parent; @@ -2215,7 +2215,7 @@ do_test e_fkey-27.3 { # is 'distinct' from the old or not. # drop_all_tables -do_test e_fkey-26.1 { +do_test e_fkey-52.1 { execsql { CREATE TABLE zeus(a INTEGER COLLATE NOCASE, b, PRIMARY KEY(a, b)); CREATE TABLE apollo(c, d, @@ -2229,31 +2229,31 @@ do_test e_fkey-26.1 { SELECT * FROM apollo; } } {ABC xyz} -do_test e_fkey-26.2 { +do_test e_fkey-52.2 { execsql { UPDATE zeus SET a = 1, b = 1; SELECT * FROM apollo; } } {1 1} -do_test e_fkey-26.3 { +do_test e_fkey-52.3 { execsql { UPDATE zeus SET a = 1, b = 1; SELECT typeof(c), c, typeof(d), d FROM apollo; } } {integer 1 integer 1} -do_test e_fkey-26.4 { +do_test e_fkey-52.4 { execsql { UPDATE zeus SET a = '1'; SELECT typeof(c), c, typeof(d), d FROM apollo; } } {integer 1 integer 1} -do_test e_fkey-26.5 { +do_test e_fkey-52.5 { execsql { UPDATE zeus SET b = '1'; SELECT typeof(c), c, typeof(d), d FROM apollo; } } {integer 1 text 1} -do_test e_fkey-26.6 { +do_test e_fkey-52.6 { execsql { UPDATE zeus SET b = NULL; SELECT typeof(c), c, typeof(d), d FROM apollo; @@ -2269,7 +2269,7 @@ do_test e_fkey-26.6 { # that is distinct from its previous value. # drop_all_tables -do_test e_fkey-13.1 { +do_test e_fkey-53.1 { execsql { CREATE TABLE parent(x PRIMARY KEY); CREATE TABLE child(y REFERENCES parent ON UPDATE SET NULL); @@ -2277,13 +2277,13 @@ do_test e_fkey-13.1 { INSERT INTO child VALUES('key'); } } {} -do_test e_fkey-13.2 { +do_test e_fkey-53.2 { execsql { UPDATE parent SET x = 'key'; SELECT IFNULL(y, 'null') FROM child; } } {key} -do_test e_fkey-13.3 { +do_test e_fkey-53.3 { execsql { UPDATE parent SET x = 'key2'; SELECT IFNULL(y, 'null') FROM child; @@ -2325,12 +2325,12 @@ foreach {tn zCreateTbl lRes} { B "CREATE TABLE t1(a, b, FOREIGN KEY(c,b) REFERENCES t2(d))" {1 {number of columns in foreign key does not match the number of columns in the referenced table}} } { - do_test e_fkey-5.$tn.off { + do_test e_fkey-54.$tn.off { drop_all_tables execsql {PRAGMA foreign_keys = OFF} catchsql $zCreateTbl } $lRes - do_test e_fkey-5.$tn.on { + do_test e_fkey-54.$tn.on { drop_all_tables execsql {PRAGMA foreign_keys = ON} catchsql $zCreateTbl @@ -2343,7 +2343,7 @@ foreach {tn zCreateTbl lRes} { proc test_efkey_6 {tn zAlter isError} { drop_all_tables - do_test e_fkey-6.$tn.1 " + do_test e_fkey-56.$tn.1 " execsql { CREATE TABLE tbl(a, b) } [list catchsql $zAlter] " [lindex {{0 {}} {1 {Cannot add a REFERENCES column with non-NULL default value}}} $isError] @@ -2364,7 +2364,7 @@ test_efkey_6 3 "ALTER TABLE tbl ADD COLUMN c DEFAULT 0 REFERENCES xx" 1 # # Test that these adjustments are visible in the sqlite_master table. # -do_test e_fkey-7.1 { +do_test e_fkey-56.1 { drop_all_tables execsql { CREATE TABLE 'p 1 "parent one"'(a REFERENCES 'p 1 "parent one"', b, PRIMARY KEY(b)); @@ -2381,10 +2381,10 @@ do_test e_fkey-7.1 { -- CREATE TABLE q(a, b, PRIMARY KEY(b)); } } {} -do_test e_fkey-7.2 { +do_test e_fkey-56.2 { execsql { ALTER TABLE 'p 1 "parent one"' RENAME TO p } } {} -do_test e_fkey-7.3 { +do_test e_fkey-56.3 { execsql { UPDATE p SET a = 'xxx', b = 'xxx'; SELECT * FROM p; @@ -2393,7 +2393,7 @@ do_test e_fkey-7.3 { SELECT * FROM c3; } } {xxx xxx 1 xxx 1 xxx 1 xxx} -do_test e_fkey-7.4 { +do_test e_fkey-56.4 { execsql { SELECT sql FROM sqlite_master WHERE type = 'table'} } [list \ {CREATE TABLE "p"(a REFERENCES "p", b, PRIMARY KEY(b))} \ @@ -2409,7 +2409,7 @@ do_test e_fkey-7.4 { # Check that a DROP TABLE does an implicit DELETE FROM. Which does not # cause any triggers to fire, but does fire foreign key actions. # -do_test e_fkey-8.1 { +do_test e_fkey-57.1 { drop_all_tables execsql { CREATE TABLE p(a, b, PRIMARY KEY(a, b)); @@ -2436,7 +2436,7 @@ do_test e_fkey-8.1 { } } {} -do_test e_fkey-8.2 { +do_test e_fkey-57.2 { execsql { INSERT INTO p VALUES('a', 'b'); INSERT INTO c1 VALUES('a', 'b'); @@ -2447,19 +2447,19 @@ do_test e_fkey-8.2 { SELECT * FROM c1; } } {{} {}} -do_test e_fkey-8.3 { +do_test e_fkey-57.3 { execsql { SELECT * FROM c2 } } {{} {}} -do_test e_fkey-8.4 { +do_test e_fkey-57.4 { execsql { SELECT * FROM c3 } } {} -do_test e_fkey-8.5 { +do_test e_fkey-57.5 { execsql { SELECT * FROM log } } {} -do_test e_fkey-8.6 { +do_test e_fkey-57.6 { execsql ROLLBACK } {} -do_test e_fkey-8.7 { +do_test e_fkey-57.7 { execsql { BEGIN; DELETE FROM p; @@ -2474,7 +2474,7 @@ do_test e_fkey-8.7 { # If an IMMEDIATE foreign key fails as a result of a DROP TABLE, the # DROP TABLE command fails. # -do_test e_fkey-9.1 { +do_test e_fkey-58.1 { execsql { DELETE FROM c1; DELETE FROM c2; @@ -2483,16 +2483,16 @@ do_test e_fkey-9.1 { execsql { INSERT INTO c5 VALUES('a', 'b') } catchsql { DROP TABLE p } } {1 {foreign key constraint failed}} -do_test e_fkey-9.2 { +do_test e_fkey-58.2 { execsql { SELECT * FROM p } } {a b} -do_test e_fkey-9.3 { +do_test e_fkey-58.3 { catchsql { BEGIN; DROP TABLE p; } } {1 {foreign key constraint failed}} -do_test e_fkey-9.4 { +do_test e_fkey-58.4 { execsql { SELECT * FROM p; SELECT * FROM c5; @@ -2506,28 +2506,28 @@ do_test e_fkey-9.4 { # If a DEFERRED foreign key fails as a result of a DROP TABLE, attempting # to commit the transaction fails unless the violation is fixed. # -do_test e_fkey-10.1 { +do_test e_fkey-59.1 { execsql { DELETE FROM c1 ; DELETE FROM c2 ; DELETE FROM c3 ; DELETE FROM c4 ; DELETE FROM c5 ; DELETE FROM c6 ; DELETE FROM c7 } } {} -do_test e_fkey-10.2 { +do_test e_fkey-59.2 { execsql { INSERT INTO c7 VALUES('a', 'b') } execsql { BEGIN; DROP TABLE p; } } {} -do_test e_fkey-10.3 { +do_test e_fkey-59.3 { catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-10.4 { +do_test e_fkey-59.4 { execsql { CREATE TABLE p(a, b, PRIMARY KEY(a, b)) } catchsql COMMIT } {1 {foreign key constraint failed}} -do_test e_fkey-10.5 { +do_test e_fkey-59.5 { execsql { INSERT INTO p VALUES('a', 'b') } execsql COMMIT } {} @@ -2539,7 +2539,7 @@ do_test e_fkey-10.5 { # "DELETE FROM tbl" are ignored. # drop_all_tables -do_test e_fkey-11.1 { +do_test e_fkey-60.1 { execsql { PRAGMA foreign_keys = OFF; @@ -2554,11 +2554,11 @@ do_test e_fkey-11.1 { INSERT INTO c3 VALUES(1, 2); } } {} -do_test e_fkey-11.2 { +do_test e_fkey-60.2 { execsql { PRAGMA foreign_keys = ON } catchsql { DELETE FROM p } } {1 {no such table: main.nosuchtable}} -do_test e_fkey-11.3 { +do_test e_fkey-60.3 { execsql { BEGIN; DROP TABLE p; @@ -2566,15 +2566,15 @@ do_test e_fkey-11.3 { ROLLBACK; } } {{} 2} -do_test e_fkey-11.4 { +do_test e_fkey-60.4 { execsql { CREATE TABLE nosuchtable(x PRIMARY KEY) } catchsql { DELETE FROM p } } {1 {foreign key mismatch}} -do_test e_fkey-11.5 { +do_test e_fkey-60.5 { execsql { DROP TABLE c1 } catchsql { DELETE FROM p } } {1 {foreign key mismatch}} -do_test e_fkey-11.6 { +do_test e_fkey-60.6 { execsql { DROP TABLE c2 } execsql { DELETE FROM p } } {} @@ -2590,21 +2590,21 @@ do_test e_fkey-11.6 { # 2. Modifying foreign key definitions when a parent table is RENAMEd. # 3. Running an implicit DELETE FROM command as part of DROP TABLE. # -do_test e_fkey-12.1.1 { +do_test e_fkey-61.1.1 { drop_all_tables execsql { CREATE TABLE t1(a, b) } catchsql { ALTER TABLE t1 ADD COLUMN c DEFAULT 'xxx' REFERENCES t2 } } {1 {Cannot add a REFERENCES column with non-NULL default value}} -do_test e_fkey-12.1.2 { +do_test e_fkey-61.1.2 { execsql { PRAGMA foreign_keys = OFF } execsql { ALTER TABLE t1 ADD COLUMN c DEFAULT 'xxx' REFERENCES t2 } execsql { SELECT sql FROM sqlite_master WHERE name = 't1' } } {{CREATE TABLE t1(a, b, c DEFAULT 'xxx' REFERENCES t2)}} -do_test e_fkey-12.1.3 { +do_test e_fkey-61.1.3 { execsql { PRAGMA foreign_keys = ON } } {} -do_test e_fkey-12.2.1 { +do_test e_fkey-61.2.1 { drop_all_tables execsql { CREATE TABLE p(a UNIQUE); @@ -2615,18 +2615,18 @@ do_test e_fkey-12.2.1 { ROLLBACK; } } {{CREATE TABLE c(b REFERENCES "parent"(a))}} -do_test e_fkey-12.2.2 { +do_test e_fkey-61.2.2 { execsql { PRAGMA foreign_keys = OFF; ALTER TABLE p RENAME TO parent; SELECT sql FROM sqlite_master WHERE name = 'c'; } } {{CREATE TABLE c(b REFERENCES p(a))}} -do_test e_fkey-12.2.3 { +do_test e_fkey-61.2.3 { execsql { PRAGMA foreign_keys = ON } } {} -do_test e_fkey-12.3.1 { +do_test e_fkey-61.3.1 { drop_all_tables execsql { CREATE TABLE p(a UNIQUE); @@ -2639,14 +2639,14 @@ do_test e_fkey-12.3.1 { ROLLBACK; } } {{}} -do_test e_fkey-12.3.2 { +do_test e_fkey-61.3.2 { execsql { PRAGMA foreign_keys = OFF; DROP TABLE p; SELECT * FROM c; } } {x} -do_test e_fkey-12.3.3 { +do_test e_fkey-61.3.3 { execsql { PRAGMA foreign_keys = ON } } {} @@ -2663,13 +2663,13 @@ do_test e_fkey-12.3.3 { # foreach zMatch [list SIMPLE PARTIAL FULL Simple parTIAL FuLL ] { drop_all_tables - do_test e_fkey-1.$zMatch.1 { + do_test e_fkey-62.$zMatch.1 { execsql " CREATE TABLE p(a, b, c, PRIMARY KEY(b, c)); CREATE TABLE c(d, e, f, FOREIGN KEY(e, f) REFERENCES p MATCH $zMatch); " } {} - do_test e_fkey-1.$zMatch.2 { + do_test e_fkey-62.$zMatch.2 { execsql { INSERT INTO p VALUES(1, 2, 3) } # MATCH SIMPLE behaviour: Allow any child key that contains one or more @@ -2694,14 +2694,14 @@ foreach zMatch [list SIMPLE PARTIAL FULL Simple parTIAL FuLL ] { # that it is possible to create both immediate and deferred constraints. # drop_all_tables -do_test e_fkey-2.1 { +do_test e_fkey-62.1 { catchsql { SET CONSTRAINTS ALL IMMEDIATE } } {1 {near "SET": syntax error}} -do_test e_fkey-2.2 { +do_test e_fkey-62.2 { catchsql { SET CONSTRAINTS ALL DEFERRED } } {1 {near "SET": syntax error}} -do_test e_fkey-2.3 { +do_test e_fkey-62.3 { execsql { CREATE TABLE p(a, b, PRIMARY KEY(a, b)); CREATE TABLE cd(c, d, @@ -2711,16 +2711,16 @@ do_test e_fkey-2.3 { BEGIN; } } {} -do_test e_fkey-2.4 { +do_test e_fkey-62.4 { catchsql { INSERT INTO ci VALUES('x', 'y') } } {1 {foreign key constraint failed}} -do_test e_fkey-2.5 { +do_test e_fkey-62.5 { catchsql { INSERT INTO cd VALUES('x', 'y') } } {0 {}} -do_test e_fkey-2.6 { +do_test e_fkey-62.6 { catchsql { COMMIT } } {1 {foreign key constraint failed}} -do_test e_fkey-2.7 { +do_test e_fkey-62.7 { execsql { DELETE FROM cd; COMMIT; @@ -2777,36 +2777,36 @@ proc test_on_update_recursion {limit} { " } -do_test e_fkey-3.1.1 { +do_test e_fkey-63.1.1 { test_on_delete_recursion $SQLITE_MAX_TRIGGER_DEPTH } {0 0} -do_test e_fkey-3.1.2 { +do_test e_fkey-63.1.2 { test_on_delete_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] } {1 {too many levels of trigger recursion}} -do_test e_fkey-3.1.3 { +do_test e_fkey-63.1.3 { sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 test_on_delete_recursion 5 } {0 0} -do_test e_fkey-3.1.4 { +do_test e_fkey-63.1.4 { test_on_delete_recursion 6 } {1 {too many levels of trigger recursion}} -do_test e_fkey-3.1.5 { +do_test e_fkey-63.1.5 { sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 } {5} -do_test e_fkey-3.2.1 { +do_test e_fkey-63.2.1 { test_on_update_recursion $SQLITE_MAX_TRIGGER_DEPTH } {0 0} -do_test e_fkey-3.2.2 { +do_test e_fkey-63.2.2 { test_on_update_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] } {1 {too many levels of trigger recursion}} -do_test e_fkey-3.2.3 { +do_test e_fkey-63.2.3 { sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 test_on_update_recursion 5 } {0 0} -do_test e_fkey-3.2.4 { +do_test e_fkey-63.2.4 { test_on_update_recursion 6 } {1 {too many levels of trigger recursion}} -do_test e_fkey-3.2.5 { +do_test e_fkey-63.2.5 { sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 } {5} @@ -2820,7 +2820,7 @@ foreach recursive_triggers_setting [list 0 1 ON OFF] { drop_all_tables execsql "PRAGMA recursive_triggers = $recursive_triggers_setting" - do_test e_fkey-4.$recursive_triggers_setting.1 { + do_test e_fkey-64.$recursive_triggers_setting.1 { execsql { CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t1 ON DELETE CASCADE); INSERT INTO t1 VALUES(1, NULL); @@ -2831,10 +2831,10 @@ foreach recursive_triggers_setting [list 0 1 ON OFF] { SELECT count(*) FROM t1; } } {5} - do_test e_fkey-4.$recursive_triggers_setting.2 { + do_test e_fkey-64.$recursive_triggers_setting.2 { execsql { SELECT count(*) FROM t1 WHERE a = 1 } } {1} - do_test e_fkey-4.$recursive_triggers_setting.3 { + do_test e_fkey-64.$recursive_triggers_setting.3 { execsql { DELETE FROM t1 WHERE a = 1; SELECT count(*) FROM t1; diff --git a/test/e_fts3.test b/test/e_fts3.test new file mode 100644 index 0000000..1c7afc2 --- /dev/null +++ b/test/e_fts3.test @@ -0,0 +1,469 @@ +# 2009 November 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests to verify the "testable statements" in the +# fts3.in document. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl + +# Procs used to make the tests in this file easier to read. +# +proc ddl_test {tn ddl} { + uplevel [list do_write_test e_fts3-$tn sqlite_master $ddl] +} +proc write_test {tn tbl sql} { + uplevel [list do_write_test e_fts3-$tn $tbl $sql] +} +proc read_test {tn sql result} { + uplevel [list do_select_test e_fts3-$tn $sql $result] +} +proc error_test {tn sql result} { + uplevel [list do_error_test e_fts3-$tn $sql $result] +} + + +#------------------------------------------------------------------------- +# The body of the following [foreach] block contains test cases to verify +# that the example code in fts3.html works as expected. The tests run three +# times, with different values for DO_MALLOC_TEST. +# +# DO_MALLOC_TEST=0: Run tests with no OOM errors. +# DO_MALLOC_TEST=1: Run tests with transient OOM errors. +# DO_MALLOC_TEST=2: Run tests with persistent OOM errors. +# +foreach DO_MALLOC_TEST [lrange {0 1 2} 0 end] { + +# Reset the database and database connection. If this iteration of the +# [foreach] loop is testing with OOM errors, disable the lookaside buffer. +# +db close +file delete -force test.db test.db-journal +sqlite3 db test.db +if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 } + +########################################################################## +# Test the example CREATE VIRTUAL TABLE statements in section 1.1 +# of fts3.in. +# +ddl_test 1.1.1.1 {CREATE VIRTUAL TABLE data USING fts3()} +read_test 1.1.1.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} + +ddl_test 1.1.2.1 { + CREATE VIRTUAL TABLE pages USING fts3(title, keywords, body) +} +read_test 1.1.2.2 { + PRAGMA table_info(pages) +} {0 title {} 0 {} 0 1 keywords {} 0 {} 0 2 body {} 0 {} 0} + +ddl_test 1.1.3.1 { + CREATE VIRTUAL TABLE mail USING fts3( + subject VARCHAR(256) NOT NULL, + body TEXT CHECK(length(body)<10240) + ) +} +read_test 1.1.3.2 { + PRAGMA table_info(mail) +} {0 subject {} 0 {} 0 1 body {} 0 {} 0} + +# A very large string. Used to test if the constraint on column "body" of +# table "mail" is enforced (it should not be - FTS3 tables do not support +# constraints). +set largetext [string repeat "the quick brown fox " 5000] +write_test 1.1.3.3 mail_content { INSERT INTO mail VALUES(NULL, $largetext) } +read_test 1.1.3.4 { + SELECT subject IS NULL, length(body) FROM mail +} [list 1 100000] + +ddl_test 1.1.4.1 { + CREATE VIRTUAL TABLE papers USING fts3(author, document, tokenize=porter) +} +read_test 1.1.4.2 { + PRAGMA table_info(papers) +} {0 author {} 0 {} 0 1 document {} 0 {} 0} + +ddl_test 1.1.5.1 { + CREATE VIRTUAL TABLE simpledata USING fts3(tokenize=simple) +} +read_test 1.1.5.2 { + PRAGMA table_info(simpledata) +} {0 content {} 0 {} 0} + +ifcapable icu { + ddl_test 1.1.6.1 { + CREATE VIRTUAL TABLE names USING fts3(a, b, tokenize=icu en_AU) + } + read_test 1.1.6.2 { + PRAGMA table_info(names) + } {0 a {} 0 {} 0 1 b {} 0 {} 0} +} + +ddl_test 1.1.7.1 {DROP TABLE data} +ddl_test 1.1.7.2 {DROP TABLE pages} +ddl_test 1.1.7.3 {DROP TABLE mail} +ddl_test 1.1.7.4 {DROP TABLE papers} +ddl_test 1.1.7.5 {DROP TABLE simpledata} +read_test 1.1.7.6 {SELECT * FROM sqlite_master} {} + +# The following is not one of the examples in section 1.1. It tests +# specifying an FTS3 table with no module arguments using a slightly +# different syntax. +ddl_test 1.1.8.1 {CREATE VIRTUAL TABLE data USING fts3;} +read_test 1.1.8.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} +ddl_test 1.1.8.3 {DROP TABLE data} + +########################################################################## +# Test the examples in section 1.2 (populating fts3 tables) +# +ddl_test 1.2.1.1 { + CREATE VIRTUAL TABLE pages USING fts3(title, body); +} +write_test 1.2.1.2 pages_content { + INSERT INTO pages(docid, title, body) + VALUES(53, 'Home Page', 'SQLite is a software...'); +} +read_test 1.2.1.3 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...}} + +write_test 1.2.1.4 pages_content { + INSERT INTO pages(title, body) + VALUES('Download', 'All SQLite source code...'); +} +read_test 1.2.1.5 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...} 54 Download {All SQLite source code...}} + +write_test 1.2.1.6 pages_content { + UPDATE pages SET title = 'Download SQLite' WHERE rowid = 54 +} +read_test 1.2.1.7 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...} 54 {Download SQLite} {All SQLite source code...}} + +write_test 1.2.1.8 pages_content { DELETE FROM pages } +read_test 1.2.1.9 { SELECT docid, * FROM pages } {} + +do_error_test fts3-1.2.1.10 { + INSERT INTO pages(rowid, docid, title, body) VALUES(1, 2, 'A title', 'A document body'); +} {SQL logic error or missing database} + +# Test the optimize() function example: +ddl_test 1.2.2.1 { CREATE VIRTUAL TABLE docs USING fts3 } +write_test 1.2.2.2 docs_content { + INSERT INTO docs VALUES('Others translate the first clause as'); +} +write_test 1.2.2.3 docs_content { + INSERT INTO docs VALUES('"which is for Solomon," meaning that'); +} +write_test 1.2.2.4 docs_content { + INSERT INTO docs VALUES('the book is dedicated to Solomon.'); +} +read_test 1.2.2.5 { SELECT count(*) FROM docs_segdir } {3} +write_test 1.2.2.6 docs_segdir { + SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0; +} +read_test 1.2.2.7 { SELECT count(*) FROM docs_segdir } {1} +ddl_test 1.2.2.8 { DROP TABLE docs } + +########################################################################## +# Test the examples in section 1.3 (querying FTS3 tables) +# +ddl_test 1.3.1.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } +read_test 1.3.1.2 { + SELECT * FROM mail WHERE rowid = 15; -- Fast. Rowid lookup. + SELECT * FROM mail WHERE body MATCH 'sqlite'; -- Fast. Full-text query. + SELECT * FROM mail WHERE mail MATCH 'search'; -- Fast. Full-text query. + SELECT * FROM mail WHERE rowid BETWEEN 15 AND 20; -- Slow. Linear scan. + SELECT * FROM mail WHERE subject = 'database'; -- Slow. Linear scan. + SELECT * FROM mail WHERE subject MATCH 'database'; -- Fast. Full-text query. +} {} +ddl_test 1.3.1.3 { DROP TABLE mail } + +ddl_test 1.3.2.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } + +write_test 1.3.2.2 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(1, 'software feedback', 'found it too slow') +} +write_test 1.3.2.3 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(2, 'software feedback', 'no feedback') +} +write_test 1.3.2.4 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(3, 'slow lunch order', 'was a software problem') +} +read_test 1.3.2.5 { + SELECT * FROM mail WHERE subject MATCH 'software' +} {{software feedback} {found it too slow} {software feedback} {no feedback}} +read_test 1.3.2.6 { + SELECT * FROM mail WHERE body MATCH 'feedback' +} {{software feedback} {no feedback}} +read_test 1.3.2.7 { + SELECT * FROM mail WHERE mail MATCH 'software' +} {{software feedback} {found it too slow} {software feedback} {no feedback} {slow lunch order} {was a software problem}} +read_test 1.3.2.7 { + SELECT * FROM mail WHERE mail MATCH 'slow' +} {{software feedback} {found it too slow} {slow lunch order} {was a software problem}} +ddl_test 1.3.2.8 { DROP TABLE mail } + +ddl_test 1.3.3.1 { CREATE VIRTUAL TABLE docs USING fts3(content) } +read_test 1.3.3.2 { SELECT * FROM docs WHERE docs MATCH 'sqlite' } {} +read_test 1.3.3.3 { SELECT * FROM docs WHERE docs.docs MATCH 'sqlite' } {} +read_test 1.3.3.4 { SELECT * FROM docs WHERE main.docs.docs MATCH 'sqlite' } {} +do_error_test e_fts3-1.3.3.5 { + SELECT * FROM docs WHERE main.docs MATCH 'sqlite' +} {no such column: main.docs} +ddl_test 1.3.2.8 { DROP TABLE docs } + +########################################################################## +# Test the examples in section 3 (full-text index queries). +# +ddl_test 1.4.1.1 { CREATE VIRTUAL TABLE docs USING fts3(title, body) } +foreach {tn title body} { + 2 "linux driver" "a device" + 3 "driver" "linguistic trick" + 4 "problems" "linux problems" + 5 "linux" "big problems" + 6 "linux driver" "a device driver problem" + 7 "good times" "applications for linux" + 8 "not so good" "linux applications" + 9 "alternative" "linoleum appliances" + 10 "no L I N" "to be seen" +} { + write_test 1.4.1.$tn docs_content { INSERT INTO docs VALUES($title,$body) } + set R($tn) [list $title $body] +} + +read_test 1.4.1.11 { + SELECT * FROM docs WHERE docs MATCH 'linux' +} [concat $R(2) $R(4) $R(5) $R(6) $R(7) $R(8)] +read_test 1.4.1.12 { + SELECT * FROM docs WHERE docs MATCH 'lin*' +} [concat $R(2) $R(3) $R(4) $R(5) $R(6) $R(7) $R(8) $R(9)] +read_test 1.4.1.13 { + SELECT * FROM docs WHERE docs MATCH 'title:linux problems' +} [concat $R(5)] +read_test 1.4.1.14 { + SELECT * FROM docs WHERE body MATCH 'title:linux driver' +} [concat $R(6)] +read_test 1.4.1.15 { + SELECT * FROM docs WHERE docs MATCH '"linux applications"' +} [concat $R(8)] +read_test 1.4.1.16 { + SELECT * FROM docs WHERE docs MATCH '"lin* app*"' +} [concat $R(8) $R(9)] +ddl_test 1.4.1.17 { DROP TABLE docs } +unset R + +ddl_test 1.4.2.1 { CREATE VIRTUAL TABLE docs USING fts3() } +write_test 1.4.2.2 docs_content { + INSERT INTO docs VALUES( + 'SQLite is an ACID compliant embedded relational database management system') +} +foreach {tn query hit} { +3 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR database'} 1 +4 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/6 sqlite'} 1 +5 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/5 sqlite'} 0 +6 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/2 "ACID compliant"'} 1 +7 {SELECT * FROM docs WHERE docs MATCH '"ACID compliant" NEAR/2 sqlite'} 1 +8 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR/2 acid NEAR/2 relational'} 1 +9 {SELECT * FROM docs WHERE docs MATCH 'acid NEAR/2 sqlite NEAR/2 relational'} 0 +} { + set res [db eval {SELECT * FROM docs WHERE $hit}] + read_test 1.4.2.$tn $query $res +} +ddl_test 1.4.2.10 { DROP TABLE docs } + +########################################################################## +# Test the example in section 3.1 (set operators with enhanced syntax). +# +set sqlite_fts3_enable_parentheses 1 +ddl_test 1.5.1.1 { CREATE VIRTUAL TABLE docs USING fts3() } +foreach {tn docid content} { + 2 1 "a database is a software system" + 3 2 "sqlite is a software system" + 4 3 "sqlite is a database" +} { + set R($docid) $content + write_test 1.5.1.$tn docs_content { + INSERT INTO docs(docid, content) VALUES($docid, $content) + } +} +read_test 1.5.1.4 { + SELECT * FROM docs WHERE docs MATCH 'sqlite AND database' +} [list $R(3)] +read_test 1.5.1.5 { + SELECT * FROM docs WHERE docs MATCH 'database sqlite' +} [list $R(3)] +read_test 1.5.1.6 { + SELECT * FROM docs WHERE docs MATCH 'sqlite OR database' +} [list $R(1) $R(2) $R(3)] +read_test 1.5.1.7 { + SELECT * FROM docs WHERE docs MATCH 'database NOT sqlite' +} [list $R(1)] +read_test 1.5.1.8 { + SELECT * FROM docs WHERE docs MATCH 'database and sqlite' +} {} + +write_test 1.5.2.1 docs_content { + INSERT INTO docs + SELECT 'sqlite is also a library' UNION ALL + SELECT 'library software' +} +read_test 1.5.2.2 { + SELECT docid FROM docs WHERE docs MATCH 'sqlite AND database OR library' +} {3 4 5} +read_test 1.5.2.3 { + SELECT docid FROM docs WHERE docs MATCH 'sqlite AND database' + UNION + SELECT docid FROM docs WHERE docs MATCH 'library' +} {3 4 5} +write_test 1.5.2.4 docs_content { + INSERT INTO docs + SELECT 'the sqlite library runs on linux' UNION ALL + SELECT 'as does the sqlite database (on linux)' UNION ALL + SELECT 'the sqlite database is accessed by the sqlite library' +} +read_test 1.5.2.2 { + SELECT docid FROM docs + WHERE docs MATCH '("sqlite database" OR "sqlite library") AND linux'; +} {6 7} +read_test 1.5.2.3 { + SELECT docid FROM docs WHERE docs MATCH 'linux' + INTERSECT + SELECT docid FROM ( + SELECT docid FROM docs WHERE docs MATCH '"sqlite library"' + UNION + SELECT docid FROM docs WHERE docs MATCH '"sqlite database"' + ); +} {6 7} + +########################################################################## +# Test the examples in section 3.2 (set operators with standard syntax). +# These tests reuse the table populated by the block above. +# +set sqlite_fts3_enable_parentheses 0 +read_test 1.6.1.1 { + SELECT * FROM docs WHERE docs MATCH 'sqlite -database' +} {{sqlite is a software system} {sqlite is also a library} {the sqlite library runs on linux}} +read_test 1.6.1.2 { + SELECT * FROM docs WHERE docs MATCH 'sqlite OR database library' +} {{sqlite is also a library} {the sqlite library runs on linux} {the sqlite database is accessed by the sqlite library}} + +set sqlite_fts3_enable_parentheses 1 +read_test 1.6.1.3 { + SELECT * FROM docs WHERE docs MATCH 'sqlite OR database library' +} {{sqlite is a software system} {sqlite is a database} {sqlite is also a library} {the sqlite library runs on linux} {as does the sqlite database (on linux)} {the sqlite database is accessed by the sqlite library}} +read_test 1.6.1.4 { + SELECT * FROM docs WHERE docs MATCH '(sqlite OR database) library' +} {{sqlite is also a library} {the sqlite library runs on linux} {the sqlite database is accessed by the sqlite library}} +set sqlite_fts3_enable_parentheses 0 +ddl_test 1.6.1.5 { DROP TABLE docs } + +########################################################################## +# Test the examples in section 4 (auxillary functions). +# +ddl_test 1.7.1.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } + +write_test 1.7.1.2 mail_content { + INSERT INTO mail VALUES( + 'hello world', 'This message is a hello world message.'); +} +write_test 1.7.1.3 mail_content { + INSERT INTO mail VALUES( + 'urgent: serious', 'This mail is seen as a more serious mail'); +} + +read_test 1.7.1.4 { + SELECT offsets(mail) FROM mail WHERE mail MATCH 'world'; +} {{0 0 6 5 1 0 24 5}} +read_test 1.7.1.5 { + SELECT offsets(mail) FROM mail WHERE mail MATCH 'message' +} {{1 0 5 7 1 0 30 7}} +read_test 1.7.1.6 { + SELECT offsets(mail) FROM mail WHERE mail MATCH '"serious mail"' +} {{1 0 28 7 1 1 36 4}} + +ddl_test 1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() } + +write_test 3.2.2 text_content { + INSERT INTO text VALUES(' + During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr. + '); +} + +read_test 1.7.2.3 { + SELECT snippet(text) FROM text WHERE text MATCH 'cold' +} {{... elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum ...}} + +read_test 1.7.2.4 { + SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"' +} {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}} + +########################################################################## +# Test the example in section 5 (custom tokenizers). +# +ddl_test 1.8.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) } +write_test 1.8.1.2 simple_content { + INSERT INTO simple VALUES('Right now they''re very frustrated') +} +read_test 1.8.1.3 {SELECT docid FROM simple WHERE simple MATCH 'Frustrated'} {1} +read_test 1.8.1.4 {SELECT docid FROM simple WHERE simple MATCH 'Frustration'} {} + +ddl_test 1.8.2.1 { CREATE VIRTUAL TABLE porter USING fts3(tokenize=porter) } +write_test 1.8.2.2 porter_content { + INSERT INTO porter VALUES('Right now they''re very frustrated') +} +read_test 1.8.2.4 { + SELECT docid FROM porter WHERE porter MATCH 'Frustration' +} {1} + +} +# End of tests of example code in fts3.html +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test that errors in the arguments passed to the snippet and offsets +# functions are handled correctly. +# +set DO_MALLOC_TEST 0 +ddl_test 2.1.1 { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } +write_test 2.1.2 t1_content { + INSERT INTO t1 VALUES('one two three', x'A1B2C3D4E5F6'); +} +error_test 2.1.3 { + SELECT offsets(a) FROM t1 WHERE a MATCH 'one' +} {illegal first argument to offsets} +error_test 2.1.4 { + SELECT offsets(b) FROM t1 WHERE a MATCH 'one' +} {illegal first argument to offsets} +error_test 2.1.5 { + SELECT optimize(a) FROM t1 LIMIT 1 +} {illegal first argument to optimize} +error_test 2.1.6 { + SELECT snippet(a) FROM t1 WHERE a MATCH 'one' +} {illegal first argument to snippet} +error_test 2.1.7 { + SELECT snippet() FROM t1 WHERE a MATCH 'one' +} {unable to use function snippet in the requested context} +error_test 2.1.8 { + SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one' +} {wrong number of arguments to function snippet()} + +finish_test diff --git a/test/fts3.test b/test/fts3.test index 2e65389..b22ec88 100644 --- a/test/fts3.test +++ b/test/fts3.test @@ -42,6 +42,8 @@ set ISQUICK 1 set EXCLUDE { fts3.test + fts3malloc.test + fts3rnd.test } # Files to include in the test. If this list is empty then everything diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl new file mode 100644 index 0000000..f954f5c --- /dev/null +++ b/test/fts3_common.tcl @@ -0,0 +1,413 @@ +# 2009 November 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains common code used the fts3 tests. At one point +# equivalent functionality was implemented in C code. But it is easier +# to use Tcl. +# + +#------------------------------------------------------------------------- +# USAGE: fts3_integrity_check TBL +# +# This proc is used to verify that the full-text index is consistent with +# the contents of the fts3 table. In other words, it checks that the +# data in the %_contents table matches that in the %_segdir and %_segments +# tables. +# +# This is not an efficient procedure. It uses a lot of memory and a lot +# of CPU. But it is better than not checking at all. +# +# The procedure is: +# +# 1) Read the entire full-text index from the %_segdir and %_segments +# tables into memory. For each entry in the index, the following is +# done: +# +# set C($iDocid,$iCol,$iPosition) $zTerm +# +# 2) Iterate through each column of each row of the %_content table. +# Tokenize all documents, and check that for each token there is +# a corresponding entry in the $C array. After checking a token, +# [unset] the $C array entry. +# +# 3) Check that array $C is now empty. +# +# +proc fts3_integrity_check {tbl} { + + fts3_read2 $tbl 1 A + + foreach zTerm [array names A] { + foreach doclist $A($zTerm) { + set docid 0 + while {[string length $doclist]>0} { + set iCol 0 + set iPos 0 + set lPos [list] + set lCol [list] + + # First varint of a doclist-entry is the docid. Delta-compressed + # with respect to the docid of the previous entry. + # + incr docid [gobble_varint doclist] + if {[info exists D($zTerm,$docid)]} { + while {[set iDelta [gobble_varint doclist]] != 0} {} + continue + } + set D($zTerm,$docid) 1 + + # Gobble varints until the 0x00 that terminates the doclist-entry + # is found. + while {[set iDelta [gobble_varint doclist]] > 0} { + if {$iDelta == 1} { + set iCol [gobble_varint doclist] + set iPos 0 + } else { + incr iPos $iDelta + incr iPos -2 + set C($docid,$iCol,$iPos) $zTerm + } + } + } + } + } + + foreach key [array names C] { + #puts "$key -> $C($key)" + } + + + db eval "SELECT * FROM ${tbl}_content" E { + set iCol 0 + set iDoc $E(docid) + foreach col [lrange $E(*) 1 end] { + set c $E($col) + set sql {SELECT fts3_tokenizer_test('simple', $c)} + + foreach {pos term dummy} [db one $sql] { + if {![info exists C($iDoc,$iCol,$pos)]} { + set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing" + lappend errors $es + } else { + if {$C($iDoc,$iCol,$pos) != "$term"} { + set es "Error at docid=$iDoc col=$iCol pos=$pos. Index " + append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\"" + lappend errors $es + } + unset C($iDoc,$iCol,$pos) + } + } + incr iCol + } + } + + foreach c [array names C] { + lappend errors "Bad index entry: $c -> $C($c)" + } + + if {[info exists errors]} { return [join $errors "\n"] } + return "ok" +} + +# USAGE: fts3_terms TBL WHERE +# +# Argument TBL must be the name of an FTS3 table. Argument WHERE is an +# SQL expression that will be used as the WHERE clause when scanning +# the %_segdir table. As in the following query: +# +# "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}" +# +# This function returns a list of all terms present in the segments +# selected by the statement above. +# +proc fts3_terms {tbl where} { + fts3_read $tbl $where a + return [lsort [array names a]] +} + + +# USAGE: fts3_doclist TBL TERM WHERE +# +# Argument TBL must be the name of an FTS3 table. TERM is a term that may +# or may not be present in the table. Argument WHERE is used to select a +# subset of the b-tree segments in the associated full-text index as +# described above for [fts3_terms]. +# +# This function returns the results of merging the doclists associated +# with TERM in the selected segments. Each doclist is an element of the +# returned list. Each doclist is formatted as follows: +# +# [$docid ?$col[$off1 $off2...]?...] +# +# The formatting is odd for a Tcl command in order to be compatible with +# the original C-language implementation. If argument WHERE is "1", then +# any empty doclists are omitted from the returned list. +# +proc fts3_doclist {tbl term where} { + fts3_read $tbl $where a + + + foreach doclist $a($term) { + set docid 0 + + while {[string length $doclist]>0} { + set iCol 0 + set iPos 0 + set lPos [list] + set lCol [list] + incr docid [gobble_varint doclist] + + while {[set iDelta [gobble_varint doclist]] > 0} { + if {$iDelta == 1} { + lappend lCol [list $iCol $lPos] + set iPos 0 + set lPos [list] + set iCol [gobble_varint doclist] + } else { + incr iPos $iDelta + incr iPos -2 + lappend lPos $iPos + } + } + + if {[llength $lPos]>0} { + lappend lCol [list $iCol $lPos] + } + + if {$where != "1" || [llength $lCol]>0} { + set ret($docid) $lCol + } else { + unset -nocomplain ret($docid) + } + } + } + + set lDoc [list] + foreach docid [lsort -integer [array names ret]] { + set lCol [list] + set cols "" + foreach col $ret($docid) { + foreach {iCol lPos} $col {} + append cols " $iCol\[[join $lPos { }]\]" + } + lappend lDoc "\[${docid}${cols}\]" + } + + join $lDoc " " +} + +########################################################################### + +proc gobble_varint {varname} { + upvar $varname blob + set n [read_varint $blob ret] + set blob [string range $blob $n end] + return $ret +} +proc gobble_string {varname nLength} { + upvar $varname blob + set ret [string range $blob 0 [expr $nLength-1]] + set blob [string range $blob $nLength end] + return $ret +} + +# The argument is a blob of data representing an FTS3 segment leaf. +# Return a list consisting of alternating terms (strings) and doclists +# (blobs of data). +# +proc fts3_readleaf {blob} { + set zPrev "" + set terms [list] + + while {[string length $blob] > 0} { + set nPrefix [gobble_varint blob] + set nSuffix [gobble_varint blob] + + set zTerm [string range $zPrev 0 [expr $nPrefix-1]] + append zTerm [gobble_string blob $nSuffix] + set doclist [gobble_string blob [gobble_varint blob]] + + lappend terms $zTerm $doclist + set zPrev $zTerm + } + + return $terms +} + +proc fts3_read2 {tbl where varname} { + upvar $varname a + array unset a + db eval " SELECT start_block, leaves_end_block, root + FROM ${tbl}_segdir WHERE $where + ORDER BY level ASC, idx DESC + " { + if {$start_block == 0} { + foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } + } else { + db eval " SELECT block + FROM ${tbl}_segments + WHERE blockid>=$start_block AND blockid<=$leaves_end_block + ORDER BY blockid + " { + foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } + + } + } + } +} + +proc fts3_read {tbl where varname} { + upvar $varname a + array unset a + db eval " SELECT start_block, leaves_end_block, root + FROM ${tbl}_segdir WHERE $where + ORDER BY level DESC, idx ASC + " { + if {$start_block == 0} { + foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } + } else { + db eval " SELECT block + FROM ${tbl}_segments + WHERE blockid>=$start_block AND blockid<$leaves_end_block + ORDER BY blockid + " { + foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } + + } + } + } +} + +########################################################################## + +#------------------------------------------------------------------------- +# This proc is used to test a single SELECT statement. Parameter $name is +# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter +# $sql is passed the text of the SELECT statement. Parameter $result is +# set to the expected output if the SELECT statement is successfully +# executed using [db eval]. +# +# Example: +# +# do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2} +# +# If global variable DO_MALLOC_TEST is set to a non-zero value, or if +# it is not defined at all, then OOM testing is performed on the SELECT +# statement. Each OOM test case is said to pass if either (a) executing +# the SELECT statement succeeds and the results match those specified +# by parameter $result, or (b) TCL throws an "out of memory" error. +# +# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement +# is executed just once. In this case the test case passes if the results +# match the expected results passed via parameter $result. +# +proc do_select_test {name sql result} { + doPassiveTest $name $sql [list 0 $result] +} + +proc do_error_test {name sql error} { + doPassiveTest $name $sql [list 1 $error] +} + +proc doPassiveTest {name sql catchres} { + if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } + + if {$::DO_MALLOC_TEST} { + set answers [list {1 {out of memory}} $catchres] + set modes [list 100000 transient 1 persistent] + } else { + set answers [list $catchres] + set modes [list 0 nofail] + } + set str [join $answers " OR "] + + foreach {nRepeat zName} $modes { + for {set iFail 1} 1 {incr iFail} { + if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} + + set res [catchsql $sql] + if {[lsearch -exact $answers $res]>=0} { + set res $str + } + do_test $name.$zName.$iFail [list set {} $res] $str + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] + if {$nFail==0} break + } + } +} + + +#------------------------------------------------------------------------- +# Test a single write to the database. In this case a "write" is a +# DELETE, UPDATE or INSERT statement. +# +# If OOM testing is performed, there are several acceptable outcomes: +# +# 1) The write succeeds. No error is returned. +# +# 2) An "out of memory" exception is thrown and: +# +# a) The statement has no effect, OR +# b) The current transaction is rolled back, OR +# c) The statement succeeds. This can only happen if the connection +# is in auto-commit mode (after the statement is executed, so this +# includes COMMIT statements). +# +# If the write operation eventually succeeds, zero is returned. If a +# transaction is rolled back, non-zero is returned. +# +# Parameter $name is the name to use for the test case (or test cases). +# The second parameter, $tbl, should be the name of the database table +# being modified. Parameter $sql contains the SQL statement to test. +# +proc do_write_test {name tbl sql} { + if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } + + # Figure out an statement to get a checksum for table $tbl. + db eval "SELECT * FROM $tbl" V break + set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl" + + # Calculate the initial table checksum. + set cksum1 [db one $cksumsql] + + if {$::DO_MALLOC_TEST } { + set answers [list {1 {out of memory}} {0 {}}] + if {$::DO_MALLOC_TEST==1} { + set modes {100000 transient} + } else { + set modes {1 persistent} + } + } else { + set answers [list {0 {}}] + set modes [list 0 nofail] + } + set str [join $answers " OR "] + + foreach {nRepeat zName} $modes { + for {set iFail 1} 1 {incr iFail} { + if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} + + set res [uplevel [list catchsql $sql]] + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] + if {$nFail==0} { + do_test $name.$zName.$iFail [list set {} $res] {0 {}} + return + } else { + if {[lsearch $answers $res]>=0} { + set res $str + } + do_test $name.$zName.$iFail [list set {} $res] $str + set cksum2 [db one $cksumsql] + if {$cksum1 != $cksum2} return + } + } + } +} diff --git a/test/fts3aa.test b/test/fts3aa.test index 46304fb..cc2daba 100644 --- a/test/fts3aa.test +++ b/test/fts3aa.test @@ -146,6 +146,7 @@ do_test fts3aa-3.3 { execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'} } {1 5 9 13 17 21 25 29} +breakpoint do_test fts3aa-4.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'} } {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31} @@ -195,6 +196,7 @@ do_test fts3aa-6.2 { do_test fts3aa-6.3 { execsql {SELECT content FROM t1 WHERE rowid = -1} } {{three four}} +breakpoint do_test fts3aa-6.4 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'four'} } {-1 0 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31} diff --git a/test/fts3ab.test b/test/fts3ab.test index 86124f7..e1f3bdc 100644 --- a/test/fts3ab.test +++ b/test/fts3ab.test @@ -115,6 +115,7 @@ for {set i 1} {$i<=15} {incr i} { db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);" } +breakpoint do_test fts3ab-4.1 { execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'} } {1 3 5 7 9 11 13 15} diff --git a/test/fts3ad.test b/test/fts3ad.test index 420b5b2..e373339 100644 --- a/test/fts3ad.test +++ b/test/fts3ad.test @@ -61,5 +61,46 @@ do_test fts3ad-1.6 { } } {3 {The value is 123456789}} +do_test fts3ad-2.1 { + execsql { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize porter); + INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping'); + SELECT rowid FROM t1 WHERE content MATCH 'run jump'; + } +} {1} +do_test fts3ad-2.2 { + execsql { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize= porter); + INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping'); + SELECT rowid FROM t1 WHERE content MATCH 'run jump'; + } +} {1} +do_test fts3ad-2.3 { + execsql { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize= simple); + INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping'); + SELECT rowid FROM t1 WHERE content MATCH 'run jump'; + } +} {} +do_test fts3ad-2.4 { + execsql { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize= porter); + INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping'); + SELECT rowid FROM t1 WHERE content MATCH 'run jump'; + } +} {1} +do_test fts3ad-2.5 { + execsql { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize = porter); + INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping'); + SELECT rowid FROM t1 WHERE content MATCH 'run jump'; + } +} {1} + finish_test diff --git a/test/fts3ae.test b/test/fts3ae.test index 949a72b..b872688 100644 --- a/test/fts3ae.test +++ b/test/fts3ae.test @@ -58,7 +58,7 @@ db eval { DELETE FROM t1 WHERE rowid = 22; } -do_test fts3af-1.1 { +do_test fts3ae-1.1 { execsql {SELECT COUNT(*) FROM t1} } {14} diff --git a/test/fts3ag.test b/test/fts3ag.test index f4f9c8f..34ef949 100644 --- a/test/fts3ag.test +++ b/test/fts3ag.test @@ -78,6 +78,7 @@ do_test fts3ag-1.10 { # Test that docListOrMerge() correctly handles reaching the end of one # doclist before it reaches the end of the other. do_test fts3ag-1.11 { +breakpoint execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR also'} } {1 2} do_test fts3ag-1.12 { diff --git a/test/fts3an.test b/test/fts3an.test index f19d573..77ca9e5 100644 --- a/test/fts3an.test +++ b/test/fts3an.test @@ -169,7 +169,7 @@ db eval { INSERT INTO t3(rowid, c) VALUES(1, $text); INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row'); } -for {set i 0} {$i<100} {incr i} { +for {set i 0} {$i<68} {incr i} { db eval {INSERT INTO t3(rowid, c) VALUES(3+$i, $bigtext)} lappend ret 192 } diff --git a/test/fts3b.test b/test/fts3b.test index 17ee0da..9bde3a2 100644 --- a/test/fts3b.test +++ b/test/fts3b.test @@ -208,11 +208,23 @@ do_test fts3b-4.8 { } } {1 {SQL logic error or missing database}} -# Don't allow update of docid, to match rowid behaviour. do_test fts3b-4.9 { - catchsql { + execsql { SELECT docid FROM t4 WHERE t4 MATCH 'testing' } +} {12} +do_test fts3b-4.10 { + execsql { UPDATE t4 SET docid = 14 WHERE docid = 12; + SELECT docid FROM t4 WHERE t4 MATCH 'testing'; } -} {1 {SQL logic error or missing database}} +} {14} +do_test fts3b-4.11 { + execsql { SELECT * FROM t4 WHERE rowid = 14; } +} {{still testing}} +do_test fts3b-4.12 { + execsql { SELECT * FROM t4 WHERE rowid = 12; } +} {} +do_test fts3b-4.13 { + execsql { SELECT docid FROM t4 WHERE t4 MATCH 'still'; } +} {14} finish_test diff --git a/test/fts3c.test b/test/fts3c.test index 2c73d4b..6b63264 100644 --- a/test/fts3c.test +++ b/test/fts3c.test @@ -12,11 +12,10 @@ # and then uses them to do some basic tests that FTS3 is internally # working as expected. # -# $Id: fts3c.test,v 1.1 2008/07/03 19:53:22 shess Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { @@ -24,126 +23,25 @@ ifcapable !fts3 { return } -#************************************************************************* -# Probe to see if support for these functions is compiled in. -# TODO(shess): Change main.mk to do the right thing and remove this test. -db eval { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts3(c); - INSERT INTO t1 (docid, c) VALUES (1, 'x'); -} - -set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} -set r {1 {unable to use function dump_terms in the requested context}} -if {[catchsql $s]==$r} { - finish_test - return -} - -#************************************************************************* -# Test that the new functions give appropriate errors. -do_test fts3c-0.0 { - catchsql { - SELECT dump_terms(t1, 1) FROM t1 LIMIT 1; - } -} {1 {dump_terms: incorrect arguments}} - -do_test fts3c-0.1 { - catchsql { - SELECT dump_terms(t1, 0, 0, 0) FROM t1 LIMIT 1; - } -} {1 {dump_terms: incorrect arguments}} - -do_test fts3c-0.2 { - catchsql { - SELECT dump_terms(1, t1) FROM t1 LIMIT 1; - } -} {1 {unable to use function dump_terms in the requested context}} - -do_test fts3c-0.3 { - catchsql { - SELECT dump_terms(t1, 16, 16) FROM t1 LIMIT 1; - } -} {1 {dump_terms: segment not found}} - -do_test fts3c-0.4 { - catchsql { - SELECT dump_doclist(t1) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.5 { - catchsql { - SELECT dump_doclist(t1, NULL) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: empty second argument}} - -do_test fts3c-0.6 { - catchsql { - SELECT dump_doclist(t1, '') FROM t1 LIMIT 1; - } -} {1 {dump_doclist: empty second argument}} - -do_test fts3c-0.7 { - catchsql { - SELECT dump_doclist(t1, 'a', 0) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.8 { - catchsql { - SELECT dump_doclist(t1, 'a', 0, 0, 0) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.9 { - catchsql { - SELECT dump_doclist(t1, 'a', 16, 16) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: segment not found}} - #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { - # TODO(shess): Figure out why uplevel in do_test can't catch - # $level and $index directly. - set ::level $level - set ::index $index - do_test $test.terms { - execsql { - SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $terms] + set where "level = $level AND idx = $index" + do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { - do_test $test.terms { - execsql { - SELECT dump_terms(t1) FROM t1 LIMIT 1; - } - } [list $terms] + do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { - # TODO(shess): Again, why can't the non-:: versions work? - set ::term $term - set ::level $level - set ::index $index - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $doclist] + set where "level = $level AND idx = $index" + do_test $test [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { - set ::term $term - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; - } - } [list $doclist] + do_test $test [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* diff --git a/test/fts3d.test b/test/fts3d.test index bec488d..715980d 100644 --- a/test/fts3d.test +++ b/test/fts3d.test @@ -11,11 +11,10 @@ # This file implements regression tests for SQLite library. The focus # of this script is testing the FTS3 module's optimize() function. # -# $Id: fts3d.test,v 1.2 2008/07/15 21:32:07 shess Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { @@ -23,64 +22,25 @@ ifcapable !fts3 { return } -#************************************************************************* -# Probe to see if support for the FTS3 dump_* functions is compiled in. -# TODO(shess): Change main.mk to do the right thing and remove this test. -db eval { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts3(c); - INSERT INTO t1 (docid, c) VALUES (1, 'x'); -} - -set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} -set r {1 {unable to use function dump_terms in the requested context}} -if {[catchsql $s]==$r} { - finish_test - return -} - #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { - # TODO(shess): Figure out why uplevel in do_test can't catch - # $level and $index directly. - set ::level $level - set ::index $index - do_test $test.terms { - execsql { - SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $terms] + set where "level = $level AND idx = $index" + do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { - do_test $test.terms { - execsql { - SELECT dump_terms(t1) FROM t1 LIMIT 1; - } - } [list $terms] + do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { - # TODO(shess): Again, why can't the non-:: versions work? - set ::term $term - set ::level $level - set ::index $index - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $doclist] + set where "level = $level AND idx = $index" + do_test $test.doclist [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { - set ::term $term - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; - } - } [list $doclist] + do_test $test.doclist [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* @@ -293,6 +253,7 @@ check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. +breakpoint do_test fts3d-4.5 { execsql { SELECT OPTIMIZE(t1) FROM t1 LIMIT 1; diff --git a/test/fts3malloc.test b/test/fts3malloc.test new file mode 100644 index 0000000..920dc0c --- /dev/null +++ b/test/fts3malloc.test @@ -0,0 +1,293 @@ +# 2009 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests to verify that malloc() errors that occur +# within the FTS3 module code are handled correctly. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +source $testdir/malloc_common.tcl +source $testdir/fts3_common.tcl + +# Ensure the lookaside buffer is disabled for these tests. +# +sqlite3 db test.db +sqlite3_db_config_lookaside db 0 0 0 + +set sqlite_fts3_enable_parentheses 1 +set DO_MALLOC_TEST 1 + +# Test organization: +# +# fts3_malloc-1.*: Test OOM during CREATE and DROP table statements. +# fts3_malloc-2.*: Test OOM during SELECT operations. +# fts3_malloc-3.*: Test OOM during SELECT operations with a larger database. +# fts3_malloc-4.*: Test OOM during database write operations. +# +# + + +proc normal_list {l} { + set ret [list] + foreach elem $l {lappend ret $elem} + set ret +} + + +do_write_test fts3_malloc-1.1 sqlite_master { + CREATE VIRTUAL TABLE ft1 USING fts3(a, b) +} +do_write_test fts3_malloc-1.2 sqlite_master { + CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]); +} +do_write_test fts3_malloc-1.3 sqlite_master { + CREATE VIRTUAL TABLE ft3 USING fts3('a', "b"); +} +do_write_test fts3_malloc-1.4 sqlite_master { + CREATE VIRTUAL TABLE ft4 USING fts3(`a`, 'fred''s column'); +} +do_error_test fts3_malloc-1.5 { + CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown) +} {unknown tokenizer: unknown} +do_write_test fts3_malloc-1.6 sqlite_master { + CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter) +} + +# Test the xConnect/xDisconnect methods: +#db eval { ATTACH 'test2.db' AS aux } +#do_write_test fts3_malloc-1.6 aux.sqlite_master { +# CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); +#} +#do_write_test fts3_malloc-1.6 aux.sqlite_master { +# CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); +#} + + + +do_test fts3_malloc-2.0 { + execsql { + DROP TABLE ft1; + DROP TABLE ft2; + DROP TABLE ft3; + DROP TABLE ft4; + DROP TABLE ft6; + } + execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) } + for {set ii 1} {$ii < 32} {incr ii} { + set a [list] + set b [list] + if {$ii & 0x01} {lappend a one ; lappend b neung} + if {$ii & 0x02} {lappend a two ; lappend b song } + if {$ii & 0x04} {lappend a three ; lappend b sahm } + if {$ii & 0x08} {lappend a four ; lappend b see } + if {$ii & 0x10} {lappend a five ; lappend b hah } + execsql { INSERT INTO ft VALUES($a, $b) } + } +} {} + +foreach {tn sql result} { + 1 "SELECT count(*) FROM sqlite_master" {5} + 2 "SELECT * FROM ft WHERE docid = 1" {one neung} + 3 "SELECT * FROM ft WHERE docid = 2" {two song} + 4 "SELECT * FROM ft WHERE docid = 3" {{one two} {neung song}} + + 5 "SELECT a FROM ft" { + {one} {two} {one two} + {three} {one three} {two three} + {one two three} {four} {one four} + {two four} {one two four} {three four} + {one three four} {two three four} {one two three four} + {five} {one five} {two five} + {one two five} {three five} {one three five} + {two three five} {one two three five} {four five} + {one four five} {two four five} {one two four five} + {three four five} {one three four five} {two three four five} + {one two three four five} + } + + 6 "SELECT a FROM ft WHERE a MATCH 'one'" { + {one} {one two} {one three} {one two three} + {one four} {one two four} {one three four} {one two three four} + {one five} {one two five} {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 7 "SELECT a FROM ft WHERE a MATCH 'o*'" { + {one} {one two} {one three} {one two three} + {one four} {one two four} {one three four} {one two three four} + {one five} {one two five} {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 8 "SELECT a FROM ft WHERE a MATCH 'o* t*'" { + {one two} {one three} {one two three} + {one two four} {one three four} {one two three four} + {one two five} {one three five} {one two three five} + {one two four five} {one three four five} {one two three four five} + } + + 9 "SELECT a FROM ft WHERE a MATCH '\"o* t*\"'" { + {one two} {one three} {one two three} + {one two four} {one three four} {one two three four} + {one two five} {one three five} {one two three five} + {one two four five} {one three four five} {one two three four five} + } + + 10 {SELECT a FROM ft WHERE a MATCH '"o* f*"'} { + {one four} {one five} {one four five} + } + + 11 {SELECT a FROM ft WHERE a MATCH '"one two three"'} { + {one two three} + {one two three four} + {one two three five} + {one two three four five} + } + + 12 {SELECT a FROM ft WHERE a MATCH '"two three four"'} { + {two three four} + {one two three four} + {two three four five} + {one two three four five} + } + + 12 {SELECT a FROM ft WHERE a MATCH '"two three" five'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 13 {SELECT a FROM ft WHERE ft MATCH '"song sahm" hah'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 14 {SELECT a FROM ft WHERE b MATCH 'neung'} { + {one} {one two} + {one three} {one two three} + {one four} {one two four} + {one three four} {one two three four} + {one five} {one two five} + {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 15 {SELECT a FROM ft WHERE b MATCH '"neung song sahm"'} { + {one two three} {one two three four} + {one two three five} {one two three four five} + } + + 16 {SELECT a FROM ft WHERE b MATCH 'hah "song sahm"'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 17 {SELECT a FROM ft WHERE b MATCH 'song OR sahm'} { + {two} {one two} {three} + {one three} {two three} {one two three} + {two four} {one two four} {three four} + {one three four} {two three four} {one two three four} + {two five} {one two five} {three five} + {one three five} {two three five} {one two three five} + {two four five} {one two four five} {three four five} + {one three four five} {two three four five} {one two three four five} + } + + 18 {SELECT a FROM ft WHERE a MATCH 'three NOT two'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 19 {SELECT a FROM ft WHERE b MATCH 'sahm NOT song'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} { + {one two three} {one two three four} + {one two three five} {one two three four five} + } + +} { + set result [normal_list $result] + do_select_test fts3_malloc-2.$tn $sql $result +} + +do_test fts3_malloc-3.0 { + execsql BEGIN + for {set ii 32} {$ii < 1024} {incr ii} { + set a [list] + set b [list] + if {$ii & 0x0001} {lappend a one ; lappend b neung } + if {$ii & 0x0002} {lappend a two ; lappend b song } + if {$ii & 0x0004} {lappend a three ; lappend b sahm } + if {$ii & 0x0008} {lappend a four ; lappend b see } + if {$ii & 0x0010} {lappend a five ; lappend b hah } + if {$ii & 0x0020} {lappend a six ; lappend b hok } + if {$ii & 0x0040} {lappend a seven ; lappend b jet } + if {$ii & 0x0080} {lappend a eight ; lappend b bairt } + if {$ii & 0x0100} {lappend a nine ; lappend b gow } + if {$ii & 0x0200} {lappend a ten ; lappend b sip } + execsql { INSERT INTO ft VALUES($a, $b) } + } + execsql COMMIT +} {} +foreach {tn sql result} { + 1 "SELECT count(*) FROM ft" {1023} + + 2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" { + {one two three four five six seven eight} + {one two three four five six seven eight nine} + {one two three four five six seven eight ten} + {one two three four five six seven eight nine ten} + } + + 3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} { + 512 262144 + } + + 4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} { + 128 66368 + } +} { + set result [normal_list $result] + do_select_test fts3_malloc-3.$tn $sql $result +} + +do_test fts3_malloc-4.0 { + execsql { DELETE FROM ft WHERE docid>=32 } +} {} +foreach {tn sql} { + 1 "DELETE FROM ft WHERE ft MATCH 'one'" + 2 "DELETE FROM ft WHERE ft MATCH 'three'" + 3 "DELETE FROM ft WHERE ft MATCH 'five'" +} { + do_write_test fts3_malloc-4.1.$tn ft_content $sql +} +do_test fts3_malloc-4.2 { + execsql { SELECT a FROM ft } +} {two four {two four}} + + +finish_test + diff --git a/test/fts3rnd.test b/test/fts3rnd.test new file mode 100644 index 0000000..0fb1512 --- /dev/null +++ b/test/fts3rnd.test @@ -0,0 +1,292 @@ +# 2009 December 03 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Brute force (random data) tests for FTS3. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl + +set nVocab 100 +set lVocab [list] + +# Generate a vocabulary of nVocab words. Each word is 3 characters long. +# +set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z} +for {set i 0} {$i < $nVocab} {incr i} { + set word [lindex $lChar [expr int(rand()*26)]] + append word [lindex $lChar [expr int(rand()*26)]] + append word [lindex $lChar [expr int(rand()*26)]] + lappend lVocab $word +} + +proc random_term {} { + lindex $::lVocab [expr {int(rand()*$::nVocab)}] +} + +# Return a document consisting of $nWord arbitrarily selected terms +# from the $::lVocab list. +# +proc generate_doc {nWord} { + set doc [list] + for {set i 0} {$i < $nWord} {incr i} { + lappend doc [random_term] + } + return $doc +} + + + +# Primitives to update the table. +# +unset -nocomplain t1 +proc insert_row {rowid} { + set a [generate_doc [expr int((rand()*100))]] + set b [generate_doc [expr int((rand()*100))]] + set c [generate_doc [expr int((rand()*100))]] + execsql { INSERT INTO t1(docid, a, b, c) VALUES($rowid, $a, $b, $c) } + set ::t1($rowid) [list $a $b $c] +} +proc delete_row {rowid} { + execsql { DELETE FROM t1 WHERE rowid = $rowid } + catch {unset ::t1($rowid)} +} +proc update_row {rowid} { + set cols {a b c} + set iCol [expr int(rand()*3)] + set doc [generate_doc [expr int((rand()*100))]] + lset ::t1($rowid) $iCol $doc + execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid" +} + +proc simple_phrase {zPrefix} { + set ret [list] + set pattern "*[string map {* \[a-z\]} $zPrefix]*" + foreach {key value} [array get ::t1] { + if {[string match $pattern $value]} { lappend ret $key } + } + lsort -integer $ret +} +proc simple_near {termlist nNear} { + set ret [list] + + foreach {key value} [array get ::t1] { + foreach v $value { + + set l [lsearch -exact -all $v [lindex $termlist 0]] + foreach T [lrange $termlist 1 end] { + set l2 [list] + foreach i $l { + set iStart [expr $i - $nNear - 1] + set iEnd [expr $i + $nNear + 1] + if {$iStart < 0} {set iStart 0} + foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] { + incr i2 $iStart + if {$i2 != $i} { lappend l2 $i2 } + } + } + set l [lsort -uniq -integer $l2] + } + + if {[llength $l]} { +#puts "MATCH($key): $v" + lappend ret $key + } + } + } + + lsort -unique -integer $ret +} + +# The following three procs: +# +# setup_not A B +# setup_or A B +# setup_and A B +# +# each take two arguments. Both arguments must be lists of integer values +# sorted by value. The return value is the list produced by evaluating +# the equivalent of "A op B", where op is the FTS3 operator NOT, OR or +# AND. +# +proc setop_not {A B} { + foreach b $B { set n($b) {} } + set ret [list] + foreach a $A { if {![info exists n($a)]} {lappend ret $a} } + return $ret +} +proc setop_or {A B} { + lsort -integer -uniq [concat $A $B] +} +proc setop_and {A B} { + foreach b $B { set n($b) {} } + set ret [list] + foreach a $A { if {[info exists n($a)]} {lappend ret $a} } + return $ret +} + +set sqlite_fts3_enable_parentheses 1 + +foreach nodesize {50 500 1000 2000} { + catch { array unset ::t1 } + + # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. + # + db transaction { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c, test:$nodesize)" + for {set i 0} {$i < 100} {incr i} { insert_row $i } + } + + for {set iTest 1} {$iTest <= 100} {incr iTest} { + + # Delete one row, update one row and insert one row. + # + set rows [array names ::t1] + set nRow [llength $rows] + set iUpdate [lindex $rows [expr {int(rand()*$nRow)}]] + set iDelete $iUpdate + while {$iDelete == $iUpdate} { + set iDelete [lindex $rows [expr {int(rand()*$nRow)}]] + } + set iInsert $iUpdate + while {[info exists ::t1($iInsert)]} { + set iInsert [expr {int(rand()*1000000)}] + } + db transaction { + insert_row $iInsert + update_row $iUpdate + delete_row $iDelete + } + + # Pick 10 terms from the vocabulary. Check that the results of querying + # the database for the set of documents containing each of these terms + # is the same as the result obtained by scanning the contents of the Tcl + # array for each term. + # + for {set i 0} {$i < 10} {incr i} { + set term [random_term] + do_test fts3rnd-1.$nodesize.$iTest.1.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $term } + } [simple_phrase $term] + } + + # This time, use the first two characters of each term as a term prefix + # to query for. Test that querying the Tcl array produces the same results + # as querying the FTS3 table for the prefix. + # + for {set i 0} {$i < 10} {incr i} { + set prefix [string range [random_term] 0 1] + set match "${prefix}*" + do_test fts3rnd-1.$nodesize.$iTest.2.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_phrase $match] + } + + # Similar to the above, except for phrase queries. + # + for {set i 0} {$i < 10} {incr i} { + set term [list [random_term] [random_term]] + set match "\"$term\"" + do_test fts3rnd-1.$nodesize.$iTest.3.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_phrase $term] + } + + # Three word phrases. + # + for {set i 0} {$i < 10} {incr i} { + set term [list [random_term] [random_term] [random_term]] + set match "\"$term\"" + do_test fts3rnd-1.$nodesize.$iTest.4.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_phrase $term] + } + + # Three word phrases made up of term-prefixes. + # + for {set i 0} {$i < 10} {incr i} { + set query "[string range [random_term] 0 1]* " + append query "[string range [random_term] 0 1]* " + append query "[string range [random_term] 0 1]*" + + set match "\"$query\"" + do_test fts3rnd-1.$nodesize.$iTest.5.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_phrase $query] + } + + # A NEAR query with terms as the arguments. + # + for {set i 0} {$i < 10} {incr i} { + set terms [list [random_term] [random_term]] + set match [join $terms " NEAR "] + do_test fts3rnd-1.$nodesize.$iTest.6.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_near $terms 10] + } + + # A 3-way NEAR query with terms as the arguments. + # + for {set i 0} {$i < 10} {incr i} { + set terms [list [random_term] [random_term] [random_term]] + set nNear 11 + set match [join $terms " NEAR/$nNear "] + set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }] + do_test fts3rnd-1.$nodesize.$iTest.7.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [simple_near $terms $nNear] + } + + # Set operations on simple term queries. + # + foreach {tn op proc} { + 8 OR setop_or + 9 NOT setop_not + 10 AND setop_and + } { + for {set i 0} {$i < 10} {incr i} { + set term1 [random_term] + set term2 [random_term] + set match "$term1 $op $term2" + do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [$proc [simple_phrase $term1] [simple_phrase $term2]] + } + } + + # Set operations on NEAR queries. + # + foreach {tn op proc} { + 8 OR setop_or + 9 NOT setop_not + 10 AND setop_and + } { + for {set i 0} {$i < 10} {incr i} { + set term1 [random_term] + set term2 [random_term] + set term3 [random_term] + set term4 [random_term] + set match "$term1 NEAR $term2 $op $term3 NEAR $term4" + do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + } [$proc \ + [simple_near [list $term1 $term2] 10] \ + [simple_near [list $term3 $term4] 10] + ] + } + } + } +} + +finish_test diff --git a/test/func2.test b/test/func2.test new file mode 100644 index 0000000..08ad857 --- /dev/null +++ b/test/func2.test @@ -0,0 +1,511 @@ +# 2009 November 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing built-in functions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Test plan: +# +# func2-1.*: substr implementation (ascii) +# func2-2.*: substr implementation (utf8) +# func2-3.*: substr implementation (blob) +# + +proc bin_to_hex {blob} { + set bytes {} + binary scan $blob \c* bytes + set bytes2 [list] + foreach b $bytes {lappend bytes2 [format %02X [expr $b & 0xFF]]} + join $bytes2 {} +} + +#---------------------------------------------------------------------------- +# Test cases func2-1.*: substr implementation (ascii) +# + +do_test func2-1.1 { + execsql {SELECT 'Supercalifragilisticexpialidocious'} +} {Supercalifragilisticexpialidocious} + +# substr(x,y), substr(x,y,z) +do_test func2-1.2.1 { + catchsql {SELECT SUBSTR()} +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-1.2.2 { + catchsql {SELECT SUBSTR('Supercalifragilisticexpialidocious')} +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-1.2.3 { + catchsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 1,1,1)} +} {1 {wrong number of arguments to function SUBSTR()}} + +# p1 is 1-indexed +do_test func2-1.3 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0)} +} {Supercalifragilisticexpialidocious} +do_test func2-1.4 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 1)} +} {Supercalifragilisticexpialidocious} +do_test func2-1.5 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2)} +} {upercalifragilisticexpialidocious} +do_test func2-1.6 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 30)} +} {cious} +do_test func2-1.7 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 34)} +} {s} +do_test func2-1.8 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 35)} +} {{}} +do_test func2-1.9 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 36)} +} {{}} + +# if p1<0, start from right +do_test func2-1.10 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -0)} +} {Supercalifragilisticexpialidocious} +do_test func2-1.11 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -1)} +} {s} +do_test func2-1.12 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -2)} +} {us} +do_test func2-1.13 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -30)} +} {rcalifragilisticexpialidocious} +do_test func2-1.14 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -34)} +} {Supercalifragilisticexpialidocious} +do_test func2-1.15 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -35)} +} {Supercalifragilisticexpialidocious} +do_test func2-1.16 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -36)} +} {Supercalifragilisticexpialidocious} + +# p1 is 1-indexed, p2 length to return +do_test func2-1.17.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0, 1)} +} {{}} +do_test func2-1.17.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0, 2)} +} {S} +do_test func2-1.18 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 1, 1)} +} {S} +do_test func2-1.19.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, 0)} +} {{}} +do_test func2-1.19.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, 1)} +} {u} +do_test func2-1.19.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, 2)} +} {up} +do_test func2-1.20 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 30, 1)} +} {c} +do_test func2-1.21 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 34, 1)} +} {s} +do_test func2-1.22 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 35, 1)} +} {{}} +do_test func2-1.23 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 36, 1)} +} {{}} + +# if p1<0, start from right, p2 length to return +do_test func2-1.24 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -0, 1)} +} {{}} +do_test func2-1.25.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -1, 0)} +} {{}} +do_test func2-1.25.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -1, 1)} +} {s} +do_test func2-1.25.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -1, 2)} +} {s} +do_test func2-1.26 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -2, 1)} +} {u} +do_test func2-1.27 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -30, 1)} +} {r} +do_test func2-1.28.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -34, 0)} +} {{}} +do_test func2-1.28.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -34, 1)} +} {S} +do_test func2-1.28.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -34, 2)} +} {Su} +do_test func2-1.29.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -35, 1)} +} {{}} +do_test func2-1.29.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -35, 2)} +} {S} +do_test func2-1.30.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -36, 0)} +} {{}} +do_test func2-1.30.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -36, 1)} +} {{}} +do_test func2-1.30.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -36, 2)} +} {{}} +do_test func2-1.30.3 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', -36, 3)} +} {S} + +# p1 is 1-indexed, p2 length to return, p2<0 return p2 chars before p1 +do_test func2-1.31.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0, 0)} +} {{}} +do_test func2-1.31.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0, -1)} +} {{}} +do_test func2-1.31.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 0, -2)} +} {{}} +do_test func2-1.32.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 1, 0)} +} {{}} +do_test func2-1.32.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 1, -1)} +} {{}} +do_test func2-1.33.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, 0)} +} {{}} +do_test func2-1.33.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, -1)} +} {S} +do_test func2-1.33.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 2, -2)} +} {S} +do_test func2-1.34.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 3, 0)} +} {{}} +do_test func2-1.34.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 3, -1)} +} {u} +do_test func2-1.34.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 3, -2)} +} {Su} +do_test func2-1.35.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 30, -1)} +} {o} +do_test func2-1.35.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 30, -2)} +} {do} +do_test func2-1.36 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 34, -1)} +} {u} +do_test func2-1.37 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 35, -1)} +} {s} +do_test func2-1.38.0 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 36, 0)} +} {{}} +do_test func2-1.38.1 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 36, -1)} +} {{}} +do_test func2-1.38.2 { + execsql {SELECT SUBSTR('Supercalifragilisticexpialidocious', 36, -2)} +} {s} + + +#---------------------------------------------------------------------------- +# Test cases func2-2.*: substr implementation (utf8) +# + +# Only do the following tests if TCL has UTF-8 capabilities +# +if {"\u1234"!="u1234"} { + +do_test func2-2.1.1 { + execsql "SELECT 'hi\u1234ho'" +} "hi\u1234ho" + +# substr(x,y), substr(x,y,z) +do_test func2-2.1.2 { + catchsql "SELECT SUBSTR()" +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-2.1.3 { + catchsql "SELECT SUBSTR('hi\u1234ho')" +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-2.1.4 { + catchsql "SELECT SUBSTR('hi\u1234ho', 1,1,1)" +} {1 {wrong number of arguments to function SUBSTR()}} + +do_test func2-2.2.0 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 0)" +} {{}} +do_test func2-2.2.1 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 1)" +} {{}} +do_test func2-2.2.2 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 2)" +} "h" +do_test func2-2.2.3 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 3)" +} "hi" +do_test func2-2.2.4 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 4)" +} "hi\u1234" +do_test func2-2.2.5 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 5)" +} "hi\u1234h" +do_test func2-2.2.6 { + execsql "SELECT SUBSTR('hi\u1234ho', 0, 6)" +} "hi\u1234ho" + +do_test func2-2.3.0 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 0)" +} {{}} +do_test func2-2.3.1 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 1)" +} "h" +do_test func2-2.3.2 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 2)" +} "hi" +do_test func2-2.3.3 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 3)" +} "hi\u1234" +do_test func2-2.3.4 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 4)" +} "hi\u1234h" +do_test func2-2.3.5 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 5)" +} "hi\u1234ho" +do_test func2-2.3.6 { + execsql "SELECT SUBSTR('hi\u1234ho', 1, 6)" +} "hi\u1234ho" + +do_test func2-2.4.0 { + execsql "SELECT SUBSTR('hi\u1234ho', 3, 0)" +} {{}} +do_test func2-2.4.1 { + execsql "SELECT SUBSTR('hi\u1234ho', 3, 1)" +} "\u1234" +do_test func2-2.4.2 { + execsql "SELECT SUBSTR('hi\u1234ho', 3, 2)" +} "\u1234h" + +do_test func2-2.5.0 { + execsql "SELECT SUBSTR('\u1234', 0, 0)" +} {{}} +do_test func2-2.5.1 { + execsql "SELECT SUBSTR('\u1234', 0, 1)" +} {{}} +do_test func2-2.5.2 { + execsql "SELECT SUBSTR('\u1234', 0, 2)" +} "\u1234" +do_test func2-2.5.3 { + execsql "SELECT SUBSTR('\u1234', 0, 3)" +} "\u1234" + +do_test func2-2.6.0 { + execsql "SELECT SUBSTR('\u1234', 1, 0)" +} {{}} +do_test func2-2.6.1 { + execsql "SELECT SUBSTR('\u1234', 1, 1)" +} "\u1234" +do_test func2-2.6.2 { + execsql "SELECT SUBSTR('\u1234', 1, 2)" +} "\u1234" +do_test func2-2.6.3 { + execsql "SELECT SUBSTR('\u1234', 1, 3)" +} "\u1234" + +do_test func2-2.7.0 { + execsql "SELECT SUBSTR('\u1234', 2, 0)" +} {{}} +do_test func2-2.7.1 { + execsql "SELECT SUBSTR('\u1234', 2, 1)" +} {{}} +do_test func2-2.7.2 { + execsql "SELECT SUBSTR('\u1234', 2, 2)" +} {{}} + +do_test func2-2.8.0 { + execsql "SELECT SUBSTR('\u1234', -1, 0)" +} {{}} +do_test func2-2.8.1 { + execsql "SELECT SUBSTR('\u1234', -1, 1)" +} "\u1234" +do_test func2-2.8.2 { + execsql "SELECT SUBSTR('\u1234', -1, 2)" +} "\u1234" +do_test func2-2.8.3 { + execsql "SELECT SUBSTR('\u1234', -1, 3)" +} "\u1234" + +} ;# End \u1234!=u1234 + +#---------------------------------------------------------------------------- +# Test cases func2-3.*: substr implementation (blob) +# + +ifcapable {!bloblit} { + finish_test + return +} + +do_test func2-3.1.1 { + set blob [execsql "SELECT x'1234'"] + bin_to_hex [lindex $blob 0] +} "1234" + +# substr(x,y), substr(x,y,z) +do_test func2-3.1.2 { + catchsql {SELECT SUBSTR()} +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-3.1.3 { + catchsql {SELECT SUBSTR(x'1234')} +} {1 {wrong number of arguments to function SUBSTR()}} +do_test func2-3.1.4 { + catchsql {SELECT SUBSTR(x'1234', 1,1,1)} +} {1 {wrong number of arguments to function SUBSTR()}} + +do_test func2-3.2.0 { + set blob [execsql "SELECT SUBSTR(x'1234', 0, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.2.1 { + set blob [execsql "SELECT SUBSTR(x'1234', 0, 1)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.2.2 { + set blob [execsql "SELECT SUBSTR(x'1234', 0, 2)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.2.3 { + set blob [execsql "SELECT SUBSTR(x'1234', 0, 3)"] + bin_to_hex [lindex $blob 0] +} "1234" + +do_test func2-3.3.0 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.3.1 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, 1)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.3.2 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, 2)"] + bin_to_hex [lindex $blob 0] +} "1234" +do_test func2-3.3.3 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, 3)"] + bin_to_hex [lindex $blob 0] +} "1234" + +do_test func2-3.4.0 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.4.1 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, 1)"] + bin_to_hex [lindex $blob 0] +} "34" +do_test func2-3.4.2 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, 2)"] + bin_to_hex [lindex $blob 0] +} "34" +do_test func2-3.4.3 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, 3)"] + bin_to_hex [lindex $blob 0] +} "34" + +do_test func2-3.5.0 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.5.1 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, 1)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.5.2 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, 2)"] + bin_to_hex [lindex $blob 0] +} "1234" +do_test func2-3.5.3 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, 3)"] + bin_to_hex [lindex $blob 0] +} "1234" + +do_test func2-3.6.0 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.6.1 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, -1)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.6.2 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, -2)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.6.3 { + set blob [execsql "SELECT SUBSTR(x'1234', -1, -3)"] + bin_to_hex [lindex $blob 0] +} "12" + +do_test func2-3.7.0 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.7.1 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, -1)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.7.2 { + set blob [execsql "SELECT SUBSTR(x'1234', -2, -2)"] + bin_to_hex [lindex $blob 0] +} {} + +do_test func2-3.8.0 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.8.1 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, -1)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.8.2 { + set blob [execsql "SELECT SUBSTR(x'1234', 1, -2)"] + bin_to_hex [lindex $blob 0] +} {} + +do_test func2-3.9.0 { + set blob [execsql "SELECT SUBSTR(x'1234', 2, 0)"] + bin_to_hex [lindex $blob 0] +} {} +do_test func2-3.9.1 { + set blob [execsql "SELECT SUBSTR(x'1234', 2, -1)"] + bin_to_hex [lindex $blob 0] +} "12" +do_test func2-3.9.2 { + set blob [execsql "SELECT SUBSTR(x'1234', 2, -2)"] + bin_to_hex [lindex $blob 0] +} "12" + +finish_test diff --git a/test/intarray.test b/test/intarray.test new file mode 100644 index 0000000..2aba080 --- /dev/null +++ b/test/intarray.test @@ -0,0 +1,109 @@ +# 2009 November 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for the "intarray" object implemented +# in test_intarray.c. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !vtab { + return +} + +do_test intarray-1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + } + for {set i 1} {$i<=999} {incr i} { + set b [format {x%03d} $i] + db eval {INSERT INTO t1(a,b) VALUES($i,$b)} + } + db eval { + CREATE TABLE t2(x INTEGER PRIMARY KEY, y); + INSERT INTO t2 SELECT * FROM t1; + SELECT b FROM t1 WHERE a IN (12,34,56,78) ORDER BY a + } +} {x012 x034 x056 x078} + +do_test intarray-1.1 { + set ia1 [sqlite3_intarray_create db ia1] + set ia2 [sqlite3_intarray_create db ia2] + set ia3 [sqlite3_intarray_create db ia3] + set ia4 [sqlite3_intarray_create db ia4] + db eval { + SELECT type, name FROM sqlite_temp_master + ORDER BY name + } +} {table ia1 table ia2 table ia3 table ia4} + +do_test intarray-1.2 { + db eval { + SELECT b FROM t1 WHERE a IN ia3 ORDER BY a + } +} {} + +do_test intarray-1.3 { + sqlite3_intarray_bind $ia3 45 123 678 + db eval { + SELECT b FROM t1 WHERE a IN ia3 ORDER BY a + } +} {x045 x123 x678} + +do_test intarray-1.4 { + db eval { + SELECT count(b) FROM t1 WHERE a NOT IN ia3 ORDER BY a + } +} {996} + +#explain {SELECT b FROM t1 WHERE a NOT IN ia3} + +do_test intarray-1.5 { + set cmd sqlite3_intarray_bind + lappend cmd $ia1 + for {set i 1} {$i<=999} {incr i} { + lappend cmd $i + lappend cmd [expr {$i+1000}] + lappend cmd [expr {$i+2000}] + } + eval $cmd + db eval { + REPLACE INTO t1 SELECT * FROM t2; + DELETE FROM t1 WHERE a NOT IN ia1; + SELECT count(*) FROM t1; + } +} {999} + +do_test intarray-1.6 { + db eval { + DELETE FROM t1 WHERE a IN ia1; + SELECT count(*) FROM t1; + } +} {0} + +do_test intarray-2.1 { + db eval { + CREATE TEMP TABLE t3(p,q); + INSERT INTO t3 SELECT * FROM t2; + SELECT count(*) FROM t3 WHERE p IN ia1; + } +} {999} + +do_test intarray-2.2 { + set ia5 [sqlite3_intarray_create db ia5] + db eval { + SELECT count(*) FROM t3 WHERE p IN ia1; + } +} {999} + +finish_test diff --git a/test/printf.test b/test/printf.test index 19f857c..100ce96 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3506,6 +3506,43 @@ do_test printf-4.3 { do_test printf-4.4 { sqlite3_mprintf_str {%d %d A NULL pointer in %%Q: %Q} 1 2 } {1 2 A NULL pointer in %Q: NULL} +do_test printf-4.5 { + sqlite3_mprintf_str {%d %d A quoted string: '%.10q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y''all'} +do_test printf-4.6 { + sqlite3_mprintf_str {%d %d A quoted string: '%.9q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y''all'} +do_test printf-4.7 { + sqlite3_mprintf_str {%d %d A quoted string: '%.8q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y''all'} +do_test printf-4.8 { + sqlite3_mprintf_str {%d %d A quoted string: '%.7q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y''al'} +do_test printf-4.9 { + sqlite3_mprintf_str {%d %d A quoted string: '%.6q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y''a'} +do_test printf-4.10 { + sqlite3_mprintf_str {%d %d A quoted string: '%.5q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y'''} +do_test printf-4.11 { + sqlite3_mprintf_str {%d %d A quoted string: '%.4q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi Y'} +do_test printf-4.12 { + sqlite3_mprintf_str {%d %d A quoted string: '%.3q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi '} +do_test printf-4.13 { + sqlite3_mprintf_str {%d %d A quoted string: '%.2q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'Hi'} +do_test printf-4.14 { + sqlite3_mprintf_str {%d %d A quoted string: '%.1q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: 'H'} +do_test printf-4.15 { + sqlite3_mprintf_str {%d %d A quoted string: '%.0q'} 1 2 {Hi Y'all} +} {1 2 A quoted string: ''} +do_test printf-4.16 { + sqlite3_mprintf_str {%d A quoted string: '%.*q'} 1 6 {Hi Y'all} +} {1 A quoted string: 'Hi Y''a'} + do_test printf-5.1 { set x [sqlite3_mprintf_str {%d %d %100000s} 0 0 {Hello}] diff --git a/test/quick.test b/test/quick.test index a56455a..431b829 100644 --- a/test/quick.test +++ b/test/quick.test @@ -57,7 +57,9 @@ set EXCLUDE { crash6.test crash7.test delete3.test + e_fts3.test fts3.test + fts3rnd.test fkey_malloc.test fuzz.test fuzz3.test diff --git a/test/tester.tcl b/test/tester.tcl index 7651017..73a7c30 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -412,7 +412,7 @@ proc execsql {sql {db db}} { # proc catchsql {sql {db db}} { # puts "SQL = $sql" - set r [catch {$db eval $sql} msg] + set r [catch [list uplevel [list $db eval $sql]] msg] lappend r $msg return $r } diff --git a/test/tkt-94c04eaadb.test b/test/tkt-94c04eaadb.test index f9e7aad..cce8a98 100644 --- a/test/tkt-94c04eaadb.test +++ b/test/tkt-94c04eaadb.test @@ -67,6 +67,7 @@ db close db2 close sqlite3async_start sqlite3async_wait +sqlite3async_control halt never sqlite3async_shutdown finish_test diff --git a/test/trace.test b/test/trace.test index 762c1d7..f40993d 100644 --- a/test/trace.test +++ b/test/trace.test @@ -168,4 +168,70 @@ ifcapable trigger { } {{UPDATE t1 SET a=a+1;} {-- TRIGGER r1t1} {-- TRIGGER r1t2} {-- TRIGGER r1t1} {-- TRIGGER r1t2} {-- TRIGGER r1t1} {-- TRIGGER r1t2}} } +# With 3.6.21, we add the ability to expand host parameters in the trace +# output. Test this feature. +# +do_test trace-6.1 { + set ::t6int [expr {3+3}] + set ::t6real [expr {1.5*4.0}] + set ::t6str {test-six y'all} + db eval {SELECT x'3031323334' AS x} {set ::t6blob $x} + unset -nocomplain t6null + set TRACE_OUT {} + execsql {SELECT $::t6int, $::t6real, $t6str, $t6blob, $t6null} +} {6 6.0 {test-six y'all} 01234 {}} +do_test trace-6.2 { + set TRACE_OUT +} {{SELECT 6, 6.0, 'test-six y''all', x'3031323334', NULL}} +do_test trace-6.3 { + set TRACE_OUT {} + execsql {SELECT $::t6int, ?1, $::t6int} +} {6 6 6} +do_test trace-6.4 { + set TRACE_OUT +} {{SELECT 6, 6, 6}} +do_test trace-6.5 { + execsql {CREATE TABLE t6([$::t6int],"?1"); INSERT INTO t6 VALUES(1,2)} + set TRACE_OUT {} + execsql {SELECT '$::t6int', [$::t6int], $::t6int, ?1, "?1", $::t6int FROM t6} +} {{$::t6int} 1 6 6 2 6} +do_test trace-6.6 { + set TRACE_OUT +} {{SELECT '$::t6int', [$::t6int], 6, 6, "?1", 6 FROM t6}} + +# Do these same tests with a UTF16 database. +# +do_test trace-6.100 { + db close + sqlite3 db :memory: + db eval { + PRAGMA encoding=UTF16be; + CREATE TABLE t6([$::t6str],"?1"); + INSERT INTO t6 VALUES(1,2); + } + db trace trace_proc + set TRACE_OUT {} + execsql {SELECT '$::t6str', [$::t6str], $::t6str, ?1, "?1", $::t6str FROM t6} +} {{$::t6str} 1 {test-six y'all} {test-six y'all} 2 {test-six y'all}} +do_test trace-6.101 { + set TRACE_OUT +} {{SELECT '$::t6str', [$::t6str], 'test-six y''all', 'test-six y''all', "?1", 'test-six y''all' FROM t6}} + +do_test trace-6.200 { + db close + sqlite3 db :memory: + db eval { + PRAGMA encoding=UTF16le; + CREATE TABLE t6([$::t6str],"?1"); + INSERT INTO t6 VALUES(1,2); + } + db trace trace_proc + set TRACE_OUT {} + execsql {SELECT '$::t6str', [$::t6str], $::t6str, ?1, "?1", $::t6str FROM t6} +} {{$::t6str} 1 {test-six y'all} {test-six y'all} 2 {test-six y'all}} +do_test trace-6.101 { + set TRACE_OUT +} {{SELECT '$::t6str', [$::t6str], 'test-six y''all', 'test-six y''all', "?1", 'test-six y''all' FROM t6}} + + finish_test diff --git a/test/triggerC.test b/test/triggerC.test index a86269f..c1967be 100644 --- a/test/triggerC.test +++ b/test/triggerC.test @@ -784,8 +784,76 @@ do_test triggerC-9.2 { SELECT a FROM t9 ORDER BY a; } } {1 2 3 4} - +# At one point (between versions 3.6.18 and 3.6.20 inclusive), an UPDATE +# that fired a BEFORE trigger that itself updated the same row as the +# statement causing it to fire was causing a strange side-effect: The +# values updated by the statement within the trigger were being overwritten +# by the values in the new.* array, even if those values were not +# themselves written by the parent UPDATE statement. +# +# Technically speaking this was not a bug. The SQLite documentation says +# that if a BEFORE UPDATE or BEFORE DELETE trigger modifies or deletes the +# row that the parent statement is operating on the results are undefined. +# But as of 3.6.21 behaviour is restored to the way it was in versions +# 3.6.17 and earlier to avoid causing unnecessary difficulties. +# +do_test triggerC-10.1 { + execsql { + CREATE TABLE t10(a, updatecnt DEFAULT 0); + CREATE TRIGGER t10_bu BEFORE UPDATE OF a ON t10 BEGIN + UPDATE t10 SET updatecnt = updatecnt+1 WHERE rowid = old.rowid; + END; + INSERT INTO t10(a) VALUES('hello'); + } + + # Before the problem was fixed, table t10 would contain the tuple + # (world, 0) after running the following script (because the value + # 1 written to column "updatecnt" was clobbered by the old value 0). + # + execsql { + UPDATE t10 SET a = 'world'; + SELECT * FROM t10; + } +} {world 1} + +do_test triggerC-10.2 { + execsql { + UPDATE t10 SET a = 'tcl', updatecnt = 5; + SELECT * FROM t10; + } +} {tcl 5} + +do_test triggerC-10.3 { + execsql { + CREATE TABLE t11( + c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, + c11, c12, c13, c14, c15, c16, c17, c18, c19, c20, + c21, c22, c23, c24, c25, c26, c27, c28, c29, c30, + c31, c32, c33, c34, c35, c36, c37, c38, c39, c40 + ); + + CREATE TRIGGER t11_bu BEFORE UPDATE OF c1 ON t11 BEGIN + UPDATE t11 SET c31 = c31+1, c32=c32+1 WHERE rowid = old.rowid; + END; + + INSERT INTO t11 VALUES( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 + ); + } + + # Before the problem was fixed, table t10 would contain the tuple + # (world, 0) after running the following script (because the value + # 1 written to column "updatecnt" was clobbered by the old value 0). + # + execsql { + UPDATE t11 SET c4=35, c33=22, c1=5; + SELECT * FROM t11; + } +} {5 2 3 35 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 32 33 22 34 35 36 37 38 39 40} finish_test diff --git a/test/vtabE.test b/test/vtabE.test new file mode 100644 index 0000000..22ec018 --- /dev/null +++ b/test/vtabE.test @@ -0,0 +1,48 @@ +# 2009 November 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# The focus of this file making sure the register cache logic works +# correctly with virtual tables. Ticket [16fbf14cb2]. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !vtab { + finish_test + return +} + +register_tclvar_module [sqlite3_connection_pointer db] + +unset -nocomplain vtabE +set vtabE(vtabE1) 11 +set vtabE(vtabE2) 22 +unset -nocomplain vtabE1 +set vtabE1(w) x +set vtabE1(y) z +unset -nocomplain vtabE2 +set vtabE2(a) b +set vtabE2(c) d + +do_test vtabE-1 { + db eval { + CREATE VIRTUAL TABLE t1 USING tclvar; + CREATE VIRTUAL TABLE t2 USING tclvar; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + SELECT t1.*, t2.*, abs(t3.b + abs(t2.value + abs(t1.value))) + FROM t1 LEFT JOIN t2 ON t2.name = t1.arrayname + LEFT JOIN t3 ON t3.a=t2.value + WHERE t1.name = 'vtabE' + ORDER BY t1.value, t2.value; + } +} {vtabE vtabE1 11 vtabE1 w x {} vtabE vtabE1 11 vtabE1 y z {} vtabE vtabE2 22 vtabE2 a b {} vtabE vtabE2 22 vtabE2 c d {}} diff --git a/test/where8.test b/test/where8.test index a8dd5ed..04146fe 100644 --- a/test/where8.test +++ b/test/where8.test @@ -642,6 +642,7 @@ foreach idxsql { } { do_test where8-4.$A.$B.1 { + unset -nocomplain R set R [execsql $sql] if {![info exists results($B)]} { set results($B) $R diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 99dc0d3..c14df4e 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -87,7 +87,7 @@ foreach hdr { btree.h btreeInt.h fts3.h - fts3_expr.h + fts3Int.h fts3_hash.h fts3_tokenizer.h hash.h @@ -251,6 +251,7 @@ foreach file { vdbemem.c vdbeaux.c vdbeapi.c + vdbetrace.c vdbe.c vdbeblob.c journal.c @@ -295,6 +296,8 @@ foreach file { fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c + fts3_write.c + fts3_snippet.c rtree.c icu.c diff --git a/tool/shell1.test b/tool/shell1.test new file mode 100644 index 0000000..6e45474 --- /dev/null +++ b/tool/shell1.test @@ -0,0 +1,707 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell1.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell1-1.*: Basic command line option handling. +# shell1-2.*: Basic "dot" command token parsing. +# shell1-3.*: Basic test that "dot" command can be called. +# + +package require sqlite3 + +set CLI "./sqlite" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +proc catchcmd {db cmd} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite3 db test.db + +#---------------------------------------------------------------------------- +# Test cases shell1-1.*: Basic command line option handling. +# + +# invalid option +do_test shell1-1.1.1 { + set res [catchcmd "-bad test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: unknown option: -bad} $res] +} {1 1} +# error on extra options +do_test shell1-1.1.2 { + set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "select 4"} $res] +} {1 1} +# error on extra options +do_test shell1-1.3.2 { + set res [catchcmd "-bad FOO test.db BAD" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -help +do_test shell1-1.2.1 { + set res [catchcmd "-help test.db" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Usage} $res] \ + [regexp {\-init} $res] \ + [regexp {\-version} $res] +} {1 1 1 1} + +# -init filename read/process named file +do_test shell1-1.3.1 { + catchcmd "-init FOO test.db" "" +} {0 {}} +do_test shell1-1.3.2 { + set res [catchcmd "-init FOO test.db .quit BAD" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: too many options: "BAD"} $res] +} {1 1} + +# -echo print commands before execution +do_test shell1-1.4.1 { + catchcmd "-echo test.db" "" +} {0 {}} + +# -[no]header turn headers on or off +do_test shell1-1.5.1 { + catchcmd "-header test.db" "" +} {0 {}} +do_test shell1-1.5.2 { + catchcmd "-noheader test.db" "" +} {0 {}} + +# -bail stop after hitting an error +do_test shell1-1.6.1 { + catchcmd "-bail test.db" "" +} {0 {}} + +# -interactive force interactive I/O +do_test shell1-1.7.1 { + set res [catchcmd "-interactive test.db" ".quit"] + set rc [lindex $res 0] + list $rc \ + [regexp {SQLite version} $res] \ + [regexp {Enter SQL statements} $res] +} {0 1 1} + +# -batch force batch I/O +do_test shell1-1.8.1 { + catchcmd "-batch test.db" "" +} {0 {}} + +# -column set output mode to 'column' +do_test shell1-1.9.1 { + catchcmd "-column test.db" "" +} {0 {}} + +# -csv set output mode to 'csv' +do_test shell1-1.10.1 { + catchcmd "-csv test.db" "" +} {0 {}} + +# -html set output mode to HTML +do_test shell1-1.11.1 { + catchcmd "-html test.db" "" +} {0 {}} + +# -line set output mode to 'line' +do_test shell1-1.12.1 { + catchcmd "-line test.db" "" +} {0 {}} + +# -list set output mode to 'list' +do_test shell1-1.13.1 { + catchcmd "-list test.db" "" +} {0 {}} + +# -separator 'x' set output field separator (|) +do_test shell1-1.14.1 { + catchcmd "-separator 'x' test.db" "" +} {0 {}} +do_test shell1-1.14.2 { + catchcmd "-separator x test.db" "" +} {0 {}} +do_test shell1-1.14.3 { + set res [catchcmd "-separator" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument for option: -separator} $res] +} {1 1} + +# -nullvalue 'text' set text string for NULL values +do_test shell1-1.15.1 { + catchcmd "-nullvalue 'x' test.db" "" +} {0 {}} +do_test shell1-1.15.2 { + catchcmd "-nullvalue x test.db" "" +} {0 {}} +do_test shell1-1.15.3 { + set res [catchcmd "-nullvalue" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: missing argument for option: -nullvalue} $res] +} {1 1} + +# -version show SQLite version +do_test shell1-1.16.1 { + catchcmd "-version test.db" "" +} {0 3.6.20} + +#---------------------------------------------------------------------------- +# Test cases shell1-2.*: Basic "dot" command token parsing. +# + +# check first token handling +do_test shell1-2.1.1 { + catchcmd " test.db" ".foo" +} {1 {Error: unknown command or invalid arguments: "foo". Enter ".help" for help}} +do_test shell1-2.1.2 { + catchcmd " test.db" ".\"foo OFF\"" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.1.3 { + catchcmd " test.db" ".\'foo OFF\'" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} + +# unbalanced quotes +do_test shell1-2.2.1 { + catchcmd " test.db" ".\"foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.2 { + catchcmd " test.db" ".\'foo OFF" +} {1 {Error: unknown command or invalid arguments: "foo OFF". Enter ".help" for help}} +do_test shell1-2.2.3 { + catchcmd " test.db" ".explain \"OFF" +} {0 {}} +do_test shell1-2.2.4 { + catchcmd " test.db" ".explain \'OFF" +} {0 {}} +do_test shell1-2.2.5 { + catchcmd " test.db" ".mode \"insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.2.6 { + catchcmd " test.db" ".mode \'insert FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# check multiple tokens, and quoted tokens +do_test shell1-2.3.1 { + catchcmd " test.db" ".explain 1" +} {0 {}} +do_test shell1-2.3.2 { + catchcmd " test.db" ".explain on" +} {0 {}} +do_test shell1-2.3.3 { + catchcmd " test.db" ".explain \"1 2 3\"" +} {0 {}} +do_test shell1-2.3.4 { + catchcmd " test.db" ".explain \"OFF\"" +} {0 {}} +do_test shell1-2.3.5 { + catchcmd " test.db" ".\'explain\' \'OFF\'" +} {0 {}} +do_test shell1-2.3.6 { + catchcmd " test.db" ".explain \'OFF\'" +} {0 {}} +do_test shell1-2.3.7 { + catchcmd " test.db" ".\'explain\' \'OFF\'" +} {0 {}} + +# check quoted args are unquoted +do_test shell1-2.4.1 { + catchcmd " test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-2.4.2 { + catchcmd " test.db" ".mode csv" +} {0 {}} +do_test shell1-2.4.2 { + catchcmd " test.db" ".mode \"csv\"" +} {0 {}} + + +#---------------------------------------------------------------------------- +# Test cases shell1-3.*: Basic test that "dot" command can be called. +# + +# .backup ?DB? FILE Backup DB (default "main") to FILE +do_test shell1-3.1.1 { + catchcmd " test.db" ".backup" +} {1 {Error: unknown command or invalid arguments: "backup". Enter ".help" for help}} +do_test shell1-3.1.2 { + # catchcmd " test.db" ".backup FOO" + #TBD!!! this asserts currently +} {} +do_test shell1-3.1.3 { + catchcmd " test.db" ".backup FOO BAR" +} {1 {Error: unknown database FOO}} +do_test shell1-3.1.4 { + # too many arguments + catchcmd " test.db" ".backup FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "backup". Enter ".help" for help}} + +# .bail ON|OFF Stop after hitting an error. Default OFF +do_test shell1-3.2.1 { + catchcmd " test.db" ".bail" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} +do_test shell1-3.2.2 { + catchcmd " test.db" ".bail ON" +} {0 {}} +do_test shell1-3.2.3 { + catchcmd " test.db" ".bail OFF" +} {0 {}} +do_test shell1-3.2.4 { + # too many arguments + catchcmd " test.db" ".bail OFF BAD" +} {1 {Error: unknown command or invalid arguments: "bail". Enter ".help" for help}} + +# .databases List names and files of attached databases +do_test shell1-3.3.1 { + set res [catchcmd " test.db" ".databases"] + regexp {0.*main.*test\.db} $res +} {1} +do_test shell1-3.3.2 { + # too many arguments + catchcmd " test.db" ".databases BAD" +} {1 {Error: unknown command or invalid arguments: "databases". Enter ".help" for help}} + +# .dump ?TABLE? ... Dump the database in an SQL text format +# If TABLE specified, only dump tables matching +# LIKE pattern TABLE. +do_test shell1-3.4.1 { + set res [catchcmd " test.db" ".dump"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.2 { + set res [catchcmd " test.db" ".dump FOO"] + list [regexp {BEGIN TRANSACTION;} $res] \ + [regexp {COMMIT;} $res] +} {1 1} +do_test shell1-3.4.3 { + # too many arguments + catchcmd " test.db" ".dump FOO BAD" +} {1 {Error: unknown command or invalid arguments: "dump". Enter ".help" for help}} + +# .echo ON|OFF Turn command echo on or off +do_test shell1-3.5.1 { + catchcmd " test.db" ".echo" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} +do_test shell1-3.5.2 { + catchcmd " test.db" ".echo ON" +} {0 {}} +do_test shell1-3.5.3 { + catchcmd " test.db" ".echo OFF" +} {0 {}} +do_test shell1-3.5.4 { + # too many arguments + catchcmd " test.db" ".echo OFF BAD" +} {1 {Error: unknown command or invalid arguments: "echo". Enter ".help" for help}} + +# .exit Exit this program +do_test shell1-3.6.1 { + catchcmd " test.db" ".exit" +} {0 {}} +do_test shell1-3.6.2 { + # too many arguments + catchcmd " test.db" ".exit BAD" +} {1 {Error: unknown command or invalid arguments: "exit". Enter ".help" for help}} + +# .explain ON|OFF Turn output mode suitable for EXPLAIN on or off. +do_test shell1-3.7.1 { + catchcmd " test.db" ".explain" + # explain is the exception to the booleans. without an option, it turns it on. +} {0 {}} +do_test shell1-3.7.2 { + catchcmd " test.db" ".explain ON" +} {0 {}} +do_test shell1-3.7.3 { + catchcmd " test.db" ".explain OFF" +} {0 {}} +do_test shell1-3.7.4 { + # too many arguments + catchcmd " test.db" ".explain OFF BAD" +} {1 {Error: unknown command or invalid arguments: "explain". Enter ".help" for help}} + +# .genfkey ?OPTIONS? Options are: +# --no-drop: Do not drop old fkey triggers. +# --ignore-errors: Ignore tables with fkey errors +# --exec: Execute generated SQL immediately +# See file tool/genfkey.README in the source +# distribution for further information. +do_test shell1-3.8.1 { + catchcmd " test.db" ".genfkey" +} {0 {}} +do_test shell1-3.8.2 { + catchcmd " test.db" ".genfkey FOO" +} {1 {unknown option: FOO}} + +# .header(s) ON|OFF Turn display of headers on or off +do_test shell1-3.9.1 { + catchcmd " test.db" ".header" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} +do_test shell1-3.9.2 { + catchcmd " test.db" ".header ON" +} {0 {}} +do_test shell1-3.9.3 { + catchcmd " test.db" ".header OFF" +} {0 {}} +do_test shell1-3.9.4 { + # too many arguments + catchcmd " test.db" ".header OFF BAD" +} {1 {Error: unknown command or invalid arguments: "header". Enter ".help" for help}} + +do_test shell1-3.9.5 { + catchcmd " test.db" ".headers" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} +do_test shell1-3.9.6 { + catchcmd " test.db" ".headers ON" +} {0 {}} +do_test shell1-3.9.7 { + catchcmd " test.db" ".headers OFF" +} {0 {}} +do_test shell1-3.9.8 { + # too many arguments + catchcmd " test.db" ".headers OFF BAD" +} {1 {Error: unknown command or invalid arguments: "headers". Enter ".help" for help}} + +# .help Show this message +do_test shell1-3.10.1 { + set res [catchcmd " test.db" ".help"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} +do_test shell1-3.10.2 { + # we allow .help to take extra args (it is help after all) + set res [catchcmd " test.db" ".help BAD"] + # look for a few of the possible help commands + list [regexp {.help} $res] \ + [regexp {.quit} $res] \ + [regexp {.show} $res] +} {1 1 1} + +# .import FILE TABLE Import data from FILE into TABLE +do_test shell1-3.11.1 { + catchcmd " test.db" ".import" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd " test.db" ".import FOO" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} +do_test shell1-3.11.2 { + catchcmd " test.db" ".import FOO BAR" +} {1 {Error: no such table: BAR}} +do_test shell1-3.11.3 { + # too many arguments + catchcmd " test.db" ".import FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "import". Enter ".help" for help}} + +# .indices ?TABLE? Show names of all indices +# If TABLE specified, only show indices for tables +# matching LIKE pattern TABLE. +do_test shell1-3.12.1 { + catchcmd " test.db" ".indices" +} {0 {}} +do_test shell1-3.12.2 { + catchcmd " test.db" ".indices FOO" +} {0 {}} +do_test shell1-3.12.3 { + # too many arguments + catchcmd " test.db" ".indices FOO BAD" +} {1 {Error: unknown command or invalid arguments: "indices". Enter ".help" for help}} + +# .mode MODE ?TABLE? Set output mode where MODE is one of: +# csv Comma-separated values +# column Left-aligned columns. (See .width) +# html HTML code +# insert SQL insert statements for TABLE +# line One value per line +# list Values delimited by .separator string +# tabs Tab-separated values +# tcl TCL list elements +do_test shell1-3.13.1 { + catchcmd " test.db" ".mode" +} {1 {Error: unknown command or invalid arguments: "mode". Enter ".help" for help}} +do_test shell1-3.13.2 { + catchcmd " test.db" ".mode FOO" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.3 { + catchcmd " test.db" ".mode csv" +} {0 {}} +do_test shell1-3.13.4 { + catchcmd " test.db" ".mode column" +} {0 {}} +do_test shell1-3.13.5 { + catchcmd " test.db" ".mode html" +} {0 {}} +do_test shell1-3.13.6 { + catchcmd " test.db" ".mode insert" +} {0 {}} +do_test shell1-3.13.7 { + catchcmd " test.db" ".mode line" +} {0 {}} +do_test shell1-3.13.8 { + catchcmd " test.db" ".mode list" +} {0 {}} +do_test shell1-3.13.9 { + catchcmd " test.db" ".mode tabs" +} {0 {}} +do_test shell1-3.13.10 { + catchcmd " test.db" ".mode tcl" +} {0 {}} +do_test shell1-3.13.11 { + # too many arguments + catchcmd " test.db" ".mode tcl BAD" +} {1 {Error: invalid arguments: "BAD". Enter ".help" for help}} + +# don't allow partial mode type matches +do_test shell1-3.13.12 { + catchcmd " test.db" ".mode l" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.13 { + catchcmd " test.db" ".mode li" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} +do_test shell1-3.13.14 { + catchcmd " test.db" ".mode lin" +} {1 {Error: mode should be one of: column csv html insert line list tabs tcl}} + +# .nullvalue STRING Print STRING in place of NULL values +do_test shell1-3.14.1 { + catchcmd " test.db" ".nullvalue" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} +do_test shell1-3.14.2 { + catchcmd " test.db" ".nullvalue FOO" +} {0 {}} +do_test shell1-3.14.3 { + # too many arguments + catchcmd " test.db" ".nullvalue FOO BAD" +} {1 {Error: unknown command or invalid arguments: "nullvalue". Enter ".help" for help}} + +# .output FILENAME Send output to FILENAME +do_test shell1-3.15.1 { + catchcmd " test.db" ".output" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} +do_test shell1-3.15.2 { + catchcmd " test.db" ".output FOO" +} {0 {}} +do_test shell1-3.15.3 { + # too many arguments + catchcmd " test.db" ".output FOO BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .output stdout Send output to the screen +do_test shell1-3.16.1 { + catchcmd " test.db" ".output stdout" +} {0 {}} +do_test shell1-3.16.2 { + # too many arguments + catchcmd " test.db" ".output stdout BAD" +} {1 {Error: unknown command or invalid arguments: "output". Enter ".help" for help}} + +# .prompt MAIN CONTINUE Replace the standard prompts +do_test shell1-3.17.1 { + catchcmd " test.db" ".prompt" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} +do_test shell1-3.17.2 { + catchcmd " test.db" ".prompt FOO" +} {0 {}} +do_test shell1-3.17.3 { + catchcmd " test.db" ".prompt FOO BAR" +} {0 {}} +do_test shell1-3.17.4 { + # too many arguments + catchcmd " test.db" ".prompt FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "prompt". Enter ".help" for help}} + +# .quit Exit this program +do_test shell1-3.18.1 { + catchcmd " test.db" ".quit" +} {0 {}} +do_test shell1-3.18.2 { + # too many arguments + catchcmd " test.db" ".quit BAD" +} {1 {Error: unknown command or invalid arguments: "quit". Enter ".help" for help}} + +# .read FILENAME Execute SQL in FILENAME +do_test shell1-3.19.1 { + catchcmd " test.db" ".read" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} +do_test shell1-3.19.2 { + file delete -force FOO + catchcmd " test.db" ".read FOO" +} {1 {Error: cannot open "FOO"}} +do_test shell1-3.19.3 { + # too many arguments + catchcmd " test.db" ".read FOO BAD" +} {1 {Error: unknown command or invalid arguments: "read". Enter ".help" for help}} + +# .restore ?DB? FILE Restore content of DB (default "main") from FILE +do_test shell1-3.20.1 { + catchcmd " test.db" ".restore" +} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} +do_test shell1-3.20.2 { + # catchcmd " test.db" ".restore FOO" + #TBD!!! this asserts currently +} {} +do_test shell1-3.20.3 { + catchcmd " test.db" ".restore FOO BAR" +} {1 {Error: unknown database FOO}} +do_test shell1-3.20.4 { + # too many arguments + catchcmd " test.db" ".restore FOO BAR BAD" +} {1 {Error: unknown command or invalid arguments: "restore". Enter ".help" for help}} + +# .schema ?TABLE? Show the CREATE statements +# If TABLE specified, only show tables matching +# LIKE pattern TABLE. +do_test shell1-3.21.1 { + catchcmd " test.db" ".schema" +} {0 {}} +do_test shell1-3.21.2 { + catchcmd " test.db" ".schema FOO" +} {0 {}} +do_test shell1-3.21.3 { + # too many arguments + catchcmd " test.db" ".schema FOO BAD" +} {1 {Error: unknown command or invalid arguments: "schema". Enter ".help" for help}} + +# .separator STRING Change separator used by output mode and .import +do_test shell1-3.22.1 { + catchcmd " test.db" ".separator" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} +do_test shell1-3.22.2 { + catchcmd " test.db" ".separator FOO" +} {0 {}} +do_test shell1-3.22.3 { + # too many arguments + catchcmd " test.db" ".separator FOO BAD" +} {1 {Error: unknown command or invalid arguments: "separator". Enter ".help" for help}} + +# .show Show the current values for various settings +do_test shell1-3.23.1 { + set res [catchcmd " test.db" ".show"] + list [regexp {echo:} $res] \ + [regexp {explain:} $res] \ + [regexp {headers:} $res] \ + [regexp {mode:} $res] \ + [regexp {nullvalue:} $res] \ + [regexp {output:} $res] \ + [regexp {separator:} $res] \ + [regexp {width:} $res] +} {1 1 1 1 1 1 1 1} +do_test shell1-3.23.2 { + # too many arguments + catchcmd " test.db" ".show BAD" +} {1 {Error: unknown command or invalid arguments: "show". Enter ".help" for help}} + +# .tables ?TABLE? List names of tables +# If TABLE specified, only list tables matching +# LIKE pattern TABLE. +do_test shell1-3.24.1 { + catchcmd " test.db" ".tables" +} {0 {}} +do_test shell1-3.24.2 { + catchcmd " test.db" ".tables FOO" +} {0 {}} +do_test shell1-3.24.3 { + # too many arguments + catchcmd " test.db" ".tables FOO BAD" +} {1 {Error: unknown command or invalid arguments: "tables". Enter ".help" for help}} + +# .timeout MS Try opening locked tables for MS milliseconds +do_test shell1-3.25.1 { + catchcmd " test.db" ".timeout" +} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} +do_test shell1-3.25.2 { + catchcmd " test.db" ".timeout zzz" + # this should be treated the same as a '0' timeout +} {0 {}} +do_test shell1-3.25.3 { + catchcmd " test.db" ".timeout 1" +} {0 {}} +do_test shell1-3.25.4 { + # too many arguments + catchcmd " test.db" ".timeout 1 BAD" +} {1 {Error: unknown command or invalid arguments: "timeout". Enter ".help" for help}} + +# .width NUM NUM ... Set column widths for "column" mode +do_test shell1-3.26.1 { + catchcmd " test.db" ".width" +} {1 {Error: unknown command or invalid arguments: "width". Enter ".help" for help}} +do_test shell1-3.26.2 { + catchcmd " test.db" ".width xxx" + # this should be treated the same as a '0' width for col 1 +} {0 {}} +do_test shell1-3.26.3 { + catchcmd " test.db" ".width xxx yyy" + # this should be treated the same as a '0' width for col 1 and 2 +} {0 {}} +do_test shell1-3.26.4 { + catchcmd " test.db" ".width 1 1" + # this should be treated the same as a '1' width for col 1 and 2 +} {0 {}} + +# .timer ON|OFF Turn the CPU timer measurement on or off +do_test shell1-3.27.1 { + catchcmd " test.db" ".timer" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} +do_test shell1-3.27.2 { + catchcmd " test.db" ".timer ON" +} {0 {}} +do_test shell1-3.27.3 { + catchcmd " test.db" ".timer OFF" +} {0 {}} +do_test shell1-3.27.4 { + # too many arguments + catchcmd " test.db" ".timer OFF BAD" +} {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} + +# diff --git a/tool/shell2.test b/tool/shell2.test new file mode 100644 index 0000000..65693b1 --- /dev/null +++ b/tool/shell2.test @@ -0,0 +1,85 @@ +# 2009 Nov 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the CLI shell tool. +# +# $Id: shell2.test,v 1.7 2009/07/17 16:54:48 shaneh Exp $ +# + +# Test plan: +# +# shell2-1.*: Misc. test of various tickets and reported errors. +# + +package require sqlite3 + +set CLI "./sqlite" + +proc do_test {name cmd expected} { + puts -nonewline "$name ..." + set res [uplevel $cmd] + if {$res eq $expected} { + puts Ok + } else { + puts Error + puts " Got: $res" + puts " Expected: $expected" + exit + } +} + +proc execsql {sql} { + uplevel [list db eval $sql] +} + +proc catchsql {sql} { + set rc [catch {uplevel [list db eval $sql]} msg] + list $rc $msg +} + +proc catchcmd {db cmd} { + global CLI + set out [open cmds.txt w] + puts $out $cmd + close $out + set line "exec $CLI $db < cmds.txt" + set rc [catch { eval $line } msg] + list $rc $msg +} + +file delete -force test.db test.db.journal +sqlite3 db test.db + + +#---------------------------------------------------------------------------- +# shell2-1.*: Misc. test of various tickets and reported errors. +# + +# Batch mode not creating databases. +# Reported on mailing list by Ken Zalewski. +# Ticket [aeff892c57]. +do_test shell2-1.1.1 { + file delete -force foo.db + set rc [ catchcmd "-batch foo.db" "CREATE TABLE t1(a);" ] + set fexist [file exist foo.db] + list $rc $fexist +} {{0 {}} 1} + +# Shell silently ignores extra parameters. +# Ticket [f5cb008a65]. +do_test shell2-1.2.1 { + set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] + list $rc \ + [regexp {Error: too many options: "select 4"} $msg] +} {1 1} + + +