commit of sqlite 3.6.0

Makefile.in

@@ -0,0 +1,756 @@
+#!/usr/make
+#
+# Makefile for SQLITE
+#
+# This makefile is suppose to be configured automatically using the
+# autoconf.  But if that does not work for you, you can configure
+# the makefile manually.  Just set the parameters below to values that
+# work well for your system.
+#
+# If the configure script does not work out-of-the-box, you might
+# be able to get it to work by giving it some hints.  See the comment
+# at the beginning of configure.in for additional information.
+#
+
+# The toplevel directory of the source tree.  This is the directory
+# that contains this "Makefile.in" and the "configure.in" script.
+#
+TOP = @srcdir@
+
+# C Compiler and options for use in building executables that
+# will run on the platform that is doing the build.
+#
+BCC = @BUILD_CC@ @BUILD_CFLAGS@
+
+# C Compile and options for use in building executables that 
+# will run on the target platform.  (BCC and TCC are usually the
+# same unless your are cross-compiling.)
+#
+TCC = @CC@ @CPPFLAGS@ @CFLAGS@ -I. -I${TOP}/src
+
+# Define this for the autoconf-based build, so that the code knows it can
+# include the generated config.h
+# 
+TCC += -D_HAVE_SQLITE_CONFIG_H
+
+# Define -DNDEBUG to compile without debugging (i.e., for production usage)
+# Omitting the define will cause extra debugging code to be inserted and
+# includes extra comments when "EXPLAIN stmt" is used.
+#
+TCC += @TARGET_DEBUG@ @XTHREADCONNECT@
+
+# Compiler options needed for programs that use the TCL library.
+#
+TCC += @TCL_INCLUDE_SPEC@
+
+# The library that programs using TCL must link against.
+#
+LIBTCL = @TCL_LIB_SPEC@ @TCL_LIBS@
+
+# Compiler options needed for programs that use the readline() library.
+#
+READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
+
+# The library that programs using readline() must link against.
+#
+LIBREADLINE = @TARGET_READLINE_LIBS@
+
+# Should the database engine be compiled threadsafe
+#
+TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
+
+# Do threads override each others locks by default (1), or do we test (-1)
+#
+TCC += -DSQLITE_THREAD_OVERRIDE_LOCK=@THREADSOVERRIDELOCKS@
+
+# Any target libraries which libsqlite must be linked against
+# 
+TLIBS = @LIBS@
+
+# Flags controlling use of the in memory btree implementation
+#
+# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
+# default to file, 2 to default to memory, and 3 to force temporary
+# tables to always be in memory.
+#
+TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
+
+# Enable/disable loadable extensions based on configuration
+TCC += @LOADEXTENSION_FLAGS@
+
+# Version numbers and release number for the SQLite being compiled.
+#
+VERSION = @VERSION@
+VERSION_NUMBER = @VERSION_NUMBER@
+RELEASE = @RELEASE@
+
+# Filename extensions
+#
+BEXE = @BUILD_EXEEXT@
+TEXE = @TARGET_EXEEXT@
+
+# The following variable is "1" if the configure script was able to locate
+# the tclConfig.sh file.  It is an empty string otherwise.  When this
+# variable is "1", the TCL extension library (libtclsqlite3.so) is built
+# and installed.
+#
+HAVE_TCL = @HAVE_TCL@
+
+# This is the command to use for tclsh - normally just "tclsh", but we may
+# know the specific version we want to use
+#
+TCLSH_CMD = @TCLSH_CMD@
+
+# The suffix used on shared libraries.  Ex:  ".dll", ".so", ".dylib"
+#
+SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
+
+# If gcov support was enabled by the configure script, add the appropriate
+# flags here.  It's not always as easy as just having the user add the right
+# CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
+# Supposedly GCC does the right thing if you use --coverage, but in 
+# practice it still fails.  See:
+#
+# http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
+#
+# for more info.
+#
+GCOV_CFLAGS1 = -fprofile-arcs -ftest-coverage
+GCOV_LDFLAGS1 = -lgcov
+USE_GCOV = @USE_GCOV@
+LTCOMPILE_EXTRAS += $(GCOV_CFLAGS$(USE_GCOV))
+LTLINK_EXTRAS += $(GCOV_LDFLAGS$(USE_GCOV))
+
+
+# The directory into which to store package information for
+
+# Some standard variables and programs
+#
+prefix = @prefix@
+exec_prefix = @exec_prefix@
+libdir = @libdir@
+bindir = @bindir@
+includedir = @includedir@
+INSTALL = @INSTALL@
+LIBTOOL = ./libtool
+ALLOWRELEASE = @ALLOWRELEASE@
+
+# libtool compile/link/install
+LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS)
+LTLINK = $(LIBTOOL) --mode=link $(TCC) @LDFLAGS@ $(LTLINK_EXTRAS)
+LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL)
+
+# nawk compatible awk.
+NAWK = @AWK@
+
+# You should not have to change anything below this line
+###############################################################################
+
+# Object files for the SQLite library (non-amalgamation).
+#
+OBJS0 = alter.lo analyze.lo attach.lo auth.lo bitvec.lo btmutex.lo \
+        btree.lo build.lo callback.lo complete.lo date.lo \
+        delete.lo expr.lo fault.lo func.lo global.lo \
+        hash.lo journal.lo insert.lo loadext.lo \
+        main.lo malloc.lo mem1.lo mem2.lo mem3.lo mem4.lo mem5.lo mutex.lo \
+        mutex_os2.lo mutex_unix.lo mutex_w32.lo \
+        opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \
+        pager.lo parse.lo pragma.lo prepare.lo printf.lo random.lo \
+        select.lo status.lo table.lo tokenize.lo trigger.lo update.lo \
+        util.lo vacuum.lo \
+        vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbefifo.lo vdbemem.lo \
+        where.lo utf.lo legacy.lo vtab.lo
+
+# Object files for the amalgamation.
+#
+OBJS1 = sqlite3.lo
+
+# Determine the real value of LIBOBJ based on the 'configure' script
+#
+USE_AMALGAMATION = @USE_AMALGAMATION@
+LIBOBJ = $(OBJS$(USE_AMALGAMATION))
+
+
+# All of the source code files.
+#
+SRC = \
+  $(TOP)/src/alter.c \
+  $(TOP)/src/analyze.c \
+  $(TOP)/src/attach.c \
+  $(TOP)/src/auth.c \
+  $(TOP)/src/bitvec.c \
+  $(TOP)/src/btmutex.c \
+  $(TOP)/src/btree.c \
+  $(TOP)/src/btree.h \
+  $(TOP)/src/btreeInt.h \
+  $(TOP)/src/build.c \
+  $(TOP)/src/callback.c \
+  $(TOP)/src/complete.c \
+  $(TOP)/src/date.c \
+  $(TOP)/src/delete.c \
+  $(TOP)/src/expr.c \
+  $(TOP)/src/fault.c \
+  $(TOP)/src/func.c \
+  $(TOP)/src/global.c \
+  $(TOP)/src/hash.c \
+  $(TOP)/src/hash.h \
+  $(TOP)/src/hwtime.h \
+  $(TOP)/src/insert.c \
+  $(TOP)/src/journal.c \
+  $(TOP)/src/legacy.c \
+  $(TOP)/src/loadext.c \
+  $(TOP)/src/main.c \
+  $(TOP)/src/malloc.c \
+  $(TOP)/src/mem1.c \
+  $(TOP)/src/mem2.c \
+  $(TOP)/src/mem3.c \
+  $(TOP)/src/mem4.c \
+  $(TOP)/src/mem5.c \
+  $(TOP)/src/mutex.c \
+  $(TOP)/src/mutex.h \
+  $(TOP)/src/mutex_os2.c \
+  $(TOP)/src/mutex_unix.c \
+  $(TOP)/src/mutex_w32.c \
+  $(TOP)/src/os.c \
+  $(TOP)/src/os.h \
+  $(TOP)/src/os_common.h \
+  $(TOP)/src/os_unix.c \
+  $(TOP)/src/os_win.c \
+  $(TOP)/src/os_os2.c \
+  $(TOP)/src/pager.c \
+  $(TOP)/src/pager.h \
+  $(TOP)/src/parse.y \
+  $(TOP)/src/pragma.c \
+  $(TOP)/src/prepare.c \
+  $(TOP)/src/printf.c \
+  $(TOP)/src/random.c \
+  $(TOP)/src/select.c \
+  $(TOP)/src/status.c \
+  $(TOP)/src/shell.c \
+  $(TOP)/src/sqlite.h.in \
+  $(TOP)/src/sqlite3ext.h \
+  $(TOP)/src/sqliteInt.h \
+  $(TOP)/src/sqliteLimit.h \
+  $(TOP)/src/table.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/src/tokenize.c \
+  $(TOP)/src/trigger.c \
+  $(TOP)/src/utf.c \
+  $(TOP)/src/update.c \
+  $(TOP)/src/util.c \
+  $(TOP)/src/vacuum.c \
+  $(TOP)/src/vdbe.c \
+  $(TOP)/src/vdbe.h \
+  $(TOP)/src/vdbeapi.c \
+  $(TOP)/src/vdbeaux.c \
+  $(TOP)/src/vdbeblob.c \
+  $(TOP)/src/vdbefifo.c \
+  $(TOP)/src/vdbemem.c \
+  $(TOP)/src/vdbeInt.h \
+  $(TOP)/src/vtab.c \
+  $(TOP)/src/where.c
+
+# Generated source code files
+#
+SRC += \
+  keywordhash.h \
+  opcodes.c \
+  opcodes.h \
+  parse.c \
+  parse.h \
+  config.h \
+  sqlite3.h
+
+# Source code for extensions
+#
+SRC += \
+  $(TOP)/ext/fts1/fts1.c \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.c \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_porter.c \
+  $(TOP)/ext/fts1/fts1_tokenizer.h \
+  $(TOP)/ext/fts1/fts1_tokenizer1.c
+SRC += \
+  $(TOP)/ext/fts2/fts2.c \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.c \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_icu.c \
+  $(TOP)/ext/fts2/fts2_porter.c \
+  $(TOP)/ext/fts2/fts2_tokenizer.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.c \
+  $(TOP)/ext/fts2/fts2_tokenizer1.c
+SRC += \
+  $(TOP)/ext/fts3/fts3.c \
+  $(TOP)/ext/fts3/fts3.h \
+  $(TOP)/ext/fts3/fts3_hash.c \
+  $(TOP)/ext/fts3/fts3_hash.h \
+  $(TOP)/ext/fts3/fts3_icu.c \
+  $(TOP)/ext/fts3/fts3_porter.c \
+  $(TOP)/ext/fts3/fts3_tokenizer.h \
+  $(TOP)/ext/fts3/fts3_tokenizer.c \
+  $(TOP)/ext/fts3/fts3_tokenizer1.c
+SRC += \
+  $(TOP)/ext/icu/icu.c
+SRC += \
+  $(TOP)/ext/rtree/rtree.h \
+  $(TOP)/ext/rtree/rtree.c
+
+# Source code to the library files needed by the test fixture
+#
+TESTSRC2 = \
+  $(TOP)/src/attach.c \
+  $(TOP)/src/bitvec.c \
+  $(TOP)/src/btree.c \
+  $(TOP)/src/build.c \
+  $(TOP)/src/date.c \
+  $(TOP)/src/expr.c \
+  $(TOP)/src/func.c \
+  $(TOP)/src/insert.c \
+  $(TOP)/src/malloc.c \
+  $(TOP)/src/os.c \
+  $(TOP)/src/os_os2.c \
+  $(TOP)/src/os_unix.c \
+  $(TOP)/src/os_win.c \
+  $(TOP)/src/pager.c \
+  $(TOP)/src/pragma.c \
+  $(TOP)/src/prepare.c \
+  $(TOP)/src/printf.c \
+  $(TOP)/src/random.c \
+  $(TOP)/src/select.c \
+  $(TOP)/src/tokenize.c \
+  $(TOP)/src/utf.c \
+  $(TOP)/src/util.c \
+  $(TOP)/src/vdbe.c \
+  $(TOP)/src/vdbeapi.c \
+  $(TOP)/src/vdbeaux.c \
+  $(TOP)/src/vdbemem.c \
+  $(TOP)/src/where.c \
+  parse.c
+
+# Source code to the actual test files.
+#
+TESTSRC = \
+  $(TOP)/src/test1.c \
+  $(TOP)/src/test2.c \
+  $(TOP)/src/test3.c \
+  $(TOP)/src/test4.c \
+  $(TOP)/src/test5.c \
+  $(TOP)/src/test6.c \
+  $(TOP)/src/test7.c \
+  $(TOP)/src/test8.c \
+  $(TOP)/src/test9.c \
+  $(TOP)/src/test_autoext.c \
+  $(TOP)/src/test_async.c \
+  $(TOP)/src/test_btree.c \
+  $(TOP)/src/test_config.c \
+  $(TOP)/src/test_devsym.c \
+  $(TOP)/src/test_func.c \
+  $(TOP)/src/test_hexio.c \
+  $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
+  $(TOP)/src/test_mutex.c \
+  $(TOP)/src/test_onefile.c \
+  $(TOP)/src/test_osinst.c \
+  $(TOP)/src/test_schema.c \
+  $(TOP)/src/test_server.c \
+  $(TOP)/src/test_tclvar.c \
+  $(TOP)/src/test_thread.c
+
+# Header files used by all library source files.
+#
+HDR = \
+   sqlite3.h  \
+   $(TOP)/src/btree.h \
+   $(TOP)/src/btreeInt.h \
+   $(TOP)/src/hash.h \
+   $(TOP)/src/hwtime.h \
+   $(TOP)/src/sqliteLimit.h \
+   $(TOP)/src/mutex.h \
+   opcodes.h \
+   $(TOP)/src/os.h \
+   $(TOP)/src/os_common.h \
+   $(TOP)/src/sqlite3ext.h \
+   $(TOP)/src/sqliteInt.h  \
+   $(TOP)/src/vdbe.h \
+   $(TOP)/src/vdbeInt.h \
+   parse.h \
+   config.h
+
+# Header files used by extensions
+#
+HDR += \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_tokenizer.h
+HDR += \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.h
+HDR += \
+  $(TOP)/ext/fts3/fts3.h \
+  $(TOP)/ext/fts3/fts3_hash.h \
+  $(TOP)/ext/fts3/fts3_tokenizer.h
+HDR += \
+  $(TOP)/ext/rtree/rtree.h
+
+# If using the amalgamation, use sqlite3.c directly to build the test
+# fixture.  Otherwise link against libsqlite3.la.  (This distinction is
+# necessary because the test fixture requires non-API symbols which are
+# hidden when the library is built via the amalgamation).
+#
+TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
+TESTFIXTURE_SRC1 = sqlite3.c
+TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
+
+
+# This is the default Makefile target.  The objects listed here
+# are what get build when you type just "make" with no arguments.
+#
+all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)
+
+Makefile: $(TOP)/Makefile.in
+	./config.status
+
+# Generate the file "last_change" which contains the date of change
+# of the most recently modified source code file
+#
+last_change:	$(SRC)
+	cat $(SRC) | grep '$$Id: ' | sort -k 5 | tail -1 \
+          | $(NAWK) '{print $$5,$$6}' >last_change
+
+libsqlite3.la:	$(LIBOBJ)
+	$(LTLINK) -o $@ $(LIBOBJ) $(TLIBS) \
+		${ALLOWRELEASE} -rpath "$(libdir)" -version-info "8:6:8"
+
+libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
+	$(LTLINK) -o $@ tclsqlite.lo \
+		$(LIBOBJ) @TCL_STUB_LIB_SPEC@ $(TLIBS) \
+                -rpath "$(libdir)/sqlite" \
+		-version-info "8:6:8"
+
+sqlite3$(TEXE):	$(TOP)/src/shell.c libsqlite3.la sqlite3.h
+	$(LTLINK) $(READLINE_FLAGS) \
+		-o $@ $(TOP)/src/shell.c libsqlite3.la \
+		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
+
+# This target creates a directory named "tsrc" and fills it with
+# copies of all of the C source code and header files needed to
+# build on the target system.  Some of the C source code and header
+# files are automatically generated.  This target takes care of
+# all that automatic generation.
+#
+.target_source:	$(SRC)
+	rm -rf tsrc
+	mkdir -p tsrc
+	cp $(SRC) tsrc
+	rm tsrc/sqlite.h.in tsrc/parse.y
+	touch .target_source
+
+sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
+
+# Rules to build the LEMON compiler generator
+#
+lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
+	$(BCC) -o $@ $(TOP)/tool/lemon.c
+	cp $(TOP)/tool/lempar.c .
+
+
+# Rule to build the amalgamation
+#
+sqlite3.lo:	sqlite3.c
+	$(LTCOMPILE) -c sqlite3.c
+
+# Rules to build individual files
+#
+alter.lo:	$(TOP)/src/alter.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/alter.c
+
+analyze.lo:	$(TOP)/src/analyze.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/analyze.c
+
+attach.lo:	$(TOP)/src/attach.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/attach.c
+
+auth.lo:	$(TOP)/src/auth.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/auth.c
+
+bitvec.lo:	$(TOP)/src/bitvec.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/bitvec.c
+
+btmutex.lo:	$(TOP)/src/btmutex.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/btmutex.c
+
+btree.lo:	$(TOP)/src/btree.c $(HDR) $(TOP)/src/pager.h
+	$(LTCOMPILE) -c $(TOP)/src/btree.c
+
+build.lo:	$(TOP)/src/build.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/build.c
+
+callback.lo:	$(TOP)/src/callback.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/callback.c
+
+complete.lo:	$(TOP)/src/complete.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/complete.c
+
+date.lo:	$(TOP)/src/date.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/date.c
+
+delete.lo:	$(TOP)/src/delete.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/delete.c
+
+expr.lo:	$(TOP)/src/expr.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/expr.c
+
+fault.lo:	$(TOP)/src/fault.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/fault.c
+
+func.lo:	$(TOP)/src/func.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/func.c
+
+global.lo:	$(TOP)/src/global.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/global.c
+
+hash.lo:	$(TOP)/src/hash.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/hash.c
+
+insert.lo:	$(TOP)/src/insert.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/insert.c
+
+journal.lo:	$(TOP)/src/journal.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/journal.c
+
+legacy.lo:	$(TOP)/src/legacy.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/legacy.c
+
+loadext.lo:	$(TOP)/src/loadext.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/loadext.c
+
+main.lo:	$(TOP)/src/main.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/main.c
+
+malloc.lo:	$(TOP)/src/malloc.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/malloc.c
+
+mem1.lo:	$(TOP)/src/mem1.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem1.c
+
+mem2.lo:	$(TOP)/src/mem2.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem2.c
+
+mem3.lo:	$(TOP)/src/mem3.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem3.c
+
+mem4.lo:	$(TOP)/src/mem4.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem4.c
+
+mem5.lo:	$(TOP)/src/mem5.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c
+
+mutex.lo:	$(TOP)/src/mutex.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex.c
+
+mutex_os2.lo:	$(TOP)/src/mutex_os2.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_os2.c
+
+mutex_unix.lo:	$(TOP)/src/mutex_unix.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_unix.c
+
+mutex_w32.lo:	$(TOP)/src/mutex_w32.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_w32.c
+
+pager.lo:	$(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h
+	$(LTCOMPILE) -c $(TOP)/src/pager.c
+
+opcodes.lo:	opcodes.c
+	$(LTCOMPILE) -c opcodes.c
+
+opcodes.c:	opcodes.h $(TOP)/mkopcodec.awk
+	sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c
+
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
+	cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
+
+os.lo:	$(TOP)/src/os.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/os.c
+
+os_unix.lo:	$(TOP)/src/os_unix.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/os_unix.c
+
+os_win.lo:	$(TOP)/src/os_win.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/os_win.c
+
+os_os2.lo:	$(TOP)/src/os_os2.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/os_os2.c
+
+parse.lo:	parse.c $(HDR)
+	$(LTCOMPILE) -c parse.c
+
+parse.h:	parse.c
+
+parse.c:	$(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
+	cp $(TOP)/src/parse.y .
+	./lemon$(BEXE) $(OPTS) parse.y
+	mv parse.h parse.h.temp
+	$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
+
+pragma.lo:	$(TOP)/src/pragma.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/pragma.c
+
+prepare.lo:	$(TOP)/src/prepare.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/prepare.c
+
+printf.lo:	$(TOP)/src/printf.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/printf.c
+
+random.lo:	$(TOP)/src/random.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/random.c
+
+select.lo:	$(TOP)/src/select.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/select.c
+
+status.lo:	$(TOP)/src/status.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/status.c
+
+sqlite3.h:	$(TOP)/src/sqlite.h.in 
+	sed -e s/--VERS--/$(RELEASE)/ $(TOP)/src/sqlite.h.in | \
+	sed -e s/--VERSION-NUMBER--/$(VERSION_NUMBER)/ >sqlite3.h
+
+table.lo:	$(TOP)/src/table.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/table.c
+
+tclsqlite.lo:	$(TOP)/src/tclsqlite.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/tclsqlite.c
+
+tokenize.lo:	$(TOP)/src/tokenize.c keywordhash.h $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/tokenize.c
+
+keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
+	$(BCC) -o mkkeywordhash$(BEXE) $(OPTS) $(TOP)/tool/mkkeywordhash.c
+	./mkkeywordhash$(BEXE) >keywordhash.h
+
+trigger.lo:	$(TOP)/src/trigger.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/trigger.c
+
+update.lo:	$(TOP)/src/update.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/update.c
+
+utf.lo:	$(TOP)/src/utf.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/utf.c
+
+util.lo:	$(TOP)/src/util.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/util.c
+
+vacuum.lo:	$(TOP)/src/vacuum.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vacuum.c
+
+vdbe.lo:	$(TOP)/src/vdbe.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbe.c
+
+vdbeapi.lo:	$(TOP)/src/vdbeapi.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbeapi.c
+
+vdbeaux.lo:	$(TOP)/src/vdbeaux.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbeaux.c
+
+vdbeblob.lo:	$(TOP)/src/vdbeblob.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbeblob.c
+
+vdbefifo.lo:	$(TOP)/src/vdbefifo.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbefifo.c
+
+vdbemem.lo:	$(TOP)/src/vdbemem.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vdbemem.c
+
+vtab.lo:	$(TOP)/src/vtab.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/vtab.c
+
+where.lo:	$(TOP)/src/where.c $(HDR)
+	$(LTCOMPILE) -c $(TOP)/src/where.c
+
+tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
+	$(LTCOMPILE) -DTCLSH=1 -o $@ -c $(TOP)/src/tclsqlite.c
+
+tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
+	$(LTCOMPILE) -DTCL_USE_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
+
+tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
+	$(LTLINK) -o $@ tclsqlite-shell.lo \
+		 libsqlite3.la $(LIBTCL)
+
+testfixture$(TEXE):	$(TESTFIXTURE_SRC)
+	$(LTLINK) -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_NO_SYNC=1\
+		-DSQLITE_CRASH_TEST=1 \
+                -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE $(TEMP_STORE) \
+		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
+
+
+fulltest:	testfixture$(TEXE) sqlite3$(TEXE)
+	./testfixture$(TEXE) $(TOP)/test/all.test
+
+test:	testfixture$(TEXE) sqlite3$(TEXE)
+	./testfixture$(TEXE) $(TOP)/test/veryquick.test
+
+sqlite3_analyzer$(TEXE):	$(TESTFIXTURE_SRC) $(TOP)/tool/spaceanal.tcl
+	sed \
+	  -e '/^#/d' \
+	  -e 's,\\,\\\\,g' \
+	  -e 's,",\\",g' \
+	  -e 's,^,",' \
+	  -e 's,$$,\\n",' \
+	  $(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
+	$(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \
+		-DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \
+		$(TEMP_STORE) -o $@ $(TESTFIXTURE_SRC) $(LIBTCL)
+
+
+install:	sqlite3$(BEXE) libsqlite3.la sqlite3.h ${HAVE_TCL:1=tcl_install}
+	$(INSTALL) -d $(DESTDIR)$(libdir)
+	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
+	$(INSTALL) -d $(DESTDIR)$(bindir)
+	$(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir)
+	$(INSTALL) -d $(DESTDIR)$(includedir)
+	$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
+	$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
+	$(INSTALL) -d $(DESTDIR)$(libdir)/pkgconfig; 
+	$(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(libdir)/pkgconfig; 
+
+tcl_install:	libtclsqlite3.la
+	$(TCLSH_CMD) $(TOP)/tclinstaller.tcl $(VERSION)
+
+clean:	
+	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
+	rm -f sqlite3.h opcodes.*
+	rm -rf .libs .deps tsrc
+	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
+	rm -f mkkeywordhash$(BEXE) keywordhash.h
+	rm -f $(PUBLISH)
+	rm -f *.da *.bb *.bbg gmon.out
+	rm -f testfixture$(TEXE) test.db
+	rm -f common.tcl
+	rm -f sqlite3.dll sqlite3.lib sqlite3.def
+	rm -f sqlite3.c .target_source
+
+distclean:	clean
+	rm -f config.log config.status libtool Makefile sqlite3.pc
+
+#
+# Windows section
+#
+dll: sqlite3.dll
+
+REAL_LIBOBJ = $(LIBOBJ:%.lo=.libs/%.o)
+
+$(REAL_LIBOBJ): $(LIBOBJ)
+
+sqlite3.def: $(REAL_LIBOBJ)
+	echo 'EXPORTS' >sqlite3.def
+	nm $(REAL_LIBOBJ) | grep ' T ' | grep ' _sqlite3_' \
+		| sed 's/^.* _//' >>sqlite3.def
+
+sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def
+	$(TCC) -shared -o $@ sqlite3.def \
+		-Wl,"--strip-all" $(REAL_LIBOBJ)

Makefile.linux-gcc

@@ -0,0 +1,128 @@
+#!/usr/make
+#
+# Makefile for SQLITE
+#
+# This is a template makefile for SQLite.  Most people prefer to
+# use the autoconf generated "configure" script to generate the
+# makefile automatically.  But that does not work for everybody
+# and in every situation.  If you are having problems with the
+# "configure" script, you might want to try this makefile as an
+# alternative.  Create a copy of this file, edit the parameters
+# below and type "make".
+#
+
+#### The toplevel directory of the source tree.  This is the directory
+#    that contains this "Makefile.in" and the "configure.in" script.
+#
+TOP = ../sqlite
+
+#### C Compiler and options for use in building executables that
+#    will run on the platform that is doing the build.
+#
+BCC = gcc -g -O2
+#BCC = /opt/ancic/bin/c89 -0
+
+#### If the target operating system supports the "usleep()" system
+#    call, then define the HAVE_USLEEP macro for all C modules.
+#
+#USLEEP = 
+USLEEP = -DHAVE_USLEEP=1
+
+#### If you want the SQLite library to be safe for use within a 
+#    multi-threaded program, then define the following macro
+#    appropriately:
+#
+#THREADSAFE = -DTHREADSAFE=1
+THREADSAFE = -DTHREADSAFE=0
+
+#### Specify any extra linker options needed to make the library
+#    thread safe
+#
+#THREADLIB = -lpthread
+THREADLIB = 
+
+#### Specify any extra libraries needed to access required functions.
+#
+#TLIBS = -lrt    # fdatasync on Solaris 8
+TLIBS = 
+
+#### Leave SQLITE_DEBUG undefined for maximum speed.  Use SQLITE_DEBUG=1
+#    to check for memory leaks.  Use SQLITE_DEBUG=2 to print a log of all
+#    malloc()s and free()s in order to track down memory leaks.
+#    
+#    SQLite uses some expensive assert() statements in the inner loop.
+#    You can make the library go almost twice as fast if you compile
+#    with -DNDEBUG=1
+#
+#OPTS = -DSQLITE_DEBUG=2
+#OPTS = -DSQLITE_DEBUG=1
+#OPTS = 
+OPTS = -DNDEBUG=1
+OPTS += -DHAVE_FDATASYNC=1
+
+#### The suffix to add to executable files.  ".exe" for windows.
+#    Nothing for unix.
+#
+#EXE = .exe
+EXE =
+
+#### C Compile and options for use in building executables that 
+#    will run on the target platform.  This is usually the same
+#    as BCC, unless you are cross-compiling.
+#
+TCC = gcc -O6
+#TCC = gcc -g -O0 -Wall
+#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
+#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
+#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
+
+#### Tools used to build a static library.
+#
+AR = ar cr
+#AR = /opt/mingw/bin/i386-mingw32-ar cr
+RANLIB = ranlib
+#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
+
+MKSHLIB = gcc -shared
+SO = so
+SHPREFIX = lib
+# SO = dll
+# SHPREFIX =
+
+#### Extra compiler options needed for programs that use the TCL library.
+#
+#TCL_FLAGS =
+#TCL_FLAGS = -DSTATIC_BUILD=1
+TCL_FLAGS = -I/home/drh/tcltk/8.4linux
+#TCL_FLAGS = -I/home/drh/tcltk/8.4win -DSTATIC_BUILD=1
+#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
+
+#### Linker options needed to link against the TCL library.
+#
+#LIBTCL = -ltcl -lm -ldl
+LIBTCL = /home/drh/tcltk/8.4linux/libtcl8.4g.a -lm -ldl
+#LIBTCL = /home/drh/tcltk/8.4win/libtcl84s.a -lmsvcrt
+#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
+
+#### Additional objects for SQLite library when TCL support is enabled.
+#TCLOBJ =
+TCLOBJ = tclsqlite.o
+
+#### Compiler options needed for programs that use the readline() library.
+#
+READLINE_FLAGS =
+#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
+
+#### Linker options needed by programs using readline() must link against.
+#
+LIBREADLINE =
+#LIBREADLINE = -static -lreadline -ltermcap
+
+#### Which "awk" program provides nawk compatibilty
+#
+# NAWK = nawk
+NAWK = awk
+
+# You should not have to change anything below this line
+###############################################################################
+include $(TOP)/main.mk

README

@@ -0,0 +1,34 @@
+This directory contains source code to 
+
+    SQLite: An Embeddable SQL Database Engine
+
+To compile the project, first create a directory in which to place
+the build products.  It is recommended, but not required, that the
+build directory be separate from the source directory.  Cd into the
+build directory and then from the build directory run the configure
+script found at the root of the source tree.  Then run "make".
+
+For example:
+
+    tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
+    mkdir bld                ;#  Build will occur in a sibling directory
+    cd bld                   ;#  Change to the build directory
+    ../sqlite/configure      ;#  Run the configure script
+    make                     ;#  Run the makefile.
+    make install             ;#  (Optional) Install the build products
+
+The configure script uses autoconf 2.61 and libtool.  If the configure
+script does not work out for you, there is a generic makefile named
+"Makefile.linux-gcc" in the top directory of the source tree that you
+can copy and edit to suit your needs.  Comments on the generic makefile
+show what changes are needed.
+
+The linux binaries on the website are created using the generic makefile,
+not the configure script.
+The windows binaries on the website are created using MinGW32 configured
+as a cross-compiler running under Linux.  For details, see the ./publish.sh
+script at the top-level of the source tree.
+
+Contacts:
+
+   http://www.sqlite.org/

VERSION

@@ -0,0 +1 @@
+3.6.0

addopcodes.awk

@@ -0,0 +1,32 @@
+#!/usr/bin/awk
+#
+# This script appends additional token codes to the end of the
+# parse.h file that lemon generates.  These extra token codes are
+# not used by the parser.  But they are used by the tokenizer and/or
+# the code generator.
+#
+#
+BEGIN {
+  max = 0
+}
+/^#define TK_/ {
+  print $0
+  if( max<$3 ) max = $3
+}
+END {
+  printf "#define TK_%-29s %4d\n", "TO_TEXT",         max+1
+  printf "#define TK_%-29s %4d\n", "TO_BLOB",         max+2
+  printf "#define TK_%-29s %4d\n", "TO_NUMERIC",      max+3
+  printf "#define TK_%-29s %4d\n", "TO_INT",          max+4
+  printf "#define TK_%-29s %4d\n", "TO_REAL",         max+5
+  printf "#define TK_%-29s %4d\n", "END_OF_FILE",     max+6
+  printf "#define TK_%-29s %4d\n", "ILLEGAL",         max+7
+  printf "#define TK_%-29s %4d\n", "SPACE",           max+8
+  printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", max+9
+  printf "#define TK_%-29s %4d\n", "COMMENT",         max+10
+  printf "#define TK_%-29s %4d\n", "FUNCTION",        max+11
+  printf "#define TK_%-29s %4d\n", "COLUMN",          max+12
+  printf "#define TK_%-29s %4d\n", "AGG_FUNCTION",    max+13
+  printf "#define TK_%-29s %4d\n", "AGG_COLUMN",      max+14
+  printf "#define TK_%-29s %4d\n", "CONST_FUNC",      max+15
+}

config.h.in

@@ -0,0 +1,98 @@
+/*
+** 2008 March 6
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Configuration header template to be filled in by 'configure' script
+**
+** @(#) $Id: config.h.in,v 1.2 2008/05/06 02:28:06 mlcreech Exp $
+*/
+#ifndef _CONFIG_H_
+#define _CONFIG_H_
+
+
+/*****************************
+** Data types
+*****************************/
+
+/* Define as 1 if you have the int8_t type */
+#undef HAVE_INT8_T
+
+/* Define as 1 if you have the int16_t type */
+#undef HAVE_INT16_T
+
+/* Define as 1 if you have the int32_t type */
+#undef HAVE_INT32_T
+
+/* Define as 1 if you have the int64_t type */
+#undef HAVE_INT64_T
+
+/* Define as 1 if you have the intptr_t type */
+#undef HAVE_INTPTR_T
+
+/* Define as 1 if you have the uint8_t type */
+#undef HAVE_UINT8_T
+
+/* Define as 1 if you have the uint16_t type */
+#undef HAVE_UINT16_T
+
+/* Define as 1 if you have the uint32_t type */
+#undef HAVE_UINT32_T
+
+/* Define as 1 if you have the uint64_t type */
+#undef HAVE_UINT64_T
+
+/* Define as 1 if you have the uintptr_t type */
+#undef HAVE_UINTPTR_T
+
+
+/*****************************
+** Header Files
+*****************************/
+
+/* Define as 1 if you have the sys/types.h header */
+#undef HAVE_SYS_TYPES_H
+
+/* Define as 1 if you have the stdlib.h header */
+#undef HAVE_STDLIB_H
+
+/* Define as 1 if you have the stdint.h header */
+#undef HAVE_STDINT_H
+
+/* Define as 1 if you have the inttypes.h header */
+#undef HAVE_INTTYPES_H
+
+
+/*****************************
+** Functions
+*****************************/
+
+/* Define as 1 if you have the usleep() function */
+#undef HAVE_USLEEP
+
+/* Define as 1 if you have the fdatasync() function */
+#undef HAVE_FDATASYNC
+
+/* Define as 1 if you have the gmtime_r() function */
+#undef HAVE_GMTIME_R
+
+/* Define as 1 if you have the localtime_r() function */
+#undef HAVE_LOCALTIME_R
+
+
+/*****************************
+** Large file support
+*****************************/
+
+#undef _FILE_OFFSET_BITS
+#undef _LARGE_FILES
+
+
+/* End of header */
+#endif

configure.ac

@@ -0,0 +1,644 @@
+#
+# The build process allows for using a cross-compiler.  But the default
+# action is to target the same platform that we are running on.  The
+# configure script needs to discover the following properties of the 
+# build and target systems:
+#
+#    srcdir
+#
+#        The is the name of the directory that contains the
+#        "configure" shell script.  All source files are
+#        located relative to this directory.
+#
+#    bindir
+#
+#        The name of the directory where executables should be
+#        written by the "install" target of the makefile.
+#
+#    program_prefix
+#
+#        Add this prefix to the names of all executables that run
+#        on the target machine.  Default: ""
+#
+#    ENABLE_SHARED
+#
+#        True if shared libraries should be generated.
+#
+#    BUILD_CC
+#
+#        The name of a command that is used to convert C
+#        source files into executables that run on the build
+#        platform.
+#
+#    BUILD_CFLAGS
+#
+#        Switches that the build compiler needs in order to construct
+#        command-line programs.
+#
+#    BUILD_LIBS
+#
+#        Libraries that the build compiler needs in order to construct
+#        command-line programs.
+#
+#    BUILD_EXEEXT
+#
+#        The filename extension for executables on the build
+#        platform.  "" for Unix and ".exe" for Windows.
+#
+#    TCL_*
+#
+#        Lots of values are read in from the tclConfig.sh script,
+#        if that script is available.  This values are used for
+#        constructing and installing the TCL extension.
+#
+#    TARGET_READLINE_LIBS
+#
+#        This is the library directives passed to the target linker
+#        that cause the executable to link against the readline library.
+#        This might be a switch like "-lreadline" or pathnames of library
+#        file like "../../src/libreadline.a".
+#
+#    TARGET_READLINE_INC
+#
+#        This variables define the directory that contain header
+#        files for the readline library.  If the compiler is able 
+#        to find <readline.h> on its own, then this can be blank.
+#
+#    TARGET_EXEEXT
+#
+#        The filename extension for executables on the
+#        target platform.  "" for Unix and ".exe" for windows.
+#
+# The generated configure script will make an attempt to guess
+# at all of the above parameters.  You can override any of
+# the guesses by setting the environment variable named
+# "config_AAAA" where "AAAA" is the name of the parameter
+# described above.  (Exception: srcdir cannot be set this way.)
+# If you have a file that sets one or more of these environment
+# variables, you can invoke configure as follows:
+#
+#           configure --with-hints=FILE
+#
+# where FILE is the name of the file that sets the environment
+# variables.  FILE should be an absolute pathname.
+#
+# This configure.in file is easy to reuse on other projects.  Just
+# change the argument to AC_INIT().  And disable any features that
+# you don't need (for example BLT) by erasing or commenting out
+# the corresponding code.
+#
+AC_INIT(sqlite, m4_esyscmd([cat VERSION | tr -d '\n']))
+
+dnl Put the RCS revision string after AC_INIT so that it will also
+dnl show in in configure.
+# The following RCS revision string applies to configure.in
+# $Revision: 1.46 $
+
+#########
+# Programs needed
+#
+AC_PROG_LIBTOOL
+AC_PROG_INSTALL
+AC_PROG_AWK
+
+#########
+# Enable large file support (if special flags are necessary)
+#
+AC_SYS_LARGEFILE
+
+#########
+# Check for needed/wanted data types
+AC_CHECK_TYPES([int8_t, int16_t, int32_t, int64_t, intptr_t, uint8_t,
+                uint16_t, uint32_t, uint64_t, uintptr_t])
+
+#########
+# Check for needed/wanted headers
+AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h])
+
+#########
+# Figure out whether or not we have these functions
+#
+AC_CHECK_FUNCS([usleep fdatasync localtime_r gmtime_r])
+
+#########
+# By default, we use the amalgamation (this may be changed below...)
+#
+USE_AMALGAMATION=1
+
+#########
+# See whether we can run specific tclsh versions known to work well;
+# if not, then we fall back to plain tclsh.
+# TODO: try other versions before falling back?
+# 
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.4 tclsh], none)
+if test "$TCLSH_CMD" = "none"; then
+  # If we can't find a local tclsh, then building the amalgamation will fail.
+  # We act as though --disable-amalgamation has been used.
+  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
+  USE_AMALGAMATION=0
+  TCLSH_CMD="tclsh"
+fi
+AC_SUBST(TCLSH_CMD)
+
+
+#########
+# Set up an appropriate program prefix
+#
+if test "$program_prefix" = "NONE"; then
+  program_prefix=""
+fi
+AC_SUBST(program_prefix)
+
+VERSION=[`cat $srcdir/VERSION | sed 's/^\([0-9]*\.*[0-9]*\).*/\1/'`]
+echo "Version set to $VERSION"
+AC_SUBST(VERSION)
+RELEASE=`cat $srcdir/VERSION`
+echo "Release set to $RELEASE"
+AC_SUBST(RELEASE)
+VERSION_NUMBER=[`cat $srcdir/VERSION  \
+                           | sed 's/[^0-9]/ /g' \
+                | awk '{printf "%d%03d%03d",$1,$2,$3}'`]
+echo "Version number set to $VERSION_NUMBER"
+AC_SUBST(VERSION_NUMBER)
+
+#########
+# Check to see if the --with-hints=FILE option is used.  If there is none,
+# then check for a files named "$host.hints" and ../$hosts.hints where
+# $host is the hostname of the build system.  If still no hints are
+# found, try looking in $system.hints and ../$system.hints where
+# $system is the result of uname -s.
+#
+AC_ARG_WITH(hints,
+  AC_HELP_STRING([--with-hints=FILE],[Read configuration options from FILE]),
+  hints=$withval)
+if test "$hints" = ""; then
+  host=`hostname | sed 's/\..*//'`
+  if test -r $host.hints; then
+    hints=$host.hints
+  else
+     if test -r ../$host.hints; then
+       hints=../$host.hints
+     fi
+  fi
+fi
+if test "$hints" = ""; then
+  sys=`uname -s`
+  if test -r $sys.hints; then
+    hints=$sys.hints
+  else
+     if test -r ../$sys.hints; then
+       hints=../$sys.hints
+     fi
+  fi
+fi
+if test "$hints" != ""; then
+  AC_MSG_RESULT(reading hints from $hints)
+  . $hints
+fi
+
+#########
+# Locate a compiler for the build machine.  This compiler should
+# generate command-line programs that run on the build machine.
+#
+if test x"$cross_compiling" = xno; then
+	BUILD_CC=$CC
+	BUILD_CFLAGS=$CFLAGS
+else
+	if test "${BUILD_CC+set}" != set; then
+		AC_CHECK_PROGS(BUILD_CC, gcc cc cl)
+	fi
+	if test "${BUILD_CFLAGS+set}" != set; then
+		BUILD_CFLAGS="-g"
+	fi
+fi
+AC_SUBST(BUILD_CC)
+
+##########
+# Do we want to support multithreaded use of sqlite
+#
+AC_ARG_ENABLE(threadsafe, 
+AC_HELP_STRING([--enable-threadsafe],[Support threadsafe operation]),,enable_threadsafe=yes)
+AC_MSG_CHECKING([whether to support threadsafe operation])
+if test "$enable_threadsafe" = "no"; then
+  SQLITE_THREADSAFE=0
+  AC_MSG_RESULT([no])
+else
+  SQLITE_THREADSAFE=1
+  AC_MSG_RESULT([yes])
+fi
+AC_SUBST(SQLITE_THREADSAFE)
+
+if test "$SQLITE_THREADSAFE" = "1"; then
+  AC_SEARCH_LIBS(pthread_create, pthread)
+fi
+
+##########
+# Do we want to allow a connection created in one thread to be used
+# in another thread.  This does not work on many Linux systems (ex: RedHat 9)
+# due to bugs in the threading implementations.  This is thus off by default.
+#
+AC_ARG_ENABLE(cross-thread-connections, 
+AC_HELP_STRING([--enable-cross-thread-connections],[Allow connection sharing across threads]),,enable_xthreadconnect=no)
+AC_MSG_CHECKING([whether to allow connections to be shared across threads])
+if test "$enable_xthreadconnect" = "no"; then
+  XTHREADCONNECT=''
+  AC_MSG_RESULT([no])
+else
+  XTHREADCONNECT='-DSQLITE_ALLOW_XTHREAD_CONNECT=1'
+  AC_MSG_RESULT([yes])
+fi
+AC_SUBST(XTHREADCONNECT)
+
+##########
+# Do we want to set threadsOverrideEachOthersLocks variable to be 1 (true) by
+# default. Normally, a test at runtime is performed to determine the
+# appropriate value of this variable. Use this option only if you're sure that
+# threads can safely override each others locks in all runtime situations.
+#
+AC_ARG_ENABLE(threads-override-locks, 
+AC_HELP_STRING([--enable-threads-override-locks],[Threads can override each others locks]),,enable_threads_override_locks=no)
+AC_MSG_CHECKING([whether threads can override each others locks])
+if test "$enable_threads_override_locks" = "no"; then
+  THREADSOVERRIDELOCKS='-1'
+  AC_MSG_RESULT([no])
+else
+  THREADSOVERRIDELOCKS='1'
+  AC_MSG_RESULT([yes])
+fi
+AC_SUBST(THREADSOVERRIDELOCKS)
+
+##########
+# Do we want to support release
+#
+AC_ARG_ENABLE(releasemode, 
+AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
+AC_MSG_CHECKING([whether to support shared library linked as release mode or not])
+if test "$enable_releasemode" = "no"; then
+  ALLOWRELEASE=""
+  AC_MSG_RESULT([no])
+else
+  ALLOWRELEASE="-release `cat $srcdir/VERSION`"
+  AC_MSG_RESULT([yes])
+fi
+AC_SUBST(ALLOWRELEASE)
+
+##########
+# Do we want temporary databases in memory
+#
+AC_ARG_ENABLE(tempstore, 
+AC_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
+AC_MSG_CHECKING([whether to use an in-ram database for temporary tables])
+case "$enable_tempstore" in
+  never ) 
+    TEMP_STORE=0
+    AC_MSG_RESULT([never])
+  ;;
+  no ) 
+    TEMP_STORE=1
+    AC_MSG_RESULT([no])
+  ;;
+  always ) 
+     TEMP_STORE=3
+    AC_MSG_RESULT([always])
+  ;;
+  yes ) 
+     TEMP_STORE=3
+    AC_MSG_RESULT([always])
+  ;;
+  * ) 
+    TEMP_STORE=1
+    AC_MSG_RESULT([yes])
+  ;;
+esac
+
+AC_SUBST(TEMP_STORE)
+
+###########
+# Lots of things are different if we are compiling for Windows using
+# the CYGWIN environment.  So check for that special case and handle
+# things accordingly.
+#
+AC_MSG_CHECKING([if executables have the .exe suffix])
+if test "$config_BUILD_EXEEXT" = ".exe"; then
+  CYGWIN=yes
+  AC_MSG_RESULT(yes)
+else
+  AC_MSG_RESULT(unknown)
+fi
+if test "$CYGWIN" != "yes"; then
+  AC_CYGWIN
+fi
+if test "$CYGWIN" = "yes"; then
+  BUILD_EXEEXT=.exe
+else
+  BUILD_EXEEXT=$EXEEXT
+fi
+if test x"$cross_compiling" = xno; then
+  TARGET_EXEEXT=$BUILD_EXEEXT
+else
+  TARGET_EXEEXT=$config_TARGET_EXEEXT
+fi
+if test "$TARGET_EXEEXT" = ".exe"; then
+  if test $OS2_SHELL ; then
+    SQLITE_OS_UNIX=0
+    SQLITE_OS_WIN=0
+    SQLITE_OS_OS2=1
+    CFLAGS="$CFLAGS -DSQLITE_OS_OS2=1"
+    if test "$ac_compiler_gnu" = "yes" ; then
+      CFLAGS="$CFLAGS -Zomf -Zexe -Zmap"
+      BUILD_CFLAGS="$BUILD_CFLAGS -Zomf -Zexe"
+    fi
+  else
+    SQLITE_OS_UNIX=0
+    SQLITE_OS_WIN=1
+    SQLITE_OS_OS2=0
+    tclsubdir=win
+    CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1"
+  fi
+else
+  SQLITE_OS_UNIX=1
+  SQLITE_OS_WIN=0
+  SQLITE_OS_OS2=0
+  tclsubdir=unix
+  CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1"
+fi
+
+AC_SUBST(BUILD_EXEEXT)
+AC_SUBST(SQLITE_OS_UNIX)
+AC_SUBST(SQLITE_OS_WIN)
+AC_SUBST(SQLITE_OS_OS2)
+AC_SUBST(TARGET_EXEEXT)
+
+##########
+# Figure out all the parameters needed to compile against Tcl.
+#
+# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
+# macros in the in the tcl.m4 file of the standard TCL distribution.
+# Those macros could not be used directly since we have to make some
+# minor changes to accomodate systems that do not have TCL installed.
+#
+AC_ARG_ENABLE(tcl, AC_HELP_STRING([--disable-tcl],[do not build TCL extension]),
+      [use_tcl=$enableval],[use_tcl=yes])
+if test "${use_tcl}" = "yes" ; then
+  AC_ARG_WITH(tcl, AC_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
+  AC_MSG_CHECKING([for Tcl configuration])
+  AC_CACHE_VAL(ac_cv_c_tclconfig,[
+    # First check to see if --with-tcl was specified.
+    if test x"${with_tclconfig}" != x ; then
+      if test -f "${with_tclconfig}/tclConfig.sh" ; then
+        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
+      else
+        AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
+      fi
+    fi
+    # then check for a private Tcl installation
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            ../tcl \
+            `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../tcl \
+            `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../../tcl \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+    # check in a few common install locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            `ls -d ${libdir} 2>/dev/null` \
+            `ls -d /usr/local/lib 2>/dev/null` \
+            `ls -d /usr/contrib/lib 2>/dev/null` \
+            `ls -d /usr/lib 2>/dev/null`
+      do
+        if test -f "$i/tclConfig.sh" ; then
+           ac_cv_c_tclconfig=`(cd $i; pwd)`
+           break
+        fi
+      done
+    fi
+
+    # check in a few other private locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+         ${srcdir}/../tcl \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+  ])
+
+  if test x"${ac_cv_c_tclconfig}" = x ; then
+    use_tcl=no
+    AC_MSG_WARN(Can't find Tcl configuration definitions)
+    AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***)
+    AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***)
+  else
+    TCL_BIN_DIR=${ac_cv_c_tclconfig}
+    AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh)
+
+    AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh])
+    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
+      AC_MSG_RESULT([loading])
+      . $TCL_BIN_DIR/tclConfig.sh
+    else
+      AC_MSG_RESULT([file not found])
+    fi
+    
+    #
+    # If the TCL_BIN_DIR is the build directory (not the install directory),
+    # then set the common variable name to the value of the build variables.
+    # For example, the variable TCL_LIB_SPEC will be set to the value
+    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
+    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
+    # installed and uninstalled version of Tcl.
+    #
+    
+    if test -f $TCL_BIN_DIR/Makefile ; then
+      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
+      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
+      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
+    fi
+    
+    #
+    # eval is required to do the TCL_DBGX substitution
+    #
+    
+    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
+    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
+    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
+    
+    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
+    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
+    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
+    
+    AC_SUBST(TCL_VERSION)
+    AC_SUBST(TCL_BIN_DIR)
+    AC_SUBST(TCL_SRC_DIR)
+    AC_SUBST(TCL_LIBS)
+    AC_SUBST(TCL_INCLUDE_SPEC)
+    
+    AC_SUBST(TCL_LIB_FILE)
+    AC_SUBST(TCL_LIB_FLAG)
+    AC_SUBST(TCL_LIB_SPEC)
+    
+    AC_SUBST(TCL_STUB_LIB_FILE)
+    AC_SUBST(TCL_STUB_LIB_FLAG)
+    AC_SUBST(TCL_STUB_LIB_SPEC)
+  fi
+fi
+if test "${use_tcl}" = "no" ; then
+  HAVE_TCL=""
+else
+  HAVE_TCL=1
+fi
+AC_SUBST(HAVE_TCL)
+
+##########
+# Figure out what C libraries are required to compile programs
+# that use "readline()" library.
+#
+TARGET_READLINE_LIBS=""
+TARGET_READLINE_INC=""
+TARGET_HAVE_READLINE=0
+AC_ARG_ENABLE([readline],
+	[AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])],
+	[with_readline=$enableval],
+	[with_readline=auto])
+
+if test x"$with_readline" != xno; then
+	found="yes"
+
+	AC_ARG_WITH([readline-lib],
+		[AC_HELP_STRING([--with-readline-lib],[specify readline library])],
+		[with_readline_lib=$withval],
+		[with_readline_lib="auto"])
+	if test "x$with_readline_lib" = xauto; then
+		save_LIBS="$LIBS"
+		LIBS=""
+		AC_SEARCH_LIBS(tgetent, [readline ncurses curses termcap], [term_LIBS="$LIBS"], [term_LIBS=""])
+		AC_CHECK_LIB([readline], [readline], [TARGET_READLINE_LIBS="-lreadline"], [found="no"])
+		TARGET_READLINE_LIBS="$TARGET_READLINE_LIBS $term_LIBS"
+		LIBS="$save_LIBS"
+	else
+		TARGET_READLINE_LIBS="$with_readline_lib"
+	fi
+
+	AC_ARG_WITH([readline-inc],
+		[AC_HELP_STRING([--with-readline-inc],[specify readline include paths])],
+		[with_readline_inc=$withval],
+		[with_readline_inc="auto"])
+	if test "x$with_readline_inc" = xauto; then
+		AC_CHECK_HEADER(readline.h, [found="yes"], [
+			found="no"
+			if test "$cross_compiling" != yes; then
+				for dir in /usr /usr/local /usr/local/readline /usr/contrib /mingw; do
+					for subdir in include include/readline; do
+						AC_CHECK_FILE($dir/$subdir/readline.h, found=yes)
+						if test "$found" = "yes"; then
+							TARGET_READLINE_INC="-I$dir/$subdir"
+							break
+						fi
+					done
+					test "$found" = "yes" && break
+				done
+			fi
+		])
+	else
+		TARGET_READLINE_INC="$with_readline_inc"
+	fi
+
+	if test x"$found" = xno; then
+		TARGET_READLINE_LIBS=""
+		TARGET_READLINE_INC=""
+		TARGET_HAVE_READLINE=0
+	else
+		TARGET_HAVE_READLINE=1
+	fi
+fi
+
+AC_SUBST(TARGET_READLINE_LIBS)
+AC_SUBST(TARGET_READLINE_INC)
+AC_SUBST(TARGET_HAVE_READLINE)
+
+##########
+# Figure out what C libraries are required to compile programs
+# that use "fdatasync()" function.
+#
+AC_SEARCH_LIBS(fdatasync, [rt])
+
+#########
+# check for debug enabled
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]),
+      [use_debug=$enableval],[use_debug=no])
+if test "${use_debug}" = "yes" ; then
+  TARGET_DEBUG="-DSQLITE_DEBUG=1"
+else
+  TARGET_DEBUG="-DNDEBUG"
+fi
+AC_SUBST(TARGET_DEBUG)
+
+#########
+# See whether we should use the amalgamation to build
+AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
+      [Disable the amalgamation and instead build all files separately]),
+      [use_amalgamation=$enableval],[use_amalgamation=yes])
+if test "${use_amalgamation}" != "yes" ; then
+  USE_AMALGAMATION=0
+fi
+AC_SUBST(USE_AMALGAMATION)
+
+#########
+# See whether we should allow loadable extensions
+AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--enable-load-extension],
+      [Enable loading of external extensions]),
+      [use_loadextension=$enableval],[use_loadextension=no])
+if test "${use_loadextension}" = "yes" ; then
+  LOADEXTENSION_FLAGS=""
+else
+  LOADEXTENSION_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
+fi
+AC_SUBST(LOADEXTENSION_FLAGS)
+
+#########
+# See whether we should allow loadable extensions
+AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
+      [Enable coverage testing using gcov]),
+      [use_gcov=$enableval],[use_gcov=no])
+if test "${use_gcov}" = "yes" ; then
+  USE_GCOV=1
+else
+  USE_GCOV=0
+fi
+AC_SUBST(USE_GCOV)
+
+
+#########
+# Output the config header
+AC_CONFIG_HEADERS(config.h)
+
+#########
+# Generate the output files.
+#
+AC_SUBST(BUILD_CFLAGS)
+AC_OUTPUT([
+Makefile
+sqlite3.pc
+])

ext/README.txt

@@ -0,0 +1,2 @@
+Version loadable extensions to SQLite are found in subfolders
+of this folder.

ext/fts1/README.txt

@@ -0,0 +1,2 @@
+This folder contains source code to the first full-text search
+extension for SQLite.

ext/fts1/ft_hash.c

@@ -0,0 +1,404 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ft_hash.h"
+
+void *malloc_and_zero(int n){
+  void *p = malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants HASH_INT, HASH_POINTER,
+** HASH_BINARY, or HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.  CopyKey only makes
+** sense for HASH_STRING and HASH_BINARY and is ignored
+** for other key classes.
+*/
+void HashInit(Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=HASH_STRING && keyClass<=HASH_BINARY );
+  pNew->keyClass = keyClass;
+#if 0
+  if( keyClass==HASH_POINTER || keyClass==HASH_INT ) copyKey = 0;
+#endif
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+  pNew->xMalloc = malloc_and_zero;
+  pNew->xFree = free;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void HashClear(Hash *pH){
+  HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      pH->xFree(elem->pKey);
+    }
+    pH->xFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is HASH_INT
+*/
+static int intHash(const void *pKey, int nKey){
+  return nKey ^ (nKey<<8) ^ (nKey>>8);
+}
+static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  return n2 - n1;
+}
+#endif
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is HASH_POINTER
+*/
+static int ptrHash(const void *pKey, int nKey){
+  uptr x = Addr(pKey);
+  return x ^ (x<<8) ^ (x>>8);
+}
+static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( pKey1==pKey2 ) return 0;
+  if( pKey1<pKey2 ) return -1;
+  return 1;
+}
+#endif
+
+/*
+** Hash and comparison functions when the mode is HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+#if 0  /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case HASH_INT:     return &intHash;
+    case HASH_POINTER: return &ptrHash;
+    case HASH_STRING:  return &strHash;
+    case HASH_BINARY:  return &binHash;;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==HASH_BINARY );
+    return &binHash;
+  }
+#endif
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+#if 0 /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case HASH_INT:     return &intCompare;
+    case HASH_POINTER: return &ptrCompare;
+    case HASH_STRING:  return &strCompare;
+    case HASH_BINARY:  return &binCompare;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==HASH_BINARY );
+    return &binCompare;
+  }
+#endif
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  Hash *pH,              /* The complete hash table */
+  struct _ht *pEntry,    /* The entry into which pNew is inserted */
+  HashElem *pNew         /* The element to be inserted */
+){
+  HashElem *pHead;       /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(Hash *pH, int new_size){
+  struct _ht *new_ht;            /* The new hash table */
+  HashElem *elem, *next_elem;    /* For looping over existing elements */
+  int (*xHash)(const void*,int); /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
+  if( new_ht==0 ) return;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static HashElem *findElementGivenHash(
+  const Hash *pH,     /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  HashElem *elem;                /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  Hash *pH,         /* The pH containing "elem" */
+  HashElem* elem,   /* The element to be removed from the pH */
+  int h             /* Hash value for the element */
+){
+  struct _ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    pH->xFree(elem->pKey);
+  }
+  pH->xFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *HashFind(const Hash *pH, const void *pKey, int nKey){
+  int h;             /* A hash on key */
+  HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
+  int hraw;             /* Raw hash value of the key */
+  int h;                /* the hash of the key modulo hash table size */
+  HashElem *elem;       /* Used to loop thru the element list */
+  HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = pH->xMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      pH->xFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      pH->xFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}

ext/fts1/ft_hash.h

@@ -0,0 +1,111 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _HASH_H_
+#define _HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct Hash Hash;
+typedef struct HashElem HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  HashElem *first;        /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _ht {            /* the hash table */
+    int count;               /* Number of entries with this hash */
+    HashElem *chain;         /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct HashElem {
+  HashElem *next, *prev;   /* Next and previous elements in the table */
+  void *data;              /* Data associated with this element */
+  void *pKey; int nKey;    /* Key associated with this element */
+};
+
+/*
+** There are 4 different modes of operation for a hash table:
+**
+**   HASH_INT         nKey is used as the key and pKey is ignored.
+**
+**   HASH_POINTER     pKey is used as the key and nKey is ignored.
+**
+**   HASH_STRING      pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   HASH_BINARY      pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made for HASH_STRING and HASH_BINARY
+** if the copyKey parameter to HashInit is 1.  
+*/
+/* #define HASH_INT       1 // NOT USED */
+/* #define HASH_POINTER   2 // NOT USED */
+#define HASH_STRING    3
+#define HASH_BINARY    4
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void HashInit(Hash*, int keytype, int copyKey);
+void *HashInsert(Hash*, const void *pKey, int nKey, void *pData);
+void *HashFind(const Hash*, const void *pKey, int nKey);
+void HashClear(Hash*);
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   Hash h;
+**   HashElem *p;
+**   ...
+**   for(p=HashFirst(&h); p; p=HashNext(p)){
+**     SomeStructure *pData = HashData(p);
+**     // do something with pData
+**   }
+*/
+#define HashFirst(H)  ((H)->first)
+#define HashNext(E)   ((E)->next)
+#define HashData(E)   ((E)->data)
+#define HashKey(E)    ((E)->pKey)
+#define HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define HashCount(H)  ((H)->count)
+
+#endif /* _HASH_H_ */

ext/fts1/fts1.h

@@ -0,0 +1,11 @@
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3Fts1Init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */

ext/fts1/fts1_hash.c

@@ -0,0 +1,369 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include "fts1_hash.h"
+
+static void *malloc_and_zero(int n){
+  void *p = malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants 
+** FTS1_HASH_BINARY or FTS1_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+void sqlite3Fts1HashInit(fts1Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=FTS1_HASH_STRING && keyClass<=FTS1_HASH_BINARY );
+  pNew->keyClass = keyClass;
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+  pNew->xMalloc = malloc_and_zero;
+  pNew->xFree = free;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3Fts1HashClear(fts1Hash *pH){
+  fts1HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    fts1HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      pH->xFree(elem->pKey);
+    }
+    pH->xFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS1_HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS1_HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+  if( keyClass==FTS1_HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==FTS1_HASH_BINARY );
+    return &binHash;
+  }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+  if( keyClass==FTS1_HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==FTS1_HASH_BINARY );
+    return &binCompare;
+  }
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  fts1Hash *pH,            /* The complete hash table */
+  struct _fts1ht *pEntry,  /* The entry into which pNew is inserted */
+  fts1HashElem *pNew       /* The element to be inserted */
+){
+  fts1HashElem *pHead;     /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(fts1Hash *pH, int new_size){
+  struct _fts1ht *new_ht;          /* The new hash table */
+  fts1HashElem *elem, *next_elem;  /* For looping over existing elements */
+  int (*xHash)(const void*,int);   /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _fts1ht *)pH->xMalloc( new_size*sizeof(struct _fts1ht) );
+  if( new_ht==0 ) return;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static fts1HashElem *findElementGivenHash(
+  const fts1Hash *pH, /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  fts1HashElem *elem;            /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _fts1ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  fts1Hash *pH,         /* The pH containing "elem" */
+  fts1HashElem* elem,   /* The element to be removed from the pH */
+  int h                 /* Hash value for the element */
+){
+  struct _fts1ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    pH->xFree(elem->pKey);
+  }
+  pH->xFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    fts1HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts1HashFind(const fts1Hash *pH, const void *pKey, int nKey){
+  int h;                 /* A hash on key */
+  fts1HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3Fts1HashInsert(
+  fts1Hash *pH,        /* The hash table to insert into */
+  const void *pKey,    /* The key */
+  int nKey,            /* Number of bytes in the key */
+  void *data           /* The data */
+){
+  int hraw;                 /* Raw hash value of the key */
+  int h;                    /* the hash of the key modulo hash table size */
+  fts1HashElem *elem;       /* Used to loop thru the element list */
+  fts1HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (fts1HashElem*)pH->xMalloc( sizeof(fts1HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = pH->xMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      pH->xFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      pH->xFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */

ext/fts1/fts1_hash.h

@@ -0,0 +1,112 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS1_HASH_H_
+#define _FTS1_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct fts1Hash fts1Hash;
+typedef struct fts1HashElem fts1HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct fts1Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  fts1HashElem *first;    /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _fts1ht {        /* the hash table */
+    int count;               /* Number of entries with this hash */
+    fts1HashElem *chain;     /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct fts1HashElem {
+  fts1HashElem *next, *prev; /* Next and previous elements in the table */
+  void *data;                /* Data associated with this element */
+  void *pKey; int nKey;      /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+**   FTS1_HASH_STRING        pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   FTS1_HASH_BINARY        pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts1HashInit is 1.  
+*/
+#define FTS1_HASH_STRING    1
+#define FTS1_HASH_BINARY    2
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3Fts1HashInit(fts1Hash*, int keytype, int copyKey);
+void *sqlite3Fts1HashInsert(fts1Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts1HashFind(const fts1Hash*, const void *pKey, int nKey);
+void sqlite3Fts1HashClear(fts1Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts1HashInit   sqlite3Fts1HashInit
+#define fts1HashInsert sqlite3Fts1HashInsert
+#define fts1HashFind   sqlite3Fts1HashFind
+#define fts1HashClear  sqlite3Fts1HashClear
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   fts1Hash h;
+**   fts1HashElem *p;
+**   ...
+**   for(p=fts1HashFirst(&h); p; p=fts1HashNext(p)){
+**     SomeStructure *pData = fts1HashData(p);
+**     // do something with pData
+**   }
+*/
+#define fts1HashFirst(H)  ((H)->first)
+#define fts1HashNext(E)   ((E)->next)
+#define fts1HashData(E)   ((E)->data)
+#define fts1HashKey(E)    ((E)->pKey)
+#define fts1HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts1HashCount(H)  ((H)->count)
+
+#endif /* _FTS1_HASH_H_ */

ext/fts1/fts1_porter.c

@@ -0,0 +1,643 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts1_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+  sqlite3_tokenizer base;      /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *zInput;          /* input we are tokenizing */
+  int nInput;                  /* size of the input */
+  int iOffset;                 /* current position in zInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nAllocated;              /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module porterTokenizerModule;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  porter_tokenizer *t;
+  t = (porter_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is zInput[0..nInput-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int porterOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput, int nInput,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  porter_tokenizer_cursor *c;
+
+  c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->zInput = zInput;
+  if( zInput==0 ){
+    c->nInput = 0;
+  }else if( nInput<0 ){
+    c->nInput = (int)strlen(zInput);
+  }else{
+    c->nInput = nInput;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  free(c->zToken);
+  free(c);
+  return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+   0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+   1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.  
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order.  So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return j;
+  return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return 1-j;
+  return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants.  Then every word can be
+** represented as:
+**
+**           [C] (VC){m} [V]
+**
+** In prose:  A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel.  "m" is the
+** number of vowel consonant pairs.  This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more.  In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order.  So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 1;
+  while( isConsonant(z) ){ z++; }
+  return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here.  So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+  return
+    z[0]!=0 && isConsonant(z) &&
+    z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+    z[1]!=0 && isVowel(z+1) &&
+    z[2]!=0 && isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the 
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order.  zTo
+** is in normal order. 
+**
+** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
+** match.  Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+  char **pz,             /* The word being stemmed (Reversed) */
+  const char *zFrom,     /* If the ending matches this... (Reversed) */
+  const char *zTo,       /* ... change the ending to this (not reversed) */
+  int (*xCond)(const char*)   /* Condition that must be true */
+){
+  char *z = *pz;
+  while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+  if( *zFrom!=0 ) return 0;
+  if( xCond && !xCond(z) ) return 1;
+  while( *zTo ){
+    *(--z) = *(zTo++);
+  }
+  *pz = z;
+  return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate.  The input word is copied into the output with
+** US-ASCII case folding.  If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, mx, j;
+  int hasDigit = 0;
+  for(i=0; i<nIn; i++){
+    int c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zOut[i] = c - 'A' + 'a';
+    }else{
+      if( c>='0' && c<='9' ) hasDigit = 1;
+      zOut[i] = c;
+    }
+  }
+  mx = hasDigit ? 3 : 10;
+  if( nIn>mx*2 ){
+    for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+      zOut[j] = zOut[i];
+    }
+    i = j;
+  }
+  zOut[i] = 0;
+  *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes.  Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case.  Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word.  If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end.  For long words without digits, 10 bytes
+** are taken from each end.  US-ASCII case folding still applies.
+** 
+** If the input word contains not digits but does characters not 
+** in [a-zA-Z] then no stemming is attempted and this routine just 
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word.  So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, j, c;
+  char zReverse[28];
+  char *z, *z2;
+  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+    /* The word is too big or too small for the porter stemmer.
+    ** Fallback to the copy stemmer */
+    copy_stemmer(zIn, nIn, zOut, pnOut);
+    return;
+  }
+  for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+    c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zReverse[j] = c + 'a' - 'A';
+    }else if( c>='a' && c<='z' ){
+      zReverse[j] = c;
+    }else{
+      /* The use of a character not in [a-zA-Z] means that we fallback
+      ** to the copy stemmer */
+      copy_stemmer(zIn, nIn, zOut, pnOut);
+      return;
+    }
+  }
+  memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+  z = &zReverse[j+1];
+
+
+  /* Step 1a */
+  if( z[0]=='s' ){
+    if(
+     !stem(&z, "sess", "ss", 0) &&
+     !stem(&z, "sei", "i", 0)  &&
+     !stem(&z, "ss", "ss", 0)
+    ){
+      z++;
+    }
+  }
+
+  /* Step 1b */  
+  z2 = z;
+  if( stem(&z, "dee", "ee", m_gt_0) ){
+    /* Do nothing.  The work was all in the test */
+  }else if( 
+     (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+      && z!=z2
+  ){
+     if( stem(&z, "ta", "ate", 0) ||
+         stem(&z, "lb", "ble", 0) ||
+         stem(&z, "zi", "ize", 0) ){
+       /* Do nothing.  The work was all in the test */
+     }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+       z++;
+     }else if( m_eq_1(z) && star_oh(z) ){
+       *(--z) = 'e';
+     }
+  }
+
+  /* Step 1c */
+  if( z[0]=='y' && hasVowel(z+1) ){
+    z[0] = 'i';
+  }
+
+  /* Step 2 */
+  switch( z[1] ){
+   case 'a':
+     stem(&z, "lanoita", "ate", m_gt_0) ||
+     stem(&z, "lanoit", "tion", m_gt_0);
+     break;
+   case 'c':
+     stem(&z, "icne", "ence", m_gt_0) ||
+     stem(&z, "icna", "ance", m_gt_0);
+     break;
+   case 'e':
+     stem(&z, "rezi", "ize", m_gt_0);
+     break;
+   case 'g':
+     stem(&z, "igol", "log", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "ilb", "ble", m_gt_0) ||
+     stem(&z, "illa", "al", m_gt_0) ||
+     stem(&z, "iltne", "ent", m_gt_0) ||
+     stem(&z, "ile", "e", m_gt_0) ||
+     stem(&z, "ilsuo", "ous", m_gt_0);
+     break;
+   case 'o':
+     stem(&z, "noitazi", "ize", m_gt_0) ||
+     stem(&z, "noita", "ate", m_gt_0) ||
+     stem(&z, "rota", "ate", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "msila", "al", m_gt_0) ||
+     stem(&z, "ssenevi", "ive", m_gt_0) ||
+     stem(&z, "ssenluf", "ful", m_gt_0) ||
+     stem(&z, "ssensuo", "ous", m_gt_0);
+     break;
+   case 't':
+     stem(&z, "itila", "al", m_gt_0) ||
+     stem(&z, "itivi", "ive", m_gt_0) ||
+     stem(&z, "itilib", "ble", m_gt_0);
+     break;
+  }
+
+  /* Step 3 */
+  switch( z[0] ){
+   case 'e':
+     stem(&z, "etaci", "ic", m_gt_0) ||
+     stem(&z, "evita", "", m_gt_0)   ||
+     stem(&z, "ezila", "al", m_gt_0);
+     break;
+   case 'i':
+     stem(&z, "itici", "ic", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "laci", "ic", m_gt_0) ||
+     stem(&z, "luf", "", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "ssen", "", m_gt_0);
+     break;
+  }
+
+  /* Step 4 */
+  switch( z[1] ){
+   case 'a':
+     if( z[0]=='l' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'c':
+     if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
+       z += 4;
+     }
+     break;
+   case 'e':
+     if( z[0]=='r' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'i':
+     if( z[0]=='c' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'l':
+     if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+       z += 4;
+     }
+     break;
+   case 'n':
+     if( z[0]=='t' ){
+       if( z[2]=='a' ){
+         if( m_gt_1(z+3) ){
+           z += 3;
+         }
+       }else if( z[2]=='e' ){
+         stem(&z, "tneme", "", m_gt_1) ||
+         stem(&z, "tnem", "", m_gt_1) ||
+         stem(&z, "tne", "", m_gt_1);
+       }
+     }
+     break;
+   case 'o':
+     if( z[0]=='u' ){
+       if( m_gt_1(z+2) ){
+         z += 2;
+       }
+     }else if( z[3]=='s' || z[3]=='t' ){
+       stem(&z, "noi", "", m_gt_1);
+     }
+     break;
+   case 's':
+     if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 't':
+     stem(&z, "eta", "", m_gt_1) ||
+     stem(&z, "iti", "", m_gt_1);
+     break;
+   case 'u':
+     if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 'v':
+   case 'z':
+     if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+  }
+
+  /* Step 5a */
+  if( z[0]=='e' ){
+    if( m_gt_1(z+1) ){
+      z++;
+    }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+      z++;
+    }
+  }
+
+  /* Step 5b */
+  if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+    z++;
+  }
+
+  /* z[] is now the stemmed word in reverse order.  Flip it back
+  ** around into forward order and return.
+  */
+  *pnOut = i = strlen(z);
+  zOut[i] = 0;
+  while( *z ){
+    zOut[--i] = *(z++);
+  }
+}
+
+/*
+** Characters that can be part of a token.  We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token.  In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char isIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+};
+#define idChar(C)  (((ch=C)&0x80)!=0 || (ch>0x2f && isIdChar[ch-0x30]))
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !isIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
+  const char **pzToken,               /* OUT: *pzToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  const char *z = c->zInput;
+
+  while( c->iOffset<c->nInput ){
+    int iStartOffset, ch;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int n = c->iOffset-iStartOffset;
+      if( n>c->nAllocated ){
+        c->nAllocated = n+20;
+        c->zToken = realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
+      }
+      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+      *pzToken = c->zToken;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+  0,
+  porterCreate,
+  porterDestroy,
+  porterOpen,
+  porterClose,
+  porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts1PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */

ext/fts1/fts1_tokenizer.h

@@ -0,0 +1,90 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS1_TOKENIZER_H_
+#define _FTS1_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface.
+*/
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+
+struct sqlite3_tokenizer_module {
+  int iVersion;                  /* currently 0 */
+
+  /*
+  ** Create and destroy a tokenizer.  argc/argv are passed down from
+  ** the fulltext virtual table creation to allow customization.
+  */
+  int (*xCreate)(int argc, const char *const*argv,
+                 sqlite3_tokenizer **ppTokenizer);
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Tokenize a particular input.  Call xOpen() to prepare to
+  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
+  ** xClose() to free any internal state.  The pInput passed to
+  ** xOpen() must exist until the cursor is closed.  The ppToken
+  ** result from xNext() is only valid until the next call to xNext()
+  ** or until xClose() is called.
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
+               const char *pInput, int nBytes,
+               sqlite3_tokenizer_cursor **ppCursor);
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+               const char **ppToken, int *pnBytes,
+               int *piStartOffset, int *piEndOffset, int *piPosition);
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+/*
+** Get the module for a tokenizer which generates tokens based on a
+** set of non-token characters.  The default is to break tokens at any
+** non-alnum character, though the set of delimiters can also be
+** specified by the first argv argument to xCreate().
+*/
+/* TODO(shess) This doesn't belong here.  Need some sort of
+** registration process.
+*/
+void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+#endif /* _FTS1_TOKENIZER_H_ */

ext/fts1/fts1_tokenizer1.c

@@ -0,0 +1,221 @@
+/*
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts1_tokenizer.h"
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  char delim[128];             /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  int iOffset;                 /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *pToken;                /* storage for current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module simpleTokenizerModule;
+
+static int isDelim(simple_tokenizer *t, unsigned char c){
+  return c<0x80 && t->delim[c];
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    int i, n = strlen(argv[1]);
+    for(i=0; i<n; i++){
+      unsigned char ch = argv[1][i];
+      /* We explicitly don't support UTF-8 delimiters for now. */
+      if( ch>=0x80 ){
+        free(t);
+        return SQLITE_ERROR;
+      }
+      t->delim[ch] = 1;
+    }
+  } else {
+    /* Mark non-alphanumeric ASCII characters as delimiters */
+    int i;
+    for(i=1; i<0x80; i++){
+      t->delim[i] = !isalnum(i);
+    }
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *pInput, int nBytes,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->pInput = pInput;
+  if( pInput==0 ){
+    c->nBytes = 0;
+  }else if( nBytes<0 ){
+    c->nBytes = (int)strlen(pInput);
+  }else{
+    c->nBytes = nBytes;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->pToken = NULL;               /* no space allocated, yet. */
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  free(c->pToken);
+  free(c);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  unsigned char *p = (unsigned char *)c->pInput;
+
+  while( c->iOffset<c->nBytes ){
+    int iStartOffset;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int i, n = c->iOffset-iStartOffset;
+      if( n>c->nTokenAllocated ){
+        c->nTokenAllocated = n+20;
+        c->pToken = realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
+      }
+      for(i=0; i<n; i++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        unsigned char ch = p[iStartOffset+i];
+        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+      }
+      *ppToken = c->pToken;
+      *pnBytes = n;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts1SimpleTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */

ext/fts1/fulltext.h

@@ -0,0 +1,11 @@
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int fulltext_init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */

ext/fts1/simple_tokenizer.c

@@ -0,0 +1,174 @@
+/*
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+#include <assert.h>
+#if !defined(__APPLE__)
+#include <malloc.h>
+#else
+#include <stdlib.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "tokenizer.h"
+
+/* Duplicate a string; the caller must free() the returned string.
+ * (We don't use strdup() since it's not part of the standard C library and
+ * may not be available everywhere.) */
+/* TODO(shess) Copied from fulltext.c, consider util.c for such
+** things. */
+static char *string_dup(const char *s){
+  char *str = malloc(strlen(s) + 1);
+  strcpy(str, s);
+  return str;
+}
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  const char *zDelim;          /* token delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  const char *pCurrent;        /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nTokenBytes;             /* actual size of current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
+
+static int simpleCreate(
+  int argc, const char **argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    t->zDelim = string_dup(argv[1]);
+  } else {
+    /* Build a string excluding alphanumeric ASCII characters */
+    char zDelim[0x80];               /* nul-terminated, so nul not a member */
+    int i, j;
+    for(i=1, j=0; i<0x80; i++){
+      if( !isalnum(i) ){
+        zDelim[j++] = i;
+      }
+    }
+    zDelim[j++] = '\0';
+    assert( j<=sizeof(zDelim) );
+    t->zDelim = string_dup(zDelim);
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
+
+  free((void *) t->zDelim);
+  free(t);
+
+  return SQLITE_OK;
+}
+
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,
+  const char *pInput, int nBytes,
+  sqlite3_tokenizer_cursor **ppCursor
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
+  c->pInput = pInput;
+  c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
+  c->pCurrent = c->pInput;        /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nTokenBytes = 0;
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+
+  if( NULL!=c->zToken ){
+    free(c->zToken);
+  }
+  free(c);
+
+  return SQLITE_OK;
+}
+
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,
+  const char **ppToken, int *pnBytes,
+  int *piStartOffset, int *piEndOffset, int *piPosition
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  int ii;
+
+  while( c->pCurrent-c->pInput<c->nBytes ){
+    int n = (int) strcspn(c->pCurrent, t->zDelim);
+    if( n>0 ){
+      if( n+1>c->nTokenAllocated ){
+        c->zToken = realloc(c->zToken, n+1);
+      }
+      for(ii=0; ii<n; ii++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        char ch = c->pCurrent[ii];
+        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
+      }
+      c->zToken[n] = '\0';
+      *ppToken = c->zToken;
+      *pnBytes = n;
+      *piStartOffset = (int) (c->pCurrent-c->pInput);
+      *piEndOffset = *piStartOffset+n;
+      *piPosition = c->iToken++;
+      c->pCurrent += n + 1;
+
+      return SQLITE_OK;
+    }
+    c->pCurrent += n + 1;
+    /* TODO(shess) could strspn() to skip delimiters en masse.  Needs
+    ** to happen in two places, though, which is annoying.
+    */
+  }
+  return SQLITE_DONE;
+}
+
+static sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+void get_simple_tokenizer_module(
+  sqlite3_tokenizer_module **ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}

ext/fts1/tokenizer.h

@@ -0,0 +1,89 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _TOKENIZER_H_
+#define _TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface.
+*/
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+
+struct sqlite3_tokenizer_module {
+  int iVersion;                  /* currently 0 */
+
+  /*
+  ** Create and destroy a tokenizer.  argc/argv are passed down from
+  ** the fulltext virtual table creation to allow customization.
+  */
+  int (*xCreate)(int argc, const char **argv,
+                 sqlite3_tokenizer **ppTokenizer);
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Tokenize a particular input.  Call xOpen() to prepare to
+  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
+  ** xClose() to free any internal state.  The pInput passed to
+  ** xOpen() must exist until the cursor is closed.  The ppToken
+  ** result from xNext() is only valid until the next call to xNext()
+  ** or until xClose() is called.
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
+               const char *pInput, int nBytes,
+               sqlite3_tokenizer_cursor **ppCursor);
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+               const char **ppToken, int *pnBytes,
+               int *piStartOffset, int *piEndOffset, int *piPosition);
+};
+
+struct sqlite3_tokenizer {
+  sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+/*
+** Get the module for a tokenizer which generates tokens based on a
+** set of non-token characters.  The default is to break tokens at any
+** non-alnum character, though the set of delimiters can also be
+** specified by the first argv argument to xCreate().
+*/
+/* TODO(shess) This doesn't belong here.  Need some sort of
+** registration process.
+*/
+void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
+
+#endif /* _TOKENIZER_H_ */

ext/fts2/README.tokenizers

@@ -0,0 +1,133 @@
+
+1. FTS2 Tokenizers
+
+  When creating a new full-text table, FTS2 allows the user to select
+  the text tokenizer implementation to be used when indexing text
+  by specifying a "tokenizer" clause as part of the CREATE VIRTUAL TABLE
+  statement:
+
+    CREATE VIRTUAL TABLE <table-name> USING fts2(
+      <columns ...> [, tokenizer <tokenizer-name> [<tokenizer-args>]]
+    );
+
+  The built-in tokenizers (valid values to pass as <tokenizer name>) are
+  "simple" and "porter".
+
+  <tokenizer-args> should consist of zero or more white-space separated
+  arguments to pass to the selected tokenizer implementation. The 
+  interpretation of the arguments, if any, depends on the individual 
+  tokenizer.
+
+2. Custom Tokenizers
+
+  FTS2 allows users to provide custom tokenizer implementations. The 
+  interface used to create a new tokenizer is defined and described in 
+  the fts2_tokenizer.h source file.
+
+  Registering a new FTS2 tokenizer is similar to registering a new 
+  virtual table module with SQLite. The user passes a pointer to a
+  structure containing pointers to various callback functions that
+  make up the implementation of the new tokenizer type. For tokenizers,
+  the structure (defined in fts2_tokenizer.h) is called
+  "sqlite3_tokenizer_module".
+
+  FTS2 does not expose a C-function that users call to register new
+  tokenizer types with a database handle. Instead, the pointer must
+  be encoded as an SQL blob value and passed to FTS2 through the SQL
+  engine by evaluating a special scalar function, "fts2_tokenizer()".
+  The fts2_tokenizer() function may be called with one or two arguments,
+  as follows:
+
+    SELECT fts2_tokenizer(<tokenizer-name>);
+    SELECT fts2_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
+  
+  Where <tokenizer-name> is a string identifying the tokenizer and
+  <sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
+  structure encoded as an SQL blob. If the second argument is present,
+  it is registered as tokenizer <tokenizer-name> and a copy of it
+  returned. If only one argument is passed, a pointer to the tokenizer
+  implementation currently registered as <tokenizer-name> is returned,
+  encoded as a blob. Or, if no such tokenizer exists, an SQL exception
+  (error) is raised.
+
+  SECURITY: If the fts2 extension is used in an environment where potentially
+    malicious users may execute arbitrary SQL (i.e. gears), they should be
+    prevented from invoking the fts2_tokenizer() function, possibly using the
+    authorisation callback.
+
+  See "Sample code" below for an example of calling the fts2_tokenizer()
+  function from C code.
+
+3. ICU Library Tokenizers
+
+  If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor 
+  symbol defined, then there exists a built-in tokenizer named "icu" 
+  implemented using the ICU library. The first argument passed to the
+  xCreate() method (see fts2_tokenizer.h) of this tokenizer may be
+  an ICU locale identifier. For example "tr_TR" for Turkish as used
+  in Turkey, or "en_AU" for English as used in Australia. For example:
+
+    "CREATE VIRTUAL TABLE thai_text USING fts2(text, tokenizer icu th_TH)"
+
+  The ICU tokenizer implementation is very simple. It splits the input
+  text according to the ICU rules for finding word boundaries and discards
+  any tokens that consist entirely of white-space. This may be suitable
+  for some applications in some locales, but not all. If more complex
+  processing is required, for example to implement stemming or 
+  discard punctuation, this can be done by creating a tokenizer 
+  implementation that uses the ICU tokenizer as part of its implementation.
+
+  When using the ICU tokenizer this way, it is safe to overwrite the
+  contents of the strings returned by the xNext() method (see
+  fts2_tokenizer.h).
+
+4. Sample code.
+
+  The following two code samples illustrate the way C code should invoke
+  the fts2_tokenizer() scalar function:
+
+      int registerTokenizer(
+        sqlite3 *db, 
+        char *zName, 
+        const sqlite3_tokenizer_module *p
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+      
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+        sqlite3_step(pStmt);
+      
+        return sqlite3_finalize(pStmt);
+      }
+      
+      int queryTokenizer(
+        sqlite3 *db, 
+        char *zName,  
+        const sqlite3_tokenizer_module **pp
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts2_tokenizer(?)";
+      
+        *pp = 0;
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        if( SQLITE_ROW==sqlite3_step(pStmt) ){
+          if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+            memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+          }
+        }
+      
+        return sqlite3_finalize(pStmt);
+      }

ext/fts2/README.txt

@@ -0,0 +1,4 @@
+This folder contains source code to the second full-text search
+extension for SQLite.  While the API is the same, this version uses a
+substantially different storage schema from fts1, so tables will need
+to be rebuilt.

ext/fts2/fts2.h

@@ -0,0 +1,26 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** FTS2 library.  All it does is declare the sqlite3Fts2Init() interface.
+*/
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3Fts2Init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */

ext/fts2/fts2_hash.c

@@ -0,0 +1,369 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fts2_hash.h"
+
+static void *malloc_and_zero(int n){
+  void *p = malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants 
+** FTS2_HASH_BINARY or FTS2_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+void sqlite3Fts2HashInit(fts2Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=FTS2_HASH_STRING && keyClass<=FTS2_HASH_BINARY );
+  pNew->keyClass = keyClass;
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+  pNew->xMalloc = malloc_and_zero;
+  pNew->xFree = free;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3Fts2HashClear(fts2Hash *pH){
+  fts2HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    fts2HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      pH->xFree(elem->pKey);
+    }
+    pH->xFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS2_HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS2_HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+  if( keyClass==FTS2_HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==FTS2_HASH_BINARY );
+    return &binHash;
+  }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+  if( keyClass==FTS2_HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==FTS2_HASH_BINARY );
+    return &binCompare;
+  }
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  fts2Hash *pH,            /* The complete hash table */
+  struct _fts2ht *pEntry,  /* The entry into which pNew is inserted */
+  fts2HashElem *pNew       /* The element to be inserted */
+){
+  fts2HashElem *pHead;     /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(fts2Hash *pH, int new_size){
+  struct _fts2ht *new_ht;          /* The new hash table */
+  fts2HashElem *elem, *next_elem;  /* For looping over existing elements */
+  int (*xHash)(const void*,int);   /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _fts2ht *)pH->xMalloc( new_size*sizeof(struct _fts2ht) );
+  if( new_ht==0 ) return;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static fts2HashElem *findElementGivenHash(
+  const fts2Hash *pH, /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  fts2HashElem *elem;            /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _fts2ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  fts2Hash *pH,         /* The pH containing "elem" */
+  fts2HashElem* elem,   /* The element to be removed from the pH */
+  int h                 /* Hash value for the element */
+){
+  struct _fts2ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    pH->xFree(elem->pKey);
+  }
+  pH->xFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    fts2HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){
+  int h;                 /* A hash on key */
+  fts2HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3Fts2HashInsert(
+  fts2Hash *pH,        /* The hash table to insert into */
+  const void *pKey,    /* The key */
+  int nKey,            /* Number of bytes in the key */
+  void *data           /* The data */
+){
+  int hraw;                 /* Raw hash value of the key */
+  int h;                    /* the hash of the key modulo hash table size */
+  fts2HashElem *elem;       /* Used to loop thru the element list */
+  fts2HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (fts2HashElem*)pH->xMalloc( sizeof(fts2HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = pH->xMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      pH->xFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      pH->xFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

ext/fts2/fts2_hash.h

@@ -0,0 +1,112 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS2_HASH_H_
+#define _FTS2_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct fts2Hash fts2Hash;
+typedef struct fts2HashElem fts2HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct fts2Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  fts2HashElem *first;    /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _fts2ht {        /* the hash table */
+    int count;               /* Number of entries with this hash */
+    fts2HashElem *chain;     /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct fts2HashElem {
+  fts2HashElem *next, *prev; /* Next and previous elements in the table */
+  void *data;                /* Data associated with this element */
+  void *pKey; int nKey;      /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+**   FTS2_HASH_STRING        pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   FTS2_HASH_BINARY        pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts2HashInit is 1.  
+*/
+#define FTS2_HASH_STRING    1
+#define FTS2_HASH_BINARY    2
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3Fts2HashInit(fts2Hash*, int keytype, int copyKey);
+void *sqlite3Fts2HashInsert(fts2Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts2HashFind(const fts2Hash*, const void *pKey, int nKey);
+void sqlite3Fts2HashClear(fts2Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts2HashInit   sqlite3Fts2HashInit
+#define fts2HashInsert sqlite3Fts2HashInsert
+#define fts2HashFind   sqlite3Fts2HashFind
+#define fts2HashClear  sqlite3Fts2HashClear
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   fts2Hash h;
+**   fts2HashElem *p;
+**   ...
+**   for(p=fts2HashFirst(&h); p; p=fts2HashNext(p)){
+**     SomeStructure *pData = fts2HashData(p);
+**     // do something with pData
+**   }
+*/
+#define fts2HashFirst(H)  ((H)->first)
+#define fts2HashNext(E)   ((E)->next)
+#define fts2HashData(E)   ((E)->data)
+#define fts2HashKey(E)    ((E)->pKey)
+#define fts2HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts2HashCount(H)  ((H)->count)
+
+#endif /* _FTS2_HASH_H_ */

ext/fts2/fts2_icu.c

@@ -0,0 +1,257 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements a tokenizer for fts2 based on the ICU library.
+** 
+** $Id: fts2_icu.c,v 1.1 2007/06/22 15:21:16 danielk1977 Exp $
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+#ifdef SQLITE_ENABLE_ICU
+
+#include <assert.h>
+#include <string.h>
+#include "fts2_tokenizer.h"
+
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+#include <unicode/utf16.h>
+
+typedef struct IcuTokenizer IcuTokenizer;
+typedef struct IcuCursor IcuCursor;
+
+struct IcuTokenizer {
+  sqlite3_tokenizer base;
+  char *zLocale;
+};
+
+struct IcuCursor {
+  sqlite3_tokenizer_cursor base;
+
+  UBreakIterator *pIter;      /* ICU break-iterator object */
+  int nChar;                  /* Number of UChar elements in pInput */
+  UChar *aChar;               /* Copy of input using utf-16 encoding */
+  int *aOffset;               /* Offsets of each character in utf-8 input */
+
+  int nBuffer;
+  char *zBuffer;
+
+  int iToken;
+};
+
+/*
+** Create a new tokenizer instance.
+*/
+static int icuCreate(
+  int argc,                            /* Number of entries in argv[] */
+  const char * const *argv,            /* Tokenizer creation arguments */
+  sqlite3_tokenizer **ppTokenizer      /* OUT: Created tokenizer */
+){
+  IcuTokenizer *p;
+  int n = 0;
+
+  if( argc>0 ){
+    n = strlen(argv[0])+1;
+  }
+  p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+  if( !p ){
+    return SQLITE_NOMEM;
+  }
+  memset(p, 0, sizeof(IcuTokenizer));
+
+  if( n ){
+    p->zLocale = (char *)&p[1];
+    memcpy(p->zLocale, argv[0], n);
+  }
+
+  *ppTokenizer = (sqlite3_tokenizer *)p;
+
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int icuOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput,                    /* Input string */
+  int nInput,                            /* Length of zInput in bytes */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  IcuCursor *pCsr;
+
+  const int32_t opt = U_FOLD_CASE_DEFAULT;
+  UErrorCode status = U_ZERO_ERROR;
+  int nChar;
+
+  UChar32 c;
+  int iInput = 0;
+  int iOut = 0;
+
+  *ppCursor = 0;
+
+  nChar = nInput+1;
+  pCsr = (IcuCursor *)sqlite3_malloc(
+      sizeof(IcuCursor) +                /* IcuCursor */
+      nChar * sizeof(UChar) +            /* IcuCursor.aChar[] */
+      (nChar+1) * sizeof(int)            /* IcuCursor.aOffset[] */
+  );
+  if( !pCsr ){
+    return SQLITE_NOMEM;
+  }
+  memset(pCsr, 0, sizeof(IcuCursor));
+  pCsr->aChar = (UChar *)&pCsr[1];
+  pCsr->aOffset = (int *)&pCsr->aChar[nChar];
+
+  pCsr->aOffset[iOut] = iInput;
+  U8_NEXT(zInput, iInput, nInput, c); 
+  while( c>0 ){
+    int isError = 0;
+    c = u_foldCase(c, opt);
+    U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
+    if( isError ){
+      sqlite3_free(pCsr);
+      return SQLITE_ERROR;
+    }
+    pCsr->aOffset[iOut] = iInput;
+
+    if( iInput<nInput ){
+      U8_NEXT(zInput, iInput, nInput, c);
+    }else{
+      c = 0;
+    }
+  }
+
+  pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
+  if( !U_SUCCESS(status) ){
+    sqlite3_free(pCsr);
+    return SQLITE_ERROR;
+  }
+  pCsr->nChar = iOut;
+
+  ubrk_first(pCsr->pIter);
+  *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to icuOpen().
+*/
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+  ubrk_close(pCsr->pIter);
+  sqlite3_free(pCsr->zBuffer);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.
+*/
+static int icuNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+
+  int iStart = 0;
+  int iEnd = 0;
+  int nByte = 0;
+
+  while( iStart==iEnd ){
+    UChar32 c;
+
+    iStart = ubrk_current(pCsr->pIter);
+    iEnd = ubrk_next(pCsr->pIter);
+    if( iEnd==UBRK_DONE ){
+      return SQLITE_DONE;
+    }
+
+    while( iStart<iEnd ){
+      int iWhite = iStart;
+      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
+      if( u_isspace(c) ){
+        iStart = iWhite;
+      }else{
+        break;
+      }
+    }
+    assert(iStart<=iEnd);
+  }
+
+  do {
+    UErrorCode status = U_ZERO_ERROR;
+    if( nByte ){
+      char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
+      if( !zNew ){
+        return SQLITE_NOMEM;
+      }
+      pCsr->zBuffer = zNew;
+      pCsr->nBuffer = nByte;
+    }
+
+    u_strToUTF8(
+        pCsr->zBuffer, pCsr->nBuffer, &nByte,    /* Output vars */
+        &pCsr->aChar[iStart], iEnd-iStart,       /* Input vars */
+        &status                                  /* Output success/failure */
+    );
+  } while( nByte>pCsr->nBuffer );
+
+  *ppToken = pCsr->zBuffer;
+  *pnBytes = nByte;
+  *piStartOffset = pCsr->aOffset[iStart];
+  *piEndOffset = pCsr->aOffset[iEnd];
+  *piPosition = pCsr->iToken++;
+
+  return SQLITE_OK;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module icuTokenizerModule = {
+  0,                           /* iVersion */
+  icuCreate,                   /* xCreate  */
+  icuDestroy,                  /* xCreate  */
+  icuOpen,                     /* xOpen    */
+  icuClose,                    /* xClose   */
+  icuNext,                     /* xNext    */
+};
+
+/*
+** Set *ppModule to point at the implementation of the ICU tokenizer.
+*/
+void sqlite3Fts2IcuTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &icuTokenizerModule;
+}
+
+#endif /* defined(SQLITE_ENABLE_ICU) */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

ext/fts2/fts2_porter.c

@@ -0,0 +1,642 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts2_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+  sqlite3_tokenizer base;      /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *zInput;          /* input we are tokenizing */
+  int nInput;                  /* size of the input */
+  int iOffset;                 /* current position in zInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nAllocated;              /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module porterTokenizerModule;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  porter_tokenizer *t;
+  t = (porter_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is zInput[0..nInput-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int porterOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput, int nInput,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  porter_tokenizer_cursor *c;
+
+  c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->zInput = zInput;
+  if( zInput==0 ){
+    c->nInput = 0;
+  }else if( nInput<0 ){
+    c->nInput = (int)strlen(zInput);
+  }else{
+    c->nInput = nInput;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  free(c->zToken);
+  free(c);
+  return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+   0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+   1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.  
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order.  So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return j;
+  return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return 1-j;
+  return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants.  Then every word can be
+** represented as:
+**
+**           [C] (VC){m} [V]
+**
+** In prose:  A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel.  "m" is the
+** number of vowel consonant pairs.  This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more.  In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order.  So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 1;
+  while( isConsonant(z) ){ z++; }
+  return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here.  So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+  return
+    z[0]!=0 && isConsonant(z) &&
+    z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+    z[1]!=0 && isVowel(z+1) &&
+    z[2]!=0 && isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the 
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order.  zTo
+** is in normal order. 
+**
+** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
+** match.  Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+  char **pz,             /* The word being stemmed (Reversed) */
+  const char *zFrom,     /* If the ending matches this... (Reversed) */
+  const char *zTo,       /* ... change the ending to this (not reversed) */
+  int (*xCond)(const char*)   /* Condition that must be true */
+){
+  char *z = *pz;
+  while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+  if( *zFrom!=0 ) return 0;
+  if( xCond && !xCond(z) ) return 1;
+  while( *zTo ){
+    *(--z) = *(zTo++);
+  }
+  *pz = z;
+  return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate.  The input word is copied into the output with
+** US-ASCII case folding.  If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, mx, j;
+  int hasDigit = 0;
+  for(i=0; i<nIn; i++){
+    int c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zOut[i] = c - 'A' + 'a';
+    }else{
+      if( c>='0' && c<='9' ) hasDigit = 1;
+      zOut[i] = c;
+    }
+  }
+  mx = hasDigit ? 3 : 10;
+  if( nIn>mx*2 ){
+    for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+      zOut[j] = zOut[i];
+    }
+    i = j;
+  }
+  zOut[i] = 0;
+  *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes.  Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case.  Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word.  If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end.  For long words without digits, 10 bytes
+** are taken from each end.  US-ASCII case folding still applies.
+** 
+** If the input word contains not digits but does characters not 
+** in [a-zA-Z] then no stemming is attempted and this routine just 
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word.  So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, j, c;
+  char zReverse[28];
+  char *z, *z2;
+  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+    /* The word is too big or too small for the porter stemmer.
+    ** Fallback to the copy stemmer */
+    copy_stemmer(zIn, nIn, zOut, pnOut);
+    return;
+  }
+  for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+    c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zReverse[j] = c + 'a' - 'A';
+    }else if( c>='a' && c<='z' ){
+      zReverse[j] = c;
+    }else{
+      /* The use of a character not in [a-zA-Z] means that we fallback
+      ** to the copy stemmer */
+      copy_stemmer(zIn, nIn, zOut, pnOut);
+      return;
+    }
+  }
+  memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+  z = &zReverse[j+1];
+
+
+  /* Step 1a */
+  if( z[0]=='s' ){
+    if(
+     !stem(&z, "sess", "ss", 0) &&
+     !stem(&z, "sei", "i", 0)  &&
+     !stem(&z, "ss", "ss", 0)
+    ){
+      z++;
+    }
+  }
+
+  /* Step 1b */  
+  z2 = z;
+  if( stem(&z, "dee", "ee", m_gt_0) ){
+    /* Do nothing.  The work was all in the test */
+  }else if( 
+     (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+      && z!=z2
+  ){
+     if( stem(&z, "ta", "ate", 0) ||
+         stem(&z, "lb", "ble", 0) ||
+         stem(&z, "zi", "ize", 0) ){
+       /* Do nothing.  The work was all in the test */
+     }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+       z++;
+     }else if( m_eq_1(z) && star_oh(z) ){
+       *(--z) = 'e';
+     }
+  }
+
+  /* Step 1c */
+  if( z[0]=='y' && hasVowel(z+1) ){
+    z[0] = 'i';
+  }
+
+  /* Step 2 */
+  switch( z[1] ){
+   case 'a':
+     stem(&z, "lanoita", "ate", m_gt_0) ||
+     stem(&z, "lanoit", "tion", m_gt_0);
+     break;
+   case 'c':
+     stem(&z, "icne", "ence", m_gt_0) ||
+     stem(&z, "icna", "ance", m_gt_0);
+     break;
+   case 'e':
+     stem(&z, "rezi", "ize", m_gt_0);
+     break;
+   case 'g':
+     stem(&z, "igol", "log", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "ilb", "ble", m_gt_0) ||
+     stem(&z, "illa", "al", m_gt_0) ||
+     stem(&z, "iltne", "ent", m_gt_0) ||
+     stem(&z, "ile", "e", m_gt_0) ||
+     stem(&z, "ilsuo", "ous", m_gt_0);
+     break;
+   case 'o':
+     stem(&z, "noitazi", "ize", m_gt_0) ||
+     stem(&z, "noita", "ate", m_gt_0) ||
+     stem(&z, "rota", "ate", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "msila", "al", m_gt_0) ||
+     stem(&z, "ssenevi", "ive", m_gt_0) ||
+     stem(&z, "ssenluf", "ful", m_gt_0) ||
+     stem(&z, "ssensuo", "ous", m_gt_0);
+     break;
+   case 't':
+     stem(&z, "itila", "al", m_gt_0) ||
+     stem(&z, "itivi", "ive", m_gt_0) ||
+     stem(&z, "itilib", "ble", m_gt_0);
+     break;
+  }
+
+  /* Step 3 */
+  switch( z[0] ){
+   case 'e':
+     stem(&z, "etaci", "ic", m_gt_0) ||
+     stem(&z, "evita", "", m_gt_0)   ||
+     stem(&z, "ezila", "al", m_gt_0);
+     break;
+   case 'i':
+     stem(&z, "itici", "ic", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "laci", "ic", m_gt_0) ||
+     stem(&z, "luf", "", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "ssen", "", m_gt_0);
+     break;
+  }
+
+  /* Step 4 */
+  switch( z[1] ){
+   case 'a':
+     if( z[0]=='l' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'c':
+     if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
+       z += 4;
+     }
+     break;
+   case 'e':
+     if( z[0]=='r' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'i':
+     if( z[0]=='c' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'l':
+     if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+       z += 4;
+     }
+     break;
+   case 'n':
+     if( z[0]=='t' ){
+       if( z[2]=='a' ){
+         if( m_gt_1(z+3) ){
+           z += 3;
+         }
+       }else if( z[2]=='e' ){
+         stem(&z, "tneme", "", m_gt_1) ||
+         stem(&z, "tnem", "", m_gt_1) ||
+         stem(&z, "tne", "", m_gt_1);
+       }
+     }
+     break;
+   case 'o':
+     if( z[0]=='u' ){
+       if( m_gt_1(z+2) ){
+         z += 2;
+       }
+     }else if( z[3]=='s' || z[3]=='t' ){
+       stem(&z, "noi", "", m_gt_1);
+     }
+     break;
+   case 's':
+     if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 't':
+     stem(&z, "eta", "", m_gt_1) ||
+     stem(&z, "iti", "", m_gt_1);
+     break;
+   case 'u':
+     if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 'v':
+   case 'z':
+     if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+  }
+
+  /* Step 5a */
+  if( z[0]=='e' ){
+    if( m_gt_1(z+1) ){
+      z++;
+    }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+      z++;
+    }
+  }
+
+  /* Step 5b */
+  if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+    z++;
+  }
+
+  /* z[] is now the stemmed word in reverse order.  Flip it back
+  ** around into forward order and return.
+  */
+  *pnOut = i = strlen(z);
+  zOut[i] = 0;
+  while( *z ){
+    zOut[--i] = *(z++);
+  }
+}
+
+/*
+** Characters that can be part of a token.  We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token.  In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
+  const char **pzToken,               /* OUT: *pzToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  const char *z = c->zInput;
+
+  while( c->iOffset<c->nInput ){
+    int iStartOffset, ch;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int n = c->iOffset-iStartOffset;
+      if( n>c->nAllocated ){
+        c->nAllocated = n+20;
+        c->zToken = realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
+      }
+      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+      *pzToken = c->zToken;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+  0,
+  porterCreate,
+  porterDestroy,
+  porterOpen,
+  porterClose,
+  porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts2PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

ext/fts2/fts2_tokenizer.c

@@ -0,0 +1,371 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+#include "fts2_hash.h"
+#include "fts2_tokenizer.h"
+#include <assert.h>
+
+/*
+** Implementation of the SQL scalar function for accessing the underlying 
+** hash table. This function may be called as follows:
+**
+**   SELECT <function-name>(<key-name>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
+*/
+static void scalarFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts2Hash *pHash;
+  void *pPtr = 0;
+  const unsigned char *zName;
+  int nName;
+
+  assert( argc==1 || argc==2 );
+
+  pHash = (fts2Hash *)sqlite3_user_data(context);
+
+  zName = sqlite3_value_text(argv[0]);
+  nName = sqlite3_value_bytes(argv[0])+1;
+
+  if( argc==2 ){
+    void *pOld;
+    int n = sqlite3_value_bytes(argv[1]);
+    if( n!=sizeof(pPtr) ){
+      sqlite3_result_error(context, "argument type mismatch", -1);
+      return;
+    }
+    pPtr = *(void **)sqlite3_value_blob(argv[1]);
+    pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
+    if( pOld==pPtr ){
+      sqlite3_result_error(context, "out of memory", -1);
+      return;
+    }
+  }else{
+    pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
+    if( !pPtr ){
+      char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+      sqlite3_result_error(context, zErr, -1);
+      sqlite3_free(zErr);
+      return;
+    }
+  }
+
+  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
+
+#ifdef SQLITE_TEST
+
+#include <tcl.h>
+#include <string.h>
+
+/*
+** Implementation of a special SQL scalar function for testing tokenizers 
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two arguments:
+**
+**   SELECT <function-name>(<key-name>, <input-string>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the 
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example, 
+** using the built-in "simple" tokenizer:
+**
+**   SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+**   "{0 i I 1 dont don't 2 see see 3 how how}"
+**   
+*/
+static void testFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts2Hash *pHash;
+  sqlite3_tokenizer_module *p;
+  sqlite3_tokenizer *pTokenizer = 0;
+  sqlite3_tokenizer_cursor *pCsr = 0;
+
+  const char *zErr = 0;
+
+  const char *zName;
+  int nName;
+  const char *zInput;
+  int nInput;
+
+  const char *zArg = 0;
+
+  const char *zToken;
+  int nToken;
+  int iStart;
+  int iEnd;
+  int iPos;
+
+  Tcl_Obj *pRet;
+
+  assert( argc==2 || argc==3 );
+
+  nName = sqlite3_value_bytes(argv[0]);
+  zName = (const char *)sqlite3_value_text(argv[0]);
+  nInput = sqlite3_value_bytes(argv[argc-1]);
+  zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+  if( argc==3 ){
+    zArg = (const char *)sqlite3_value_text(argv[1]);
+  }
+
+  pHash = (fts2Hash *)sqlite3_user_data(context);
+  p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
+
+  if( !p ){
+    char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+    sqlite3_result_error(context, zErr, -1);
+    sqlite3_free(zErr);
+    return;
+  }
+
+  pRet = Tcl_NewObj();
+  Tcl_IncrRefCount(pRet);
+
+  if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
+    zErr = "error in xCreate()";
+    goto finish;
+  }
+  pTokenizer->pModule = p;
+  if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+    zErr = "error in xOpen()";
+    goto finish;
+  }
+  pCsr->pTokenizer = pTokenizer;
+
+  while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+    zToken = &zInput[iStart];
+    nToken = iEnd-iStart;
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+  }
+
+  if( SQLITE_OK!=p->xClose(pCsr) ){
+    zErr = "error in xClose()";
+    goto finish;
+  }
+  if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+    zErr = "error in xDestroy()";
+    goto finish;
+  }
+
+finish:
+  if( zErr ){
+    sqlite3_result_error(context, zErr, -1);
+  }else{
+    sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+  }
+  Tcl_DecrRefCount(pRet);
+}
+
+static
+int registerTokenizer(
+  sqlite3 *db, 
+  char *zName, 
+  const sqlite3_tokenizer_module *p
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+  sqlite3_step(pStmt);
+
+  return sqlite3_finalize(pStmt);
+}
+
+static
+int queryTokenizer(
+  sqlite3 *db, 
+  char *zName,  
+  const sqlite3_tokenizer_module **pp
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts2_tokenizer(?)";
+
+  *pp = 0;
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+      memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+    }
+  }
+
+  return sqlite3_finalize(pStmt);
+}
+
+void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts2_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts2_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+**     SELECT fts2_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int rc;
+  const sqlite3_tokenizer_module *p1;
+  const sqlite3_tokenizer_module *p2;
+  sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+  /* Test the query function */
+  sqlite3Fts2SimpleTokenizerModule(&p1);
+  rc = queryTokenizer(db, "simple", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p1==p2 );
+  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_ERROR );
+  assert( p2==0 );
+  assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+  /* Test the storage function */
+  rc = registerTokenizer(db, "nosuchtokenizer", p1);
+  assert( rc==SQLITE_OK );
+  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p2==p1 );
+
+  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
+#endif
+
+/*
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialised to use string keys, and to take a private copy 
+** of the key when a value is inserted. i.e. by a call similar to:
+**
+**    sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header 
+** comment above struct HashTableVtab) to the database schema. Both 
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
+*/
+int sqlite3Fts2InitHashTable(
+  sqlite3 *db, 
+  fts2Hash *pHash, 
+  const char *zName
+){
+  int rc = SQLITE_OK;
+  void *p = (void *)pHash;
+  const int any = SQLITE_ANY;
+  char *zTest = 0;
+  char *zTest2 = 0;
+
+#ifdef SQLITE_TEST
+  void *pdb = (void *)db;
+  zTest = sqlite3_mprintf("%s_test", zName);
+  zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+  if( !zTest || !zTest2 ){
+    rc = SQLITE_NOMEM;
+  }
+#endif
+
+  if( rc!=SQLITE_OK
+   || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+#ifdef SQLITE_TEST
+   || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+#endif
+  );
+
+  sqlite3_free(zTest);
+  sqlite3_free(zTest2);
+  return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

ext/fts2/fts2_tokenizer.h

@@ -0,0 +1,145 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS2_TOKENIZER_H_
+#define _FTS2_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts2 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an 
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts2 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+  /*
+  ** Structure version. Should always be set to 0.
+  */
+  int iVersion;
+
+  /*
+  ** Create a new tokenizer. The values in the argv[] array are the
+  ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+  ** TABLE statement that created the fts2 table. For example, if
+  ** the following SQL is executed:
+  **
+  **   CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2)
+  **
+  ** then argc is set to 2, and the argv[] array contains pointers
+  ** to the strings "arg1" and "arg2".
+  **
+  ** This method should return either SQLITE_OK (0), or an SQLite error 
+  ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+  ** to point at the newly created tokenizer structure. The generic
+  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** this callback. The caller will do so.
+  */
+  int (*xCreate)(
+    int argc,                           /* Size of argv array */
+    const char *const*argv,             /* Tokenizer argument strings */
+    sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
+  );
+
+  /*
+  ** Destroy an existing tokenizer. The fts2 module calls this method
+  ** exactly once for each successful call to xCreate().
+  */
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Create a tokenizer cursor to tokenize an input buffer. The caller
+  ** is responsible for ensuring that the input buffer remains valid
+  ** until the cursor is closed (using the xClose() method). 
+  */
+  int (*xOpen)(
+    sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
+    const char *pInput, int nBytes,      /* Input buffer */
+    sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
+  );
+
+  /*
+  ** Destroy an existing tokenizer cursor. The fts2 module calls this 
+  ** method exactly once for each successful call to xOpen().
+  */
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+  /*
+  ** Retrieve the next token from the tokenizer cursor pCursor. This
+  ** method should either return SQLITE_OK and set the values of the
+  ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+  ** the end of the buffer has been reached, or an SQLite error code.
+  **
+  ** *ppToken should be set to point at a buffer containing the 
+  ** normalized version of the token (i.e. after any case-folding and/or
+  ** stemming has been performed). *pnBytes should be set to the length
+  ** of this buffer in bytes. The input text that generated the token is
+  ** identified by the byte offsets returned in *piStartOffset and
+  ** *piEndOffset.
+  **
+  ** The buffer *ppToken is set to point at is managed by the tokenizer
+  ** implementation. It is only required to be valid until the next call
+  ** to xNext() or xClose(). 
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xNext)(
+    sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
+    const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
+    int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
+    int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
+    int *piPosition      /* OUT: Number of tokens returned before this one */
+  );
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+#endif /* _FTS2_TOKENIZER_H_ */

ext/fts2/fts2_tokenizer1.c

@@ -0,0 +1,229 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts2_tokenizer.h"
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  char delim[128];             /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  int iOffset;                 /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *pToken;                /* storage for current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module simpleTokenizerModule;
+
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+  return c<0x80 && t->delim[c];
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    int i, n = strlen(argv[1]);
+    for(i=0; i<n; i++){
+      unsigned char ch = argv[1][i];
+      /* We explicitly don't support UTF-8 delimiters for now. */
+      if( ch>=0x80 ){
+        free(t);
+        return SQLITE_ERROR;
+      }
+      t->delim[ch] = 1;
+    }
+  } else {
+    /* Mark non-alphanumeric ASCII characters as delimiters */
+    int i;
+    for(i=1; i<0x80; i++){
+      t->delim[i] = !isalnum(i);
+    }
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *pInput, int nBytes,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->pInput = pInput;
+  if( pInput==0 ){
+    c->nBytes = 0;
+  }else if( nBytes<0 ){
+    c->nBytes = (int)strlen(pInput);
+  }else{
+    c->nBytes = nBytes;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->pToken = NULL;               /* no space allocated, yet. */
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  free(c->pToken);
+  free(c);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  unsigned char *p = (unsigned char *)c->pInput;
+
+  while( c->iOffset<c->nBytes ){
+    int iStartOffset;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int i, n = c->iOffset-iStartOffset;
+      if( n>c->nTokenAllocated ){
+        c->nTokenAllocated = n+20;
+        c->pToken = realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
+      }
+      for(i=0; i<n; i++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        unsigned char ch = p[iStartOffset+i];
+        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+      }
+      *ppToken = c->pToken;
+      *pnBytes = n;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts2SimpleTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */

ext/fts2/mkfts2amal.tcl

@@ -0,0 +1,116 @@
+#!/usr/bin/tclsh
+#
+# This script builds a single C code file holding all of FTS2 code.
+# The name of the output file is fts2amal.c.  To build this file,
+# first do:
+#
+#      make target_source
+#
+# The make target above moves all of the source code files into
+# a subdirectory named "tsrc".  (This script expects to find the files
+# there and will not work if they are not found.)
+#
+# After the "tsrc" directory has been created and populated, run
+# this script:
+#
+#      tclsh mkfts2amal.tcl
+#
+# The amalgamated FTS2 code will be written into fts2amal.c
+#
+
+# Open the output file and write a header comment at the beginning
+# of the file.
+#
+set out [open fts2amal.c w]
+set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
+puts $out [subst \
+{/******************************************************************************
+** This file is an amalgamation of separate C source files from the SQLite
+** Full Text Search extension 2 (fts2).  By combining all the individual C 
+** code  files into this single large file, the entire code can be compiled 
+** as a one translation unit.  This allows many compilers to do optimizations
+** that would not be possible if the files were compiled separately.  It also
+** makes the code easier to import into other projects.
+**
+** This amalgamation was generated on $today.
+*/}]
+
+# These are the header files used by FTS2.  The first time any of these 
+# files are seen in a #include statement in the C code, include the complete
+# text of the file in-line.  The file only needs to be included once.
+#
+foreach hdr {
+   fts2.h
+   fts2_hash.h
+   fts2_tokenizer.h
+   sqlite3.h
+   sqlite3ext.h
+} {
+  set available_hdr($hdr) 1
+}
+
+# 78 stars used for comment formatting.
+set s78 \
+{*****************************************************************************}
+
+# Insert a comment into the code
+#
+proc section_comment {text} {
+  global out s78
+  set n [string length $text]
+  set nstar [expr {60 - $n}]
+  set stars [string range $s78 0 $nstar]
+  puts $out "/************** $text $stars/"
+}
+
+# Read the source file named $filename and write it into the
+# sqlite3.c output file.  If any #include statements are seen,
+# process them approprately.
+#
+proc copy_file {filename} {
+  global seen_hdr available_hdr out
+  set tail [file tail $filename]
+  section_comment "Begin file $tail"
+  set in [open $filename r]
+  while {![eof $in]} {
+    set line [gets $in]
+    if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
+      if {[info exists available_hdr($hdr)]} {
+        if {$available_hdr($hdr)} {
+          section_comment "Include $hdr in the middle of $tail"
+          copy_file tsrc/$hdr
+          section_comment "Continuing where we left off in $tail"
+        }
+      } elseif {![info exists seen_hdr($hdr)]} {
+        set seen_hdr($hdr) 1
+        puts $out $line
+      }
+    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
+      puts $out "#if 0"
+    } elseif {[regexp {^#line} $line]} {
+      # Skip #line directives.
+    } else {
+      puts $out $line
+    }
+  }
+  close $in
+  section_comment "End of $tail"
+}
+
+
+# Process the source files.  Process files containing commonly
+# used subroutines first in order to help the compiler find
+# inlining opportunities.
+#
+foreach file {
+   fts2.c
+   fts2_hash.c
+   fts2_porter.c
+   fts2_tokenizer.c
+   fts2_tokenizer1.c
+   fts2_icu.c
+} {
+  copy_file tsrc/$file
+}
+
+close $out

ext/fts3/README.tokenizers

@@ -0,0 +1,133 @@
+
+1. FTS3 Tokenizers
+
+  When creating a new full-text table, FTS3 allows the user to select
+  the text tokenizer implementation to be used when indexing text
+  by specifying a "tokenizer" clause as part of the CREATE VIRTUAL TABLE
+  statement:
+
+    CREATE VIRTUAL TABLE <table-name> USING fts3(
+      <columns ...> [, tokenizer <tokenizer-name> [<tokenizer-args>]]
+    );
+
+  The built-in tokenizers (valid values to pass as <tokenizer name>) are
+  "simple" and "porter".
+
+  <tokenizer-args> should consist of zero or more white-space separated
+  arguments to pass to the selected tokenizer implementation. The 
+  interpretation of the arguments, if any, depends on the individual 
+  tokenizer.
+
+2. Custom Tokenizers
+
+  FTS3 allows users to provide custom tokenizer implementations. The 
+  interface used to create a new tokenizer is defined and described in 
+  the fts3_tokenizer.h source file.
+
+  Registering a new FTS3 tokenizer is similar to registering a new 
+  virtual table module with SQLite. The user passes a pointer to a
+  structure containing pointers to various callback functions that
+  make up the implementation of the new tokenizer type. For tokenizers,
+  the structure (defined in fts3_tokenizer.h) is called
+  "sqlite3_tokenizer_module".
+
+  FTS3 does not expose a C-function that users call to register new
+  tokenizer types with a database handle. Instead, the pointer must
+  be encoded as an SQL blob value and passed to FTS3 through the SQL
+  engine by evaluating a special scalar function, "fts3_tokenizer()".
+  The fts3_tokenizer() function may be called with one or two arguments,
+  as follows:
+
+    SELECT fts3_tokenizer(<tokenizer-name>);
+    SELECT fts3_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
+  
+  Where <tokenizer-name> is a string identifying the tokenizer and
+  <sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
+  structure encoded as an SQL blob. If the second argument is present,
+  it is registered as tokenizer <tokenizer-name> and a copy of it
+  returned. If only one argument is passed, a pointer to the tokenizer
+  implementation currently registered as <tokenizer-name> is returned,
+  encoded as a blob. Or, if no such tokenizer exists, an SQL exception
+  (error) is raised.
+
+  SECURITY: If the fts3 extension is used in an environment where potentially
+    malicious users may execute arbitrary SQL (i.e. gears), they should be
+    prevented from invoking the fts3_tokenizer() function, possibly using the
+    authorisation callback.
+
+  See "Sample code" below for an example of calling the fts3_tokenizer()
+  function from C code.
+
+3. ICU Library Tokenizers
+
+  If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor 
+  symbol defined, then there exists a built-in tokenizer named "icu" 
+  implemented using the ICU library. The first argument passed to the
+  xCreate() method (see fts3_tokenizer.h) of this tokenizer may be
+  an ICU locale identifier. For example "tr_TR" for Turkish as used
+  in Turkey, or "en_AU" for English as used in Australia. For example:
+
+    "CREATE VIRTUAL TABLE thai_text USING fts3(text, tokenizer icu th_TH)"
+
+  The ICU tokenizer implementation is very simple. It splits the input
+  text according to the ICU rules for finding word boundaries and discards
+  any tokens that consist entirely of white-space. This may be suitable
+  for some applications in some locales, but not all. If more complex
+  processing is required, for example to implement stemming or 
+  discard punctuation, this can be done by creating a tokenizer 
+  implementation that uses the ICU tokenizer as part of its implementation.
+
+  When using the ICU tokenizer this way, it is safe to overwrite the
+  contents of the strings returned by the xNext() method (see
+  fts3_tokenizer.h).
+
+4. Sample code.
+
+  The following two code samples illustrate the way C code should invoke
+  the fts3_tokenizer() scalar function:
+
+      int registerTokenizer(
+        sqlite3 *db, 
+        char *zName, 
+        const sqlite3_tokenizer_module *p
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
+      
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+        sqlite3_step(pStmt);
+      
+        return sqlite3_finalize(pStmt);
+      }
+      
+      int queryTokenizer(
+        sqlite3 *db, 
+        char *zName,  
+        const sqlite3_tokenizer_module **pp
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts3_tokenizer(?)";
+      
+        *pp = 0;
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        if( SQLITE_ROW==sqlite3_step(pStmt) ){
+          if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+            memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+          }
+        }
+      
+        return sqlite3_finalize(pStmt);
+      }

ext/fts3/README.txt

@@ -0,0 +1,4 @@
+This folder contains source code to the second full-text search
+extension for SQLite.  While the API is the same, this version uses a
+substantially different storage schema from fts1, so tables will need
+to be rebuilt.

ext/fts3/fts3.h

@@ -0,0 +1,26 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** FTS3 library.  All it does is declare the sqlite3Fts3Init() interface.
+*/
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3Fts3Init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */

ext/fts3/fts3_hash.c

@@ -0,0 +1,374 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS3 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS3 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "sqlite3.h"
+#include "fts3_hash.h"
+
+/*
+** Malloc and Free functions
+*/
+static void *fts3HashMalloc(int n){
+  void *p = sqlite3_malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+static void fts3HashFree(void *p){
+  sqlite3_free(p);
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants 
+** FTS3_HASH_BINARY or FTS3_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
+  pNew->keyClass = keyClass;
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3Fts3HashClear(fts3Hash *pH){
+  fts3HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  fts3HashFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    fts3HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      fts3HashFree(elem->pKey);
+    }
+    fts3HashFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_STRING
+*/
+static int fts3StrHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS3_HASH_BINARY
+*/
+static int fts3BinHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "ftsHashFunction".  The function takes a
+** single parameter "keyClass".  The return value of ftsHashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of ftsHashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*ftsHashFunction(int keyClass))(const void*,int){
+  if( keyClass==FTS3_HASH_STRING ){
+    return &fts3StrHash;
+  }else{
+    assert( keyClass==FTS3_HASH_BINARY );
+    return &fts3BinHash;
+  }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){
+  if( keyClass==FTS3_HASH_STRING ){
+    return &fts3StrCompare;
+  }else{
+    assert( keyClass==FTS3_HASH_BINARY );
+    return &fts3BinCompare;
+  }
+}
+
+/* Link an element into the hash table
+*/
+static void fts3HashInsertElement(
+  fts3Hash *pH,            /* The complete hash table */
+  struct _fts3ht *pEntry,  /* The entry into which pNew is inserted */
+  fts3HashElem *pNew       /* The element to be inserted */
+){
+  fts3HashElem *pHead;     /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void fts3Rehash(fts3Hash *pH, int new_size){
+  struct _fts3ht *new_ht;          /* The new hash table */
+  fts3HashElem *elem, *next_elem;  /* For looping over existing elements */
+  int (*xHash)(const void*,int);   /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
+  if( new_ht==0 ) return;
+  fts3HashFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = ftsHashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    fts3HashInsertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static fts3HashElem *fts3FindElementByHash(
+  const fts3Hash *pH, /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  fts3HashElem *elem;            /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _fts3ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = ftsCompareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void fts3RemoveElementByHash(
+  fts3Hash *pH,         /* The pH containing "elem" */
+  fts3HashElem* elem,   /* The element to be removed from the pH */
+  int h                 /* Hash value for the element */
+){
+  struct _fts3ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    fts3HashFree(elem->pKey);
+  }
+  fts3HashFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    fts3HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
+  int h;                 /* A hash on key */
+  fts3HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = ftsHashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3Fts3HashInsert(
+  fts3Hash *pH,        /* The hash table to insert into */
+  const void *pKey,    /* The key */
+  int nKey,            /* Number of bytes in the key */
+  void *data           /* The data */
+){
+  int hraw;                 /* Raw hash value of the key */
+  int h;                    /* the hash of the key modulo hash table size */
+  fts3HashElem *elem;       /* Used to loop thru the element list */
+  fts3HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = ftsHashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = fts3FindElementByHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      fts3RemoveElementByHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = fts3HashMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      fts3HashFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    fts3Rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      fts3HashFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    fts3Rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  fts3HashInsertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

ext/fts3/fts3_hash.h

@@ -0,0 +1,110 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS3_HASH_H_
+#define _FTS3_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct fts3Hash fts3Hash;
+typedef struct fts3HashElem fts3HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct fts3Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  fts3HashElem *first;    /* The first element of the array */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _fts3ht {        /* the hash table */
+    int count;               /* Number of entries with this hash */
+    fts3HashElem *chain;     /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct fts3HashElem {
+  fts3HashElem *next, *prev; /* Next and previous elements in the table */
+  void *data;                /* Data associated with this element */
+  void *pKey; int nKey;      /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+**   FTS3_HASH_STRING        pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   FTS3_HASH_BINARY        pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.  
+*/
+#define FTS3_HASH_STRING    1
+#define FTS3_HASH_BINARY    2
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
+void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey);
+void sqlite3Fts3HashClear(fts3Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts3HashInit   sqlite3Fts3HashInit
+#define fts3HashInsert sqlite3Fts3HashInsert
+#define fts3HashFind   sqlite3Fts3HashFind
+#define fts3HashClear  sqlite3Fts3HashClear
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   fts3Hash h;
+**   fts3HashElem *p;
+**   ...
+**   for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
+**     SomeStructure *pData = fts3HashData(p);
+**     // do something with pData
+**   }
+*/
+#define fts3HashFirst(H)  ((H)->first)
+#define fts3HashNext(E)   ((E)->next)
+#define fts3HashData(E)   ((E)->data)
+#define fts3HashKey(E)    ((E)->pKey)
+#define fts3HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts3HashCount(H)  ((H)->count)
+
+#endif /* _FTS3_HASH_H_ */

ext/fts3/fts3_icu.c

@@ -0,0 +1,258 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements a tokenizer for fts3 based on the ICU library.
+** 
+** $Id: fts3_icu.c,v 1.2 2007/10/24 21:52:37 shess Exp $
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+#ifdef SQLITE_ENABLE_ICU
+
+#include <assert.h>
+#include <string.h>
+#include "fts3_tokenizer.h"
+
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+#include <unicode/utf16.h>
+
+typedef struct IcuTokenizer IcuTokenizer;
+typedef struct IcuCursor IcuCursor;
+
+struct IcuTokenizer {
+  sqlite3_tokenizer base;
+  char *zLocale;
+};
+
+struct IcuCursor {
+  sqlite3_tokenizer_cursor base;
+
+  UBreakIterator *pIter;      /* ICU break-iterator object */
+  int nChar;                  /* Number of UChar elements in pInput */
+  UChar *aChar;               /* Copy of input using utf-16 encoding */
+  int *aOffset;               /* Offsets of each character in utf-8 input */
+
+  int nBuffer;
+  char *zBuffer;
+
+  int iToken;
+};
+
+/*
+** Create a new tokenizer instance.
+*/
+static int icuCreate(
+  int argc,                            /* Number of entries in argv[] */
+  const char * const *argv,            /* Tokenizer creation arguments */
+  sqlite3_tokenizer **ppTokenizer      /* OUT: Created tokenizer */
+){
+  IcuTokenizer *p;
+  int n = 0;
+
+  if( argc>0 ){
+    n = strlen(argv[0])+1;
+  }
+  p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+  if( !p ){
+    return SQLITE_NOMEM;
+  }
+  memset(p, 0, sizeof(IcuTokenizer));
+
+  if( n ){
+    p->zLocale = (char *)&p[1];
+    memcpy(p->zLocale, argv[0], n);
+  }
+
+  *ppTokenizer = (sqlite3_tokenizer *)p;
+
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int icuOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput,                    /* Input string */
+  int nInput,                            /* Length of zInput in bytes */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  IcuCursor *pCsr;
+
+  const int32_t opt = U_FOLD_CASE_DEFAULT;
+  UErrorCode status = U_ZERO_ERROR;
+  int nChar;
+
+  UChar32 c;
+  int iInput = 0;
+  int iOut = 0;
+
+  *ppCursor = 0;
+
+  if( -1 == nInput ) nInput = strlen(nInput);
+  nChar = nInput+1;
+  pCsr = (IcuCursor *)sqlite3_malloc(
+      sizeof(IcuCursor) +                /* IcuCursor */
+      nChar * sizeof(UChar) +            /* IcuCursor.aChar[] */
+      (nChar+1) * sizeof(int)            /* IcuCursor.aOffset[] */
+  );
+  if( !pCsr ){
+    return SQLITE_NOMEM;
+  }
+  memset(pCsr, 0, sizeof(IcuCursor));
+  pCsr->aChar = (UChar *)&pCsr[1];
+  pCsr->aOffset = (int *)&pCsr->aChar[nChar];
+
+  pCsr->aOffset[iOut] = iInput;
+  U8_NEXT(zInput, iInput, nInput, c); 
+  while( c>0 ){
+    int isError = 0;
+    c = u_foldCase(c, opt);
+    U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
+    if( isError ){
+      sqlite3_free(pCsr);
+      return SQLITE_ERROR;
+    }
+    pCsr->aOffset[iOut] = iInput;
+
+    if( iInput<nInput ){
+      U8_NEXT(zInput, iInput, nInput, c);
+    }else{
+      c = 0;
+    }
+  }
+
+  pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
+  if( !U_SUCCESS(status) ){
+    sqlite3_free(pCsr);
+    return SQLITE_ERROR;
+  }
+  pCsr->nChar = iOut;
+
+  ubrk_first(pCsr->pIter);
+  *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to icuOpen().
+*/
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+  ubrk_close(pCsr->pIter);
+  sqlite3_free(pCsr->zBuffer);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.
+*/
+static int icuNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+
+  int iStart = 0;
+  int iEnd = 0;
+  int nByte = 0;
+
+  while( iStart==iEnd ){
+    UChar32 c;
+
+    iStart = ubrk_current(pCsr->pIter);
+    iEnd = ubrk_next(pCsr->pIter);
+    if( iEnd==UBRK_DONE ){
+      return SQLITE_DONE;
+    }
+
+    while( iStart<iEnd ){
+      int iWhite = iStart;
+      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
+      if( u_isspace(c) ){
+        iStart = iWhite;
+      }else{
+        break;
+      }
+    }
+    assert(iStart<=iEnd);
+  }
+
+  do {
+    UErrorCode status = U_ZERO_ERROR;
+    if( nByte ){
+      char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
+      if( !zNew ){
+        return SQLITE_NOMEM;
+      }
+      pCsr->zBuffer = zNew;
+      pCsr->nBuffer = nByte;
+    }
+
+    u_strToUTF8(
+        pCsr->zBuffer, pCsr->nBuffer, &nByte,    /* Output vars */
+        &pCsr->aChar[iStart], iEnd-iStart,       /* Input vars */
+        &status                                  /* Output success/failure */
+    );
+  } while( nByte>pCsr->nBuffer );
+
+  *ppToken = pCsr->zBuffer;
+  *pnBytes = nByte;
+  *piStartOffset = pCsr->aOffset[iStart];
+  *piEndOffset = pCsr->aOffset[iEnd];
+  *piPosition = pCsr->iToken++;
+
+  return SQLITE_OK;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module icuTokenizerModule = {
+  0,                           /* iVersion */
+  icuCreate,                   /* xCreate  */
+  icuDestroy,                  /* xCreate  */
+  icuOpen,                     /* xOpen    */
+  icuClose,                    /* xClose   */
+  icuNext,                     /* xNext    */
+};
+
+/*
+** Set *ppModule to point at the implementation of the ICU tokenizer.
+*/
+void sqlite3Fts3IcuTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &icuTokenizerModule;
+}
+
+#endif /* defined(SQLITE_ENABLE_ICU) */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

ext/fts3/fts3_porter.c

@@ -0,0 +1,642 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS3 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS3 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts3_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+  sqlite3_tokenizer base;      /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *zInput;          /* input we are tokenizing */
+  int nInput;                  /* size of the input */
+  int iOffset;                 /* current position in zInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nAllocated;              /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module porterTokenizerModule;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  porter_tokenizer *t;
+  t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
+  if( t==NULL ) return SQLITE_NOMEM;
+  memset(t, 0, sizeof(*t));
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+  sqlite3_free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is zInput[0..nInput-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int porterOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput, int nInput,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  porter_tokenizer_cursor *c;
+
+  c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->zInput = zInput;
+  if( zInput==0 ){
+    c->nInput = 0;
+  }else if( nInput<0 ){
+    c->nInput = (int)strlen(zInput);
+  }else{
+    c->nInput = nInput;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  sqlite3_free(c->zToken);
+  sqlite3_free(c);
+  return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+   0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+   1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.  
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order.  So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return j;
+  return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return 1-j;
+  return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants.  Then every word can be
+** represented as:
+**
+**           [C] (VC){m} [V]
+**
+** In prose:  A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel.  "m" is the
+** number of vowel consonant pairs.  This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more.  In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order.  So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 1;
+  while( isConsonant(z) ){ z++; }
+  return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here.  So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+  return
+    z[0]!=0 && isConsonant(z) &&
+    z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+    z[1]!=0 && isVowel(z+1) &&
+    z[2]!=0 && isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the 
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order.  zTo
+** is in normal order. 
+**
+** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
+** match.  Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+  char **pz,             /* The word being stemmed (Reversed) */
+  const char *zFrom,     /* If the ending matches this... (Reversed) */
+  const char *zTo,       /* ... change the ending to this (not reversed) */
+  int (*xCond)(const char*)   /* Condition that must be true */
+){
+  char *z = *pz;
+  while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+  if( *zFrom!=0 ) return 0;
+  if( xCond && !xCond(z) ) return 1;
+  while( *zTo ){
+    *(--z) = *(zTo++);
+  }
+  *pz = z;
+  return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate.  The input word is copied into the output with
+** US-ASCII case folding.  If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, mx, j;
+  int hasDigit = 0;
+  for(i=0; i<nIn; i++){
+    int c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zOut[i] = c - 'A' + 'a';
+    }else{
+      if( c>='0' && c<='9' ) hasDigit = 1;
+      zOut[i] = c;
+    }
+  }
+  mx = hasDigit ? 3 : 10;
+  if( nIn>mx*2 ){
+    for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+      zOut[j] = zOut[i];
+    }
+    i = j;
+  }
+  zOut[i] = 0;
+  *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes.  Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case.  Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word.  If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end.  For long words without digits, 10 bytes
+** are taken from each end.  US-ASCII case folding still applies.
+** 
+** If the input word contains not digits but does characters not 
+** in [a-zA-Z] then no stemming is attempted and this routine just 
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word.  So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, j, c;
+  char zReverse[28];
+  char *z, *z2;
+  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+    /* The word is too big or too small for the porter stemmer.
+    ** Fallback to the copy stemmer */
+    copy_stemmer(zIn, nIn, zOut, pnOut);
+    return;
+  }
+  for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+    c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zReverse[j] = c + 'a' - 'A';
+    }else if( c>='a' && c<='z' ){
+      zReverse[j] = c;
+    }else{
+      /* The use of a character not in [a-zA-Z] means that we fallback
+      ** to the copy stemmer */
+      copy_stemmer(zIn, nIn, zOut, pnOut);
+      return;
+    }
+  }
+  memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+  z = &zReverse[j+1];
+
+
+  /* Step 1a */
+  if( z[0]=='s' ){
+    if(
+     !stem(&z, "sess", "ss", 0) &&
+     !stem(&z, "sei", "i", 0)  &&
+     !stem(&z, "ss", "ss", 0)
+    ){
+      z++;
+    }
+  }
+
+  /* Step 1b */  
+  z2 = z;
+  if( stem(&z, "dee", "ee", m_gt_0) ){
+    /* Do nothing.  The work was all in the test */
+  }else if( 
+     (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+      && z!=z2
+  ){
+     if( stem(&z, "ta", "ate", 0) ||
+         stem(&z, "lb", "ble", 0) ||
+         stem(&z, "zi", "ize", 0) ){
+       /* Do nothing.  The work was all in the test */
+     }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+       z++;
+     }else if( m_eq_1(z) && star_oh(z) ){
+       *(--z) = 'e';
+     }
+  }
+
+  /* Step 1c */
+  if( z[0]=='y' && hasVowel(z+1) ){
+    z[0] = 'i';
+  }
+
+  /* Step 2 */
+  switch( z[1] ){
+   case 'a':
+     stem(&z, "lanoita", "ate", m_gt_0) ||
+     stem(&z, "lanoit", "tion", m_gt_0);
+     break;
+   case 'c':
+     stem(&z, "icne", "ence", m_gt_0) ||
+     stem(&z, "icna", "ance", m_gt_0);
+     break;
+   case 'e':
+     stem(&z, "rezi", "ize", m_gt_0);
+     break;
+   case 'g':
+     stem(&z, "igol", "log", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "ilb", "ble", m_gt_0) ||
+     stem(&z, "illa", "al", m_gt_0) ||
+     stem(&z, "iltne", "ent", m_gt_0) ||
+     stem(&z, "ile", "e", m_gt_0) ||
+     stem(&z, "ilsuo", "ous", m_gt_0);
+     break;
+   case 'o':
+     stem(&z, "noitazi", "ize", m_gt_0) ||
+     stem(&z, "noita", "ate", m_gt_0) ||
+     stem(&z, "rota", "ate", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "msila", "al", m_gt_0) ||
+     stem(&z, "ssenevi", "ive", m_gt_0) ||
+     stem(&z, "ssenluf", "ful", m_gt_0) ||
+     stem(&z, "ssensuo", "ous", m_gt_0);
+     break;
+   case 't':
+     stem(&z, "itila", "al", m_gt_0) ||
+     stem(&z, "itivi", "ive", m_gt_0) ||
+     stem(&z, "itilib", "ble", m_gt_0);
+     break;
+  }
+
+  /* Step 3 */
+  switch( z[0] ){
+   case 'e':
+     stem(&z, "etaci", "ic", m_gt_0) ||
+     stem(&z, "evita", "", m_gt_0)   ||
+     stem(&z, "ezila", "al", m_gt_0);
+     break;
+   case 'i':
+     stem(&z, "itici", "ic", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "laci", "ic", m_gt_0) ||
+     stem(&z, "luf", "", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "ssen", "", m_gt_0);
+     break;
+  }
+
+  /* Step 4 */
+  switch( z[1] ){
+   case 'a':
+     if( z[0]=='l' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'c':
+     if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
+       z += 4;
+     }
+     break;
+   case 'e':
+     if( z[0]=='r' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'i':
+     if( z[0]=='c' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'l':
+     if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+       z += 4;
+     }
+     break;
+   case 'n':
+     if( z[0]=='t' ){
+       if( z[2]=='a' ){
+         if( m_gt_1(z+3) ){
+           z += 3;
+         }
+       }else if( z[2]=='e' ){
+         stem(&z, "tneme", "", m_gt_1) ||
+         stem(&z, "tnem", "", m_gt_1) ||
+         stem(&z, "tne", "", m_gt_1);
+       }
+     }
+     break;
+   case 'o':
+     if( z[0]=='u' ){
+       if( m_gt_1(z+2) ){
+         z += 2;
+       }
+     }else if( z[3]=='s' || z[3]=='t' ){
+       stem(&z, "noi", "", m_gt_1);
+     }
+     break;
+   case 's':
+     if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 't':
+     stem(&z, "eta", "", m_gt_1) ||
+     stem(&z, "iti", "", m_gt_1);
+     break;
+   case 'u':
+     if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 'v':
+   case 'z':
+     if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+  }
+
+  /* Step 5a */
+  if( z[0]=='e' ){
+    if( m_gt_1(z+1) ){
+      z++;
+    }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+      z++;
+    }
+  }
+
+  /* Step 5b */
+  if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+    z++;
+  }
+
+  /* z[] is now the stemmed word in reverse order.  Flip it back
+  ** around into forward order and return.
+  */
+  *pnOut = i = strlen(z);
+  zOut[i] = 0;
+  while( *z ){
+    zOut[--i] = *(z++);
+  }
+}
+
+/*
+** Characters that can be part of a token.  We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token.  In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
+  const char **pzToken,               /* OUT: *pzToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  const char *z = c->zInput;
+
+  while( c->iOffset<c->nInput ){
+    int iStartOffset, ch;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int n = c->iOffset-iStartOffset;
+      if( n>c->nAllocated ){
+        c->nAllocated = n+20;
+        c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
+      }
+      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+      *pzToken = c->zToken;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+  0,
+  porterCreate,
+  porterDestroy,
+  porterOpen,
+  porterClose,
+  porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts3PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

ext/fts3/fts3_tokenizer.c

@@ -0,0 +1,371 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS3 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS3 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+#include "sqlite3ext.h"
+#ifndef SQLITE_CORE
+  SQLITE_EXTENSION_INIT1
+#endif
+
+#include "fts3_hash.h"
+#include "fts3_tokenizer.h"
+#include <assert.h>
+
+/*
+** Implementation of the SQL scalar function for accessing the underlying 
+** hash table. This function may be called as follows:
+**
+**   SELECT <function-name>(<key-name>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
+*/
+static void scalarFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3Hash *pHash;
+  void *pPtr = 0;
+  const unsigned char *zName;
+  int nName;
+
+  assert( argc==1 || argc==2 );
+
+  pHash = (fts3Hash *)sqlite3_user_data(context);
+
+  zName = sqlite3_value_text(argv[0]);
+  nName = sqlite3_value_bytes(argv[0])+1;
+
+  if( argc==2 ){
+    void *pOld;
+    int n = sqlite3_value_bytes(argv[1]);
+    if( n!=sizeof(pPtr) ){
+      sqlite3_result_error(context, "argument type mismatch", -1);
+      return;
+    }
+    pPtr = *(void **)sqlite3_value_blob(argv[1]);
+    pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
+    if( pOld==pPtr ){
+      sqlite3_result_error(context, "out of memory", -1);
+      return;
+    }
+  }else{
+    pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
+    if( !pPtr ){
+      char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+      sqlite3_result_error(context, zErr, -1);
+      sqlite3_free(zErr);
+      return;
+    }
+  }
+
+  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
+
+#ifdef SQLITE_TEST
+
+#include <tcl.h>
+#include <string.h>
+
+/*
+** Implementation of a special SQL scalar function for testing tokenizers 
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two arguments:
+**
+**   SELECT <function-name>(<key-name>, <input-string>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the 
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example, 
+** using the built-in "simple" tokenizer:
+**
+**   SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+**   "{0 i I 1 dont don't 2 see see 3 how how}"
+**   
+*/
+static void testFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts3Hash *pHash;
+  sqlite3_tokenizer_module *p;
+  sqlite3_tokenizer *pTokenizer = 0;
+  sqlite3_tokenizer_cursor *pCsr = 0;
+
+  const char *zErr = 0;
+
+  const char *zName;
+  int nName;
+  const char *zInput;
+  int nInput;
+
+  const char *zArg = 0;
+
+  const char *zToken;
+  int nToken;
+  int iStart;
+  int iEnd;
+  int iPos;
+
+  Tcl_Obj *pRet;
+
+  assert( argc==2 || argc==3 );
+
+  nName = sqlite3_value_bytes(argv[0]);
+  zName = (const char *)sqlite3_value_text(argv[0]);
+  nInput = sqlite3_value_bytes(argv[argc-1]);
+  zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+  if( argc==3 ){
+    zArg = (const char *)sqlite3_value_text(argv[1]);
+  }
+
+  pHash = (fts3Hash *)sqlite3_user_data(context);
+  p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
+
+  if( !p ){
+    char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+    sqlite3_result_error(context, zErr, -1);
+    sqlite3_free(zErr);
+    return;
+  }
+
+  pRet = Tcl_NewObj();
+  Tcl_IncrRefCount(pRet);
+
+  if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
+    zErr = "error in xCreate()";
+    goto finish;
+  }
+  pTokenizer->pModule = p;
+  if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+    zErr = "error in xOpen()";
+    goto finish;
+  }
+  pCsr->pTokenizer = pTokenizer;
+
+  while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+    zToken = &zInput[iStart];
+    nToken = iEnd-iStart;
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+  }
+
+  if( SQLITE_OK!=p->xClose(pCsr) ){
+    zErr = "error in xClose()";
+    goto finish;
+  }
+  if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+    zErr = "error in xDestroy()";
+    goto finish;
+  }
+
+finish:
+  if( zErr ){
+    sqlite3_result_error(context, zErr, -1);
+  }else{
+    sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+  }
+  Tcl_DecrRefCount(pRet);
+}
+
+static
+int registerTokenizer(
+  sqlite3 *db, 
+  char *zName, 
+  const sqlite3_tokenizer_module *p
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+  sqlite3_step(pStmt);
+
+  return sqlite3_finalize(pStmt);
+}
+
+static
+int queryTokenizer(
+  sqlite3 *db, 
+  char *zName,  
+  const sqlite3_tokenizer_module **pp
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts3_tokenizer(?)";
+
+  *pp = 0;
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+      memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+    }
+  }
+
+  return sqlite3_finalize(pStmt);
+}
+
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts3_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts3_tokenizer() function
+** can be used as designed by the C-code in the queryTokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+**     SELECT fts3_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int rc;
+  const sqlite3_tokenizer_module *p1;
+  const sqlite3_tokenizer_module *p2;
+  sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+  /* Test the query function */
+  sqlite3Fts3SimpleTokenizerModule(&p1);
+  rc = queryTokenizer(db, "simple", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p1==p2 );
+  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_ERROR );
+  assert( p2==0 );
+  assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+  /* Test the storage function */
+  rc = registerTokenizer(db, "nosuchtokenizer", p1);
+  assert( rc==SQLITE_OK );
+  rc = queryTokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p2==p1 );
+
+  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
+#endif
+
+/*
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialised to use string keys, and to take a private copy 
+** of the key when a value is inserted. i.e. by a call similar to:
+**
+**    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header 
+** comment above struct HashTableVtab) to the database schema. Both 
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
+*/
+int sqlite3Fts3InitHashTable(
+  sqlite3 *db, 
+  fts3Hash *pHash, 
+  const char *zName
+){
+  int rc = SQLITE_OK;
+  void *p = (void *)pHash;
+  const int any = SQLITE_ANY;
+  char *zTest = 0;
+  char *zTest2 = 0;
+
+#ifdef SQLITE_TEST
+  void *pdb = (void *)db;
+  zTest = sqlite3_mprintf("%s_test", zName);
+  zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+  if( !zTest || !zTest2 ){
+    rc = SQLITE_NOMEM;
+  }
+#endif
+
+  if( rc!=SQLITE_OK
+   || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+#ifdef SQLITE_TEST
+   || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+#endif
+  );
+
+  sqlite3_free(zTest);
+  sqlite3_free(zTest2);
+  return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

ext/fts3/fts3_tokenizer.h

@@ -0,0 +1,145 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an 
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+  /*
+  ** Structure version. Should always be set to 0.
+  */
+  int iVersion;
+
+  /*
+  ** Create a new tokenizer. The values in the argv[] array are the
+  ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+  ** TABLE statement that created the fts3 table. For example, if
+  ** the following SQL is executed:
+  **
+  **   CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+  **
+  ** then argc is set to 2, and the argv[] array contains pointers
+  ** to the strings "arg1" and "arg2".
+  **
+  ** This method should return either SQLITE_OK (0), or an SQLite error 
+  ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+  ** to point at the newly created tokenizer structure. The generic
+  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** this callback. The caller will do so.
+  */
+  int (*xCreate)(
+    int argc,                           /* Size of argv array */
+    const char *const*argv,             /* Tokenizer argument strings */
+    sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
+  );
+
+  /*
+  ** Destroy an existing tokenizer. The fts3 module calls this method
+  ** exactly once for each successful call to xCreate().
+  */
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Create a tokenizer cursor to tokenize an input buffer. The caller
+  ** is responsible for ensuring that the input buffer remains valid
+  ** until the cursor is closed (using the xClose() method). 
+  */
+  int (*xOpen)(
+    sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
+    const char *pInput, int nBytes,      /* Input buffer */
+    sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
+  );
+
+  /*
+  ** Destroy an existing tokenizer cursor. The fts3 module calls this 
+  ** method exactly once for each successful call to xOpen().
+  */
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+  /*
+  ** Retrieve the next token from the tokenizer cursor pCursor. This
+  ** method should either return SQLITE_OK and set the values of the
+  ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+  ** the end of the buffer has been reached, or an SQLite error code.
+  **
+  ** *ppToken should be set to point at a buffer containing the 
+  ** normalized version of the token (i.e. after any case-folding and/or
+  ** stemming has been performed). *pnBytes should be set to the length
+  ** of this buffer in bytes. The input text that generated the token is
+  ** identified by the byte offsets returned in *piStartOffset and
+  ** *piEndOffset.
+  **
+  ** The buffer *ppToken is set to point at is managed by the tokenizer
+  ** implementation. It is only required to be valid until the next call
+  ** to xNext() or xClose(). 
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xNext)(
+    sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
+    const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
+    int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
+    int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
+    int *piPosition      /* OUT: Number of tokens returned before this one */
+  );
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+#endif /* _FTS3_TOKENIZER_H_ */

ext/fts3/fts3_tokenizer1.c

@@ -0,0 +1,230 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS3 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS3 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts3_tokenizer.h"
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  char delim[128];             /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  int iOffset;                 /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *pToken;                /* storage for current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module simpleTokenizerModule;
+
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+  return c<0x80 && t->delim[c];
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
+  if( t==NULL ) return SQLITE_NOMEM;
+  memset(t, 0, sizeof(*t));
+
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    int i, n = strlen(argv[1]);
+    for(i=0; i<n; i++){
+      unsigned char ch = argv[1][i];
+      /* We explicitly don't support UTF-8 delimiters for now. */
+      if( ch>=0x80 ){
+        sqlite3_free(t);
+        return SQLITE_ERROR;
+      }
+      t->delim[ch] = 1;
+    }
+  } else {
+    /* Mark non-alphanumeric ASCII characters as delimiters */
+    int i;
+    for(i=1; i<0x80; i++){
+      t->delim[i] = !isalnum(i);
+    }
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  sqlite3_free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *pInput, int nBytes,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->pInput = pInput;
+  if( pInput==0 ){
+    c->nBytes = 0;
+  }else if( nBytes<0 ){
+    c->nBytes = (int)strlen(pInput);
+  }else{
+    c->nBytes = nBytes;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->pToken = NULL;               /* no space allocated, yet. */
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  sqlite3_free(c->pToken);
+  sqlite3_free(c);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  unsigned char *p = (unsigned char *)c->pInput;
+
+  while( c->iOffset<c->nBytes ){
+    int iStartOffset;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int i, n = c->iOffset-iStartOffset;
+      if( n>c->nTokenAllocated ){
+        c->nTokenAllocated = n+20;
+        c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
+      }
+      for(i=0; i<n; i++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        unsigned char ch = p[iStartOffset+i];
+        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+      }
+      *ppToken = c->pToken;
+      *pnBytes = n;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts3SimpleTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

ext/fts3/mkfts3amal.tcl

@@ -0,0 +1,115 @@
+#!/usr/bin/tclsh
+#
+# This script builds a single C code file holding all of FTS3 code.
+# The name of the output file is fts3amal.c.  To build this file,
+# first do:
+#
+#      make target_source
+#
+# The make target above moves all of the source code files into
+# a subdirectory named "tsrc".  (This script expects to find the files
+# there and will not work if they are not found.)
+#
+# After the "tsrc" directory has been created and populated, run
+# this script:
+#
+#      tclsh mkfts3amal.tcl
+#
+# The amalgamated FTS3 code will be written into fts3amal.c
+#
+
+# Open the output file and write a header comment at the beginning
+# of the file.
+#
+set out [open fts3amal.c w]
+set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
+puts $out [subst \
+{/******************************************************************************
+** This file is an amalgamation of separate C source files from the SQLite
+** Full Text Search extension 2 (fts3).  By combining all the individual C 
+** code  files into this single large file, the entire code can be compiled 
+** as a one translation unit.  This allows many compilers to do optimizations
+** that would not be possible if the files were compiled separately.  It also
+** makes the code easier to import into other projects.
+**
+** This amalgamation was generated on $today.
+*/}]
+
+# These are the header files used by FTS3.  The first time any of these 
+# files are seen in a #include statement in the C code, include the complete
+# text of the file in-line.  The file only needs to be included once.
+#
+foreach hdr {
+   fts3.h
+   fts3_hash.h
+   fts3_tokenizer.h
+   sqlite3.h
+   sqlite3ext.h
+} {
+  set available_hdr($hdr) 1
+}
+
+# 78 stars used for comment formatting.
+set s78 \
+{*****************************************************************************}
+
+# Insert a comment into the code
+#
+proc section_comment {text} {
+  global out s78
+  set n [string length $text]
+  set nstar [expr {60 - $n}]
+  set stars [string range $s78 0 $nstar]
+  puts $out "/************** $text $stars/"
+}
+
+# Read the source file named $filename and write it into the
+# sqlite3.c output file.  If any #include statements are seen,
+# process them approprately.
+#
+proc copy_file {filename} {
+  global seen_hdr available_hdr out
+  set tail [file tail $filename]
+  section_comment "Begin file $tail"
+  set in [open $filename r]
+  while {![eof $in]} {
+    set line [gets $in]
+    if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
+      if {[info exists available_hdr($hdr)]} {
+        if {$available_hdr($hdr)} {
+          section_comment "Include $hdr in the middle of $tail"
+          copy_file tsrc/$hdr
+          section_comment "Continuing where we left off in $tail"
+        }
+      } elseif {![info exists seen_hdr($hdr)]} {
+        set seen_hdr($hdr) 1
+        puts $out $line
+      }
+    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
+      puts $out "#if 0"
+    } elseif {[regexp {^#line} $line]} {
+      # Skip #line directives.
+    } else {
+      puts $out $line
+    }
+  }
+  close $in
+  section_comment "End of $tail"
+}
+
+
+# Process the source files.  Process files containing commonly
+# used subroutines first in order to help the compiler find
+# inlining opportunities.
+#
+foreach file {
+   fts3.c
+   fts3_hash.c
+   fts3_porter.c
+   fts3_tokenizer.c
+   fts3_tokenizer1.c
+} {
+  copy_file tsrc/$file
+}
+
+close $out

ext/icu/README.txt

@@ -0,0 +1,170 @@
+
+This directory contains source code for the SQLite "ICU" extension, an
+integration of the "International Components for Unicode" library with
+SQLite. Documentation follows.
+
+    1. Features
+    
+        1.1  SQL Scalars upper() and lower()
+        1.2  Unicode Aware LIKE Operator
+        1.3  ICU Collation Sequences
+        1.4  SQL REGEXP Operator
+    
+    2. Compilation and Usage
+    
+    3. Bugs, Problems and Security Issues
+    
+        3.1  The "case_sensitive_like" Pragma
+        3.2  The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro
+        3.3  Collation Sequence Security Issue
+
+
+1. FEATURES
+
+  1.1  SQL Scalars upper() and lower()
+
+    SQLite's built-in implementations of these two functions only 
+    provide case mapping for the 26 letters used in the English
+    language. The ICU based functions provided by this extension
+    provide case mapping, where defined, for the full range of 
+    unicode characters.
+
+    ICU provides two types of case mapping, "general" case mapping and
+    "language specific". Refer to ICU documentation for the differences
+    between the two. Specifically:
+
+       http://www.icu-project.org/userguide/caseMappings.html
+       http://www.icu-project.org/userguide/posix.html#case_mappings
+
+    To utilise "general" case mapping, the upper() or lower() scalar 
+    functions are invoked with one argument:
+
+        upper('ABC') -> 'abc'
+        lower('abc') -> 'ABC'
+
+    To access ICU "language specific" case mapping, upper() or lower()
+    should be invoked with two arguments. The second argument is the name
+    of the locale to use. Passing an empty string ("") or SQL NULL value
+    as the second argument is the same as invoking the 1 argument version
+    of upper() or lower():
+
+        lower('I', 'en_us') -> 'i'
+        lower('I', 'tr_tr') -> 'ı' (small dotless i)
+
+  1.2  Unicode Aware LIKE Operator
+
+    Similarly to the upper() and lower() functions, the built-in SQLite LIKE
+    operator understands case equivalence for the 26 letters of the English
+    language alphabet. The implementation of LIKE included in this
+    extension uses the ICU function u_foldCase() to provide case
+    independent comparisons for the full range of unicode characters.  
+
+    The U_FOLD_CASE_DEFAULT flag is passed to u_foldCase(), meaning the
+    dotless 'I' character used in the Turkish language is considered
+    to be in the same equivalence class as the dotted 'I' character
+    used by many languages (including English).
+
+  1.3  ICU Collation Sequences
+
+    A special SQL scalar function, icu_load_collation() is provided that 
+    may be used to register ICU collation sequences with SQLite. It
+    is always called with exactly two arguments, the ICU locale 
+    identifying the collation sequence to ICU, and the name of the
+    SQLite collation sequence to create. For example, to create an
+    SQLite collation sequence named "turkish" using Turkish language
+    sorting rules, the SQL statement:
+
+        SELECT icu_load_collation('tr_TR', 'turkish');
+
+    Or, for Australian English:
+
+        SELECT icu_load_collation('en_AU', 'australian');
+
+    The identifiers "turkish" and "australian" may then be used
+    as collation sequence identifiers in SQL statements:
+
+        CREATE TABLE aust_turkish_penpals(
+          australian_penpal_name TEXT COLLATE australian,
+          turkish_penpal_name    TEXT COLLATE turkish
+        );
+  
+  1.4 SQL REGEXP Operator
+
+    This extension provides an implementation of the SQL binary
+    comparision operator "REGEXP", based on the regular expression functions
+    provided by the ICU library. The syntax of the operator is as described
+    in SQLite documentation:
+
+        <string> REGEXP <re-pattern>
+
+    This extension uses the ICU defaults for regular expression matching
+    behaviour. Specifically, this means that:
+
+        * Matching is case-sensitive,
+        * Regular expression comments are not allowed within patterns, and
+        * The '^' and '$' characters match the beginning and end of the
+          <string> argument, not the beginning and end of lines within
+          the <string> argument.
+
+    Even more specifically, the value passed to the "flags" parameter
+    of ICU C function uregex_open() is 0.
+
+
+2  COMPILATION AND USAGE
+
+  The easiest way to compile and use the ICU extension is to build
+  and use it as a dynamically loadable SQLite extension. To do this
+  using gcc on *nix:
+
+    gcc -shared icu.c `icu-config --ldflags` -o libSqliteIcu.so
+
+  You may need to add "-I" flags so that gcc can find sqlite3ext.h
+  and sqlite3.h. The resulting shared lib, libSqliteIcu.so, may be
+  loaded into sqlite in the same way as any other dynamically loadable
+  extension.
+
+
+3 BUGS, PROBLEMS AND SECURITY ISSUES
+
+  3.1 The "case_sensitive_like" Pragma
+
+    This extension does not work well with the "case_sensitive_like"
+    pragma. If this pragma is used before the ICU extension is loaded,
+    then the pragma has no effect. If the pragma is used after the ICU
+    extension is loaded, then SQLite ignores the ICU implementation and
+    always uses the built-in LIKE operator.
+
+    The ICU extension LIKE operator is always case insensitive.
+
+  3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro
+
+    Passing very long patterns to the built-in SQLite LIKE operator can
+    cause a stack overflow. To curb this problem, SQLite defines the
+    SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a
+    pattern in bytes (irrespective of encoding). The default value is
+    defined in internal header file "limits.h".
+    
+    The ICU extension LIKE implementation suffers from the same 
+    problem and uses the same solution. However, since the ICU extension
+    code does not include the SQLite file "limits.h", modifying
+    the default value therein does not affect the ICU extension.
+    The default value of SQLITE_MAX_LIKE_PATTERN_LENGTH used by
+    the ICU extension LIKE operator is 50000, defined in source 
+    file "icu.c".
+
+  3.3 Collation Sequence Security Issue
+
+    Internally, SQLite assumes that indices stored in database files
+    are sorted according to the collation sequence indicated by the
+    SQL schema. Changing the definition of a collation sequence after
+    an index has been built is therefore equivalent to database
+    corruption. The SQLite library is not very well tested under
+    these conditions, and may contain potential buffer overruns
+    or other programming errors that could be exploited by a malicious
+    programmer.
+
+    If the ICU extension is used in an environment where potentially
+    malicious users may execute arbitrary SQL (i.e. gears), they
+    should be prevented from invoking the icu_load_collation() function,
+    possibly using the authorisation callback.
+

ext/icu/icu.c

@@ -0,0 +1,499 @@
+/*
+** 2007 May 6
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $
+**
+** This file implements an integration between the ICU library 
+** ("International Components for Unicode", an open-source library 
+** for handling unicode data) and SQLite. The integration uses 
+** ICU to provide the following to SQLite:
+**
+**   * An implementation of the SQL regexp() function (and hence REGEXP
+**     operator) using the ICU uregex_XX() APIs.
+**
+**   * Implementations of the SQL scalar upper() and lower() functions
+**     for case mapping.
+**
+**   * Integration of ICU and SQLite collation seqences.
+**
+**   * An implementation of the LIKE operator that uses ICU to 
+**     provide case-independent matching.
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+
+/* Include ICU headers */
+#include <unicode/utypes.h>
+#include <unicode/uregex.h>
+#include <unicode/ustring.h>
+#include <unicode/ucol.h>
+
+#include <assert.h>
+
+#ifndef SQLITE_CORE
+  #include "sqlite3ext.h"
+  SQLITE_EXTENSION_INIT1
+#else
+  #include "sqlite3.h"
+#endif
+
+/*
+** Maximum length (in bytes) of the pattern in a LIKE or GLOB
+** operator.
+*/
+#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
+# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000
+#endif
+
+/*
+** Version of sqlite3_free() that is always a function, never a macro.
+*/
+static void xFree(void *p){
+  sqlite3_free(p);
+}
+
+/*
+** Compare two UTF-8 strings for equality where the first string is
+** a "LIKE" expression. Return true (1) if they are the same and 
+** false (0) if they are different.
+*/
+static int icuLikeCompare(
+  const uint8_t *zPattern,   /* LIKE pattern */
+  const uint8_t *zString,    /* The UTF-8 string to compare against */
+  const UChar32 uEsc         /* The escape character */
+){
+  static const int MATCH_ONE = (UChar32)'_';
+  static const int MATCH_ALL = (UChar32)'%';
+
+  int iPattern = 0;       /* Current byte index in zPattern */
+  int iString = 0;        /* Current byte index in zString */
+
+  int prevEscape = 0;     /* True if the previous character was uEsc */
+
+  while( zPattern[iPattern]!=0 ){
+
+    /* Read (and consume) the next character from the input pattern. */
+    UChar32 uPattern;
+    U8_NEXT_UNSAFE(zPattern, iPattern, uPattern);
+    assert(uPattern!=0);
+
+    /* There are now 4 possibilities:
+    **
+    **     1. uPattern is an unescaped match-all character "%",
+    **     2. uPattern is an unescaped match-one character "_",
+    **     3. uPattern is an unescaped escape character, or
+    **     4. uPattern is to be handled as an ordinary character
+    */
+    if( !prevEscape && uPattern==MATCH_ALL ){
+      /* Case 1. */
+      uint8_t c;
+
+      /* Skip any MATCH_ALL or MATCH_ONE characters that follow a
+      ** MATCH_ALL. For each MATCH_ONE, skip one character in the 
+      ** test string.
+      */
+      while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){
+        if( c==MATCH_ONE ){
+          if( zString[iString]==0 ) return 0;
+          U8_FWD_1_UNSAFE(zString, iString);
+        }
+        iPattern++;
+      }
+
+      if( zPattern[iPattern]==0 ) return 1;
+
+      while( zString[iString] ){
+        if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){
+          return 1;
+        }
+        U8_FWD_1_UNSAFE(zString, iString);
+      }
+      return 0;
+
+    }else if( !prevEscape && uPattern==MATCH_ONE ){
+      /* Case 2. */
+      if( zString[iString]==0 ) return 0;
+      U8_FWD_1_UNSAFE(zString, iString);
+
+    }else if( !prevEscape && uPattern==uEsc){
+      /* Case 3. */
+      prevEscape = 1;
+
+    }else{
+      /* Case 4. */
+      UChar32 uString;
+      U8_NEXT_UNSAFE(zString, iString, uString);
+      uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT);
+      uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT);
+      if( uString!=uPattern ){
+        return 0;
+      }
+      prevEscape = 0;
+    }
+  }
+
+  return zString[iString]==0;
+}
+
+/*
+** Implementation of the like() SQL function.  This function implements
+** the build-in LIKE operator.  The first argument to the function is the
+** pattern and the second argument is the string.  So, the SQL statements:
+**
+**       A LIKE B
+**
+** is implemented as like(B, A). If there is an escape character E, 
+**
+**       A LIKE B ESCAPE E
+**
+** is mapped to like(B, A, E).
+*/
+static void icuLikeFunc(
+  sqlite3_context *context, 
+  int argc, 
+  sqlite3_value **argv
+){
+  const unsigned char *zA = sqlite3_value_text(argv[0]);
+  const unsigned char *zB = sqlite3_value_text(argv[1]);
+  UChar32 uEsc = 0;
+
+  /* Limit the length of the LIKE or GLOB pattern to avoid problems
+  ** of deep recursion and N*N behavior in patternCompare().
+  */
+  if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){
+    sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1);
+    return;
+  }
+
+
+  if( argc==3 ){
+    /* The escape character string must consist of a single UTF-8 character.
+    ** Otherwise, return an error.
+    */
+    int nE= sqlite3_value_bytes(argv[2]);
+    const unsigned char *zE = sqlite3_value_text(argv[2]);
+    int i = 0;
+    if( zE==0 ) return;
+    U8_NEXT(zE, i, nE, uEsc);
+    if( i!=nE){
+      sqlite3_result_error(context, 
+          "ESCAPE expression must be a single character", -1);
+      return;
+    }
+  }
+
+  if( zA && zB ){
+    sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
+  }
+}
+
+/*
+** This function is called when an ICU function called from within
+** the implementation of an SQL scalar function returns an error.
+**
+** The scalar function context passed as the first argument is 
+** loaded with an error message based on the following two args.
+*/
+static void icuFunctionError(
+  sqlite3_context *pCtx,       /* SQLite scalar function context */
+  const char *zName,           /* Name of ICU function that failed */
+  UErrorCode e                 /* Error code returned by ICU function */
+){
+  char zBuf[128];
+  sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
+  zBuf[127] = '\0';
+  sqlite3_result_error(pCtx, zBuf, -1);
+}
+
+/*
+** Function to delete compiled regexp objects. Registered as
+** a destructor function with sqlite3_set_auxdata().
+*/
+static void icuRegexpDelete(void *p){
+  URegularExpression *pExpr = (URegularExpression *)p;
+  uregex_close(pExpr);
+}
+
+/*
+** Implementation of SQLite REGEXP operator. This scalar function takes
+** two arguments. The first is a regular expression pattern to compile
+** the second is a string to match against that pattern. If either 
+** argument is an SQL NULL, then NULL Is returned. Otherwise, the result
+** is 1 if the string matches the pattern, or 0 otherwise.
+**
+** SQLite maps the regexp() function to the regexp() operator such
+** that the following two are equivalent:
+**
+**     zString REGEXP zPattern
+**     regexp(zPattern, zString)
+**
+** Uses the following ICU regexp APIs:
+**
+**     uregex_open()
+**     uregex_matches()
+**     uregex_close()
+*/
+static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){
+  UErrorCode status = U_ZERO_ERROR;
+  URegularExpression *pExpr;
+  UBool res;
+  const UChar *zString = sqlite3_value_text16(apArg[1]);
+
+  /* If the left hand side of the regexp operator is NULL, 
+  ** then the result is also NULL. 
+  */
+  if( !zString ){
+    return;
+  }
+
+  pExpr = sqlite3_get_auxdata(p, 0);
+  if( !pExpr ){
+    const UChar *zPattern = sqlite3_value_text16(apArg[0]);
+    if( !zPattern ){
+      return;
+    }
+    pExpr = uregex_open(zPattern, -1, 0, 0, &status);
+
+    if( U_SUCCESS(status) ){
+      sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete);
+    }else{
+      assert(!pExpr);
+      icuFunctionError(p, "uregex_open", status);
+      return;
+    }
+  }
+
+  /* Configure the text that the regular expression operates on. */
+  uregex_setText(pExpr, zString, -1, &status);
+  if( !U_SUCCESS(status) ){
+    icuFunctionError(p, "uregex_setText", status);
+    return;
+  }
+
+  /* Attempt the match */
+  res = uregex_matches(pExpr, 0, &status);
+  if( !U_SUCCESS(status) ){
+    icuFunctionError(p, "uregex_matches", status);
+    return;
+  }
+
+  /* Set the text that the regular expression operates on to a NULL
+  ** pointer. This is not really necessary, but it is tidier than 
+  ** leaving the regular expression object configured with an invalid
+  ** pointer after this function returns.
+  */
+  uregex_setText(pExpr, 0, 0, &status);
+
+  /* Return 1 or 0. */
+  sqlite3_result_int(p, res ? 1 : 0);
+}
+
+/*
+** Implementations of scalar functions for case mapping - upper() and 
+** lower(). Function upper() converts its input to upper-case (ABC).
+** Function lower() converts to lower-case (abc).
+**
+** ICU provides two types of case mapping, "general" case mapping and
+** "language specific". Refer to ICU documentation for the differences
+** between the two.
+**
+** To utilise "general" case mapping, the upper() or lower() scalar 
+** functions are invoked with one argument:
+**
+**     upper('ABC') -> 'abc'
+**     lower('abc') -> 'ABC'
+**
+** To access ICU "language specific" case mapping, upper() or lower()
+** should be invoked with two arguments. The second argument is the name
+** of the locale to use. Passing an empty string ("") or SQL NULL value
+** as the second argument is the same as invoking the 1 argument version
+** of upper() or lower().
+**
+**     lower('I', 'en_us') -> 'i'
+**     lower('I', 'tr_tr') -> 'ı' (small dotless i)
+**
+** http://www.icu-project.org/userguide/posix.html#case_mappings
+*/
+static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){
+  const UChar *zInput;
+  UChar *zOutput;
+  int nInput;
+  int nOutput;
+
+  UErrorCode status = U_ZERO_ERROR;
+  const char *zLocale = 0;
+
+  assert(nArg==1 || nArg==2);
+  if( nArg==2 ){
+    zLocale = (const char *)sqlite3_value_text(apArg[1]);
+  }
+
+  zInput = sqlite3_value_text16(apArg[0]);
+  if( !zInput ){
+    return;
+  }
+  nInput = sqlite3_value_bytes16(apArg[0]);
+
+  nOutput = nInput * 2 + 2;
+  zOutput = sqlite3_malloc(nOutput);
+  if( !zOutput ){
+    return;
+  }
+
+  if( sqlite3_user_data(p) ){
+    u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status);
+  }else{
+    u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status);
+  }
+
+  if( !U_SUCCESS(status) ){
+    icuFunctionError(p, "u_strToLower()/u_strToUpper", status);
+    return;
+  }
+
+  sqlite3_result_text16(p, zOutput, -1, xFree);
+}
+
+/*
+** Collation sequence destructor function. The pCtx argument points to
+** a UCollator structure previously allocated using ucol_open().
+*/
+static void icuCollationDel(void *pCtx){
+  UCollator *p = (UCollator *)pCtx;
+  ucol_close(p);
+}
+
+/*
+** Collation sequence comparison function. The pCtx argument points to
+** a UCollator structure previously allocated using ucol_open().
+*/
+static int icuCollationColl(
+  void *pCtx,
+  int nLeft,
+  const void *zLeft,
+  int nRight,
+  const void *zRight
+){
+  UCollationResult res;
+  UCollator *p = (UCollator *)pCtx;
+  res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2);
+  switch( res ){
+    case UCOL_LESS:    return -1;
+    case UCOL_GREATER: return +1;
+    case UCOL_EQUAL:   return 0;
+  }
+  assert(!"Unexpected return value from ucol_strcoll()");
+  return 0;
+}
+
+/*
+** Implementation of the scalar function icu_load_collation().
+**
+** This scalar function is used to add ICU collation based collation 
+** types to an SQLite database connection. It is intended to be called
+** as follows:
+**
+**     SELECT icu_load_collation(<locale>, <collation-name>);
+**
+** Where <locale> is a string containing an ICU locale identifier (i.e.
+** "en_AU", "tr_TR" etc.) and <collation-name> is the name of the
+** collation sequence to create.
+*/
+static void icuLoadCollation(
+  sqlite3_context *p, 
+  int nArg, 
+  sqlite3_value **apArg
+){
+  sqlite3 *db = (sqlite3 *)sqlite3_user_data(p);
+  UErrorCode status = U_ZERO_ERROR;
+  const char *zLocale;      /* Locale identifier - (eg. "jp_JP") */
+  const char *zName;        /* SQL Collation sequence name (eg. "japanese") */
+  UCollator *pUCollator;    /* ICU library collation object */
+  int rc;                   /* Return code from sqlite3_create_collation_x() */
+
+  assert(nArg==2);
+  zLocale = (const char *)sqlite3_value_text(apArg[0]);
+  zName = (const char *)sqlite3_value_text(apArg[1]);
+
+  if( !zLocale || !zName ){
+    return;
+  }
+
+  pUCollator = ucol_open(zLocale, &status);
+  if( !U_SUCCESS(status) ){
+    icuFunctionError(p, "ucol_open", status);
+    return;
+  }
+  assert(p);
+
+  rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, 
+      icuCollationColl, icuCollationDel
+  );
+  if( rc!=SQLITE_OK ){
+    ucol_close(pUCollator);
+    sqlite3_result_error(p, "Error registering collation function", -1);
+  }
+}
+
+/*
+** Register the ICU extension functions with database db.
+*/
+int sqlite3IcuInit(sqlite3 *db){
+  struct IcuScalar {
+    const char *zName;                        /* Function name */
+    int nArg;                                 /* Number of arguments */
+    int enc;                                  /* Optimal text encoding */
+    void *pContext;                           /* sqlite3_user_data() context */
+    void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+  } scalars[] = {
+    {"regexp",-1, SQLITE_ANY,          0, icuRegexpFunc},
+
+    {"lower",  1, SQLITE_UTF16,        0, icuCaseFunc16},
+    {"lower",  2, SQLITE_UTF16,        0, icuCaseFunc16},
+    {"upper",  1, SQLITE_UTF16, (void*)1, icuCaseFunc16},
+    {"upper",  2, SQLITE_UTF16, (void*)1, icuCaseFunc16},
+
+    {"lower",  1, SQLITE_UTF8,         0, icuCaseFunc16},
+    {"lower",  2, SQLITE_UTF8,         0, icuCaseFunc16},
+    {"upper",  1, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
+    {"upper",  2, SQLITE_UTF8,  (void*)1, icuCaseFunc16},
+
+    {"like",   2, SQLITE_UTF8,         0, icuLikeFunc},
+    {"like",   3, SQLITE_UTF8,         0, icuLikeFunc},
+
+    {"icu_load_collation",  2, SQLITE_UTF8, (void*)db, icuLoadCollation},
+  };
+
+  int rc = SQLITE_OK;
+  int i;
+
+  for(i=0; rc==SQLITE_OK && i<(sizeof(scalars)/sizeof(struct IcuScalar)); i++){
+    struct IcuScalar *p = &scalars[i];
+    rc = sqlite3_create_function(
+        db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0
+    );
+  }
+
+  return rc;
+}
+
+#if !SQLITE_CORE
+int sqlite3_extension_init(
+  sqlite3 *db, 
+  char **pzErrMsg,
+  const sqlite3_api_routines *pApi
+){
+  SQLITE_EXTENSION_INIT2(pApi)
+  return sqlite3IcuInit(db);
+}
+#endif
+
+#endif

ext/rtree/README

@@ -0,0 +1,120 @@
+
+This directory contains an SQLite extension that implements a virtual 
+table type that allows users to create, query and manipulate r-tree[1] 
+data structures inside of SQLite databases. Users create, populate 
+and query r-tree structures using ordinary SQL statements.
+
+    1.  SQL Interface
+
+        1.1  Table Creation
+        1.2  Data Manipulation
+        1.3  Data Querying
+        1.4  Introspection and Analysis
+
+    2.  Compilation and Deployment
+
+    3.  References
+
+
+1. SQL INTERFACE
+
+  1.1 Table Creation.
+
+    All r-tree virtual tables have an odd number of columns between
+    3 and 11. Unlike regular SQLite tables, r-tree tables are strongly 
+    typed. 
+
+    The leftmost column is always the pimary key and contains 64-bit 
+    integer values. Each subsequent column contains a 32-bit real
+    value. For each pair of real values, the first (leftmost) must be 
+    less than or equal to the second. R-tree tables may be 
+    constructed using the following syntax:
+
+      CREATE VIRTUAL TABLE <name> USING rtree(<column-names>)
+
+    For example:
+
+      CREATE VIRTUAL TABLE boxes USING rtree(boxno, xmin, xmax, ymin, ymax);
+      INSERT INTO boxes VALUES(1, 1.0, 3.0, 2.0, 4.0);
+
+    Constructing a virtual r-tree table <name> creates the following three
+    real tables in the database to store the data structure:
+
+      <name>_node
+      <name>_rowid
+      <name>_parent
+
+    Dropping or modifying the contents of these tables directly will
+    corrupt the r-tree structure. To delete an r-tree from a database,
+    use a regular DROP TABLE statement:
+
+      DROP TABLE <name>;
+
+    Dropping the main r-tree table automatically drops the automatically
+    created tables. 
+
+  1.2 Data Manipulation (INSERT, UPDATE, DELETE).
+
+    The usual INSERT, UPDATE or DELETE syntax is used to manipulate data
+    stored in an r-tree table. Please note the following:
+
+      * Inserting a NULL value into the primary key column has the
+        same effect as inserting a NULL into an INTEGER PRIMARY KEY
+        column of a regular table. The system automatically assigns
+        an unused integer key value to the new record. Usually, this
+        is one greater than the largest primary key value currently
+        present in the table.
+
+      * Attempting to insert a duplicate primary key value fails with
+        an SQLITE_CONSTRAINT error.
+
+      * Attempting to insert or modify a record such that the value
+        stored in the (N*2)th column is greater than that stored in
+        the (N*2+1)th column fails with an SQLITE_CONSTRAINT error.
+
+      * When a record is inserted, values are always converted to 
+        the required type (64-bit integer or 32-bit real) as if they
+        were part of an SQL CAST expression. Non-numeric strings are
+        converted to zero.
+
+  1.3 Queries.
+
+    R-tree tables may be queried using all of the same SQL syntax supported
+    by regular tables. However, some query patterns are more efficient
+    than others.
+
+    R-trees support fast lookup by primary key value (O(logN), like 
+    regular tables).
+
+    Any combination of equality and range (<, <=, >, >=) constraints
+    on spatial data columns may be used to optimize other queries. This
+    is the key advantage to using r-tree tables instead of creating 
+    indices on regular tables.
+
+  1.4 Introspection and Analysis.
+
+    TODO: Describe rtreenode() and rtreedepth() functions.
+
+
+2. COMPILATION AND USAGE
+
+  The easiest way to compile and use the RTREE extension is to build
+  and use it as a dynamically loadable SQLite extension. To do this
+  using gcc on *nix:
+
+    gcc -shared rtree.c -o libSqliteRtree.so
+
+  You may need to add "-I" flags so that gcc can find sqlite3ext.h
+  and sqlite3.h. The resulting shared lib, libSqliteRtree.so, may be
+  loaded into sqlite in the same way as any other dynamicly loadable
+  extension.
+
+
+3. REFERENCES
+
+  [1]  Atonin Guttman, "R-trees - A Dynamic Index Structure For Spatial 
+       Searching", University of California Berkeley, 1984.
+
+  [2]  Norbert Beckmann, Hans-Peter Kriegel, Ralf Schneider, Bernhard Seeger,
+       "The R*-tree: An Efficient and Robust Access Method for Points and
+       Rectangles", Universitaet Bremen, 1990.

ext/rtree/rtree.h

@@ -0,0 +1,26 @@
+/*
+** 2008 May 26
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** RTREE library.  All it does is declare the sqlite3RtreeInit() interface.
+*/
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3RtreeInit(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */

ext/rtree/rtree1.test

@@ -0,0 +1,364 @@
+# 2008 Feb 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the r-tree extension.
+#
+# $Id: rtree1.test,v 1.5 2008/07/14 15:37:01 danielk1977 Exp $
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname $argv0] .. .. test]
+}
+source [file join [file dirname [info script]] rtree_util.tcl]
+source $testdir/tester.tcl
+
+# Test plan:
+#
+#   rtree-1.*: Creating/destroying r-tree tables.
+#   rtree-2.*: Test the implicit constraints - unique rowid and
+#              (coord[N]<=coord[N+1]) for even values of N. Also
+#              automatic assigning of rowid values.
+#   rtree-3.*: Linear scans of r-tree data.
+#   rtree-4.*: Test INSERT
+#   rtree-5.*: Test DELETE
+#   rtree-6.*: Test UPDATE
+#   rtree-7.*: Test renaming an r-tree table.
+#   rtree-8.*: Test constrained scans of r-tree data.
+#
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-1.* test CREATE and DROP table statements.
+#
+
+# Test creating and dropping an rtree table.
+#
+do_test rtree-1.1.1 {
+  execsql { CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2) }
+} {}
+do_test rtree-1.1.2 {
+  execsql { SELECT name FROM sqlite_master ORDER BY name }
+} {t1 t1_node t1_parent t1_rowid}
+do_test rtree-1.1.3 {
+  execsql { 
+    DROP TABLE t1; 
+    SELECT name FROM sqlite_master ORDER BY name;
+  }
+} {}
+
+# Test creating and dropping an rtree table with an odd name in
+# an attached database.
+#
+do_test rtree-1.2.1 {
+  file delete -force test2.db
+  execsql {
+    ATTACH 'test2.db' AS aux;
+    CREATE VIRTUAL TABLE aux.'a" "b' USING rtree(ii, x1, x2, y1, y2);
+  }
+} {}
+do_test rtree-1.2.2 {
+  execsql { SELECT name FROM sqlite_master ORDER BY name }
+} {}
+do_test rtree-1.2.3 {
+  execsql { SELECT name FROM aux.sqlite_master ORDER BY name }
+} {{a" "b} {a" "b_node} {a" "b_parent} {a" "b_rowid}}
+do_test rtree-1.2.4 {
+  execsql { 
+    DROP TABLE aux.'a" "b'; 
+    SELECT name FROM aux.sqlite_master ORDER BY name;
+  }
+} {}
+
+# Test that the logic for checking the number of columns specified
+# for an rtree table. Acceptable values are odd numbers between 3 and
+# 11, inclusive.
+#
+set cols [list i1 i2 i3 i4 i5 i6 i7 i8 i9 iA iB iC iD iE iF iG iH iI iJ iK]
+for {set nCol 1} {$nCol<[llength $cols]} {incr nCol} {
+
+  set columns [join [lrange $cols 0 [expr {$nCol-1}]] ,]
+
+  set X {0 {}}
+  if {$nCol%2 == 0}  { set X {1 {Wrong number of columns for an rtree table}} }
+  if {$nCol < 3}     { set X {1 {Too few columns for an rtree table}} }
+  if {$nCol > 11}    { set X {1 {Too many columns for an rtree table}} }
+
+  do_test rtree-1.3.$nCol {
+    catchsql " 
+      CREATE VIRTUAL TABLE t1 USING rtree($columns);
+    "
+  } $X
+
+  catchsql { DROP TABLE t1 }
+}
+
+# Test that it is possible to open an existing database that contains
+# r-tree tables.
+#
+do_test rtree-1.4.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2);
+    INSERT INTO t1 VALUES(1, 5.0, 10.0);
+    INSERT INTO t1 VALUES(2, 15.0, 20.0);
+  }
+} {}
+do_test rtree-1.4.2 {
+  db close
+  sqlite3 db test.db
+  execsql { SELECT * FROM t1 ORDER BY ii }
+} {1 5.0 10.0 2 15.0 20.0}
+do_test rtree-1.4.3 {
+  execsql { DROP TABLE t1 }
+} {}
+
+# Test that it is possible to create an r-tree table with ridiculous
+# column names.
+#
+do_test rtree-1.5.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim");
+    INSERT INTO t1 VALUES(1, 2, 3);
+    SELECT "the key", "x dim.", "x2'dim" FROM t1;
+  }
+} {1 2.0 3.0}
+do_test rtree-1.5.1 {
+  execsql { DROP TABLE t1 }
+} {}
+
+# Force the r-tree constructor to fail.
+#
+do_test rtree-1.6.1 {
+  execsql { CREATE TABLE t1_rowid(a); }
+  catchsql {
+    CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim");
+  }
+} {1 {table "t1_rowid" already exists}}
+do_test rtree-1.6.1 {
+  execsql { DROP TABLE t1_rowid }
+} {}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-2.* 
+#
+do_test rtree-2.1.1 {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2);
+    SELECT * FROM t1;
+  }
+} {}
+
+do_test rtree-2.1.2 {
+  execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
+  execsql { SELECT * FROM t1 }
+} {1 1.0 3.0 2.0 4.0}
+do_test rtree-2.1.3 {
+  execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
+  execsql { SELECT rowid FROM t1 ORDER BY rowid }
+} {1 2}
+do_test rtree-2.1.3 {
+  execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
+  execsql { SELECT ii FROM t1 ORDER BY ii }
+} {1 2 3}
+
+do_test rtree-2.2.1 {
+  catchsql { INSERT INTO t1 VALUES(2, 1, 3, 2, 4) }
+} {1 {constraint failed}}
+do_test rtree-2.2.2 {
+  catchsql { INSERT INTO t1 VALUES(4, 1, 3, 4, 2) }
+} {1 {constraint failed}}
+do_test rtree-2.2.3 {
+  catchsql { INSERT INTO t1 VALUES(4, 3, 1, 2, 4) }
+} {1 {constraint failed}}
+do_test rtree-2.2.4 {
+  execsql { SELECT ii FROM t1 ORDER BY ii }
+} {1 2 3}
+
+do_test rtree-2.X {
+  execsql { DROP TABLE t1 }
+} {}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-3.* test linear scans of r-tree table data. To test
+# this we have to insert some data into an r-tree, but that is not the
+# focus of these tests.
+#
+do_test rtree-3.1.1 {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2, y1, y2);
+    SELECT * FROM t1;
+  }
+} {}
+do_test rtree-3.1.2 {
+  execsql { 
+    INSERT INTO t1 VALUES(5, 1, 3, 2, 4);
+    SELECT * FROM t1;
+  }
+} {5 1.0 3.0 2.0 4.0}
+do_test rtree-3.1.3 {
+  execsql {
+    INSERT INTO t1 VALUES(6, 2, 6, 4, 8);
+    SELECT * FROM t1;
+  }
+} {5 1.0 3.0 2.0 4.0 6 2.0 6.0 4.0 8.0}
+
+# Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)):
+do_test rtree-3.2.1 {
+  catchsql { INSERT INTO t1 VALUES(7, 2, 6, 4, 3) }
+} {1 {constraint failed}}
+do_test rtree-3.2.2 {
+  catchsql { INSERT INTO t1 VALUES(8, 2, 6, 3, 3) }
+} {0 {}}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-5.* test DELETE operations.
+#
+do_test rtree-5.1.1 {
+  execsql { CREATE VIRTUAL TABLE t2 USING rtree(ii, x1, x2) }
+} {}
+do_test rtree-5.1.2 {
+  execsql { 
+    INSERT INTO t2 VALUES(1, 10, 20);
+    INSERT INTO t2 VALUES(2, 30, 40);
+    INSERT INTO t2 VALUES(3, 50, 60);
+    SELECT * FROM t2 ORDER BY ii;
+  }
+} {1 10.0 20.0 2 30.0 40.0 3 50.0 60.0}
+do_test rtree-5.1.3 {
+  execsql { 
+    DELETE FROM t2 WHERE ii=2;
+    SELECT * FROM t2 ORDER BY ii;
+  }
+} {1 10.0 20.0 3 50.0 60.0}
+do_test rtree-5.1.4 {
+  execsql { 
+    DELETE FROM t2 WHERE ii=1;
+    SELECT * FROM t2 ORDER BY ii;
+  }
+} {3 50.0 60.0}
+do_test rtree-5.1.5 {
+  execsql { 
+    DELETE FROM t2 WHERE ii=3;
+    SELECT * FROM t2 ORDER BY ii;
+  }
+} {}
+do_test rtree-5.1.6 {
+  execsql { SELECT * FROM t2_rowid }
+} {}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-5.* test UPDATE operations.
+#
+do_test rtree-6.1.1 {
+  execsql { CREATE VIRTUAL TABLE t3 USING rtree(ii, x1, x2, y1, y2) }
+} {}
+do_test rtree-6.1.2 {
+  execsql {
+    INSERT INTO t3 VALUES(1, 2, 3, 4, 5);
+    UPDATE t3 SET x2=5;
+    SELECT * FROM t3;
+  }
+} {1 2.0 5.0 4.0 5.0}
+do_test rtree-6.1.3 {
+  execsql { UPDATE t3 SET ii = 2 }
+  execsql { SELECT * FROM t3 }
+} {2 2.0 5.0 4.0 5.0}
+
+#----------------------------------------------------------------------------
+# Test cases rtree-7.* test rename operations.
+#
+do_test rtree-7.1.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t4 USING rtree(ii, x1, x2, y1, y2, z1, z2);
+    INSERT INTO t4 VALUES(1, 2, 3, 4, 5, 6, 7);
+  }
+} {}
+do_test rtree-7.1.2 {
+  execsql { ALTER TABLE t4 RENAME TO t5 }
+  execsql { SELECT * FROM t5 }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.1.3 {
+  db close
+  sqlite3 db test.db
+  execsql { SELECT * FROM t5 }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.1.4 {
+  execsql { ALTER TABLE t5 RENAME TO 'raisara "one"'''}
+  execsql { SELECT * FROM "raisara ""one""'" }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.1.5 {
+  execsql { SELECT * FROM 'raisara "one"''' }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.1.6 {
+  execsql { ALTER TABLE "raisara ""one""'" RENAME TO "abc 123" }
+  execsql { SELECT * FROM "abc 123" }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.1.7 {
+  db close
+  sqlite3 db test.db
+  execsql { SELECT * FROM "abc 123" }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+
+# An error midway through a rename operation.
+do_test rtree-7.2.1 {
+  execsql { 
+    CREATE TABLE t4_node(a);
+  }
+  catchsql { ALTER TABLE "abc 123" RENAME TO t4 }
+} {1 {SQL logic error or missing database}}
+do_test rtree-7.2.2 {
+  execsql { SELECT * FROM "abc 123" }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.2.3 {
+  execsql { 
+    DROP TABLE t4_node;
+    CREATE TABLE t4_rowid(a);
+  }
+  catchsql { ALTER TABLE "abc 123" RENAME TO t4 }
+} {1 {SQL logic error or missing database}}
+do_test rtree-7.2.4 {
+  db close
+  sqlite3 db test.db
+  execsql { SELECT * FROM "abc 123" }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+do_test rtree-7.2.5 {
+  execsql { DROP TABLE t4_rowid }
+  execsql { ALTER TABLE "abc 123" RENAME TO t4 }
+  execsql { SELECT * FROM t4 }
+} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+
+
+#----------------------------------------------------------------------------
+# Test cases rtree-8.*
+#
+
+# Test that the function to determine if a leaf cell is part of the
+# result set works.
+do_test rtree-8.1.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t6 USING rtree(ii, x1, x2);
+    INSERT INTO t6 VALUES(1, 3, 7);
+    INSERT INTO t6 VALUES(2, 4, 6);
+  }
+} {}
+do_test rtree-8.1.2 { execsql { SELECT ii FROM t6 WHERE x1>2 } } {1 2}
+do_test rtree-8.1.3 { execsql { SELECT ii FROM t6 WHERE x1>3 } } {2}
+do_test rtree-8.1.4 { execsql { SELECT ii FROM t6 WHERE x1>4 } } {}
+do_test rtree-8.1.5 { execsql { SELECT ii FROM t6 WHERE x1>5 } } {}
+do_test rtree-8.1.6 { execsql { SELECT ii FROM t6 WHERE x1<3 } } {}
+do_test rtree-8.1.7 { execsql { SELECT ii FROM t6 WHERE x1<4 } } {1}
+do_test rtree-8.1.8 { execsql { SELECT ii FROM t6 WHERE x1<5 } } {1 2}
+
+
+finish_test

ext/rtree/rtree2.test

@@ -0,0 +1,152 @@
+# 2008 Feb 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the r-tree extension.
+#
+# $Id: rtree2.test,v 1.4 2008/07/14 15:37:01 danielk1977 Exp $
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname $argv0] .. .. test]
+} 
+source [file join [file dirname [info script]] rtree_util.tcl]
+source $testdir/tester.tcl
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+set ::NROW   1000
+set ::NDEL     10
+set ::NSELECT 100
+
+if {[info exists ISQUICK] && $ISQUICK} {
+  set ::NROW 100
+  set ::NSELECT 10
+}
+
+foreach module {rtree_i32 rtree} {
+  for {set nDim 1} {$nDim <= 5} {incr nDim} {
+  
+    do_test rtree2-$module.$nDim.1 {
+      set cols [list]
+      foreach c [list c0 c1 c2 c3 c4 c5 c6 c7 c8 c9] {
+        lappend cols "$c REAL"
+      }
+      set cols [join [lrange $cols 0 [expr {$nDim*2-1}]] ", "]
+      execsql " 
+        CREATE VIRTUAL TABLE t1 USING ${module}(ii, $cols);
+        CREATE TABLE t2 (ii, $cols);
+      "
+    } {}
+  
+    do_test rtree2-$module.$nDim.2 {
+      db transaction {
+        for {set ii 0} {$ii < $::NROW} {incr ii} {
+          #puts "Row $ii"
+          set values [list]
+          for {set jj 0} {$jj<$nDim*2} {incr jj} {
+            lappend values [expr int(rand()*1000)]
+          }
+          set values [join $values ,]
+          #puts [rtree_treedump db t1]
+          #puts "INSERT INTO t2 VALUES($ii, $values)"
+          set rc [catch {db eval "INSERT INTO t1 VALUES($ii, $values)"}]
+          if {$rc} {
+            incr ii -1
+          } else {
+            db eval "INSERT INTO t2 VALUES($ii, $values)"
+          }
+          #if {[rtree_check db t1]} {
+            #puts [rtree_treedump db t1]
+            #exit
+          #}
+        }
+      }
+  
+      set t1 [execsql {SELECT * FROM t1 ORDER BY ii}]
+      set t2 [execsql {SELECT * FROM t2 ORDER BY ii}]
+      set rc [expr {$t1 eq $t2}]
+      if {$rc != 1} {
+        puts $t1
+        puts $t2
+      }
+      set rc
+    } {1}
+  
+    do_test rtree2-$module.$nDim.3 {
+      rtree_check db t1
+    } 0
+  
+    set OPS [list < > <= >= =]
+    for {set ii 0} {$ii < $::NSELECT} {incr ii} {
+      do_test rtree2-$module.$nDim.4.$ii.1 {
+        set where [list]
+        foreach look_three_dots! {. . .} {
+          set colidx [expr int(rand()*($nDim*2+1))-1]
+          if {$colidx<0} {
+            set col ii
+          } else {
+            set col "c$colidx"
+          }
+          set op  [lindex $OPS [expr int(rand()*[llength $OPS])]]
+          set val [expr int(rand()*1000)]
+          lappend where "$col $op $val"
+        }
+        set where [join $where " AND "]
+  
+        set t1 [execsql "SELECT * FROM t1 WHERE $where ORDER BY ii"]
+        set t2 [execsql "SELECT * FROM t2 WHERE $where ORDER BY ii"]
+        set rc [expr {$t1 eq $t2}]
+        if {$rc != 1} {
+          #puts $where
+          puts $t1
+          puts $t2
+          #puts [rtree_treedump db t1]
+          #breakpoint
+          #set t1 [execsql "SELECT * FROM t1 WHERE $where ORDER BY ii"]
+          #exit
+        }
+        set rc
+      } {1}
+    }
+  
+    for {set ii 0} {$ii < $::NROW} {incr ii $::NDEL} {
+      #puts [rtree_treedump db t1]
+      do_test rtree2-$module.$nDim.5.$ii.1 {
+        execsql "DELETE FROM t2 WHERE ii <= $::ii"
+        execsql "DELETE FROM t1 WHERE ii <= $::ii"
+  
+        set t1 [execsql {SELECT * FROM t1 ORDER BY ii}]
+        set t2 [execsql {SELECT * FROM t2 ORDER BY ii}]
+        set rc [expr {$t1 eq $t2}]
+        if {$rc != 1} {
+          puts $t1
+          puts $t2
+        }
+        set rc
+      } {1}
+      do_test rtree2-$module.$nDim.5.$ii.2 {
+        rtree_check db t1
+      } {0}
+    }
+  
+    do_test rtree2-$module.$nDim.6 {
+      execsql {
+        DROP TABLE t1;
+        DROP TABLE t2;
+      }
+    } {}
+  }
+}
+
+finish_test

ext/rtree/rtree3.test

@@ -0,0 +1,74 @@
+# 2008 Feb 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing that the r-tree correctly handles
+# out-of-memory conditions.
+#
+# $Id: rtree3.test,v 1.2 2008/06/23 15:55:52 danielk1977 Exp $
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname $argv0] .. .. test]
+} 
+source $testdir/tester.tcl
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+# Only run these tests if memory debugging is turned on.
+#
+source $testdir/malloc_common.tcl
+if {!$MEMDEBUG} {
+   puts "Skipping malloc tests: not compiled with -DSQLITE_MEMDEBUG..."
+   finish_test
+   return
+}
+
+do_malloc_test rtree3-1 -sqlbody {
+  BEGIN TRANSACTION;
+  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+  INSERT INTO rt VALUES(NULL, 13, 15, 17, 19);
+  DELETE FROM rt WHERE ii = 1;
+  SELECT * FROM rt;
+  SELECT ii FROM rt WHERE ii = 2;
+  COMMIT;
+} 
+do_malloc_test rtree3-2 -sqlprep {
+  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+} -sqlbody {
+  DROP TABLE rt;
+} 
+
+
+do_malloc_test rtree3-3 -sqlprep {
+  CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2);
+  INSERT INTO rt VALUES(NULL, 3, 5, 7, 9);
+} -tclbody {
+  db eval BEGIN
+  for {set ii 0} {$ii < 100} {incr ii} {
+    set f [expr rand()]
+    db eval {INSERT INTO rt VALUES(NULL, $f*10.0, $f*10.0, $f*15.0, $f*15.0)}
+  }
+  db eval COMMIT
+  db eval BEGIN
+  for {set ii 0} {$ii < 100} {incr ii} {
+    set f [expr rand()]
+    db eval { DELETE FROM rt WHERE x1<($f*10.0) AND x1>($f*10.5) }
+  }
+  db eval COMMIT
+} 
+
+finish_test
+

ext/rtree/rtree4.test

@@ -0,0 +1,236 @@
+# 2008 May 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Randomized test cases for the rtree extension.
+#
+# $Id: rtree4.test,v 1.3 2008/06/23 15:55:52 danielk1977 Exp $
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname $argv0] .. .. test]
+} 
+source $testdir/tester.tcl
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+set ::NROW 2500
+if {[info exists ISQUICK] && $ISQUICK} {
+  set ::NROW 250
+}
+
+# Return a floating point number between -X and X.
+# 
+proc rand {X} {
+  return [expr {int((rand()-0.5)*1024.0*$X)/512.0}]
+}
+
+# Return a positive floating point number less than or equal to X
+#
+proc randincr {X} {
+  while 1 {
+    set r [expr {int(rand()*$X*32.0)/32.0}]
+    if {$r>0.0} {return $r}
+  }
+}
+
+# Scramble the $inlist into a random order.
+#
+proc scramble {inlist} {
+  set y {}
+  foreach x $inlist {
+    lappend y [list [expr {rand()}] $x]
+  }
+  set y [lsort $y]
+  set outlist {}
+  foreach x $y {
+    lappend outlist [lindex $x 1]
+  }
+  return $outlist
+}
+
+# Always use the same random seed so that the sequence of tests
+# is repeatable.
+#
+expr {srand(1234)}
+
+# Run these tests for all number of dimensions between 1 and 5.
+#
+for {set nDim 1} {$nDim<=5} {incr nDim} {
+
+  # Construct an rtree virtual table and an ordinary btree table
+  # to mirror it.  The ordinary table should be much slower (since
+  # it has to do a full table scan) but should give the exact same
+  # answers.
+  #
+  do_test rtree4-$nDim.1 {
+    set clist {}
+    set cklist {}
+    for {set i 0} {$i<$nDim} {incr i} {
+      lappend clist mn$i mx$i
+      lappend cklist "mn$i<mx$i"
+    }
+    db eval "DROP TABLE IF EXISTS rx"
+    db eval "DROP TABLE IF EXISTS bx"
+    db eval "CREATE VIRTUAL TABLE rx USING rtree(id, [join $clist ,])"
+    db eval "CREATE TABLE bx(id INTEGER PRIMARY KEY,\
+                [join $clist ,], CHECK( [join $cklist { AND }] ))"
+  } {}
+
+  # Do many insertions of small objects.  Do both overlapping and
+  # contained-within queries after each insert to verify that all
+  # is well.
+  #
+  unset -nocomplain where
+  for {set i 1} {$i<$::NROW} {incr i} {
+    # Do a random insert
+    #
+    do_test rtree-$nDim.2.$i.1 {
+      set vlist {}
+      for {set j 0} {$j<$nDim} {incr j} {
+        set mn [rand 10000]
+        set mx [expr {$mn+[randincr 50]}]
+        lappend vlist $mn $mx
+      }
+      db eval "INSERT INTO rx VALUES(NULL, [join $vlist ,])"
+      db eval "INSERT INTO bx VALUES(NULL, [join $vlist ,])"
+    } {}
+
+    # Do a contained-in query on all dimensions
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mn$j>=$mn mx$j<=$mx
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.2 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do an overlaps query on all dimensions
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mx$j>=$mn mn$j<=$mx
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.3 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do a contained-in query with surplus contraints at the beginning.
+    # This should force a full-table scan on the rtree.
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      lappend where mn$j>-10000 mx$j<10000
+    }
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mn$j>=$mn mx$j<=$mx
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.3 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do an overlaps query with surplus contraints at the beginning.
+    # This should force a full-table scan on the rtree.
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      lappend where mn$j>=-10000 mx$j<=10000
+    }
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mx$j>$mn mn$j<$mx
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.4 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do a contained-in query with surplus contraints at the end
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mn$j>=$mn mx$j<$mx
+    }
+    for {set j [expr {$nDim-1}]} {$j>=0} {incr j -1} {
+      lappend where mn$j>=-10000 mx$j<10000
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.5 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do an overlaps query with surplus contraints at the end
+    #
+    set where {}
+    for {set j [expr {$nDim-1}]} {$j>=0} {incr j -1} {
+      set mn [rand 10000]
+      set mx [expr {$mn+[randincr 500]}]
+      lappend where mx$j>$mn mn$j<=$mx
+    }
+    for {set j 0} {$j<$nDim} {incr j} {
+      lappend where mx$j>-10000 mn$j<=10000
+    }
+    set where "WHERE [join $where { AND }]"
+    do_test rtree-$nDim.2.$i.6 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do a contained-in query with surplus contraints where the 
+    # constraints appear in a random order.
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn1 [rand 10000]
+      set mn2 [expr {$mn1+[randincr 100]}]
+      set mx1 [expr {$mn2+[randincr 400]}]
+      set mx2 [expr {$mx1+[randincr 100]}]
+      lappend where mn$j>=$mn1 mn$j>$mn2 mx$j<$mx1 mx$j<=$mx2
+    }
+    set where "WHERE [join [scramble $where] { AND }]"
+    do_test rtree-$nDim.2.$i.7 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+
+    # Do an overlaps query with surplus contraints where the
+    # constraints appear in a random order.
+    #
+    set where {}
+    for {set j 0} {$j<$nDim} {incr j} {
+      set mn1 [rand 10000]
+      set mn2 [expr {$mn1+[randincr 100]}]
+      set mx1 [expr {$mn2+[randincr 400]}]
+      set mx2 [expr {$mx1+[randincr 100]}]
+      lappend where mx$j>=$mn1 mx$j>$mn2 mn$j<$mx1 mn$j<=$mx2
+    }
+    set where "WHERE [join [scramble $where] { AND }]"
+    do_test rtree-$nDim.2.$i.8 {
+      list $where [db eval "SELECT id FROM rx $where ORDER BY id"]
+    } [list $where [db eval "SELECT id FROM bx $where ORDER BY id"]]
+  }
+
+}
+
+finish_test

ext/rtree/rtree5.test

@@ -0,0 +1,80 @@
+# 2008 Jul 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the r-tree extension when it is
+# configured to store values as 32 bit integers.
+#
+# $Id: rtree5.test,v 1.1 2008/07/14 15:37:01 danielk1977 Exp $
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname $argv0] .. .. test]
+} 
+source $testdir/tester.tcl
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+do_test rtree5-1.0 {
+  execsql { CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2, y1, y2) }
+} {}
+do_test rtree5-1.1 {
+  execsql { INSERT INTO t1 VALUES(1, 5, 10, 4, 11.2) }
+} {}
+do_test rtree5-1.2 { 
+  execsql { SELECT * FROM t1 }
+} {1 5 10 4 11}
+do_test rtree5-1.3 { 
+  execsql { SELECT typeof(x1) FROM t1 }
+} {integer}
+
+do_test rtree5-1.4 { 
+  execsql { SELECT x1==5 FROM t1 }
+} {1}
+do_test rtree5-1.5 { 
+  execsql { SELECT x1==5.2 FROM t1 }
+} {0}
+do_test rtree5-1.6 { 
+  execsql { SELECT x1==5.0 FROM t1 }
+} {1}
+
+do_test rtree5-1.7 { 
+  execsql { SELECT count(*) FROM t1 WHERE x1==5 }
+} {1}
+do_test rtree5-1.8 { 
+  execsql { SELECT count(*) FROM t1 WHERE x1==5.2 }
+} {0}
+do_test rtree5-1.9 { 
+  execsql { SELECT count(*) FROM t1 WHERE x1==5.0 }
+} {1}
+
+do_test rtree5-1.10 { 
+  execsql { SELECT (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5 }
+} {2147483643 2147483647 -2147483648 -2147483643}
+do_test rtree5-1.10 { 
+  execsql { 
+    INSERT INTO t1 VALUES(2, (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5) 
+  }
+} {}
+do_test rtree5-1.12 { 
+  execsql { SELECT * FROM t1 WHERE id=2 }
+} {2 2147483643 2147483647 -2147483648 -2147483643}
+do_test rtree5-1.13 { 
+  execsql { 
+    SELECT * FROM t1 WHERE 
+        x1=2147483643 AND x2=2147483647 AND 
+        y1=-2147483648 AND y2=-2147483643
+  }
+} {2 2147483643 2147483647 -2147483648 -2147483643}
+
+finish_test

ext/rtree/rtree_perf.tcl

@@ -0,0 +1,76 @@
+
+set testdir [file join [file dirname $argv0] .. .. test]
+source $testdir/tester.tcl
+
+ifcapable !rtree {
+  finish_test
+  return
+}
+
+set NROW   10000
+set NQUERY   500
+
+puts "Generating $NROW rows of data..."
+set data [list]
+for {set ii 0} {$ii < $NROW} {incr ii} {
+  set x1 [expr {rand()*1000}]
+  set x2 [expr {$x1+rand()*50}]
+  set y1 [expr {rand()*1000}]
+  set y2 [expr {$y1+rand()*50}]
+  lappend data $x1 $x2 $y1 $y2
+}
+puts "Finished generating data"
+
+
+set sql1 {CREATE TABLE btree(ii INTEGER PRIMARY KEY, x1, x2, y1, y2)}
+set sql2 {CREATE VIRTUAL TABLE rtree USING rtree(ii, x1, x2, y1, y2)}
+puts "Creating tables:"
+puts "  $sql1"
+puts "  $sql2"
+db eval $sql1
+db eval $sql2
+
+db eval "pragma cache_size=100"
+
+puts -nonewline "Inserting into btree... "
+flush stdout
+set btree_time [time {db transaction {
+  set ii 1
+  foreach {x1 x2 y1 y2} $data {
+    db eval {INSERT INTO btree VALUES($ii, $x1, $x2, $y1, $y2)}
+    incr ii
+  }
+}}]
+puts "$btree_time"
+
+puts -nonewline "Inserting into rtree... "
+flush stdout
+set rtree_time [time {db transaction {
+  set ii 1
+  foreach {x1 x2 y1 y2} $data {
+    incr ii
+    db eval {INSERT INTO rtree VALUES($ii, $x1, $x2, $y1, $y2)}
+  }
+}}]
+puts "$rtree_time"
+
+
+puts -nonewline "Selecting from btree... "
+flush stdout
+set btree_select_time [time {
+  foreach {x1 x2 y1 y2} [lrange $data 0 [expr $NQUERY*4-1]] {
+    db eval {SELECT * FROM btree WHERE x1<$x1 AND x2>$x2 AND y1<$y1 AND y2>$y2}
+ }
+}]
+puts "$btree_select_time"
+
+puts -nonewline "Selecting from rtree... "
+flush stdout
+set rtree_select_time [time {
+  foreach {x1 x2 y1 y2} [lrange $data 0 [expr $NQUERY*4-1]] {
+    db eval {SELECT * FROM rtree WHERE x1<$x1 AND x2>$x2 AND y1<$y1 AND y2>$y2}
+  }
+}]
+puts "$rtree_select_time"
+
+

ext/rtree/rtree_util.tcl

@@ -0,0 +1,195 @@
+# 2008 Feb 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains Tcl code that may be useful for testing or
+# analyzing r-tree structures created with this module. It is
+# used by both test procedures and the r-tree viewer application.
+#
+# $Id: rtree_util.tcl,v 1.1 2008/05/26 18:41:54 danielk1977 Exp $
+#
+
+
+#--------------------------------------------------------------------------
+# PUBLIC API:
+#
+#   rtree_depth
+#   rtree_ndim
+#   rtree_node
+#   rtree_mincells
+#   rtree_check
+#   rtree_dump
+#   rtree_treedump
+#
+
+proc rtree_depth {db zTab} {
+  $db one "SELECT rtreedepth(data) FROM ${zTab}_node WHERE nodeno=1"
+}
+
+proc rtree_nodedepth {db zTab iNode} {
+  set iDepth [rtree_depth $db $zTab]
+  
+  set ii $iNode
+  while {$ii != 1} {
+    set sql "SELECT parentnode FROM ${zTab}_parent WHERE nodeno = $ii"
+    set ii [db one $sql]
+    incr iDepth -1
+  }
+  
+  return $iDepth
+}
+
+# Return the number of dimensions of the rtree.
+#
+proc rtree_ndim {db zTab} {
+  set nDim [expr {(([llength [$db eval "pragma table_info($zTab)"]]/6)-1)/2}]
+}
+
+# Return the contents of rtree node $iNode.
+#
+proc rtree_node {db zTab iNode {iPrec 6}} {
+  set nDim [rtree_ndim $db $zTab]
+  set sql "
+    SELECT rtreenode($nDim, data) FROM ${zTab}_node WHERE nodeno = $iNode
+  "
+  set node [db one $sql]
+
+  set nCell [llength $node]
+  set nCoord [expr $nDim*2]
+  for {set ii 0} {$ii < $nCell} {incr ii} {
+    for {set jj 1} {$jj <= $nCoord} {incr jj} {
+      set newval [format "%.${iPrec}f" [lindex $node $ii $jj]]
+      lset node $ii $jj $newval
+    }
+  }
+  set node
+}
+
+proc rtree_mincells {db zTab} {
+  set n [$db one "select length(data) FROM ${zTab}_node LIMIT 1"]
+  set nMax [expr {int(($n-4)/(8+[rtree_ndim $db $zTab]*2*4))}]
+  return [expr {int($nMax/3)}]
+}
+
+# An integrity check for the rtree $zTab accessible via database 
+# connection $db.
+#
+proc rtree_check {db zTab} {
+  array unset ::checked
+ 
+  # Check each r-tree node.
+  set rc [catch {
+    rtree_node_check $db $zTab 1 [rtree_depth $db $zTab]
+  } msg]
+  if {$rc && $msg ne ""} { error $msg }
+
+  # Check that the _rowid and _parent tables have the right 
+  # number of entries.
+  set nNode   [$db one "SELECT count(*) FROM ${zTab}_node"]
+  set nRow    [$db one "SELECT count(*) FROM ${zTab}"]
+  set nRowid  [$db one "SELECT count(*) FROM ${zTab}_rowid"]
+  set nParent [$db one "SELECT count(*) FROM ${zTab}_parent"]
+
+  if {$nNode != ($nParent+1)} { 
+    error "Wrong number of entries in ${zTab}_parent"
+  }
+  if {$nRow != $nRowid} { 
+    error "Wrong number of entries in ${zTab}_rowid"
+  }
+  
+  return $rc
+}
+
+proc rtree_node_check {db zTab iNode iDepth} {
+  if {[info exists ::checked($iNode)]} { error "Second ref to $iNode" }
+  set ::checked($iNode) 1
+
+  set node [rtree_node $db $zTab $iNode]
+  if {$iNode!=1 && [llength $node]==0} { error "No such node: $iNode" }
+
+  if {$iNode != 1 && [llength $node]<[rtree_mincells $db $zTab]} {
+    puts "Node $iNode: Has only [llength $node] cells"
+    error ""
+  }
+  if {$iNode == 1 && [llength $node]==1 && [rtree_depth $db $zTab]>0} {
+    set depth [rtree_depth $db $zTab]
+    puts "Node $iNode: Has only 1 child (tree depth is $depth)"
+    error ""
+  }
+
+  set nDim [expr {([llength [lindex $node 0]]-1)/2}]
+
+  if {$iDepth > 0} {
+    set d [expr $iDepth-1]
+    foreach cell $node {
+      set shouldbe [rtree_node_check $db $zTab [lindex $cell 0] $d]
+      if {$cell ne $shouldbe} {
+        puts "Node $iNode: Cell is: {$cell}, should be {$shouldbe}"
+        error ""
+      }
+    }
+  }
+
+  set mapping_table "${zTab}_parent" 
+  set mapping_sql "SELECT parentnode FROM $mapping_table WHERE rowid = \$rowid"
+  if {$iDepth==0} { 
+    set mapping_table "${zTab}_rowid"
+    set mapping_sql "SELECT nodeno FROM $mapping_table WHERE rowid = \$rowid"
+  }
+  foreach cell $node {
+    set rowid [lindex $cell 0]
+    set mapping [db one $mapping_sql]
+    if {$mapping != $iNode} {
+      puts "Node $iNode: $mapping_table entry for cell $rowid is $mapping"
+      error ""
+    }
+  }
+
+  set ret [list $iNode]
+  for {set ii 1} {$ii <= $nDim*2} {incr ii} {
+    set f [lindex $node 0 $ii]
+    foreach cell $node {
+      set f2 [lindex $cell $ii]
+      if {($ii%2)==1 && $f2<$f} {set f $f2}
+      if {($ii%2)==0 && $f2>$f} {set f $f2}
+    }
+    lappend ret $f
+  }
+  return $ret
+}
+
+proc rtree_dump {db zTab} {
+  set zRet ""
+  set nDim [expr {(([llength [$db eval "pragma table_info($zTab)"]]/6)-1)/2}]
+  set sql "SELECT nodeno, rtreenode($nDim, data) AS node FROM ${zTab}_node"
+  $db eval $sql {
+    append zRet [format "% -10s %s\n" $nodeno $node]
+  }
+  set zRet
+}
+
+proc rtree_nodetreedump {db zTab zIndent iDepth iNode} {
+  set ret ""
+  set node [rtree_node $db $zTab $iNode 1]
+  append ret [format "%-3d %s%s\n" $iNode $zIndent $node]
+  if {$iDepth>0} {
+    foreach cell $node {
+      set i [lindex $cell 0]
+      append ret [rtree_nodetreedump $db $zTab "$zIndent  " [expr $iDepth-1] $i]
+    }
+  }
+  set ret
+}
+
+proc rtree_treedump {db zTab} {
+  set d [rtree_depth $db $zTab]
+  rtree_nodetreedump $db $zTab "" $d 1
+}
+

ext/rtree/viewrtree.tcl

@@ -0,0 +1,189 @@
+
+load ./libsqlite3.dylib
+#package require sqlite3
+source [file join [file dirname $argv0] rtree_util.tcl]
+
+wm title . "SQLite r-tree viewer"
+
+if {[llength $argv]!=1} {
+  puts stderr "Usage: $argv0 <database-file>"
+  puts stderr ""
+  exit
+}
+sqlite3 db [lindex $argv 0]
+
+canvas .c -background white -width 400 -height 300 -highlightthickness 0
+
+button .b -text "Parent Node" -command {
+  set sql "SELECT parentnode FROM $::O(zTab)_parent WHERE nodeno = $::O(iNode)"
+  set ::O(iNode) [db one $sql]
+  if {$::O(iNode) eq ""} {set ::O(iNode) 1}
+  view_node
+}
+
+set O(iNode) 1
+set O(zTab) ""
+set O(listbox_captions)  [list]
+set O(listbox_itemmap)   [list]
+set O(listbox_highlight) -1
+
+listbox   .l -listvariable ::O(listbox_captions) -yscrollcommand {.ls set}
+scrollbar .ls -command {.l yview}
+label     .status -font courier -anchor w
+label     .title -anchor w -text "Node 1:" -background white -borderwidth 0
+
+
+set rtree_tables [list]
+db eval {
+  SELECT name 
+  FROM sqlite_master 
+  WHERE type='table' AND sql LIKE '%virtual%table%using%rtree%'
+} {
+  set nCol [expr [llength [db eval "pragma table_info($name)"]]/6]
+  if {$nCol != 5} {
+    puts stderr "Not viewing $name - is not 2-dimensional"
+  } else {
+    lappend rtree_tables [list Table $name]
+  }
+}
+if {$rtree_tables eq ""} {
+  puts stderr "Cannot find an r-tree table in database [lindex $argv 0]"
+  puts stderr ""
+  exit
+}
+eval tk_optionMenu .select option_var $rtree_tables
+trace add variable option_var write set_option_var
+proc set_option_var {args} {
+  set ::O(zTab) [lindex $::option_var 1]
+  set ::O(iNode) 1
+  view_node
+}
+set ::O(zTab) [lindex $::rtree_tables 0 1]
+
+bind .l <1> {listbox_click [.l nearest %y]}
+bind .l <Motion> {listbox_mouseover [.l nearest %y]}
+bind .l <Leave>  {listbox_mouseover -1}
+
+proc listbox_click {sel} {
+  if {$sel ne ""} {
+    set ::O(iNode) [lindex $::O(listbox_captions) $sel 1]
+    view_node
+  }
+}
+proc listbox_mouseover {i} {
+  set oldid [lindex $::O(listbox_itemmap) $::O(listbox_highlight)]
+  .c itemconfigure $oldid -fill ""
+
+  .l selection clear 0 end
+  .status configure -text ""
+  if {$i>=0} {
+    set id [lindex $::O(listbox_itemmap) $i]
+    .c itemconfigure $id -fill grey
+    .c lower $id
+    set ::O(listbox_highlight) $i
+    .l selection set $i
+    .status configure -text [cell_report db $::O(zTab) $::O(iNode) $i]
+  }
+}
+
+grid configure .select  -row 0 -column 0 -columnspan 2 -sticky nsew
+grid configure .b       -row 1 -column 0 -columnspan 2 -sticky nsew
+grid configure .l       -row 2 -column 0               -sticky nsew
+grid configure .status  -row 3 -column 0 -columnspan 3 -sticky nsew
+
+grid configure .title   -row 0 -column 2               -sticky nsew
+grid configure .c       -row 1 -column 2 -rowspan 2    -sticky nsew
+grid configure .ls      -row 2 -column 1               -sticky nsew
+
+grid columnconfigure . 2 -weight 1
+grid rowconfigure    . 2 -weight 1
+
+proc node_bbox {data} {
+  set xmin 0
+  set xmax 0
+  set ymin 0
+  set ymax 0
+  foreach {rowid xmin xmax ymin ymax} [lindex $data 0] break
+  foreach cell [lrange $data 1 end] {
+    foreach {rowid x1 x2 y1 y2} $cell break
+    if {$x1 < $xmin} {set xmin $x1}
+    if {$x2 > $xmax} {set xmax $x2}
+    if {$y1 < $ymin} {set ymin $y1}
+    if {$y2 > $ymax} {set ymax $y2}
+  }
+  list $xmin $xmax $ymin $ymax
+}
+
+proc view_node {} {
+  set iNode $::O(iNode)
+  set zTab $::O(zTab)
+
+  set data [rtree_node db $zTab $iNode 12]
+  set depth [rtree_nodedepth db $zTab $iNode]
+
+  .c delete all
+  set ::O(listbox_captions) [list]
+  set ::O(listbox_itemmap) [list]
+  set $::O(listbox_highlight) -1
+
+  .b configure -state normal
+  if {$iNode == 1} {.b configure -state disabled}
+  .title configure -text "Node $iNode: [cell_report db $zTab $iNode -1]"
+
+  foreach {xmin xmax ymin ymax} [node_bbox $data] break
+  set total_area 0.0
+
+  set xscale [expr {double([winfo width .c]-20)/($xmax-$xmin)}]
+  set yscale [expr {double([winfo height .c]-20)/($ymax-$ymin)}]
+
+  set xoff [expr {10.0 - $xmin*$xscale}]
+  set yoff [expr {10.0 - $ymin*$yscale}]
+
+  foreach cell $data {
+    foreach {rowid x1 x2 y1 y2} $cell break
+    set total_area [expr {$total_area + ($x2-$x1)*($y2-$y1)}]
+    set x1 [expr {$x1*$xscale + $xoff}]
+    set x2 [expr {$x2*$xscale + $xoff}]
+    set y1 [expr {$y1*$yscale + $yoff}]
+    set y2 [expr {$y2*$yscale + $yoff}]
+
+    set id [.c create rectangle $x1 $y1 $x2 $y2]
+    if {$depth>0} {
+      lappend ::O(listbox_captions) "Node $rowid"
+      lappend ::O(listbox_itemmap) $id
+    }
+  }
+}
+
+proc cell_report {db zTab iParent iCell} {
+  set data [rtree_node db $zTab $iParent 12]
+  set cell [lindex $data $iCell]
+
+  foreach {xmin xmax ymin ymax} [node_bbox $data] break
+  set total_area [expr ($xmax-$xmin)*($ymax-$ymin)]
+
+  if {$cell eq ""} {
+    set cell_area 0.0
+    foreach cell $data {
+      foreach {rowid x1 x2 y1 y2} $cell break
+      set cell_area [expr $cell_area+($x2-$x1)*($y2-$y1)]
+    }
+    set cell_area [expr $cell_area/[llength $data]]
+    set zReport [format "Size = %.1f x %.1f    Average child area = %.1f%%" \
+      [expr $xmax-$xmin] [expr $ymax-$ymin] [expr 100.0*$cell_area/$total_area]\
+    ]
+    append zReport "   Sub-tree height: [rtree_nodedepth db $zTab $iParent]"
+  } else {
+    foreach {rowid x1 x2 y1 y2} $cell break
+    set cell_area  [expr ($x2-$x1)*($y2-$y1)]
+    set zReport [format "Size = %.1f x %.1f    Area = %.1f%%" \
+      [expr $x2-$x1] [expr $y2-$y1] [expr 100.0*$cell_area/$total_area]
+    ]
+  }
+
+  return $zReport
+}
+
+view_node
+bind .c <Configure> view_node
+

install-sh

@@ -0,0 +1,251 @@
+#!/bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5 (mit/util/scripts/install.sh).
+#
+# Copyright 1991 by the Massachusetts Institute of Technology
+#
+# Permission to use, copy, modify, distribute, and sell this software and its
+# documentation for any purpose is hereby granted without fee, provided that
+# the above copyright notice appear in all copies and that both that
+# copyright notice and this permission notice appear in supporting
+# documentation, and that the name of M.I.T. not be used in advertising or
+# publicity pertaining to distribution of the software without specific,
+# written prior permission.  M.I.T. makes no representations about the
+# suitability of this software for any purpose.  It is provided "as is"
+# without express or implied warranty.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.  It can only install one file at a time, a restriction
+# shared with many OS's install programs.
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+transformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+		chmodcmd=""
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0

main.mk

@@ -0,0 +1,504 @@
+###############################################################################
+# The following macros should be defined before this script is
+# invoked:
+#
+# TOP              The toplevel directory of the source tree.  This is the
+#                  directory that contains this "Makefile.in" and the
+#                  "configure.in" script.
+#
+# BCC              C Compiler and options for use in building executables that
+#                  will run on the platform that is doing the build.
+#
+# THREADLIB        Specify any extra linker options needed to make the library
+#                  thread safe
+#
+# OPTS             Extra compiler command-line options.
+#
+# EXE              The suffix to add to executable files.  ".exe" for windows
+#                  and "" for Unix.
+#
+# TCC              C Compiler and options for use in building executables that 
+#                  will run on the target platform.  This is usually the same
+#                  as BCC, unless you are cross-compiling.
+#
+# AR               Tools used to build a static library.
+# RANLIB
+#
+# TCL_FLAGS        Extra compiler options needed for programs that use the
+#                  TCL library.
+#
+# LIBTCL           Linker options needed to link against the TCL library.
+#
+# READLINE_FLAGS   Compiler options needed for programs that use the
+#                  readline() library.
+#
+# LIBREADLINE      Linker options needed by programs using readline() must
+#                  link against.
+#
+# NAWK             Nawk compatible awk program.  Older (obsolete?) solaris
+#                  systems need this to avoid using the original AT&T AWK.
+#
+# Once the macros above are defined, the rest of this make script will
+# build the SQLite library and testing tools.
+################################################################################
+
+# This is how we compile
+#
+TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) -I$(TOP)/ext/rtree
+
+# Object files for the SQLite library.
+#
+LIBOBJ+= alter.o analyze.o attach.o auth.o bitvec.o btmutex.o btree.o build.o \
+         callback.o complete.o date.o delete.o \
+         expr.o fault.o func.o global.o hash.o insert.o journal.o loadext.o \
+         main.o malloc.o mem1.o mem2.o mem3.o mem4.o mem5.o \
+         mutex.o mutex_os2.o mutex_unix.o mutex_w32.o \
+         opcodes.o os.o os_os2.o os_unix.o os_win.o \
+         pager.o parse.o pragma.o prepare.o printf.o random.o \
+         select.o status.o table.o $(TCLOBJ) tokenize.o trigger.o \
+         update.o util.o vacuum.o \
+         vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \
+         where.o utf.o legacy.o vtab.o rtree.o
+
+EXTOBJ = icu.o
+EXTOBJ += fts1.o \
+	  fts1_hash.o \
+	  fts1_tokenizer1.o \
+	  fts1_porter.o
+EXTOBJ += fts2.o \
+	  fts2_hash.o \
+	  fts2_icu.o \
+	  fts2_porter.o \
+          fts2_tokenizer.o \
+	  fts2_tokenizer1.o
+EXTOBJ += fts3.o \
+	  fts3_hash.o \
+	  fts3_icu.o \
+	  fts3_porter.o \
+          fts3_tokenizer.o \
+	  fts3_tokenizer1.o
+EXTOBJ += rtree.o
+
+# All of the source code files.
+#
+SRC = \
+  $(TOP)/src/alter.c \
+  $(TOP)/src/analyze.c \
+  $(TOP)/src/attach.c \
+  $(TOP)/src/auth.c \
+  $(TOP)/src/bitvec.c \
+  $(TOP)/src/btmutex.c \
+  $(TOP)/src/btree.c \
+  $(TOP)/src/btree.h \
+  $(TOP)/src/btreeInt.h \
+  $(TOP)/src/build.c \
+  $(TOP)/src/callback.c \
+  $(TOP)/src/complete.c \
+  $(TOP)/src/date.c \
+  $(TOP)/src/delete.c \
+  $(TOP)/src/expr.c \
+  $(TOP)/src/fault.c \
+  $(TOP)/src/func.c \
+  $(TOP)/src/global.c \
+  $(TOP)/src/hash.c \
+  $(TOP)/src/hash.h \
+  $(TOP)/src/hwtime.h \
+  $(TOP)/src/insert.c \
+  $(TOP)/src/journal.c \
+  $(TOP)/src/legacy.c \
+  $(TOP)/src/loadext.c \
+  $(TOP)/src/main.c \
+  $(TOP)/src/malloc.c \
+  $(TOP)/src/mem1.c \
+  $(TOP)/src/mem2.c \
+  $(TOP)/src/mem3.c \
+  $(TOP)/src/mem4.c \
+  $(TOP)/src/mem5.c \
+  $(TOP)/src/mutex.c \
+  $(TOP)/src/mutex.h \
+  $(TOP)/src/mutex_os2.c \
+  $(TOP)/src/mutex_unix.c \
+  $(TOP)/src/mutex_w32.c \
+  $(TOP)/src/os.c \
+  $(TOP)/src/os.h \
+  $(TOP)/src/os_common.h \
+  $(TOP)/src/os_os2.c \
+  $(TOP)/src/os_unix.c \
+  $(TOP)/src/os_win.c \
+  $(TOP)/src/pager.c \
+  $(TOP)/src/pager.h \
+  $(TOP)/src/parse.y \
+  $(TOP)/src/pragma.c \
+  $(TOP)/src/prepare.c \
+  $(TOP)/src/printf.c \
+  $(TOP)/src/random.c \
+  $(TOP)/src/select.c \
+  $(TOP)/src/status.c \
+  $(TOP)/src/shell.c \
+  $(TOP)/src/sqlite.h.in \
+  $(TOP)/src/sqlite3ext.h \
+  $(TOP)/src/sqliteInt.h \
+  $(TOP)/src/sqliteLimit.h \
+  $(TOP)/src/table.c \
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/src/tokenize.c \
+  $(TOP)/src/trigger.c \
+  $(TOP)/src/utf.c \
+  $(TOP)/src/update.c \
+  $(TOP)/src/util.c \
+  $(TOP)/src/vacuum.c \
+  $(TOP)/src/vdbe.c \
+  $(TOP)/src/vdbe.h \
+  $(TOP)/src/vdbeapi.c \
+  $(TOP)/src/vdbeaux.c \
+  $(TOP)/src/vdbeblob.c \
+  $(TOP)/src/vdbefifo.c \
+  $(TOP)/src/vdbemem.c \
+  $(TOP)/src/vdbeInt.h \
+  $(TOP)/src/vtab.c \
+  $(TOP)/src/where.c
+
+# Source code for extensions
+#
+SRC += \
+  $(TOP)/ext/fts1/fts1.c \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.c \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_porter.c \
+  $(TOP)/ext/fts1/fts1_tokenizer.h \
+  $(TOP)/ext/fts1/fts1_tokenizer1.c
+SRC += \
+  $(TOP)/ext/fts2/fts2.c \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.c \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_icu.c \
+  $(TOP)/ext/fts2/fts2_porter.c \
+  $(TOP)/ext/fts2/fts2_tokenizer.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.c \
+  $(TOP)/ext/fts2/fts2_tokenizer1.c
+SRC += \
+  $(TOP)/ext/fts3/fts3.c \
+  $(TOP)/ext/fts3/fts3.h \
+  $(TOP)/ext/fts3/fts3_hash.c \
+  $(TOP)/ext/fts3/fts3_hash.h \
+  $(TOP)/ext/fts3/fts3_icu.c \
+  $(TOP)/ext/fts3/fts3_porter.c \
+  $(TOP)/ext/fts3/fts3_tokenizer.h \
+  $(TOP)/ext/fts3/fts3_tokenizer.c \
+  $(TOP)/ext/fts3/fts3_tokenizer1.c
+SRC += \
+  $(TOP)/ext/icu/icu.c
+SRC += \
+  $(TOP)/ext/rtree/rtree.h \
+  $(TOP)/ext/rtree/rtree.c
+
+
+# Generated source code files
+#
+SRC += \
+  keywordhash.h \
+  opcodes.c \
+  opcodes.h \
+  parse.c \
+  parse.h \
+  sqlite3.h
+
+
+# Source code to the test files.
+#
+TESTSRC = \
+  $(TOP)/src/test1.c \
+  $(TOP)/src/test2.c \
+  $(TOP)/src/test3.c \
+  $(TOP)/src/test4.c \
+  $(TOP)/src/test5.c \
+  $(TOP)/src/test6.c \
+  $(TOP)/src/test7.c \
+  $(TOP)/src/test8.c \
+  $(TOP)/src/test9.c \
+  $(TOP)/src/test_autoext.c \
+  $(TOP)/src/test_async.c \
+  $(TOP)/src/test_btree.c \
+  $(TOP)/src/test_config.c \
+  $(TOP)/src/test_devsym.c \
+  $(TOP)/src/test_func.c \
+  $(TOP)/src/test_hexio.c \
+  $(TOP)/src/test_malloc.c \
+  $(TOP)/src/test_md5.c \
+  $(TOP)/src/test_mutex.c \
+  $(TOP)/src/test_onefile.c \
+  $(TOP)/src/test_osinst.c \
+  $(TOP)/src/test_schema.c \
+  $(TOP)/src/test_server.c \
+  $(TOP)/src/test_tclvar.c \
+  $(TOP)/src/test_thread.c \
+
+#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
+#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
+
+TESTSRC2 = \
+  $(TOP)/src/attach.c $(TOP)/src/btree.c $(TOP)/src/build.c $(TOP)/src/date.c  \
+  $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c      \
+  $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c                 \
+  $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c                  \
+  $(TOP)/src/printf.c $(TOP)/src/random.c                                      \
+  $(TOP)/src/select.c $(TOP)/src/tokenize.c                                    \
+  $(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
+  $(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c
+
+# Header files used by all library source files.
+#
+HDR = \
+   $(TOP)/src/btree.h \
+   $(TOP)/src/btreeInt.h \
+   $(TOP)/src/hash.h \
+   $(TOP)/src/hwtime.h \
+   keywordhash.h \
+   $(TOP)/src/mutex.h \
+   opcodes.h \
+   $(TOP)/src/os.h \
+   $(TOP)/src/os_common.h \
+   $(TOP)/src/pager.h \
+   parse.h  \
+   sqlite3.h  \
+   $(TOP)/src/sqlite3ext.h \
+   $(TOP)/src/sqliteInt.h  \
+   $(TOP)/src/sqliteLimit.h \
+   $(TOP)/src/vdbe.h \
+   $(TOP)/src/vdbeInt.h
+
+# Header files used by extensions
+#
+EXTHDR += \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_tokenizer.h
+EXTHDR += \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.h
+EXTHDR += \
+  $(TOP)/ext/fts3/fts3.h \
+  $(TOP)/ext/fts3/fts3_hash.h \
+  $(TOP)/ext/fts3/fts3_tokenizer.h
+EXTHDR += \
+  $(TOP)/ext/rtree/rtree.h
+
+# This is the default Makefile target.  The objects listed here
+# are what get build when you type just "make" with no arguments.
+#
+all:	sqlite3.h libsqlite3.a sqlite3$(EXE)
+
+libsqlite3.a:	$(LIBOBJ)
+	$(AR) libsqlite3.a $(LIBOBJ)
+	$(RANLIB) libsqlite3.a
+
+sqlite3$(EXE):	$(TOP)/src/shell.c libsqlite3.a sqlite3.h
+	$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE)                  \
+		$(TOP)/src/shell.c                                  \
+		libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)
+
+objects: $(LIBOBJ_ORIG)
+
+# This target creates a directory named "tsrc" and fills it with
+# copies of all of the C source code and header files needed to
+# build on the target system.  Some of the C source code and header
+# files are automatically generated.  This target takes care of
+# all that automatic generation.
+#
+target_source:	$(SRC)
+	rm -rf tsrc
+	mkdir tsrc
+	cp -f $(SRC) tsrc
+	rm tsrc/sqlite.h.in tsrc/parse.y
+	touch target_source
+
+sqlite3.c:	target_source $(TOP)/tool/mksqlite3c.tcl
+	tclsh $(TOP)/tool/mksqlite3c.tcl
+	cp sqlite3.c tclsqlite3.c
+	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
+
+fts2amal.c:	target_source $(TOP)/ext/fts2/mkfts2amal.tcl
+	tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
+
+fts3amal.c:	target_source $(TOP)/ext/fts3/mkfts3amal.tcl
+	tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
+
+# Rules to build the LEMON compiler generator
+#
+lemon:	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
+	$(BCC) -o lemon $(TOP)/tool/lemon.c
+	cp $(TOP)/tool/lempar.c .
+
+# Rules to build individual *.o files from generated *.c files. This
+# applies to:
+#
+#     parse.o
+#     opcodes.o
+#
+%.o: %.c $(HDR)
+	$(TCCX) -c $<
+
+# Rules to build individual *.o files from files in the src directory.
+#
+%.o: $(TOP)/src/%.c $(HDR)
+	$(TCCX) -c $<
+
+tclsqlite.o:	$(TOP)/src/tclsqlite.c $(HDR)
+	$(TCCX) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
+
+
+
+# Rules to build opcodes.c and opcodes.h
+#
+opcodes.c:	opcodes.h $(TOP)/mkopcodec.awk
+	sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c
+
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
+	cat parse.h $(TOP)/src/vdbe.c |$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
+
+
+# Rules to build parse.c and parse.h - the outputs of lemon.
+#
+parse.h:	parse.c
+
+parse.c:	$(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
+	cp $(TOP)/src/parse.y .
+	rm -f parse.h
+	./lemon $(OPTS) parse.y
+	mv parse.h parse.h.temp
+	awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
+
+sqlite3.h:	$(TOP)/src/sqlite.h.in 
+	sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
+	    -e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
+                 $(TOP)/src/sqlite.h.in >sqlite3.h
+
+keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
+	$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
+	./mkkeywordhash >keywordhash.h
+
+
+
+# Rules to build the extension objects.
+#
+icu.o:	$(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
+
+fts2.o:	$(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
+
+fts2_hash.o:	$(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
+
+fts2_icu.o:	$(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
+
+fts2_porter.o:	$(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
+
+fts2_tokenizer.o:	$(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
+
+fts2_tokenizer1.o:	$(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
+
+fts3.o:	$(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
+
+fts3_hash.o:	$(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
+
+fts3_icu.o:	$(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
+
+fts3_porter.o:	$(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
+
+fts3_tokenizer.o:	$(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
+
+fts3_tokenizer1.o:	$(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
+
+rtree.o:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
+
+
+# Rules for building test programs and for running tests
+#
+tclsqlite3:	$(TOP)/src/tclsqlite.c libsqlite3.a
+	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
+		$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
+
+
+# Rules to build the 'testfixture' application.
+#
+TESTFIXTURE_FLAGS  = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+
+testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
+		$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c                \
+		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
+
+amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
+		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c                  \
+		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
+
+fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
+	$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS)                            \
+	-DSQLITE_ENABLE_FTS3=1                                               \
+		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
+		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
+
+fulltest:	testfixture$(EXE) sqlite3$(EXE)
+	./testfixture$(EXE) $(TOP)/test/all.test
+
+soaktest:	testfixture$(EXE) sqlite3$(EXE)
+	./testfixture$(EXE) $(TOP)/test/all.test -soak 1
+
+test:	testfixture$(EXE) sqlite3$(EXE)
+	./testfixture$(EXE) $(TOP)/test/veryquick.test
+
+sqlite3_analyzer$(EXE):	$(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
+			$(TOP)/tool/spaceanal.tcl
+	sed \
+	  -e '/^#/d' \
+	  -e 's,\\,\\\\,g' \
+	  -e 's,",\\",g' \
+	  -e 's,^,",' \
+	  -e 's,$$,\\n",' \
+	  $(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
+	$(TCCX) $(TCL_FLAGS)                  $(TESTFIXTURE_FLAGS)                                 \
+		-DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \
+		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c                    \
+		-o sqlite3_analyzer$(EXE)                                      \
+		$(LIBTCL) $(THREADLIB)
+
+TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
+$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
+	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
+
+extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
+	./testfixture$(EXE) $(TOP)/test/loadext.test
+
+
+# Standard install and cleanup targets
+#
+install:	sqlite3 libsqlite3.a sqlite3.h
+	mv sqlite3 /usr/bin
+	mv libsqlite3.a /usr/lib
+	mv sqlite3.h /usr/include
+
+clean:	
+	rm -f *.o sqlite3 libsqlite3.a sqlite3.h opcodes.*
+	rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
+	rm -f $(PUBLISH)
+	rm -f *.da *.bb *.bbg gmon.out
+	rm -rf tsrc target_source
+	rm -f testloadext.dll libtestloadext.so
+	rm -f sqlite3.c fts?amal.c

mkdll.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite into a DLL.
+#
+# Two separate DLLs are generated.  "sqlite3.dll" is the core
+# library.  "tclsqlite3.dll" contains the TCL bindings and is the
+# library that is loaded into TCL in order to run SQLite.
+#
+make sqlite3.c
+PATH=$PATH:/opt/mingw/bin
+TCLDIR=/home/drh/tcltk/846/win/846win
+TCLSTUBLIB=$TCLDIR/libtcl84stub.a
+OPTS='-DUSE_TCL_STUBS=1 -DSQLITE_THREADSAFE=1 -DBUILD_sqlite=1 -DSQLITE_OS_WIN=1'
+OPTS="$OPTS -DSQLITE_ENABLE_FTS3=1"
+CC="i386-mingw32msvc-gcc -Os $OPTS -Itsrc -I$TCLDIR"
+NM="i386-mingw32msvc-nm"
+CMD="$CC -c sqlite3.c"
+echo $CMD
+$CMD
+CMD="$CC -c tclsqlite3.c"
+echo $CMD
+$CMD
+echo 'EXPORTS' >tclsqlite3.def
+$NM tclsqlite3.o | grep ' T ' >temp1
+grep '_Init$' temp1 >temp2
+grep '_SafeInit$' temp1 >>temp2
+grep ' T _sqlite3_' temp1 >>temp2
+echo 'EXPORTS' >tclsqlite3.def
+sed 's/^.* T _//' temp2 | sort | uniq >>tclsqlite3.def
+i386-mingw32msvc-dllwrap \
+     --def tclsqlite3.def -v --export-all \
+     --driver-name i386-mingw32msvc-gcc \
+     --dlltool-name i386-mingw32msvc-dlltool \
+     --as i386-mingw32msvc-as \
+     --target i386-mingw32 \
+     -dllname tclsqlite3.dll -lmsvcrt tclsqlite3.o $TCLSTUBLIB
+$NM sqlite3.o | grep ' T ' >temp1
+echo 'EXPORTS' >sqlite3.def
+grep ' _sqlite3_' temp1 | sed 's/^.* _//' >>sqlite3.def
+i386-mingw32msvc-dllwrap \
+     --def sqlite3.def -v --export-all \
+     --driver-name i386-mingw32msvc-gcc \
+     --dlltool-name i386-mingw32msvc-dlltool \
+     --as i386-mingw32msvc-as \
+     --target i386-mingw32 \
+     -dllname sqlite3.dll -lmsvcrt sqlite3.o

mkextu.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite into a shared library on Linux.
+#
+# Two separate shared libraries are generated.  "sqlite3.so" is the core
+# library.  "tclsqlite3.so" contains the TCL bindings and is the
+# library that is loaded into TCL in order to run SQLite.
+#
+CFLAGS=-O2 -Wall
+make fts2amal.c
+echo gcc $CFLAGS -shared fts2amal.c -o fts2.so
+gcc $CFLAGS -shared fts2amal.c -o fts2.so
+strip fts2.so

mkextw.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite extensions into DLLs.
+#
+make fts2amal.c
+PATH=$PATH:/opt/mingw/bin
+OPTS='-DTHREADSAFE=1 -DBUILD_sqlite=1 -DSQLITE_OS_WIN=1'
+CC="i386-mingw32msvc-gcc -O2 $OPTS -Itsrc"
+NM="i386-mingw32msvc-nm"
+CMD="$CC -c fts2amal.c"
+echo $CMD
+$CMD
+echo 'EXPORTS' >fts2.def
+echo 'sqlite3_extension_init' >>fts2.def
+i386-mingw32msvc-dllwrap \
+     --def fts2.def -v --export-all \
+     --driver-name i386-mingw32msvc-gcc \
+     --dlltool-name i386-mingw32msvc-dlltool \
+     --as i386-mingw32msvc-as \
+     --target i386-mingw32 \
+     -dllname fts2.dll -lmsvcrt fts2amal.o
+zip fts2dll.zip fts2.dll fts2.def

mkopcodec.awk

@@ -0,0 +1,31 @@
+#!/usr/bin/awk -f
+#
+# This AWK script scans the opcodes.h file (which is itself generated by
+# another awk script) and uses the information gleaned to create the
+# opcodes.c source file.
+#
+# Opcodes.c contains strings which are the symbolic names for the various
+# opcodes used by the VDBE.  These strings are used when disassembling a
+# VDBE program during tracing or as a result of the EXPLAIN keyword.
+#
+BEGIN {
+  print "/* Automatically generated.  Do not edit */"
+  print "/* See the mkopcodec.awk script for details. */"
+  printf "#if !defined(SQLITE_OMIT_EXPLAIN)"
+  printf    " || !defined(NDEBUG)"
+  printf    " || defined(VDBE_PROFILE)"
+  print     " || defined(SQLITE_DEBUG)"
+  print "const char *sqlite3OpcodeName(int i){"
+  print " static const char *const azName[] = { \"?\","
+}
+/define OP_/ {
+  sub("OP_","",$2)
+  i++
+  printf "     /* %3d */ \"%s\",\n", $3, $2
+}
+END {
+  print "  };"
+  print "  return azName[i];"
+  print "}"
+  print "#endif"
+}

mkopcodeh.awk

@@ -0,0 +1,157 @@
+#!/usr/bin/awk -f
+#
+# Generate the file opcodes.h.
+#
+# This AWK script scans a concatenation of the parse.h output file from the
+# parser and the vdbe.c source file in order to generate the opcodes numbers
+# for all opcodes.  
+#
+# The lines of the vdbe.c that we are interested in are of the form:
+#
+#       case OP_aaaa:      /* same as TK_bbbbb */
+#
+# The TK_ comment is optional.  If it is present, then the value assigned to
+# the OP_ is the same as the TK_ value.  If missing, the OP_ value is assigned
+# a small integer that is different from every other OP_ value.
+#
+# We go to the trouble of making some OP_ values the same as TK_ values
+# as an optimization.  During parsing, things like expression operators
+# are coded with TK_ values such as TK_ADD, TK_DIVIDE, and so forth.  Later
+# during code generation, we need to generate corresponding opcodes like
+# OP_Add and OP_Divide.  By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide,
+# code to translate from one to the other is avoided.  This makes the
+# code generator run (infinitesimally) faster and more importantly it makes
+# the library footprint smaller.
+#
+# This script also scans for lines of the form:
+#
+#       case OP_aaaa:       /* no-push */
+#
+# When the no-push comment is found on an opcode, it means that that
+# opcode does not leave a result on the stack.  By identifying which
+# opcodes leave results on the stack it is possible to determine a
+# much smaller upper bound on the size of the stack.  This allows
+# a smaller stack to be allocated, which is important to embedded
+# systems with limited memory space.  This script generates a series
+# of "NOPUSH_MASK" defines that contain bitmaps of opcodes that leave
+# results on the stack.  The NOPUSH_MASK defines are used in vdbeaux.c
+# to help determine the maximum stack size.
+#
+
+
+# Remember the TK_ values from the parse.h file
+/^#define TK_/ {
+  tk[$2] = 0+$3
+}
+
+# Scan for "case OP_aaaa:" lines in the vdbe.c file
+/^case OP_/ {
+  name = $2
+  sub(/:/,"",name)
+  sub("\r","",name)
+  op[name] = -1
+  jump[name] = 0
+  out2_prerelease[name] = 0
+  in1[name] = 0
+  in2[name] = 0
+  in3[name] = 0
+  out3[name] = 0
+  for(i=3; i<NF; i++){
+    if($i=="same" && $(i+1)=="as"){
+      sym = $(i+2)
+      sub(/,/,"",sym)
+      op[name] = tk[sym]
+      used[op[name]] = 1
+      sameas[op[name]] = sym
+    }
+    x = $i
+    sub(",","",x)
+    if(x=="jump"){
+      jump[name] = 1
+    }else if(x=="out2-prerelease"){
+      out2_prerelease[name] = 1
+    }else if(x=="in1"){
+      in1[name] = 1
+    }else if(x=="in2"){
+      in2[name] = 1
+    }else if(x=="in3"){
+      in3[name] = 1
+    }else if(x=="out3"){
+      out3[name] = 1
+    }
+  }
+}
+
+# Assign numbers to all opcodes and output the result.
+END {
+  cnt = 0
+  max = 0
+  print "/* Automatically generated.  Do not edit */"
+  print "/* See the mkopcodeh.awk script for details */"
+  op["OP_Noop"] = -1;
+  op["OP_Explain"] = -1;
+  for(name in op){
+    if( op[name]<0 ){
+      cnt++
+      while( used[cnt] ) cnt++
+      op[name] = cnt
+    }
+    used[op[name]] = 1;
+    if( op[name]>max ) max = op[name]
+    printf "#define %-25s %15d", name, op[name]
+    if( sameas[op[name]] ) {
+      printf "   /* same as %-12s*/", sameas[op[name]]
+    } 
+    printf "\n"
+
+  }
+  seenUnused = 0;
+  for(i=1; i<max; i++){
+    if( !used[i] ){
+      if( !seenUnused ){
+        printf "\n/* The following opcode values are never used */\n"
+        seenUnused = 1
+      }
+      printf "#define %-25s %15d\n", sprintf( "OP_NotUsed_%-3d", i ), i
+    }
+  }
+
+  # Generate the bitvectors:
+  #
+  #  bit 0:     jump
+  #  bit 1:     pushes a result onto stack
+  #  bit 2:     output to p1.  release p1 before opcode runs
+  #
+  for(i=0; i<=max; i++) bv[i] = 0;
+  for(name in op){
+    x = op[name]
+    a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = 0
+    # a8 = a9 = a10 = a11 = a12 = a13 = a14 = a15 = 0
+    if( jump[name] ) a0 = 1;
+    if( out2_prerelease[name] ) a1 = 2;
+    if( in1[name] ) a2 = 4;
+    if( in2[name] ) a3 = 8;
+    if( in3[name] ) a4 = 16;
+    if( out3[name] ) a5 = 32;
+    # bv[x] = a0+a1+a2+a3+a4+a5+a6+a7+a8+a9+a10+a11+a12+a13+a14+a15;
+    bv[x] = a0+a1+a2+a3+a4+a5+a6+a7;
+  }
+  print "\n"
+  print "/* Properties such as \"out2\" or \"jump\" that are specified in"
+  print "** comments following the \"case\" for each opcode in the vdbe.c"
+  print "** are encoded into bitvectors as follows:"
+  print "*/"
+  print "#define OPFLG_JUMP            0x0001  /* jump:  P2 holds jmp target */"
+  print "#define OPFLG_OUT2_PRERELEASE 0x0002  /* out2-prerelease: */"
+  print "#define OPFLG_IN1             0x0004  /* in1:   P1 is an input */"
+  print "#define OPFLG_IN2             0x0008  /* in2:   P2 is an input */"
+  print "#define OPFLG_IN3             0x0010  /* in3:   P3 is an input */"
+  print "#define OPFLG_OUT3            0x0020  /* out3:  P3 is an output */"
+  print "#define OPFLG_INITIALIZER {\\"
+  for(i=0; i<=max; i++){
+    if( i%8==0 ) printf("/* %3d */",i)
+    printf " 0x%02x,", bv[i]
+    if( i%8==7 ) printf("\\\n");
+  }
+  print "}"
+}

mkso.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite into a shared library on Linux.
+#
+# Two separate shared libraries are generated.  "sqlite3.so" is the core
+# library.  "tclsqlite3.so" contains the TCL bindings and is the
+# library that is loaded into TCL in order to run SQLite.
+#
+make target_source
+cd tsrc
+rm shell.c
+TCLDIR=/home/drh/tcltk/846/linux/846linux
+TCLSTUBLIB=$TCLDIR/libtclstub8.4g.a
+OPTS='-DUSE_TCL_STUBS=1 -DNDEBUG=1 -DHAVE_DLOPEN=1'
+for i in *.c; do
+  if test $i != 'keywordhash.c'; then
+    CMD="cc -fPIC $OPTS -O2 -I. -I$TCLDIR -c $i"
+    echo $CMD
+    $CMD
+  fi
+done
+echo gcc -shared *.o $TCLSTUBLIB -o tclsqlite3.so
+gcc -shared *.o $TCLSTUBLIB -o tclsqlite3.so
+strip tclsqlite3.so
+rm tclsqlite.c tclsqlite.o
+echo gcc -shared *.o -o sqlite3.so
+gcc -shared *.o -o sqlite3.so
+strip sqlite3.so
+cd ..

publish.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite and package everything up
+# so that it is ready to move to the SQLite website.
+#
+
+# Set srcdir to the name of the directory that contains the publish.sh
+# script.
+#
+srcdir=`echo "$0" | sed 's%\(^.*\)/[^/][^/]*$%\1%'`
+
+# Get the makefile.
+#
+cp $srcdir/Makefile.linux-gcc ./Makefile
+chmod +x $srcdir/install-sh
+
+# Get the current version number - needed to help build filenames
+#
+VERS=`cat $srcdir/VERSION`
+VERSW=`sed 's/\./_/g' $srcdir/VERSION`
+echo "VERSIONS: $VERS $VERSW"
+
+# Start by building an sqlite shell for linux.
+#
+make clean
+make sqlite3.c
+CFLAGS="-Os -DSQLITE_ENABLE_FTS3=0 -DSQLITE_ENABLE_RTREE=0"
+CFLAGS="$CFLAGS -DSQLITE_THREADSAFE=0"
+echo '***** '"COMPILING sqlite3-$VERS.bin..."
+gcc $CFLAGS -Itsrc sqlite3.c tsrc/shell.c -o sqlite3 -ldl
+strip sqlite3
+mv sqlite3 sqlite3-$VERS.bin
+gzip sqlite3-$VERS.bin
+chmod 644 sqlite3-$VERS.bin.gz
+mv sqlite3-$VERS.bin.gz doc
+
+# Build a source archive useful for windows.
+#
+make target_source
+cd tsrc
+echo '***** BUILDING preprocessed source archives'
+rm fts[12]* icu*
+rm -f ../doc/sqlite-source-$VERSW.zip
+zip ../doc/sqlite-source-$VERSW.zip *
+cd ..
+cp tsrc/sqlite3.h tsrc/sqlite3ext.h .
+pwd
+zip doc/sqlite-amalgamation-$VERSW.zip sqlite3.c sqlite3.h sqlite3ext.h
+
+# Build the sqlite.so and tclsqlite.so shared libraries
+# under Linux
+#
+TCLDIR=/home/drh/tcltk/846/linux/846linux
+TCLSTUBLIB=$TCLDIR/libtclstub8.4g.a
+CFLAGS="-Os -DSQLITE_ENABLE_FTS3=3 -DSQLITE_ENABLE_RTREE=1"
+CFLAGS="$CFLAGS -DHAVE_LOCALTIME_R=1 -DHAVE_GMTIME_R=1"
+echo '***** BUILDING shared libraries for linux'
+gcc $CFLAGS -shared tclsqlite3.c $TCLSTUBLIB -o tclsqlite3.so -lpthread
+strip tclsqlite3.so
+chmod 644 tclsqlite3.so
+mv tclsqlite3.so tclsqlite-$VERS.so
+gzip tclsqlite-$VERS.so
+mv tclsqlite-$VERS.so.gz doc
+gcc $CFLAGS -shared sqlite3.c -o sqlite3.so -lpthread
+strip sqlite3.so
+chmod 644 sqlite3.so
+mv sqlite3.so sqlite-$VERS.so
+gzip sqlite-$VERS.so
+mv sqlite-$VERS.so.gz doc
+
+
+# Build the tclsqlite3.dll and sqlite3.dll shared libraries.
+#
+. $srcdir/mkdll.sh
+echo '***** PACKAGING shared libraries for windows'
+echo zip doc/tclsqlite-$VERSW.zip tclsqlite3.dll
+zip doc/tclsqlite-$VERSW.zip tclsqlite3.dll
+echo zip doc/sqlitedll-$VERSW.zip sqlite3.dll sqlite3.def
+zip doc/sqlitedll-$VERSW.zip sqlite3.dll sqlite3.def
+
+# Build the sqlite.exe executable for windows.
+#
+OPTS='-DSTATIC_BUILD=1 -DNDEBUG=1 -DSQLITE_THREADSAFE=0'
+OPTS="$OPTS -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_RTREE=1"
+i386-mingw32msvc-gcc -Os $OPTS -Itsrc -I$TCLDIR sqlite3.c tsrc/shell.c \
+      -o sqlite3.exe
+zip doc/sqlite-$VERSW.zip sqlite3.exe
+
+# Construct a tarball of the source tree
+#
+echo '***** BUILDING source archive'
+ORIGIN=`pwd`
+cd $srcdir
+cd ..
+mv sqlite sqlite-$VERS
+EXCLUDE=`find sqlite-$VERS -print | egrep '(CVS|www/|art/|doc/|contrib/)' | sed 's,^, --exclude ,'`
+#echo EXCLUDE=$EXCLUDE
+tar czf $ORIGIN/doc/sqlite-$VERS.tar.gz $EXCLUDE sqlite-$VERS
+mv sqlite-$VERS sqlite
+cd $ORIGIN
+
+#
+# Build RPMS (binary) and Source RPM
+#
+
+# Make sure we are properly setup to build RPMs
+#
+echo "%HOME %{expand:%%(cd; pwd)}" > $HOME/.rpmmacros
+echo "%_topdir %{HOME}/rpm" >> $HOME/.rpmmacros
+mkdir $HOME/rpm
+mkdir $HOME/rpm/BUILD
+mkdir $HOME/rpm/SOURCES
+mkdir $HOME/rpm/RPMS
+mkdir $HOME/rpm/SRPMS
+mkdir $HOME/rpm/SPECS
+
+# create the spec file from the template
+sed s/SQLITE_VERSION/$VERS/g $srcdir/spec.template > $HOME/rpm/SPECS/sqlite.spec
+
+# copy the source tarball to the rpm directory
+cp doc/sqlite-$VERS.tar.gz $HOME/rpm/SOURCES/.
+
+# build all the rpms
+rpm -ba $HOME/rpm/SPECS/sqlite.spec >& rpm-$vers.log
+
+# copy the RPMs into the build directory.
+mv $HOME/rpm/RPMS/i386/sqlite*-$vers*.rpm doc
+mv $HOME/rpm/SRPMS/sqlite-$vers*.rpm doc
+
+# Build the website
+#
+#cp $srcdir/../historical/* doc
+#make doc
+#cd doc
+#chmod 644 *.gz

publish_osx.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# This script is used to compile SQLite and package everything up
+# so that it is ready to move to the SQLite website.
+#
+
+# Set srcdir to the name of the directory that contains the publish.sh
+# script.
+#
+srcdir=`echo "$0" | sed 's%\(^.*\)/[^/][^/]*$%\1%'`
+
+# Get the makefile.
+#
+cp $srcdir/Makefile.linux-gcc ./Makefile
+chmod +x $srcdir/install-sh
+
+# Get the current version number - needed to help build filenames
+#
+VERS=`cat $srcdir/VERSION`
+VERSW=`sed 's/\./_/g' $srcdir/VERSION`
+echo "VERSIONS: $VERS $VERSW"
+
+# Start by building an sqlite shell for linux.
+#
+make clean
+make sqlite3.c
+CFLAGS="-Os -DSQLITE_ENABLE_FTS3=1 -DSQLITE_THREADSAFE=0"
+NAME=sqlite3-$VERS-osx-x86.bin
+echo '***** '"COMPILING $NAME..."
+gcc $CFLAGS -Itsrc sqlite3.c tsrc/shell.c -o $NAME -ldl
+strip $NAME
+chmod 644 $NAME
+gzip $NAME
+mkdir -p doc
+mv $NAME.gz doc

spec.template

@@ -0,0 +1,67 @@
+%define name sqlite
+%define version SQLITE_VERSION
+%define release 1
+
+Name: %{name}
+Summary: SQLite is a C library that implements an embeddable SQL database engine
+Version: %{version}
+Release: %{release}
+Source: %{name}-%{version}.tar.gz
+Group: System/Libraries
+URL: http://www.sqlite.org/
+License: Public Domain
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+
+%description
+SQLite is a software library that implements a self-contained, serverless,
+zero-configuration, transactional SQL database engine.
+Programs that link with the SQLite library can have SQL database access
+without running a separate RDBMS process. The distribution comes with a
+standalone command-line access program (sqlite) that can be used to
+administer an SQLite database and which serves as an example of how to
+use the SQLite library.
+
+%package -n %{name}-devel
+Summary: Header files and libraries for developing apps which will use sqlite
+Group: Development/C
+Requires: %{name} = %{version}-%{release}
+
+%description -n %{name}-devel
+The sqlite-devel package contains the header files and libraries needed
+to develop programs that use the SQLite database library.
+
+%prep
+%setup -q -n %{name}
+
+%build
+CFLAGS="%optflags -DNDEBUG=1" CXXFLAGS="%optflags -DNDEBUG=1" ./configure --prefix=%{_prefix}
+
+make
+make doc
+
+%install
+install -d $RPM_BUILD_ROOT/%{_prefix}
+install -d $RPM_BUILD_ROOT/%{_prefix}/bin
+install -d $RPM_BUILD_ROOT/%{_prefix}/include
+install -d $RPM_BUILD_ROOT/%{_prefix}/lib
+make install prefix=$RPM_BUILD_ROOT/%{_prefix}
+
+%post -p /sbin/ldconfig
+
+%postun -p /sbin/ldconfig
+
+%clean
+rm -fr $RPM_BUILD_ROOT
+
+%files
+%defattr(-, root, root)
+%{_libdir}/*.so*
+%{_bindir}/*
+
+%files -n %{name}-devel
+%defattr(-, root, root)
+%{_libdir}/pkgconfig/sqlite3.pc
+%{_libdir}/*.a
+%{_libdir}/*.la
+%{_includedir}/*
+%doc doc/*

sqlite.pc.in

@@ -0,0 +1,13 @@
+# Package Information for pkg-config
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: SQLite
+Description: SQL database engine
+Version: @VERSION@
+Libs: -L${libdir} -lsqlite
+Libs.private: @LIBS@
+Cflags: -I${includedir}

sqlite3.1

@@ -0,0 +1,229 @@
+.\"                                      Hey, EMACS: -*- nroff -*-
+.\" First parameter, NAME, should be all caps
+.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
+.\" other parameters are allowed: see man(7), man(1)
+.TH SQLITE3 1 "Mon Apr 15 23:49:17 2002"
+.\" Please adjust this date whenever revising the manpage.
+.\"
+.\" Some roff macros, for reference:
+.\" .nh        disable hyphenation
+.\" .hy        enable hyphenation
+.\" .ad l      left justify
+.\" .ad b      justify to both left and right margins
+.\" .nf        disable filling
+.\" .fi        enable filling
+.\" .br        insert line break
+.\" .sp <n>    insert n+1 empty lines
+.\" for manpage-specific macros, see man(7)
+.SH NAME
+.B sqlite3 
+\- A command line interface for SQLite version 3
+
+.SH SYNOPSIS
+.B sqlite3
+.RI [ options ]
+.RI [ databasefile ]
+.RI [ SQL ]
+
+.SH SUMMARY
+.PP
+.B sqlite3
+is a terminal-based front-end to the SQLite library that can evaluate
+queries interactively and display the results in multiple formats.
+.B sqlite3
+can also be used within shell scripts and other applications to provide
+batch processing features.
+
+.SH DESCRIPTION
+To start a
+.B sqlite3
+interactive session, invoke the
+.B sqlite3
+command and optionally provide the name of a database file.  If the
+database file does not exist, it will be created.  If the database file
+does exist, it will be opened.
+
+For example, to create a new database file named "mydata.db", create
+a table named "memos" and insert a couple of records into that table:
+.sp
+$ 
+.B sqlite3 mydata.db
+.br
+SQLite version 3.1.3
+.br
+Enter ".help" for instructions
+.br
+sqlite>
+.B create table memos(text, priority INTEGER);
+.br
+sqlite>
+.B insert into memos values('deliver project description', 10);
+.br
+sqlite>
+.B insert into memos values('lunch with Christine', 100);
+.br
+sqlite>
+.B select * from memos;
+.br
+deliver project description|10
+.br
+lunch with Christine|100
+.br
+sqlite>
+.sp
+
+If no database name is supplied, the ATTACH sql command can be used
+to attach to existing or create new database files.  ATTACH can also
+be used to attach to multiple databases within the same interactive
+session.  This is useful for migrating data between databases,
+possibly changing the schema along the way.
+
+Optionally, a SQL statement or set of SQL statements can be supplied as
+a single argument.  Multiple statements should be separated by
+semi-colons.
+
+For example:
+.sp
+$ 
+.B sqlite3 -line mydata.db 'select * from memos where priority > 20;'
+.br
+    text = lunch with Christine
+.br
+priority = 100
+.br
+.sp
+
+.SS SQLITE META-COMMANDS
+.PP
+The interactive interpreter offers a set of meta-commands that can be
+used to control the output format, examine the currently attached
+database files, or perform administrative operations upon the
+attached databases (such as rebuilding indices).   Meta-commands are
+always prefixed with a dot (.).
+
+A list of available meta-commands can be viewed at any time by issuing
+the '.help' command.  For example:
+.sp
+sqlite>
+.B .help
+.nf
+.cc |
+.databases             List names and files of attached databases
+.dump ?TABLE? ...      Dump the database in an SQL text format
+.echo ON|OFF           Turn command echo on or off
+.exit                  Exit this program
+.explain ON|OFF        Turn output mode suitable for EXPLAIN on or off.
+.header(s) ON|OFF      Turn display of headers on or off
+.help                  Show this message
+.import FILE TABLE     Import data from FILE into TABLE
+.indices TABLE         Show names of all indices on TABLE
+.mode MODE ?TABLE?     Set output mode where MODE is one of:
+                         csv      Comma-separated values
+                         column   Left-aligned columns.  (See .width)
+                         html     HTML <table> code
+                         insert   SQL insert statements for TABLE
+                         line     One value per line
+                         list     Values delimited by .separator string
+                         tabs     Tab-separated values
+                         tcl      TCL list elements
+.nullvalue STRING      Print STRING in place of NULL values
+.output FILENAME       Send output to FILENAME
+.output stdout         Send output to the screen
+.prompt MAIN CONTINUE  Replace the standard prompts
+.quit                  Exit this program
+.read FILENAME         Execute SQL in FILENAME
+.schema ?TABLE?        Show the CREATE statements
+.separator STRING      Change separator used by output mode and .import
+.show                  Show the current values for various settings
+.tables ?PATTERN?      List names of tables matching a LIKE pattern
+.timeout MS            Try opening locked tables for MS milliseconds
+.width NUM NUM ...     Set column widths for "column" mode
+sqlite>
+|cc .
+.sp
+.fi
+
+.SH OPTIONS
+.B sqlite3
+has the following options:
+.TP
+.BI \-init\ file
+Read and execute commands from
+.I file
+, which can contain a mix of SQL statements and meta-commands.
+.TP
+.B \-echo
+Print commands before execution.
+.TP
+.B \-[no]header
+Turn headers on or off.
+.TP
+.B \-column
+Query results will be displayed in a table like form, using
+whitespace characters to separate the columns and align the
+output.
+.TP
+.B \-html
+Query results will be output as simple HTML tables.
+.TP
+.B \-line
+Query results will be displayed with one value per line, rows
+separated by a blank line.  Designed to be easily parsed by
+scripts or other programs
+.TP
+.B \-list
+Query results will be displayed with the separator (|, by default)
+character between each field value.  The default.
+.TP
+.BI \-separator\  separator
+Set output field separator.  Default is '|'.
+.TP
+.BI \-nullvalue\  string
+Set string used to represent NULL values.  Default is ''
+(empty string).
+.TP
+.B \-version
+Show SQLite version.
+.TP
+.B \-help
+Show help on options and exit.
+
+
+.SH INIT FILE
+.B sqlite3
+reads an initialization file to set the configuration of the
+interactive environment.  Throughout initialization, any previously
+specified setting can be overridden.  The sequence of initialization is
+as follows:
+
+o The default configuration is established as follows:
+
+.sp
+.nf
+.cc |
+mode            = LIST
+separator       = "|"
+main prompt     = "sqlite> "
+continue prompt = "   ...> "
+|cc .
+.sp
+.fi
+
+o If the file 
+.B ~/.sqliterc
+exists, it is processed first.
+can be found in the user's home directory, it is
+read and processed.  It should generally only contain meta-commands.
+
+o If the -init option is present, the specified file is processed.
+
+o All other command line options are processed.
+
+.SH SEE ALSO
+http://www.sqlite.org/
+.br
+The sqlite-doc package
+.SH AUTHOR
+This manual page was originally written by Andreas Rottmann
+<rotty@debian.org>, for the Debian GNU/Linux system (but may be used
+by others).   It was subsequently revised by Bill Bumgarner <bbum@mac.com>.

sqlite3.def

@@ -0,0 +1,150 @@
+EXPORTS
+sqlite3_aggregate_context
+sqlite3_aggregate_count
+sqlite3_auto_extension
+sqlite3_bind_blob
+sqlite3_bind_double
+sqlite3_bind_int
+sqlite3_bind_int64
+sqlite3_bind_null
+sqlite3_bind_parameter_count
+sqlite3_bind_parameter_index
+sqlite3_bind_parameter_name
+sqlite3_bind_text
+sqlite3_bind_text16
+sqlite3_bind_value
+sqlite3_bind_zeroblob
+sqlite3_blob_bytes
+sqlite3_blob_close
+sqlite3_blob_open
+sqlite3_blob_read
+sqlite3_blob_write
+sqlite3_busy_handler
+sqlite3_busy_timeout
+sqlite3_changes
+sqlite3_clear_bindings
+sqlite3_close
+sqlite3_collation_needed
+sqlite3_collation_needed16
+sqlite3_column_blob
+sqlite3_column_bytes
+sqlite3_column_bytes16
+sqlite3_column_count
+sqlite3_column_decltype
+sqlite3_column_decltype16
+sqlite3_column_double
+sqlite3_column_int
+sqlite3_column_int64
+sqlite3_column_name
+sqlite3_column_name16
+sqlite3_column_text
+sqlite3_column_text16
+sqlite3_column_type
+sqlite3_column_value
+sqlite3_commit_hook
+sqlite3_complete
+sqlite3_complete16
+sqlite3_create_collation
+sqlite3_create_collation16
+sqlite3_create_collation_v2
+sqlite3_create_function
+sqlite3_create_function16
+sqlite3_create_module
+sqlite3_create_module_v2
+sqlite3_data_count
+sqlite3_db_handle
+sqlite3_declare_vtab
+sqlite3_enable_load_extension
+sqlite3_enable_shared_cache
+sqlite3_errcode
+sqlite3_errmsg
+sqlite3_errmsg16
+sqlite3_exec
+sqlite3_expired
+sqlite3_extended_result_codes
+sqlite3_file_control
+sqlite3_finalize
+sqlite3_free
+sqlite3_free_table
+sqlite3_get_autocommit
+sqlite3_get_auxdata
+sqlite3_get_table
+sqlite3_global_recover
+sqlite3_interrupt
+sqlite3_last_insert_rowid
+sqlite3_libversion
+sqlite3_libversion_number
+sqlite3_load_extension
+sqlite3_malloc
+sqlite3_memory_alarm
+sqlite3_memory_highwater
+sqlite3_memory_used
+sqlite3_mprintf
+sqlite3_mutex_alloc
+sqlite3_mutex_enter
+sqlite3_mutex_free
+sqlite3_mutex_leave
+sqlite3_mutex_try
+sqlite3_open
+sqlite3_open16
+sqlite3_open_v2
+sqlite3_overload_function
+sqlite3_prepare
+sqlite3_prepare16
+sqlite3_prepare16_v2
+sqlite3_prepare_v2
+sqlite3_profile
+sqlite3_progress_handler
+sqlite3_randomness
+sqlite3_realloc
+sqlite3_release_memory
+sqlite3_reset
+sqlite3_reset_auto_extension
+sqlite3_result_blob
+sqlite3_result_double
+sqlite3_result_error
+sqlite3_result_error16
+sqlite3_result_error_code
+sqlite3_result_error_nomem
+sqlite3_result_error_toobig
+sqlite3_result_int
+sqlite3_result_int64
+sqlite3_result_null
+sqlite3_result_text
+sqlite3_result_text16
+sqlite3_result_text16be
+sqlite3_result_text16le
+sqlite3_result_value
+sqlite3_result_zeroblob
+sqlite3_rollback_hook
+sqlite3_set_authorizer
+sqlite3_set_auxdata
+sqlite3_sleep
+sqlite3_snprintf
+sqlite3_soft_heap_limit
+sqlite3_sql
+sqlite3_step
+sqlite3_test_control
+sqlite3_thread_cleanup
+sqlite3_threadsafe
+sqlite3_total_changes
+sqlite3_trace
+sqlite3_transfer_bindings
+sqlite3_update_hook
+sqlite3_user_data
+sqlite3_value_blob
+sqlite3_value_bytes
+sqlite3_value_bytes16
+sqlite3_value_double
+sqlite3_value_int
+sqlite3_value_int64
+sqlite3_value_numeric_type
+sqlite3_value_text
+sqlite3_value_text16
+sqlite3_value_text16be
+sqlite3_value_text16le
+sqlite3_value_type
+sqlite3_vfs_find
+sqlite3_vfs_register
+sqlite3_vfs_unregister
+sqlite3_vmprintf

sqlite3.pc.in

@@ -0,0 +1,13 @@
+# Package Information for pkg-config
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: SQLite
+Description: SQL database engine
+Version: @VERSION@
+Libs: -L${libdir} -lsqlite3
+Libs.private: @LIBS@
+Cflags: -I${includedir}

src/alter.c

@@ -0,0 +1,620 @@
+/*
+** 2005 February 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains C code routines that used to generate VDBE code
+** that implements the ALTER TABLE command.
+**
+** $Id: alter.c,v 1.46 2008/07/15 14:47:19 drh Exp $
+*/
+#include "sqliteInt.h"
+#include <ctype.h>
+
+/*
+** The code in this file only exists if we are not omitting the
+** ALTER TABLE logic from the build.
+*/
+#ifndef SQLITE_OMIT_ALTERTABLE
+
+
+/*
+** This function is used by SQL generated to implement the 
+** ALTER TABLE command. The first argument is the text of a CREATE TABLE or
+** CREATE INDEX command. The second is a table name. The table name in 
+** the CREATE TABLE or CREATE INDEX statement is replaced with the third
+** argument and the result returned. Examples:
+**
+** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def')
+**     -> 'CREATE TABLE def(a, b, c)'
+**
+** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def')
+**     -> 'CREATE INDEX i ON def(a, b, c)'
+*/
+static void renameTableFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  unsigned char const *zSql = sqlite3_value_text(argv[0]);
+  unsigned char const *zTableName = sqlite3_value_text(argv[1]);
+
+  int token;
+  Token tname;
+  unsigned char const *zCsr = zSql;
+  int len = 0;
+  char *zRet;
+
+  sqlite3 *db = sqlite3_context_db_handle(context);
+
+  /* The principle used to locate the table name in the CREATE TABLE 
+  ** statement is that the table name is the first non-space token that
+  ** is immediately followed by a TK_LP or TK_USING token.
+  */
+  if( zSql ){
+    do {
+      if( !*zCsr ){
+        /* Ran out of input before finding an opening bracket. Return NULL. */
+        return;
+      }
+
+      /* Store the token that zCsr points to in tname. */
+      tname.z = zCsr;
+      tname.n = len;
+
+      /* Advance zCsr to the next token. Store that token type in 'token',
+      ** and its length in 'len' (to be used next iteration of this loop).
+      */
+      do {
+        zCsr += len;
+        len = sqlite3GetToken(zCsr, &token);
+      } while( token==TK_SPACE || token==TK_COMMENT );
+      assert( len>0 );
+    } while( token!=TK_LP && token!=TK_USING );
+
+    zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", tname.z - zSql, zSql, 
+       zTableName, tname.z+tname.n);
+    sqlite3_result_text(context, zRet, -1, sqlite3_free);
+  }
+}
+
+#ifndef SQLITE_OMIT_TRIGGER
+/* This function is used by SQL generated to implement the
+** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER 
+** statement. The second is a table name. The table name in the CREATE 
+** TRIGGER statement is replaced with the third argument and the result 
+** returned. This is analagous to renameTableFunc() above, except for CREATE
+** TRIGGER, not CREATE INDEX and CREATE TABLE.
+*/
+static void renameTriggerFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  unsigned char const *zSql = sqlite3_value_text(argv[0]);
+  unsigned char const *zTableName = sqlite3_value_text(argv[1]);
+
+  int token;
+  Token tname;
+  int dist = 3;
+  unsigned char const *zCsr = zSql;
+  int len = 0;
+  char *zRet;
+
+  sqlite3 *db = sqlite3_context_db_handle(context);
+
+  /* The principle used to locate the table name in the CREATE TRIGGER 
+  ** statement is that the table name is the first token that is immediatedly
+  ** preceded by either TK_ON or TK_DOT and immediatedly followed by one
+  ** of TK_WHEN, TK_BEGIN or TK_FOR.
+  */
+  if( zSql ){
+    do {
+
+      if( !*zCsr ){
+        /* Ran out of input before finding the table name. Return NULL. */
+        return;
+      }
+
+      /* Store the token that zCsr points to in tname. */
+      tname.z = zCsr;
+      tname.n = len;
+
+      /* Advance zCsr to the next token. Store that token type in 'token',
+      ** and its length in 'len' (to be used next iteration of this loop).
+      */
+      do {
+        zCsr += len;
+        len = sqlite3GetToken(zCsr, &token);
+      }while( token==TK_SPACE );
+      assert( len>0 );
+
+      /* Variable 'dist' stores the number of tokens read since the most
+      ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN 
+      ** token is read and 'dist' equals 2, the condition stated above
+      ** to be met.
+      **
+      ** Note that ON cannot be a database, table or column name, so
+      ** there is no need to worry about syntax like 
+      ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc.
+      */
+      dist++;
+      if( token==TK_DOT || token==TK_ON ){
+        dist = 0;
+      }
+    } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) );
+
+    /* Variable tname now contains the token that is the old table-name
+    ** in the CREATE TRIGGER statement.
+    */
+    zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", tname.z - zSql, zSql, 
+       zTableName, tname.z+tname.n);
+    sqlite3_result_text(context, zRet, -1, sqlite3_free);
+  }
+}
+#endif   /* !SQLITE_OMIT_TRIGGER */
+
+/*
+** Register built-in functions used to help implement ALTER TABLE
+*/
+void sqlite3AlterFunctions(sqlite3 *db){
+  sqlite3CreateFunc(db, "sqlite_rename_table", 2, SQLITE_UTF8, 0,
+                         renameTableFunc, 0, 0);
+#ifndef SQLITE_OMIT_TRIGGER
+  sqlite3CreateFunc(db, "sqlite_rename_trigger", 2, SQLITE_UTF8, 0,
+                         renameTriggerFunc, 0, 0);
+#endif
+}
+
+/*
+** Generate the text of a WHERE expression which can be used to select all
+** temporary triggers on table pTab from the sqlite_temp_master table. If
+** table pTab has no temporary triggers, or is itself stored in the 
+** temporary database, NULL is returned.
+*/
+static char *whereTempTriggers(Parse *pParse, Table *pTab){
+  Trigger *pTrig;
+  char *zWhere = 0;
+  char *tmp = 0;
+  const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */
+
+  /* If the table is not located in the temp-db (in which case NULL is 
+  ** returned, loop through the tables list of triggers. For each trigger
+  ** that is not part of the temp-db schema, add a clause to the WHERE 
+  ** expression being built up in zWhere.
+  */
+  if( pTab->pSchema!=pTempSchema ){
+    sqlite3 *db = pParse->db;
+    for( pTrig=pTab->pTrigger; pTrig; pTrig=pTrig->pNext ){
+      if( pTrig->pSchema==pTempSchema ){
+        if( !zWhere ){
+          zWhere = sqlite3MPrintf(db, "name=%Q", pTrig->name);
+        }else{
+          tmp = zWhere;
+          zWhere = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, pTrig->name);
+          sqlite3_free(tmp);
+        }
+      }
+    }
+  }
+  return zWhere;
+}
+
+/*
+** Generate code to drop and reload the internal representation of table
+** pTab from the database, including triggers and temporary triggers.
+** Argument zName is the name of the table in the database schema at
+** the time the generated code is executed. This can be different from
+** pTab->zName if this function is being called to code part of an 
+** "ALTER TABLE RENAME TO" statement.
+*/
+static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){
+  Vdbe *v;
+  char *zWhere;
+  int iDb;                   /* Index of database containing pTab */
+#ifndef SQLITE_OMIT_TRIGGER
+  Trigger *pTrig;
+#endif
+
+  v = sqlite3GetVdbe(pParse);
+  if( !v ) return;
+  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
+  iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
+  assert( iDb>=0 );
+
+#ifndef SQLITE_OMIT_TRIGGER
+  /* Drop any table triggers from the internal schema. */
+  for(pTrig=pTab->pTrigger; pTrig; pTrig=pTrig->pNext){
+    int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema);
+    assert( iTrigDb==iDb || iTrigDb==1 );
+    sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->name, 0);
+  }
+#endif
+
+  /* Drop the table and index from the internal schema */
+  sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
+
+  /* Reload the table, index and permanent trigger schemas. */
+  zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
+  if( !zWhere ) return;
+  sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
+
+#ifndef SQLITE_OMIT_TRIGGER
+  /* Now, if the table is not stored in the temp database, reload any temp 
+  ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. 
+  */
+  if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
+    sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC);
+  }
+#endif
+}
+
+/*
+** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" 
+** command. 
+*/
+void sqlite3AlterRenameTable(
+  Parse *pParse,            /* Parser context. */
+  SrcList *pSrc,            /* The table to rename. */
+  Token *pName              /* The new table name. */
+){
+  int iDb;                  /* Database that contains the table */
+  char *zDb;                /* Name of database iDb */
+  Table *pTab;              /* Table being renamed */
+  char *zName = 0;          /* NULL-terminated version of pName */ 
+  sqlite3 *db = pParse->db; /* Database connection */
+  int nTabName;             /* Number of UTF-8 characters in zTabName */
+  const char *zTabName;     /* Original name of the table */
+  Vdbe *v;
+#ifndef SQLITE_OMIT_TRIGGER
+  char *zWhere = 0;         /* Where clause to locate temp triggers */
+#endif
+  int isVirtualRename = 0;  /* True if this is a v-table with an xRename() */
+  
+  if( db->mallocFailed ) goto exit_rename_table;
+  assert( pSrc->nSrc==1 );
+  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
+
+  pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase);
+  if( !pTab ) goto exit_rename_table;
+  iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
+  zDb = db->aDb[iDb].zName;
+
+  /* Get a NULL terminated version of the new table name. */
+  zName = sqlite3NameFromToken(db, pName);
+  if( !zName ) goto exit_rename_table;
+
+  /* Check that a table or index named 'zName' does not already exist
+  ** in database iDb. If so, this is an error.
+  */
+  if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){
+    sqlite3ErrorMsg(pParse, 
+        "there is already another table or index with this name: %s", zName);
+    goto exit_rename_table;
+  }
+
+  /* Make sure it is not a system table being altered, or a reserved name
+  ** that the table is being renamed to.
+  */
+  if( strlen(pTab->zName)>6 && 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ){
+    sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName);
+    goto exit_rename_table;
+  }
+  if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){
+    goto exit_rename_table;
+  }
+
+#ifndef SQLITE_OMIT_VIEW
+  if( pTab->pSelect ){
+    sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName);
+    goto exit_rename_table;
+  }
+#endif
+
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  /* Invoke the authorization callback. */
+  if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){
+    goto exit_rename_table;
+  }
+#endif
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( sqlite3ViewGetColumnNames(pParse, pTab) ){
+    goto exit_rename_table;
+  }
+  if( IsVirtual(pTab) && pTab->pMod->pModule->xRename ){
+    isVirtualRename = 1;
+  }
+#endif
+
+  /* Begin a transaction and code the VerifyCookie for database iDb. 
+  ** Then modify the schema cookie (since the ALTER TABLE modifies the
+  ** schema). Open a statement transaction if the table is a virtual
+  ** table.
+  */
+  v = sqlite3GetVdbe(pParse);
+  if( v==0 ){
+    goto exit_rename_table;
+  }
+  sqlite3BeginWriteOperation(pParse, isVirtualRename, iDb);
+  sqlite3ChangeCookie(pParse, iDb);
+
+  /* If this is a virtual table, invoke the xRename() function if
+  ** one is defined. The xRename() callback will modify the names
+  ** of any resources used by the v-table implementation (including other
+  ** SQLite tables) that are identified by the name of the virtual table.
+  */
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( isVirtualRename ){
+    int i = ++pParse->nMem;
+    sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0);
+    sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pTab->pVtab, P4_VTAB);
+  }
+#endif
+
+  /* figure out how many UTF-8 characters are in zName */
+  zTabName = pTab->zName;
+  nTabName = sqlite3Utf8CharLen(zTabName, -1);
+
+  /* Modify the sqlite_master table to use the new table name. */
+  sqlite3NestedParse(pParse,
+      "UPDATE %Q.%s SET "
+#ifdef SQLITE_OMIT_TRIGGER
+          "sql = sqlite_rename_table(sql, %Q), "
+#else
+          "sql = CASE "
+            "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)"
+            "ELSE sqlite_rename_table(sql, %Q) END, "
+#endif
+          "tbl_name = %Q, "
+          "name = CASE "
+            "WHEN type='table' THEN %Q "
+            "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN "
+             "'sqlite_autoindex_' || %Q || substr(name,%d+18) "
+            "ELSE name END "
+      "WHERE tbl_name=%Q AND "
+          "(type='table' OR type='index' OR type='trigger');", 
+      zDb, SCHEMA_TABLE(iDb), zName, zName, zName, 
+#ifndef SQLITE_OMIT_TRIGGER
+      zName,
+#endif
+      zName, nTabName, zTabName
+  );
+
+#ifndef SQLITE_OMIT_AUTOINCREMENT
+  /* If the sqlite_sequence table exists in this database, then update 
+  ** it with the new table name.
+  */
+  if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){
+    sqlite3NestedParse(pParse,
+        "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q",
+        zDb, zName, pTab->zName);
+  }
+#endif
+
+#ifndef SQLITE_OMIT_TRIGGER
+  /* If there are TEMP triggers on this table, modify the sqlite_temp_master
+  ** table. Don't do this if the table being ALTERed is itself located in
+  ** the temp database.
+  */
+  if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
+    sqlite3NestedParse(pParse, 
+        "UPDATE sqlite_temp_master SET "
+            "sql = sqlite_rename_trigger(sql, %Q), "
+            "tbl_name = %Q "
+            "WHERE %s;", zName, zName, zWhere);
+    sqlite3_free(zWhere);
+  }
+#endif
+
+  /* Drop and reload the internal table schema. */
+  reloadTableSchema(pParse, pTab, zName);
+
+exit_rename_table:
+  sqlite3SrcListDelete(pSrc);
+  sqlite3_free(zName);
+}
+
+
+/*
+** This function is called after an "ALTER TABLE ... ADD" statement
+** has been parsed. Argument pColDef contains the text of the new
+** column definition.
+**
+** The Table structure pParse->pNewTable was extended to include
+** the new column during parsing.
+*/
+void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
+  Table *pNew;              /* Copy of pParse->pNewTable */
+  Table *pTab;              /* Table being altered */
+  int iDb;                  /* Database number */
+  const char *zDb;          /* Database name */
+  const char *zTab;         /* Table name */
+  char *zCol;               /* Null-terminated column definition */
+  Column *pCol;             /* The new column */
+  Expr *pDflt;              /* Default value for the new column */
+  sqlite3 *db;              /* The database connection; */
+
+  if( pParse->nErr ) return;
+  pNew = pParse->pNewTable;
+  assert( pNew );
+
+  db = pParse->db;
+  assert( sqlite3BtreeHoldsAllMutexes(db) );
+  iDb = sqlite3SchemaToIndex(db, pNew->pSchema);
+  zDb = db->aDb[iDb].zName;
+  zTab = pNew->zName;
+  pCol = &pNew->aCol[pNew->nCol-1];
+  pDflt = pCol->pDflt;
+  pTab = sqlite3FindTable(db, zTab, zDb);
+  assert( pTab );
+
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  /* Invoke the authorization callback. */
+  if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){
+    return;
+  }
+#endif
+
+  /* If the default value for the new column was specified with a 
+  ** literal NULL, then set pDflt to 0. This simplifies checking
+  ** for an SQL NULL default below.
+  */
+  if( pDflt && pDflt->op==TK_NULL ){
+    pDflt = 0;
+  }
+
+  /* Check that the new column is not specified as PRIMARY KEY or UNIQUE.
+  ** If there is a NOT NULL constraint, then the default value for the
+  ** column must not be NULL.
+  */
+  if( pCol->isPrimKey ){
+    sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column");
+    return;
+  }
+  if( pNew->pIndex ){
+    sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column");
+    return;
+  }
+  if( pCol->notNull && !pDflt ){
+    sqlite3ErrorMsg(pParse, 
+        "Cannot add a NOT NULL column with default value NULL");
+    return;
+  }
+
+  /* Ensure the default expression is something that sqlite3ValueFromExpr()
+  ** can handle (i.e. not CURRENT_TIME etc.)
+  */
+  if( pDflt ){
+    sqlite3_value *pVal;
+    if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){
+      db->mallocFailed = 1;
+      return;
+    }
+    if( !pVal ){
+      sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default");
+      return;
+    }
+    sqlite3ValueFree(pVal);
+  }
+
+  /* Modify the CREATE TABLE statement. */
+  zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n);
+  if( zCol ){
+    char *zEnd = &zCol[pColDef->n-1];
+    while( (zEnd>zCol && *zEnd==';') || isspace(*(unsigned char *)zEnd) ){
+      *zEnd-- = '\0';
+    }
+    sqlite3NestedParse(pParse, 
+        "UPDATE \"%w\".%s SET "
+          "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) "
+        "WHERE type = 'table' AND name = %Q", 
+      zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1,
+      zTab
+    );
+    sqlite3_free(zCol);
+  }
+
+  /* If the default value of the new column is NULL, then set the file
+  ** format to 2. If the default value of the new column is not NULL,
+  ** the file format becomes 3.
+  */
+  sqlite3MinimumFileFormat(pParse, iDb, pDflt ? 3 : 2);
+
+  /* Reload the schema of the modified table. */
+  reloadTableSchema(pParse, pTab, pTab->zName);
+}
+
+/*
+** This function is called by the parser after the table-name in
+** an "ALTER TABLE <table-name> ADD" statement is parsed. Argument 
+** pSrc is the full-name of the table being altered.
+**
+** This routine makes a (partial) copy of the Table structure
+** for the table being altered and sets Parse.pNewTable to point
+** to it. Routines called by the parser as the column definition
+** is parsed (i.e. sqlite3AddColumn()) add the new Column data to 
+** the copy. The copy of the Table structure is deleted by tokenize.c 
+** after parsing is finished.
+**
+** Routine sqlite3AlterFinishAddColumn() will be called to complete
+** coding the "ALTER TABLE ... ADD" statement.
+*/
+void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){
+  Table *pNew;
+  Table *pTab;
+  Vdbe *v;
+  int iDb;
+  int i;
+  int nAlloc;
+  sqlite3 *db = pParse->db;
+
+  /* Look up the table being altered. */
+  assert( pParse->pNewTable==0 );
+  assert( sqlite3BtreeHoldsAllMutexes(db) );
+  if( db->mallocFailed ) goto exit_begin_add_column;
+  pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase);
+  if( !pTab ) goto exit_begin_add_column;
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( IsVirtual(pTab) ){
+    sqlite3ErrorMsg(pParse, "virtual tables may not be altered");
+    goto exit_begin_add_column;
+  }
+#endif
+
+  /* Make sure this is not an attempt to ALTER a view. */
+  if( pTab->pSelect ){
+    sqlite3ErrorMsg(pParse, "Cannot add a column to a view");
+    goto exit_begin_add_column;
+  }
+
+  assert( pTab->addColOffset>0 );
+  iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+
+  /* Put a copy of the Table struct in Parse.pNewTable for the
+  ** sqlite3AddColumn() function and friends to modify.
+  */
+  pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table));
+  if( !pNew ) goto exit_begin_add_column;
+  pParse->pNewTable = pNew;
+  pNew->nRef = 1;
+  pNew->nCol = pTab->nCol;
+  assert( pNew->nCol>0 );
+  nAlloc = (((pNew->nCol-1)/8)*8)+8;
+  assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 );
+  pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc);
+  pNew->zName = sqlite3DbStrDup(db, pTab->zName);
+  if( !pNew->aCol || !pNew->zName ){
+    db->mallocFailed = 1;
+    goto exit_begin_add_column;
+  }
+  memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol);
+  for(i=0; i<pNew->nCol; i++){
+    Column *pCol = &pNew->aCol[i];
+    pCol->zName = sqlite3DbStrDup(db, pCol->zName);
+    pCol->zColl = 0;
+    pCol->zType = 0;
+    pCol->pDflt = 0;
+  }
+  pNew->pSchema = db->aDb[iDb].pSchema;
+  pNew->addColOffset = pTab->addColOffset;
+  pNew->nRef = 1;
+
+  /* Begin a transaction and increment the schema cookie.  */
+  sqlite3BeginWriteOperation(pParse, 0, iDb);
+  v = sqlite3GetVdbe(pParse);
+  if( !v ) goto exit_begin_add_column;
+  sqlite3ChangeCookie(pParse, iDb);
+
+exit_begin_add_column:
+  sqlite3SrcListDelete(pSrc);
+  return;
+}
+#endif  /* SQLITE_ALTER_TABLE */

src/analyze.c

@@ -0,0 +1,425 @@
+/*
+** 2005 July 8
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains code associated with the ANALYZE command.
+**
+** @(#) $Id: analyze.c,v 1.42 2008/03/25 09:47:35 danielk1977 Exp $
+*/
+#ifndef SQLITE_OMIT_ANALYZE
+#include "sqliteInt.h"
+
+/*
+** This routine generates code that opens the sqlite_stat1 table on cursor
+** iStatCur.
+**
+** If the sqlite_stat1 tables does not previously exist, it is created.
+** If it does previously exist, all entires associated with table zWhere
+** are removed.  If zWhere==0 then all entries are removed.
+*/
+static void openStatTable(
+  Parse *pParse,          /* Parsing context */
+  int iDb,                /* The database we are looking in */
+  int iStatCur,           /* Open the sqlite_stat1 table on this cursor */
+  const char *zWhere      /* Delete entries associated with this table */
+){
+  sqlite3 *db = pParse->db;
+  Db *pDb;
+  int iRootPage;
+  int createStat1 = 0;
+  Table *pStat;
+  Vdbe *v = sqlite3GetVdbe(pParse);
+
+  if( v==0 ) return;
+  assert( sqlite3BtreeHoldsAllMutexes(db) );
+  assert( sqlite3VdbeDb(v)==db );
+  pDb = &db->aDb[iDb];
+  if( (pStat = sqlite3FindTable(db, "sqlite_stat1", pDb->zName))==0 ){
+    /* The sqlite_stat1 tables does not exist.  Create it.  
+    ** Note that a side-effect of the CREATE TABLE statement is to leave
+    ** the rootpage of the new table in register pParse->regRoot.  This is
+    ** important because the OpenWrite opcode below will be needing it. */
+    sqlite3NestedParse(pParse,
+      "CREATE TABLE %Q.sqlite_stat1(tbl,idx,stat)",
+      pDb->zName
+    );
+    iRootPage = pParse->regRoot;
+    createStat1 = 1;  /* Cause rootpage to be taken from top of stack */
+  }else if( zWhere ){
+    /* The sqlite_stat1 table exists.  Delete all entries associated with
+    ** the table zWhere. */
+    sqlite3NestedParse(pParse,
+       "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q",
+       pDb->zName, zWhere
+    );
+    iRootPage = pStat->tnum;
+  }else{
+    /* The sqlite_stat1 table already exists.  Delete all rows. */
+    iRootPage = pStat->tnum;
+    sqlite3VdbeAddOp2(v, OP_Clear, pStat->tnum, iDb);
+  }
+
+  /* Open the sqlite_stat1 table for writing. Unless it was created
+  ** by this vdbe program, lock it for writing at the shared-cache level. 
+  ** If this vdbe did create the sqlite_stat1 table, then it must have 
+  ** already obtained a schema-lock, making the write-lock redundant.
+  */
+  if( !createStat1 ){
+    sqlite3TableLock(pParse, iDb, iRootPage, 1, "sqlite_stat1");
+  }
+  sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, 3);
+  sqlite3VdbeAddOp3(v, OP_OpenWrite, iStatCur, iRootPage, iDb);
+  sqlite3VdbeChangeP5(v, createStat1);
+}
+
+/*
+** Generate code to do an analysis of all indices associated with
+** a single table.
+*/
+static void analyzeOneTable(
+  Parse *pParse,   /* Parser context */
+  Table *pTab,     /* Table whose indices are to be analyzed */
+  int iStatCur,    /* Cursor that writes to the sqlite_stat1 table */
+  int iMem         /* Available memory locations begin here */
+){
+  Index *pIdx;     /* An index to being analyzed */
+  int iIdxCur;     /* Cursor number for index being analyzed */
+  int nCol;        /* Number of columns in the index */
+  Vdbe *v;         /* The virtual machine being built up */
+  int i;           /* Loop counter */
+  int topOfLoop;   /* The top of the loop */
+  int endOfLoop;   /* The end of the loop */
+  int addr;        /* The address of an instruction */
+  int iDb;         /* Index of database containing pTab */
+
+  v = sqlite3GetVdbe(pParse);
+  if( v==0 || pTab==0 || pTab->pIndex==0 ){
+    /* Do no analysis for tables that have no indices */
+    return;
+  }
+  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
+  iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
+  assert( iDb>=0 );
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0,
+      pParse->db->aDb[iDb].zName ) ){
+    return;
+  }
+#endif
+
+  /* Establish a read-lock on the table at the shared-cache level. */
+  sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
+
+  iIdxCur = pParse->nTab;
+  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx);
+    int regFields;    /* Register block for building records */
+    int regRec;       /* Register holding completed record */
+    int regTemp;      /* Temporary use register */
+    int regCol;       /* Content of a column from the table being analyzed */
+    int regRowid;     /* Rowid for the inserted record */
+    int regF2;
+
+    /* Open a cursor to the index to be analyzed
+    */
+    assert( iDb==sqlite3SchemaToIndex(pParse->db, pIdx->pSchema) );
+    nCol = pIdx->nColumn;
+    sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, nCol+1);
+    sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb,
+        (char *)pKey, P4_KEYINFO_HANDOFF);
+    VdbeComment((v, "%s", pIdx->zName));
+    regFields = iMem+nCol*2;
+    regTemp = regRowid = regCol = regFields+3;
+    regRec = regCol+1;
+    if( regRec>pParse->nMem ){
+      pParse->nMem = regRec;
+    }
+
+    /* Memory cells are used as follows:
+    **
+    **    mem[iMem]:             The total number of rows in the table.
+    **    mem[iMem+1]:           Number of distinct values in column 1
+    **    ...
+    **    mem[iMem+nCol]:        Number of distinct values in column N
+    **    mem[iMem+nCol+1]       Last observed value of column 1
+    **    ...
+    **    mem[iMem+nCol+nCol]:   Last observed value of column N
+    **
+    ** Cells iMem through iMem+nCol are initialized to 0.  The others
+    ** are initialized to NULL.
+    */
+    for(i=0; i<=nCol; i++){
+      sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i);
+    }
+    for(i=0; i<nCol; i++){
+      sqlite3VdbeAddOp2(v, OP_Null, 0, iMem+nCol+i+1);
+    }
+
+    /* Do the analysis.
+    */
+    endOfLoop = sqlite3VdbeMakeLabel(v);
+    sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop);
+    topOfLoop = sqlite3VdbeCurrentAddr(v);
+    sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);
+    for(i=0; i<nCol; i++){
+      sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol);
+      sqlite3VdbeAddOp3(v, OP_Ne, regCol, 0, iMem+nCol+i+1);
+      /**** TODO:  add collating sequence *****/
+      sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
+    }
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop);
+    for(i=0; i<nCol; i++){
+      sqlite3VdbeJumpHere(v, topOfLoop + 2*(i + 1));
+      sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1);
+      sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1);
+    }
+    sqlite3VdbeResolveLabel(v, endOfLoop);
+    sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop);
+    sqlite3VdbeAddOp1(v, OP_Close, iIdxCur);
+
+    /* Store the results.  
+    **
+    ** The result is a single row of the sqlite_stat1 table.  The first
+    ** two columns are the names of the table and index.  The third column
+    ** is a string composed of a list of integer statistics about the
+    ** index.  The first integer in the list is the total number of entires
+    ** in the index.  There is one additional integer in the list for each
+    ** column of the table.  This additional integer is a guess of how many
+    ** rows of the table the index will select.  If D is the count of distinct
+    ** values and K is the total number of rows, then the integer is computed
+    ** as:
+    **
+    **        I = (K+D-1)/D
+    **
+    ** If K==0 then no entry is made into the sqlite_stat1 table.  
+    ** If K>0 then it is always the case the D>0 so division by zero
+    ** is never possible.
+    */
+    addr = sqlite3VdbeAddOp1(v, OP_IfNot, iMem);
+    sqlite3VdbeAddOp4(v, OP_String8, 0, regFields, 0, pTab->zName, 0);
+    sqlite3VdbeAddOp4(v, OP_String8, 0, regFields+1, 0, pIdx->zName, 0);
+    regF2 = regFields+2;
+    sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regF2);
+    for(i=0; i<nCol; i++){
+      sqlite3VdbeAddOp4(v, OP_String8, 0, regTemp, 0, " ", 0);
+      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regF2, regF2);
+      sqlite3VdbeAddOp3(v, OP_Add, iMem, iMem+i+1, regTemp);
+      sqlite3VdbeAddOp2(v, OP_AddImm, regTemp, -1);
+      sqlite3VdbeAddOp3(v, OP_Divide, iMem+i+1, regTemp, regTemp);
+      sqlite3VdbeAddOp1(v, OP_ToInt, regTemp);
+      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regF2, regF2);
+    }
+    sqlite3VdbeAddOp4(v, OP_MakeRecord, regFields, 3, regRec, "aaa", 0);
+    sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid);
+    sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid);
+    sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
+    sqlite3VdbeJumpHere(v, addr);
+  }
+}
+
+/*
+** Generate code that will cause the most recent index analysis to
+** be laoded into internal hash tables where is can be used.
+*/
+static void loadAnalysis(Parse *pParse, int iDb){
+  Vdbe *v = sqlite3GetVdbe(pParse);
+  if( v ){
+    sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb);
+  }
+}
+
+/*
+** Generate code that will do an analysis of an entire database
+*/
+static void analyzeDatabase(Parse *pParse, int iDb){
+  sqlite3 *db = pParse->db;
+  Schema *pSchema = db->aDb[iDb].pSchema;    /* Schema of database iDb */
+  HashElem *k;
+  int iStatCur;
+  int iMem;
+
+  sqlite3BeginWriteOperation(pParse, 0, iDb);
+  iStatCur = pParse->nTab++;
+  openStatTable(pParse, iDb, iStatCur, 0);
+  iMem = pParse->nMem+1;
+  for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){
+    Table *pTab = (Table*)sqliteHashData(k);
+    analyzeOneTable(pParse, pTab, iStatCur, iMem);
+  }
+  loadAnalysis(pParse, iDb);
+}
+
+/*
+** Generate code that will do an analysis of a single table in
+** a database.
+*/
+static void analyzeTable(Parse *pParse, Table *pTab){
+  int iDb;
+  int iStatCur;
+
+  assert( pTab!=0 );
+  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
+  iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
+  sqlite3BeginWriteOperation(pParse, 0, iDb);
+  iStatCur = pParse->nTab++;
+  openStatTable(pParse, iDb, iStatCur, pTab->zName);
+  analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem+1);
+  loadAnalysis(pParse, iDb);
+}
+
+/*
+** Generate code for the ANALYZE command.  The parser calls this routine
+** when it recognizes an ANALYZE command.
+**
+**        ANALYZE                            -- 1
+**        ANALYZE  <database>                -- 2
+**        ANALYZE  ?<database>.?<tablename>  -- 3
+**
+** Form 1 causes all indices in all attached databases to be analyzed.
+** Form 2 analyzes all indices the single database named.
+** Form 3 analyzes all indices associated with the named table.
+*/
+void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){
+  sqlite3 *db = pParse->db;
+  int iDb;
+  int i;
+  char *z, *zDb;
+  Table *pTab;
+  Token *pTableName;
+
+  /* Read the database schema. If an error occurs, leave an error message
+  ** and code in pParse and return NULL. */
+  assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
+  if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
+    return;
+  }
+
+  if( pName1==0 ){
+    /* Form 1:  Analyze everything */
+    for(i=0; i<db->nDb; i++){
+      if( i==1 ) continue;  /* Do not analyze the TEMP database */
+      analyzeDatabase(pParse, i);
+    }
+  }else if( pName2==0 || pName2->n==0 ){
+    /* Form 2:  Analyze the database or table named */
+    iDb = sqlite3FindDb(db, pName1);
+    if( iDb>=0 ){
+      analyzeDatabase(pParse, iDb);
+    }else{
+      z = sqlite3NameFromToken(db, pName1);
+      if( z ){
+        pTab = sqlite3LocateTable(pParse, 0, z, 0);
+        sqlite3_free(z);
+        if( pTab ){
+          analyzeTable(pParse, pTab);
+        }
+      }
+    }
+  }else{
+    /* Form 3: Analyze the fully qualified table name */
+    iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName);
+    if( iDb>=0 ){
+      zDb = db->aDb[iDb].zName;
+      z = sqlite3NameFromToken(db, pTableName);
+      if( z ){
+        pTab = sqlite3LocateTable(pParse, 0, z, zDb);
+        sqlite3_free(z);
+        if( pTab ){
+          analyzeTable(pParse, pTab);
+        }
+      }
+    }   
+  }
+}
+
+/*
+** Used to pass information from the analyzer reader through to the
+** callback routine.
+*/
+typedef struct analysisInfo analysisInfo;
+struct analysisInfo {
+  sqlite3 *db;
+  const char *zDatabase;
+};
+
+/*
+** This callback is invoked once for each index when reading the
+** sqlite_stat1 table.  
+**
+**     argv[0] = name of the index
+**     argv[1] = results of analysis - on integer for each column
+*/
+static int analysisLoader(void *pData, int argc, char **argv, char **azNotUsed){
+  analysisInfo *pInfo = (analysisInfo*)pData;
+  Index *pIndex;
+  int i, c;
+  unsigned int v;
+  const char *z;
+
+  assert( argc==2 );
+  if( argv==0 || argv[0]==0 || argv[1]==0 ){
+    return 0;
+  }
+  pIndex = sqlite3FindIndex(pInfo->db, argv[0], pInfo->zDatabase);
+  if( pIndex==0 ){
+    return 0;
+  }
+  z = argv[1];
+  for(i=0; *z && i<=pIndex->nColumn; i++){
+    v = 0;
+    while( (c=z[0])>='0' && c<='9' ){
+      v = v*10 + c - '0';
+      z++;
+    }
+    pIndex->aiRowEst[i] = v;
+    if( *z==' ' ) z++;
+  }
+  return 0;
+}
+
+/*
+** Load the content of the sqlite_stat1 table into the index hash tables.
+*/
+int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
+  analysisInfo sInfo;
+  HashElem *i;
+  char *zSql;
+  int rc;
+
+  assert( iDb>=0 && iDb<db->nDb );
+  assert( db->aDb[iDb].pBt!=0 );
+  assert( sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
+
+  /* Clear any prior statistics */
+  for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){
+    Index *pIdx = sqliteHashData(i);
+    sqlite3DefaultRowEst(pIdx);
+  }
+
+  /* Check to make sure the sqlite_stat1 table existss */
+  sInfo.db = db;
+  sInfo.zDatabase = db->aDb[iDb].zName;
+  if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){
+     return SQLITE_ERROR;
+  }
+
+
+  /* Load new statistics out of the sqlite_stat1 table */
+  zSql = sqlite3MPrintf(db, "SELECT idx, stat FROM %Q.sqlite_stat1",
+                        sInfo.zDatabase);
+  (void)sqlite3SafetyOff(db);
+  rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0);
+  (void)sqlite3SafetyOn(db);
+  sqlite3_free(zSql);
+  return rc;
+}
+
+
+#endif /* SQLITE_OMIT_ANALYZE */

src/attach.c

@@ -0,0 +1,524 @@
+/*
+** 2003 April 6
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains code used to implement the ATTACH and DETACH commands.
+**
+** $Id: attach.c,v 1.76 2008/06/15 02:51:47 drh Exp $
+*/
+#include "sqliteInt.h"
+
+#ifndef SQLITE_OMIT_ATTACH
+/*
+** Resolve an expression that was part of an ATTACH or DETACH statement. This
+** is slightly different from resolving a normal SQL expression, because simple
+** identifiers are treated as strings, not possible column names or aliases.
+**
+** i.e. if the parser sees:
+**
+**     ATTACH DATABASE abc AS def
+**
+** it treats the two expressions as literal strings 'abc' and 'def' instead of
+** looking for columns of the same name.
+**
+** This only applies to the root node of pExpr, so the statement:
+**
+**     ATTACH DATABASE abc||def AS 'db2'
+**
+** will fail because neither abc or def can be resolved.
+*/
+static int resolveAttachExpr(NameContext *pName, Expr *pExpr)
+{
+  int rc = SQLITE_OK;
+  if( pExpr ){
+    if( pExpr->op!=TK_ID ){
+      rc = sqlite3ExprResolveNames(pName, pExpr);
+      if( rc==SQLITE_OK && !sqlite3ExprIsConstant(pExpr) ){
+        sqlite3ErrorMsg(pName->pParse, "invalid name: \"%T\"", &pExpr->span);
+        return SQLITE_ERROR;
+      }
+    }else{
+      pExpr->op = TK_STRING;
+    }
+  }
+  return rc;
+}
+
+/*
+** An SQL user-function registered to do the work of an ATTACH statement. The
+** three arguments to the function come directly from an attach statement:
+**
+**     ATTACH DATABASE x AS y KEY z
+**
+**     SELECT sqlite_attach(x, y, z)
+**
+** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the
+** third argument.
+*/
+static void attachFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int i;
+  int rc = 0;
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  const char *zName;
+  const char *zFile;
+  Db *aNew;
+  char *zErrDyn = 0;
+  char zErr[128];
+
+  zFile = (const char *)sqlite3_value_text(argv[0]);
+  zName = (const char *)sqlite3_value_text(argv[1]);
+  if( zFile==0 ) zFile = "";
+  if( zName==0 ) zName = "";
+
+  /* Check for the following errors:
+  **
+  **     * Too many attached databases,
+  **     * Transaction currently open
+  **     * Specified database name already being used.
+  */
+  if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){
+    sqlite3_snprintf(
+      sizeof(zErr), zErr, "too many attached databases - max %d", 
+      db->aLimit[SQLITE_LIMIT_ATTACHED]
+    );
+    goto attach_error;
+  }
+  if( !db->autoCommit ){
+    sqlite3_snprintf(sizeof(zErr), zErr,
+                     "cannot ATTACH database within transaction");
+    goto attach_error;
+  }
+  for(i=0; i<db->nDb; i++){
+    char *z = db->aDb[i].zName;
+    if( z && zName && sqlite3StrICmp(z, zName)==0 ){
+      sqlite3_snprintf(sizeof(zErr), zErr, 
+                       "database %s is already in use", zName);
+      goto attach_error;
+    }
+  }
+
+  /* Allocate the new entry in the db->aDb[] array and initialise the schema
+  ** hash tables.
+  */
+  if( db->aDb==db->aDbStatic ){
+    aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 );
+    if( aNew==0 ) return;
+    memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2);
+  }else{
+    aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) );
+    if( aNew==0 ) return;
+  }
+  db->aDb = aNew;
+  aNew = &db->aDb[db->nDb++];
+  memset(aNew, 0, sizeof(*aNew));
+
+  /* Open the database file. If the btree is successfully opened, use
+  ** it to obtain the database schema. At this point the schema may
+  ** or may not be initialised.
+  */
+  rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE,
+                           db->openFlags | SQLITE_OPEN_MAIN_DB,
+                           &aNew->pBt);
+  if( rc==SQLITE_OK ){
+    Pager *pPager;
+    aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt);
+    if( !aNew->pSchema ){
+      rc = SQLITE_NOMEM;
+    }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){
+      sqlite3_snprintf(sizeof(zErr), zErr, 
+        "attached databases must use the same text encoding as main database");
+      goto attach_error;
+    }
+    pPager = sqlite3BtreePager(aNew->pBt);
+    sqlite3PagerLockingMode(pPager, db->dfltLockMode);
+    sqlite3PagerJournalMode(pPager, db->dfltJournalMode);
+  }
+  aNew->zName = sqlite3DbStrDup(db, zName);
+  aNew->safety_level = 3;
+
+#if SQLITE_HAS_CODEC
+  {
+    extern int sqlite3CodecAttach(sqlite3*, int, const void*, int);
+    extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
+    int nKey;
+    char *zKey;
+    int t = sqlite3_value_type(argv[2]);
+    switch( t ){
+      case SQLITE_INTEGER:
+      case SQLITE_FLOAT:
+        zErrDyn = sqlite3DbStrDup(db, "Invalid key value");
+        rc = SQLITE_ERROR;
+        break;
+        
+      case SQLITE_TEXT:
+      case SQLITE_BLOB:
+        nKey = sqlite3_value_bytes(argv[2]);
+        zKey = (char *)sqlite3_value_blob(argv[2]);
+        sqlite3CodecAttach(db, db->nDb-1, zKey, nKey);
+        break;
+
+      case SQLITE_NULL:
+        /* No key specified.  Use the key from the main database */
+        sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey);
+        sqlite3CodecAttach(db, db->nDb-1, zKey, nKey);
+        break;
+    }
+  }
+#endif
+
+  /* If the file was opened successfully, read the schema for the new database.
+  ** If this fails, or if opening the file failed, then close the file and 
+  ** remove the entry from the db->aDb[] array. i.e. put everything back the way
+  ** we found it.
+  */
+  if( rc==SQLITE_OK ){
+    (void)sqlite3SafetyOn(db);
+    sqlite3BtreeEnterAll(db);
+    rc = sqlite3Init(db, &zErrDyn);
+    sqlite3BtreeLeaveAll(db);
+    (void)sqlite3SafetyOff(db);
+  }
+  if( rc ){
+    int iDb = db->nDb - 1;
+    assert( iDb>=2 );
+    if( db->aDb[iDb].pBt ){
+      sqlite3BtreeClose(db->aDb[iDb].pBt);
+      db->aDb[iDb].pBt = 0;
+      db->aDb[iDb].pSchema = 0;
+    }
+    sqlite3ResetInternalSchema(db, 0);
+    db->nDb = iDb;
+    if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
+      db->mallocFailed = 1;
+      sqlite3_snprintf(sizeof(zErr),zErr, "out of memory");
+    }else{
+      sqlite3_snprintf(sizeof(zErr),zErr, "unable to open database: %s", zFile);
+    }
+    goto attach_error;
+  }
+  
+  return;
+
+attach_error:
+  /* Return an error if we get here */
+  if( zErrDyn ){
+    sqlite3_result_error(context, zErrDyn, -1);
+    sqlite3_free(zErrDyn);
+  }else{
+    zErr[sizeof(zErr)-1] = 0;
+    sqlite3_result_error(context, zErr, -1);
+  }
+  if( rc ) sqlite3_result_error_code(context, rc);
+}
+
+/*
+** An SQL user-function registered to do the work of an DETACH statement. The
+** three arguments to the function come directly from a detach statement:
+**
+**     DETACH DATABASE x
+**
+**     SELECT sqlite_detach(x)
+*/
+static void detachFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zName = (const char *)sqlite3_value_text(argv[0]);
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  int i;
+  Db *pDb = 0;
+  char zErr[128];
+
+  if( zName==0 ) zName = "";
+  for(i=0; i<db->nDb; i++){
+    pDb = &db->aDb[i];
+    if( pDb->pBt==0 ) continue;
+    if( sqlite3StrICmp(pDb->zName, zName)==0 ) break;
+  }
+
+  if( i>=db->nDb ){
+    sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName);
+    goto detach_error;
+  }
+  if( i<2 ){
+    sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName);
+    goto detach_error;
+  }
+  if( !db->autoCommit ){
+    sqlite3_snprintf(sizeof(zErr), zErr,
+                     "cannot DETACH database within transaction");
+    goto detach_error;
+  }
+  if( sqlite3BtreeIsInReadTrans(pDb->pBt) ){
+    sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName);
+    goto detach_error;
+  }
+
+  sqlite3BtreeClose(pDb->pBt);
+  pDb->pBt = 0;
+  pDb->pSchema = 0;
+  sqlite3ResetInternalSchema(db, 0);
+  return;
+
+detach_error:
+  sqlite3_result_error(context, zErr, -1);
+}
+
+/*
+** This procedure generates VDBE code for a single invocation of either the
+** sqlite_detach() or sqlite_attach() SQL user functions.
+*/
+static void codeAttach(
+  Parse *pParse,       /* The parser context */
+  int type,            /* Either SQLITE_ATTACH or SQLITE_DETACH */
+  const char *zFunc,   /* Either "sqlite_attach" or "sqlite_detach */
+  int nFunc,           /* Number of args to pass to zFunc */
+  Expr *pAuthArg,      /* Expression to pass to authorization callback */
+  Expr *pFilename,     /* Name of database file */
+  Expr *pDbname,       /* Name of the database to use internally */
+  Expr *pKey           /* Database key for encryption extension */
+){
+  int rc;
+  NameContext sName;
+  Vdbe *v;
+  FuncDef *pFunc;
+  sqlite3* db = pParse->db;
+  int regArgs;
+
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  assert( db->mallocFailed || pAuthArg );
+  if( pAuthArg ){
+    char *zAuthArg = sqlite3NameFromToken(db, &pAuthArg->span);
+    if( !zAuthArg ){
+      goto attach_end;
+    }
+    rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0);
+    sqlite3_free(zAuthArg);
+    if(rc!=SQLITE_OK ){
+      goto attach_end;
+    }
+  }
+#endif /* SQLITE_OMIT_AUTHORIZATION */
+
+  memset(&sName, 0, sizeof(NameContext));
+  sName.pParse = pParse;
+
+  if( 
+      SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) ||
+      SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) ||
+      SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey))
+  ){
+    pParse->nErr++;
+    goto attach_end;
+  }
+
+  v = sqlite3GetVdbe(pParse);
+  regArgs = sqlite3GetTempRange(pParse, 4);
+  sqlite3ExprCode(pParse, pFilename, regArgs);
+  sqlite3ExprCode(pParse, pDbname, regArgs+1);
+  sqlite3ExprCode(pParse, pKey, regArgs+2);
+
+  assert( v || db->mallocFailed );
+  if( v ){
+    sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-nFunc, regArgs+3);
+    sqlite3VdbeChangeP5(v, nFunc);
+    pFunc = sqlite3FindFunction(db, zFunc, strlen(zFunc), nFunc, SQLITE_UTF8,0);
+    sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF);
+
+    /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this
+    ** statement only). For DETACH, set it to false (expire all existing
+    ** statements).
+    */
+    sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH));
+  }
+  
+attach_end:
+  sqlite3ExprDelete(pFilename);
+  sqlite3ExprDelete(pDbname);
+  sqlite3ExprDelete(pKey);
+}
+
+/*
+** Called by the parser to compile a DETACH statement.
+**
+**     DETACH pDbname
+*/
+void sqlite3Detach(Parse *pParse, Expr *pDbname){
+  codeAttach(pParse, SQLITE_DETACH, "sqlite_detach", 1, pDbname, 0, 0, pDbname);
+}
+
+/*
+** Called by the parser to compile an ATTACH statement.
+**
+**     ATTACH p AS pDbname KEY pKey
+*/
+void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){
+  codeAttach(pParse, SQLITE_ATTACH, "sqlite_attach", 3, p, p, pDbname, pKey);
+}
+#endif /* SQLITE_OMIT_ATTACH */
+
+/*
+** Register the functions sqlite_attach and sqlite_detach.
+*/
+void sqlite3AttachFunctions(sqlite3 *db){
+#ifndef SQLITE_OMIT_ATTACH
+  static const int enc = SQLITE_UTF8;
+  sqlite3CreateFunc(db, "sqlite_attach", 3, enc, 0, attachFunc, 0, 0);
+  sqlite3CreateFunc(db, "sqlite_detach", 1, enc, 0, detachFunc, 0, 0);
+#endif
+}
+
+/*
+** Initialize a DbFixer structure.  This routine must be called prior
+** to passing the structure to one of the sqliteFixAAAA() routines below.
+**
+** The return value indicates whether or not fixation is required.  TRUE
+** means we do need to fix the database references, FALSE means we do not.
+*/
+int sqlite3FixInit(
+  DbFixer *pFix,      /* The fixer to be initialized */
+  Parse *pParse,      /* Error messages will be written here */
+  int iDb,            /* This is the database that must be used */
+  const char *zType,  /* "view", "trigger", or "index" */
+  const Token *pName  /* Name of the view, trigger, or index */
+){
+  sqlite3 *db;
+
+  if( iDb<0 || iDb==1 ) return 0;
+  db = pParse->db;
+  assert( db->nDb>iDb );
+  pFix->pParse = pParse;
+  pFix->zDb = db->aDb[iDb].zName;
+  pFix->zType = zType;
+  pFix->pName = pName;
+  return 1;
+}
+
+/*
+** The following set of routines walk through the parse tree and assign
+** a specific database to all table references where the database name
+** was left unspecified in the original SQL statement.  The pFix structure
+** must have been initialized by a prior call to sqlite3FixInit().
+**
+** These routines are used to make sure that an index, trigger, or
+** view in one database does not refer to objects in a different database.
+** (Exception: indices, triggers, and views in the TEMP database are
+** allowed to refer to anything.)  If a reference is explicitly made
+** to an object in a different database, an error message is added to
+** pParse->zErrMsg and these routines return non-zero.  If everything
+** checks out, these routines return 0.
+*/
+int sqlite3FixSrcList(
+  DbFixer *pFix,       /* Context of the fixation */
+  SrcList *pList       /* The Source list to check and modify */
+){
+  int i;
+  const char *zDb;
+  struct SrcList_item *pItem;
+
+  if( pList==0 ) return 0;
+  zDb = pFix->zDb;
+  for(i=0, pItem=pList->a; i<pList->nSrc; i++, pItem++){
+    if( pItem->zDatabase==0 ){
+      pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb);
+    }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){
+      sqlite3ErrorMsg(pFix->pParse,
+         "%s %T cannot reference objects in database %s",
+         pFix->zType, pFix->pName, pItem->zDatabase);
+      return 1;
+    }
+#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
+    if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1;
+    if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1;
+#endif
+  }
+  return 0;
+}
+#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
+int sqlite3FixSelect(
+  DbFixer *pFix,       /* Context of the fixation */
+  Select *pSelect      /* The SELECT statement to be fixed to one database */
+){
+  while( pSelect ){
+    if( sqlite3FixExprList(pFix, pSelect->pEList) ){
+      return 1;
+    }
+    if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){
+      return 1;
+    }
+    if( sqlite3FixExpr(pFix, pSelect->pWhere) ){
+      return 1;
+    }
+    if( sqlite3FixExpr(pFix, pSelect->pHaving) ){
+      return 1;
+    }
+    pSelect = pSelect->pPrior;
+  }
+  return 0;
+}
+int sqlite3FixExpr(
+  DbFixer *pFix,     /* Context of the fixation */
+  Expr *pExpr        /* The expression to be fixed to one database */
+){
+  while( pExpr ){
+    if( sqlite3FixSelect(pFix, pExpr->pSelect) ){
+      return 1;
+    }
+    if( sqlite3FixExprList(pFix, pExpr->pList) ){
+      return 1;
+    }
+    if( sqlite3FixExpr(pFix, pExpr->pRight) ){
+      return 1;
+    }
+    pExpr = pExpr->pLeft;
+  }
+  return 0;
+}
+int sqlite3FixExprList(
+  DbFixer *pFix,     /* Context of the fixation */
+  ExprList *pList    /* The expression to be fixed to one database */
+){
+  int i;
+  struct ExprList_item *pItem;
+  if( pList==0 ) return 0;
+  for(i=0, pItem=pList->a; i<pList->nExpr; i++, pItem++){
+    if( sqlite3FixExpr(pFix, pItem->pExpr) ){
+      return 1;
+    }
+  }
+  return 0;
+}
+#endif
+
+#ifndef SQLITE_OMIT_TRIGGER
+int sqlite3FixTriggerStep(
+  DbFixer *pFix,     /* Context of the fixation */
+  TriggerStep *pStep /* The trigger step be fixed to one database */
+){
+  while( pStep ){
+    if( sqlite3FixSelect(pFix, pStep->pSelect) ){
+      return 1;
+    }
+    if( sqlite3FixExpr(pFix, pStep->pWhere) ){
+      return 1;
+    }
+    if( sqlite3FixExprList(pFix, pStep->pExprList) ){
+      return 1;
+    }
+    pStep = pStep->pNext;
+  }
+  return 0;
+}
+#endif

src/auth.c

@@ -0,0 +1,234 @@
+/*
+** 2003 January 11
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains code used to implement the sqlite3_set_authorizer()
+** API.  This facility is an optional feature of the library.  Embedded
+** systems that do not need this facility may omit it by recompiling
+** the library with -DSQLITE_OMIT_AUTHORIZATION=1
+**
+** $Id: auth.c,v 1.29 2007/09/18 15:55:07 drh Exp $
+*/
+#include "sqliteInt.h"
+
+/*
+** All of the code in this file may be omitted by defining a single
+** macro.
+*/
+#ifndef SQLITE_OMIT_AUTHORIZATION
+
+/*
+** Set or clear the access authorization function.
+**
+** The access authorization function is be called during the compilation
+** phase to verify that the user has read and/or write access permission on
+** various fields of the database.  The first argument to the auth function
+** is a copy of the 3rd argument to this routine.  The second argument
+** to the auth function is one of these constants:
+**
+**       SQLITE_CREATE_INDEX
+**       SQLITE_CREATE_TABLE
+**       SQLITE_CREATE_TEMP_INDEX
+**       SQLITE_CREATE_TEMP_TABLE
+**       SQLITE_CREATE_TEMP_TRIGGER
+**       SQLITE_CREATE_TEMP_VIEW
+**       SQLITE_CREATE_TRIGGER
+**       SQLITE_CREATE_VIEW
+**       SQLITE_DELETE
+**       SQLITE_DROP_INDEX
+**       SQLITE_DROP_TABLE
+**       SQLITE_DROP_TEMP_INDEX
+**       SQLITE_DROP_TEMP_TABLE
+**       SQLITE_DROP_TEMP_TRIGGER
+**       SQLITE_DROP_TEMP_VIEW
+**       SQLITE_DROP_TRIGGER
+**       SQLITE_DROP_VIEW
+**       SQLITE_INSERT
+**       SQLITE_PRAGMA
+**       SQLITE_READ
+**       SQLITE_SELECT
+**       SQLITE_TRANSACTION
+**       SQLITE_UPDATE
+**
+** The third and fourth arguments to the auth function are the name of
+** the table and the column that are being accessed.  The auth function
+** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE.  If
+** SQLITE_OK is returned, it means that access is allowed.  SQLITE_DENY
+** means that the SQL statement will never-run - the sqlite3_exec() call
+** will return with an error.  SQLITE_IGNORE means that the SQL statement
+** should run but attempts to read the specified column will return NULL
+** and attempts to write the column will be ignored.
+**
+** Setting the auth function to NULL disables this hook.  The default
+** setting of the auth function is NULL.
+*/
+int sqlite3_set_authorizer(
+  sqlite3 *db,
+  int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
+  void *pArg
+){
+  sqlite3_mutex_enter(db->mutex);
+  db->xAuth = xAuth;
+  db->pAuthArg = pArg;
+  sqlite3ExpirePreparedStatements(db);
+  sqlite3_mutex_leave(db->mutex);
+  return SQLITE_OK;
+}
+
+/*
+** Write an error message into pParse->zErrMsg that explains that the
+** user-supplied authorization function returned an illegal value.
+*/
+static void sqliteAuthBadReturnCode(Parse *pParse, int rc){
+  sqlite3ErrorMsg(pParse, "illegal return value (%d) from the "
+    "authorization function - should be SQLITE_OK, SQLITE_IGNORE, "
+    "or SQLITE_DENY", rc);
+  pParse->rc = SQLITE_ERROR;
+}
+
+/*
+** The pExpr should be a TK_COLUMN expression.  The table referred to
+** is in pTabList or else it is the NEW or OLD table of a trigger.  
+** Check to see if it is OK to read this particular column.
+**
+** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN 
+** instruction into a TK_NULL.  If the auth function returns SQLITE_DENY,
+** then generate an error.
+*/
+void sqlite3AuthRead(
+  Parse *pParse,        /* The parser context */
+  Expr *pExpr,          /* The expression to check authorization on */
+  Schema *pSchema,      /* The schema of the expression */
+  SrcList *pTabList     /* All table that pExpr might refer to */
+){
+  sqlite3 *db = pParse->db;
+  int rc;
+  Table *pTab = 0;      /* The table being read */
+  const char *zCol;     /* Name of the column of the table */
+  int iSrc;             /* Index in pTabList->a[] of table being read */
+  const char *zDBase;   /* Name of database being accessed */
+  TriggerStack *pStack; /* The stack of current triggers */
+  int iDb;              /* The index of the database the expression refers to */
+
+  if( db->xAuth==0 ) return;
+  if( pExpr->op!=TK_COLUMN ) return;
+  iDb = sqlite3SchemaToIndex(pParse->db, pSchema);
+  if( iDb<0 ){
+    /* An attempt to read a column out of a subquery or other
+    ** temporary table. */
+    return;
+  }
+  for(iSrc=0; pTabList && iSrc<pTabList->nSrc; iSrc++){
+    if( pExpr->iTable==pTabList->a[iSrc].iCursor ) break;
+  }
+  if( iSrc>=0 && pTabList && iSrc<pTabList->nSrc ){
+    pTab = pTabList->a[iSrc].pTab;
+  }else if( (pStack = pParse->trigStack)!=0 ){
+    /* This must be an attempt to read the NEW or OLD pseudo-tables
+    ** of a trigger.
+    */
+    assert( pExpr->iTable==pStack->newIdx || pExpr->iTable==pStack->oldIdx );
+    pTab = pStack->pTab;
+  }
+  if( pTab==0 ) return;
+  if( pExpr->iColumn>=0 ){
+    assert( pExpr->iColumn<pTab->nCol );
+    zCol = pTab->aCol[pExpr->iColumn].zName;
+  }else if( pTab->iPKey>=0 ){
+    assert( pTab->iPKey<pTab->nCol );
+    zCol = pTab->aCol[pTab->iPKey].zName;
+  }else{
+    zCol = "ROWID";
+  }
+  assert( iDb>=0 && iDb<db->nDb );
+  zDBase = db->aDb[iDb].zName;
+  rc = db->xAuth(db->pAuthArg, SQLITE_READ, pTab->zName, zCol, zDBase, 
+                 pParse->zAuthContext);
+  if( rc==SQLITE_IGNORE ){
+    pExpr->op = TK_NULL;
+  }else if( rc==SQLITE_DENY ){
+    if( db->nDb>2 || iDb!=0 ){
+      sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited", 
+         zDBase, pTab->zName, zCol);
+    }else{
+      sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited",pTab->zName,zCol);
+    }
+    pParse->rc = SQLITE_AUTH;
+  }else if( rc!=SQLITE_OK ){
+    sqliteAuthBadReturnCode(pParse, rc);
+  }
+}
+
+/*
+** Do an authorization check using the code and arguments given.  Return
+** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY.  If SQLITE_DENY
+** is returned, then the error count and error message in pParse are
+** modified appropriately.
+*/
+int sqlite3AuthCheck(
+  Parse *pParse,
+  int code,
+  const char *zArg1,
+  const char *zArg2,
+  const char *zArg3
+){
+  sqlite3 *db = pParse->db;
+  int rc;
+
+  /* Don't do any authorization checks if the database is initialising
+  ** or if the parser is being invoked from within sqlite3_declare_vtab.
+  */
+  if( db->init.busy || IN_DECLARE_VTAB ){
+    return SQLITE_OK;
+  }
+
+  if( db->xAuth==0 ){
+    return SQLITE_OK;
+  }
+  rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext);
+  if( rc==SQLITE_DENY ){
+    sqlite3ErrorMsg(pParse, "not authorized");
+    pParse->rc = SQLITE_AUTH;
+  }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){
+    rc = SQLITE_DENY;
+    sqliteAuthBadReturnCode(pParse, rc);
+  }
+  return rc;
+}
+
+/*
+** Push an authorization context.  After this routine is called, the
+** zArg3 argument to authorization callbacks will be zContext until
+** popped.  Or if pParse==0, this routine is a no-op.
+*/
+void sqlite3AuthContextPush(
+  Parse *pParse,
+  AuthContext *pContext, 
+  const char *zContext
+){
+  pContext->pParse = pParse;
+  if( pParse ){
+    pContext->zAuthContext = pParse->zAuthContext;
+    pParse->zAuthContext = zContext;
+  }
+}
+
+/*
+** Pop an authorization context that was previously pushed
+** by sqlite3AuthContextPush
+*/
+void sqlite3AuthContextPop(AuthContext *pContext){
+  if( pContext->pParse ){
+    pContext->pParse->zAuthContext = pContext->zAuthContext;
+    pContext->pParse = 0;
+  }
+}
+
+#endif /* SQLITE_OMIT_AUTHORIZATION */

src/bitvec.c

@@ -0,0 +1,325 @@
+/*
+** 2008 February 16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements an object that represents a fixed-length
+** bitmap.  Bits are numbered starting with 1.
+**
+** A bitmap is used to record what pages a database file have been
+** journalled during a transaction.  Usually only a few pages are
+** journalled.  So the bitmap is usually sparse and has low cardinality.
+** But sometimes (for example when during a DROP of a large table) most
+** or all of the pages get journalled.  In those cases, the bitmap becomes
+** dense.  The algorithm needs to handle both cases well.
+**
+** The size of the bitmap is fixed when the object is created.
+**
+** All bits are clear when the bitmap is created.  Individual bits
+** may be set or cleared one at a time.
+**
+** Test operations are about 100 times more common that set operations.
+** Clear operations are exceedingly rare.  There are usually between
+** 5 and 500 set operations per Bitvec object, though the number of sets can
+** sometimes grow into tens of thousands or larger.  The size of the
+** Bitvec object is the number of pages in the database file at the
+** start of a transaction, and is thus usually less than a few thousand,
+** but can be as large as 2 billion for a really big database.
+**
+** @(#) $Id: bitvec.c,v 1.6 2008/06/20 14:59:51 danielk1977 Exp $
+*/
+#include "sqliteInt.h"
+
+#define BITVEC_SZ        512
+/* Round the union size down to the nearest pointer boundary, since that's how 
+** it will be aligned within the Bitvec struct. */
+#define BITVEC_USIZE     (((BITVEC_SZ-12)/sizeof(Bitvec*))*sizeof(Bitvec*))
+#define BITVEC_NCHAR     BITVEC_USIZE
+#define BITVEC_NBIT      (BITVEC_NCHAR*8)
+#define BITVEC_NINT      (BITVEC_USIZE/4)
+#define BITVEC_MXHASH    (BITVEC_NINT/2)
+#define BITVEC_NPTR      (BITVEC_USIZE/sizeof(Bitvec *))
+
+#define BITVEC_HASH(X)   (((X)*37)%BITVEC_NINT)
+
+/*
+** A bitmap is an instance of the following structure.
+**
+** This bitmap records the existance of zero or more bits
+** with values between 1 and iSize, inclusive.
+**
+** There are three possible representations of the bitmap.
+** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight
+** bitmap.  The least significant bit is bit 1.
+**
+** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is
+** a hash table that will hold up to BITVEC_MXHASH distinct values.
+**
+** Otherwise, the value i is redirected into one of BITVEC_NPTR
+** sub-bitmaps pointed to by Bitvec.u.apSub[].  Each subbitmap
+** handles up to iDivisor separate values of i.  apSub[0] holds
+** values between 1 and iDivisor.  apSub[1] holds values between
+** iDivisor+1 and 2*iDivisor.  apSub[N] holds values between
+** N*iDivisor+1 and (N+1)*iDivisor.  Each subbitmap is normalized
+** to hold deal with values between 1 and iDivisor.
+*/
+struct Bitvec {
+  u32 iSize;      /* Maximum bit index */
+  u32 nSet;       /* Number of bits that are set */
+  u32 iDivisor;   /* Number of bits handled by each apSub[] entry */
+  union {
+    u8 aBitmap[BITVEC_NCHAR];    /* Bitmap representation */
+    u32 aHash[BITVEC_NINT];      /* Hash table representation */
+    Bitvec *apSub[BITVEC_NPTR];  /* Recursive representation */
+  } u;
+};
+
+/*
+** Create a new bitmap object able to handle bits between 0 and iSize,
+** inclusive.  Return a pointer to the new object.  Return NULL if 
+** malloc fails.
+*/
+Bitvec *sqlite3BitvecCreate(u32 iSize){
+  Bitvec *p;
+  assert( sizeof(*p)==BITVEC_SZ );
+  p = sqlite3MallocZero( sizeof(*p) );
+  if( p ){
+    p->iSize = iSize;
+  }
+  return p;
+}
+
+/*
+** Check to see if the i-th bit is set.  Return true or false.
+** If p is NULL (if the bitmap has not been created) or if
+** i is out of range, then return false.
+*/
+int sqlite3BitvecTest(Bitvec *p, u32 i){
+  if( p==0 ) return 0;
+  if( i>p->iSize || i==0 ) return 0;
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    return (p->u.aBitmap[i/8] & (1<<(i&7)))!=0;
+  }
+  if( p->iDivisor>0 ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    return sqlite3BitvecTest(p->u.apSub[bin], i);
+  }else{
+    u32 h = BITVEC_HASH(i);
+    while( p->u.aHash[h] ){
+      if( p->u.aHash[h]==i ) return 1;
+      h++;
+      if( h>=BITVEC_NINT ) h = 0;
+    }
+    return 0;
+  }
+}
+
+/*
+** Set the i-th bit.  Return 0 on success and an error code if
+** anything goes wrong.
+*/
+int sqlite3BitvecSet(Bitvec *p, u32 i){
+  u32 h;
+  assert( p!=0 );
+  assert( i>0 );
+  assert( i<=p->iSize );
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    p->u.aBitmap[i/8] |= 1 << (i&7);
+    return SQLITE_OK;
+  }
+  if( p->iDivisor ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    if( p->u.apSub[bin]==0 ){
+      sqlite3BeginBenignMalloc();
+      p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor );
+      sqlite3EndBenignMalloc();
+      if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM;
+    }
+    return sqlite3BitvecSet(p->u.apSub[bin], i);
+  }
+  h = BITVEC_HASH(i);
+  while( p->u.aHash[h] ){
+    if( p->u.aHash[h]==i ) return SQLITE_OK;
+    h++;
+    if( h==BITVEC_NINT ) h = 0;
+  }
+  p->nSet++;
+  if( p->nSet>=BITVEC_MXHASH ){
+    int j, rc;
+    u32 aiValues[BITVEC_NINT];
+    memcpy(aiValues, p->u.aHash, sizeof(aiValues));
+    memset(p->u.apSub, 0, sizeof(p->u.apSub[0])*BITVEC_NPTR);
+    p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR;
+    rc = sqlite3BitvecSet(p, i);
+    for(j=0; j<BITVEC_NINT; j++){
+      if( aiValues[j] ) rc |= sqlite3BitvecSet(p, aiValues[j]);
+    }
+    return rc;
+  }
+  p->u.aHash[h] = i;
+  return SQLITE_OK;
+}
+
+/*
+** Clear the i-th bit.  Return 0 on success and an error code if
+** anything goes wrong.
+*/
+void sqlite3BitvecClear(Bitvec *p, u32 i){
+  assert( p!=0 );
+  assert( i>0 );
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    p->u.aBitmap[i/8] &= ~(1 << (i&7));
+  }else if( p->iDivisor ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    if( p->u.apSub[bin] ){
+      sqlite3BitvecClear(p->u.apSub[bin], i);
+    }
+  }else{
+    int j;
+    u32 aiValues[BITVEC_NINT];
+    memcpy(aiValues, p->u.aHash, sizeof(aiValues));
+    memset(p->u.aHash, 0, sizeof(p->u.aHash[0])*BITVEC_NINT);
+    p->nSet = 0;
+    for(j=0; j<BITVEC_NINT; j++){
+      if( aiValues[j] && aiValues[j]!=i ){
+        sqlite3BitvecSet(p, aiValues[j]);
+      }
+    }
+  }
+}
+
+/*
+** Destroy a bitmap object.  Reclaim all memory used.
+*/
+void sqlite3BitvecDestroy(Bitvec *p){
+  if( p==0 ) return;
+  if( p->iDivisor ){
+    int i;
+    for(i=0; i<BITVEC_NPTR; i++){
+      sqlite3BitvecDestroy(p->u.apSub[i]);
+    }
+  }
+  sqlite3_free(p);
+}
+
+#ifndef SQLITE_OMIT_BUILTIN_TEST
+/*
+** Let V[] be an array of unsigned characters sufficient to hold
+** up to N bits.  Let I be an integer between 0 and N.  0<=I<N.
+** Then the following macros can be used to set, clear, or test
+** individual bits within V.
+*/
+#define SETBIT(V,I)      V[I>>3] |= (1<<(I&7))
+#define CLEARBIT(V,I)    V[I>>3] &= ~(1<<(I&7))
+#define TESTBIT(V,I)     (V[I>>3]&(1<<(I&7)))!=0
+
+/*
+** This routine runs an extensive test of the Bitvec code.
+**
+** The input is an array of integers that acts as a program
+** to test the Bitvec.  The integers are opcodes followed
+** by 0, 1, or 3 operands, depending on the opcode.  Another
+** opcode follows immediately after the last operand.
+**
+** There are 6 opcodes numbered from 0 through 5.  0 is the
+** "halt" opcode and causes the test to end.
+**
+**    0          Halt and return the number of errors
+**    1 N S X    Set N bits beginning with S and incrementing by X
+**    2 N S X    Clear N bits beginning with S and incrementing by X
+**    3 N        Set N randomly chosen bits
+**    4 N        Clear N randomly chosen bits
+**    5 N S X    Set N bits from S increment X in array only, not in bitvec
+**
+** The opcodes 1 through 4 perform set and clear operations are performed
+** on both a Bitvec object and on a linear array of bits obtained from malloc.
+** Opcode 5 works on the linear array only, not on the Bitvec.
+** Opcode 5 is used to deliberately induce a fault in order to
+** confirm that error detection works.
+**
+** At the conclusion of the test the linear array is compared
+** against the Bitvec object.  If there are any differences,
+** an error is returned.  If they are the same, zero is returned.
+**
+** If a memory allocation error occurs, return -1.
+*/
+int sqlite3BitvecBuiltinTest(int sz, int *aOp){
+  Bitvec *pBitvec = 0;
+  unsigned char *pV = 0;
+  int rc = -1;
+  int i, nx, pc, op;
+
+  /* Allocate the Bitvec to be tested and a linear array of
+  ** bits to act as the reference */
+  pBitvec = sqlite3BitvecCreate( sz );
+  pV = sqlite3_malloc( (sz+7)/8 + 1 );
+  if( pBitvec==0 || pV==0 ) goto bitvec_end;
+  memset(pV, 0, (sz+7)/8 + 1);
+
+  /* Run the program */
+  pc = 0;
+  while( (op = aOp[pc])!=0 ){
+    switch( op ){
+      case 1:
+      case 2:
+      case 5: {
+        nx = 4;
+        i = aOp[pc+2] - 1;
+        aOp[pc+2] += aOp[pc+3];
+        break;
+      }
+      case 3:
+      case 4: 
+      default: {
+        nx = 2;
+        sqlite3_randomness(sizeof(i), &i);
+        break;
+      }
+    }
+    if( (--aOp[pc+1]) > 0 ) nx = 0;
+    pc += nx;
+    i = (i & 0x7fffffff)%sz;
+    if( (op & 1)!=0 ){
+      SETBIT(pV, (i+1));
+      if( op!=5 ){
+        if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end;
+      }
+    }else{
+      CLEARBIT(pV, (i+1));
+      sqlite3BitvecClear(pBitvec, i+1);
+    }
+  }
+
+  /* Test to make sure the linear array exactly matches the
+  ** Bitvec object.  Start with the assumption that they do
+  ** match (rc==0).  Change rc to non-zero if a discrepancy
+  ** is found.
+  */
+  rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1)
+          + sqlite3BitvecTest(pBitvec, 0);
+  for(i=1; i<=sz; i++){
+    if(  (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){
+      rc = i;
+      break;
+    }
+  }
+
+  /* Free allocated structure */
+bitvec_end:
+  sqlite3_free(pV);
+  sqlite3BitvecDestroy(pBitvec);
+  return rc;
+}
+#endif /* SQLITE_OMIT_BUILTIN_TEST */

src/btmutex.c

@@ -0,0 +1,317 @@
+/*
+** 2007 August 27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** $Id: btmutex.c,v 1.10 2008/07/14 19:39:17 drh Exp $
+**
+** This file contains code used to implement mutexes on Btree objects.
+** This code really belongs in btree.c.  But btree.c is getting too
+** big and we want to break it down some.  This packaged seemed like
+** a good breakout.
+*/
+#include "btreeInt.h"
+#if SQLITE_THREADSAFE && !defined(SQLITE_OMIT_SHARED_CACHE)
+
+
+/*
+** Enter a mutex on the given BTree object.
+**
+** If the object is not sharable, then no mutex is ever required
+** and this routine is a no-op.  The underlying mutex is non-recursive.
+** But we keep a reference count in Btree.wantToLock so the behavior
+** of this interface is recursive.
+**
+** To avoid deadlocks, multiple Btrees are locked in the same order
+** by all database connections.  The p->pNext is a list of other
+** Btrees belonging to the same database connection as the p Btree
+** which need to be locked after p.  If we cannot get a lock on
+** p, then first unlock all of the others on p->pNext, then wait
+** for the lock to become available on p, then relock all of the
+** subsequent Btrees that desire a lock.
+*/
+void sqlite3BtreeEnter(Btree *p){
+  Btree *pLater;
+
+  /* Some basic sanity checking on the Btree.  The list of Btrees
+  ** connected by pNext and pPrev should be in sorted order by
+  ** Btree.pBt value. All elements of the list should belong to
+  ** the same connection. Only shared Btrees are on the list. */
+  assert( p->pNext==0 || p->pNext->pBt>p->pBt );
+  assert( p->pPrev==0 || p->pPrev->pBt<p->pBt );
+  assert( p->pNext==0 || p->pNext->db==p->db );
+  assert( p->pPrev==0 || p->pPrev->db==p->db );
+  assert( p->sharable || (p->pNext==0 && p->pPrev==0) );
+
+  /* Check for locking consistency */
+  assert( !p->locked || p->wantToLock>0 );
+  assert( p->sharable || p->wantToLock==0 );
+
+  /* We should already hold a lock on the database connection */
+  assert( sqlite3_mutex_held(p->db->mutex) );
+
+  if( !p->sharable ) return;
+  p->wantToLock++;
+  if( p->locked ) return;
+
+#ifndef SQLITE_MUTEX_NOOP
+  /* In most cases, we should be able to acquire the lock we
+  ** want without having to go throught the ascending lock
+  ** procedure that follows.  Just be sure not to block.
+  */
+  if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){
+    p->locked = 1;
+    return;
+  }
+
+  /* To avoid deadlock, first release all locks with a larger
+  ** BtShared address.  Then acquire our lock.  Then reacquire
+  ** the other BtShared locks that we used to hold in ascending
+  ** order.
+  */
+  for(pLater=p->pNext; pLater; pLater=pLater->pNext){
+    assert( pLater->sharable );
+    assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt );
+    assert( !pLater->locked || pLater->wantToLock>0 );
+    if( pLater->locked ){
+      sqlite3_mutex_leave(pLater->pBt->mutex);
+      pLater->locked = 0;
+    }
+  }
+  sqlite3_mutex_enter(p->pBt->mutex);
+  p->locked = 1;
+  for(pLater=p->pNext; pLater; pLater=pLater->pNext){
+    if( pLater->wantToLock ){
+      sqlite3_mutex_enter(pLater->pBt->mutex);
+      pLater->locked = 1;
+    }
+  }
+#endif /* SQLITE_MUTEX_NOOP */
+}
+
+/*
+** Exit the recursive mutex on a Btree.
+*/
+void sqlite3BtreeLeave(Btree *p){
+  if( p->sharable ){
+    assert( p->wantToLock>0 );
+    p->wantToLock--;
+    if( p->wantToLock==0 ){
+      assert( p->locked );
+      sqlite3_mutex_leave(p->pBt->mutex);
+      p->locked = 0;
+    }
+  }
+}
+
+#ifndef NDEBUG
+/*
+** Return true if the BtShared mutex is held on the btree.  
+**
+** This routine makes no determination one why or another if the
+** database connection mutex is held.
+**
+** This routine is used only from within assert() statements.
+*/
+int sqlite3BtreeHoldsMutex(Btree *p){
+  return (p->sharable==0 ||
+             (p->locked && p->wantToLock && sqlite3_mutex_held(p->pBt->mutex)));
+}
+#endif
+
+
+#ifndef SQLITE_OMIT_INCRBLOB
+/*
+** Enter and leave a mutex on a Btree given a cursor owned by that
+** Btree.  These entry points are used by incremental I/O and can be
+** omitted if that module is not used.
+*/
+void sqlite3BtreeEnterCursor(BtCursor *pCur){
+  sqlite3BtreeEnter(pCur->pBtree);
+}
+void sqlite3BtreeLeaveCursor(BtCursor *pCur){
+  sqlite3BtreeLeave(pCur->pBtree);
+}
+#endif /* SQLITE_OMIT_INCRBLOB */
+
+
+/*
+** Enter the mutex on every Btree associated with a database
+** connection.  This is needed (for example) prior to parsing
+** a statement since we will be comparing table and column names
+** against all schemas and we do not want those schemas being
+** reset out from under us.
+**
+** There is a corresponding leave-all procedures.
+**
+** Enter the mutexes in accending order by BtShared pointer address
+** to avoid the possibility of deadlock when two threads with
+** two or more btrees in common both try to lock all their btrees
+** at the same instant.
+*/
+void sqlite3BtreeEnterAll(sqlite3 *db){
+  int i;
+  Btree *p, *pLater;
+  assert( sqlite3_mutex_held(db->mutex) );
+  for(i=0; i<db->nDb; i++){
+    p = db->aDb[i].pBt;
+    if( p && p->sharable ){
+      p->wantToLock++;
+      if( !p->locked ){
+        assert( p->wantToLock==1 );
+        while( p->pPrev ) p = p->pPrev;
+        while( p->locked && p->pNext ) p = p->pNext;
+        for(pLater = p->pNext; pLater; pLater=pLater->pNext){
+          if( pLater->locked ){
+            sqlite3_mutex_leave(pLater->pBt->mutex);
+            pLater->locked = 0;
+          }
+        }
+        while( p ){
+          sqlite3_mutex_enter(p->pBt->mutex);
+          p->locked++;
+          p = p->pNext;
+        }
+      }
+    }
+  }
+}
+void sqlite3BtreeLeaveAll(sqlite3 *db){
+  int i;
+  Btree *p;
+  assert( sqlite3_mutex_held(db->mutex) );
+  for(i=0; i<db->nDb; i++){
+    p = db->aDb[i].pBt;
+    if( p && p->sharable ){
+      assert( p->wantToLock>0 );
+      p->wantToLock--;
+      if( p->wantToLock==0 ){
+        assert( p->locked );
+        sqlite3_mutex_leave(p->pBt->mutex);
+        p->locked = 0;
+      }
+    }
+  }
+}
+
+#ifndef NDEBUG
+/*
+** Return true if the current thread holds the database connection
+** mutex and all required BtShared mutexes.
+**
+** This routine is used inside assert() statements only.
+*/
+int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){
+  int i;
+  if( !sqlite3_mutex_held(db->mutex) ){
+    return 0;
+  }
+  for(i=0; i<db->nDb; i++){
+    Btree *p;
+    p = db->aDb[i].pBt;
+    if( p && p->sharable &&
+         (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){
+      return 0;
+    }
+  }
+  return 1;
+}
+#endif /* NDEBUG */
+
+/*
+** Add a new Btree pointer to a BtreeMutexArray. 
+** if the pointer can possibly be shared with
+** another database connection.
+**
+** The pointers are kept in sorted order by pBtree->pBt.  That
+** way when we go to enter all the mutexes, we can enter them
+** in order without every having to backup and retry and without
+** worrying about deadlock.
+**
+** The number of shared btrees will always be small (usually 0 or 1)
+** so an insertion sort is an adequate algorithm here.
+*/
+void sqlite3BtreeMutexArrayInsert(BtreeMutexArray *pArray, Btree *pBtree){
+  int i, j;
+  BtShared *pBt;
+  if( pBtree==0 || pBtree->sharable==0 ) return;
+#ifndef NDEBUG
+  {
+    for(i=0; i<pArray->nMutex; i++){
+      assert( pArray->aBtree[i]!=pBtree );
+    }
+  }
+#endif
+  assert( pArray->nMutex>=0 );
+  assert( pArray->nMutex<sizeof(pArray->aBtree)/sizeof(pArray->aBtree[0])-1 );
+  pBt = pBtree->pBt;
+  for(i=0; i<pArray->nMutex; i++){
+    assert( pArray->aBtree[i]!=pBtree );
+    if( pArray->aBtree[i]->pBt>pBt ){
+      for(j=pArray->nMutex; j>i; j--){
+        pArray->aBtree[j] = pArray->aBtree[j-1];
+      }
+      pArray->aBtree[i] = pBtree;
+      pArray->nMutex++;
+      return;
+    }
+  }
+  pArray->aBtree[pArray->nMutex++] = pBtree;
+}
+
+/*
+** Enter the mutex of every btree in the array.  This routine is
+** called at the beginning of sqlite3VdbeExec().  The mutexes are
+** exited at the end of the same function.
+*/
+void sqlite3BtreeMutexArrayEnter(BtreeMutexArray *pArray){
+  int i;
+  for(i=0; i<pArray->nMutex; i++){
+    Btree *p = pArray->aBtree[i];
+    /* Some basic sanity checking */
+    assert( i==0 || pArray->aBtree[i-1]->pBt<p->pBt );
+    assert( !p->locked || p->wantToLock>0 );
+
+    /* We should already hold a lock on the database connection */
+    assert( sqlite3_mutex_held(p->db->mutex) );
+
+    p->wantToLock++;
+    if( !p->locked && p->sharable ){
+      sqlite3_mutex_enter(p->pBt->mutex);
+      p->locked = 1;
+    }
+  }
+}
+
+/*
+** Leave the mutex of every btree in the group.
+*/
+void sqlite3BtreeMutexArrayLeave(BtreeMutexArray *pArray){
+  int i;
+  for(i=0; i<pArray->nMutex; i++){
+    Btree *p = pArray->aBtree[i];
+    /* Some basic sanity checking */
+    assert( i==0 || pArray->aBtree[i-1]->pBt<p->pBt );
+    assert( p->locked || !p->sharable );
+    assert( p->wantToLock>0 );
+
+    /* We should already hold a lock on the database connection */
+    assert( sqlite3_mutex_held(p->db->mutex) );
+
+    p->wantToLock--;
+    if( p->wantToLock==0 && p->locked ){
+      sqlite3_mutex_leave(p->pBt->mutex);
+      p->locked = 0;
+    }
+  }
+}
+
+
+#endif  /* SQLITE_THREADSAFE && !SQLITE_OMIT_SHARED_CACHE */

src/btree.h

@@ -0,0 +1,219 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This header file defines the interface that the sqlite B-Tree file
+** subsystem.  See comments in the source code for a detailed description
+** of what each interface routine does.
+**
+** @(#) $Id: btree.h,v 1.102 2008/07/11 21:02:54 drh Exp $
+*/
+#ifndef _BTREE_H_
+#define _BTREE_H_
+
+/* TODO: This definition is just included so other modules compile. It
+** needs to be revisited.
+*/
+#define SQLITE_N_BTREE_META 10
+
+/*
+** If defined as non-zero, auto-vacuum is enabled by default. Otherwise
+** it must be turned on for each database using "PRAGMA auto_vacuum = 1".
+*/
+#ifndef SQLITE_DEFAULT_AUTOVACUUM
+  #define SQLITE_DEFAULT_AUTOVACUUM 0
+#endif
+
+#define BTREE_AUTOVACUUM_NONE 0        /* Do not do auto-vacuum */
+#define BTREE_AUTOVACUUM_FULL 1        /* Do full auto-vacuum */
+#define BTREE_AUTOVACUUM_INCR 2        /* Incremental vacuum */
+
+/*
+** Forward declarations of structure
+*/
+typedef struct Btree Btree;
+typedef struct BtCursor BtCursor;
+typedef struct BtShared BtShared;
+typedef struct BtreeMutexArray BtreeMutexArray;
+
+/*
+** This structure records all of the Btrees that need to hold
+** a mutex before we enter sqlite3VdbeExec().  The Btrees are
+** are placed in aBtree[] in order of aBtree[]->pBt.  That way,
+** we can always lock and unlock them all quickly.
+*/
+struct BtreeMutexArray {
+  int nMutex;
+  Btree *aBtree[SQLITE_MAX_ATTACHED+1];
+};
+
+
+int sqlite3BtreeOpen(
+  const char *zFilename,   /* Name of database file to open */
+  sqlite3 *db,             /* Associated database connection */
+  Btree **,                /* Return open Btree* here */
+  int flags,               /* Flags */
+  int vfsFlags             /* Flags passed through to VFS open */
+);
+
+/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the
+** following values.
+**
+** NOTE:  These values must match the corresponding PAGER_ values in
+** pager.h.
+*/
+#define BTREE_OMIT_JOURNAL  1  /* Do not use journal.  No argument */
+#define BTREE_NO_READLOCK   2  /* Omit readlocks on readonly files */
+#define BTREE_MEMORY        4  /* In-memory DB.  No argument */
+#define BTREE_READONLY      8  /* Open the database in read-only mode */
+#define BTREE_READWRITE    16  /* Open for both reading and writing */
+#define BTREE_CREATE       32  /* Create the database if it does not exist */
+
+int sqlite3BtreeClose(Btree*);
+int sqlite3BtreeSetCacheSize(Btree*,int);
+int sqlite3BtreeSetSafetyLevel(Btree*,int,int);
+int sqlite3BtreeSyncDisabled(Btree*);
+int sqlite3BtreeSetPageSize(Btree*,int,int);
+int sqlite3BtreeGetPageSize(Btree*);
+int sqlite3BtreeMaxPageCount(Btree*,int);
+int sqlite3BtreeGetReserve(Btree*);
+int sqlite3BtreeSetAutoVacuum(Btree *, int);
+int sqlite3BtreeGetAutoVacuum(Btree *);
+int sqlite3BtreeBeginTrans(Btree*,int);
+int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster);
+int sqlite3BtreeCommitPhaseTwo(Btree*);
+int sqlite3BtreeCommit(Btree*);
+int sqlite3BtreeRollback(Btree*);
+int sqlite3BtreeBeginStmt(Btree*);
+int sqlite3BtreeCommitStmt(Btree*);
+int sqlite3BtreeRollbackStmt(Btree*);
+int sqlite3BtreeCreateTable(Btree*, int*, int flags);
+int sqlite3BtreeIsInTrans(Btree*);
+int sqlite3BtreeIsInStmt(Btree*);
+int sqlite3BtreeIsInReadTrans(Btree*);
+void *sqlite3BtreeSchema(Btree *, int, void(*)(void *));
+int sqlite3BtreeSchemaLocked(Btree *);
+int sqlite3BtreeLockTable(Btree *, int, u8);
+
+const char *sqlite3BtreeGetFilename(Btree *);
+const char *sqlite3BtreeGetDirname(Btree *);
+const char *sqlite3BtreeGetJournalname(Btree *);
+int sqlite3BtreeCopyFile(Btree *, Btree *);
+
+int sqlite3BtreeIncrVacuum(Btree *);
+
+/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR
+** of the following flags:
+*/
+#define BTREE_INTKEY     1    /* Table has only 64-bit signed integer keys */
+#define BTREE_ZERODATA   2    /* Table has keys only - no data */
+#define BTREE_LEAFDATA   4    /* Data stored in leaves only.  Implies INTKEY */
+
+int sqlite3BtreeDropTable(Btree*, int, int*);
+int sqlite3BtreeClearTable(Btree*, int);
+int sqlite3BtreeGetMeta(Btree*, int idx, u32 *pValue);
+int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value);
+void sqlite3BtreeTripAllCursors(Btree*, int);
+
+struct UnpackedRecord;  /* Forward declaration.  Definition in vdbeaux.c. */
+
+int sqlite3BtreeCursor(
+  Btree*,                              /* BTree containing table to open */
+  int iTable,                          /* Index of root page */
+  int wrFlag,                          /* 1 for writing.  0 for read-only */
+  struct KeyInfo*,                     /* First argument to compare function */
+  BtCursor *pCursor                    /* Space to write cursor structure */
+);
+int sqlite3BtreeCursorSize(void);
+
+int sqlite3BtreeCloseCursor(BtCursor*);
+int sqlite3BtreeMoveto(
+  BtCursor*,
+  const void *pKey,
+  struct UnpackedRecord *pUnKey,
+  i64 nKey,
+  int bias,
+  int *pRes
+);
+int sqlite3BtreeCursorHasMoved(BtCursor*, int*);
+int sqlite3BtreeDelete(BtCursor*);
+int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey,
+                                  const void *pData, int nData,
+                                  int nZero, int bias);
+int sqlite3BtreeFirst(BtCursor*, int *pRes);
+int sqlite3BtreeLast(BtCursor*, int *pRes);
+int sqlite3BtreeNext(BtCursor*, int *pRes);
+int sqlite3BtreeEof(BtCursor*);
+int sqlite3BtreeFlags(BtCursor*);
+int sqlite3BtreePrevious(BtCursor*, int *pRes);
+int sqlite3BtreeKeySize(BtCursor*, i64 *pSize);
+int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*);
+sqlite3 *sqlite3BtreeCursorDb(const BtCursor*);
+const void *sqlite3BtreeKeyFetch(BtCursor*, int *pAmt);
+const void *sqlite3BtreeDataFetch(BtCursor*, int *pAmt);
+int sqlite3BtreeDataSize(BtCursor*, u32 *pSize);
+int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*);
+
+char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*);
+struct Pager *sqlite3BtreePager(Btree*);
+
+int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*);
+void sqlite3BtreeCacheOverflow(BtCursor *);
+
+#ifdef SQLITE_TEST
+int sqlite3BtreeCursorInfo(BtCursor*, int*, int);
+void sqlite3BtreeCursorList(Btree*);
+#endif
+
+/*
+** If we are not using shared cache, then there is no need to
+** use mutexes to access the BtShared structures.  So make the
+** Enter and Leave procedures no-ops.
+*/
+#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE
+  void sqlite3BtreeEnter(Btree*);
+  void sqlite3BtreeLeave(Btree*);
+#ifndef NDEBUG
+  /* This routine is used inside assert() statements only. */
+  int sqlite3BtreeHoldsMutex(Btree*);
+#endif
+  void sqlite3BtreeEnterCursor(BtCursor*);
+  void sqlite3BtreeLeaveCursor(BtCursor*);
+  void sqlite3BtreeEnterAll(sqlite3*);
+  void sqlite3BtreeLeaveAll(sqlite3*);
+#ifndef NDEBUG
+  /* This routine is used inside assert() statements only. */
+  int sqlite3BtreeHoldsAllMutexes(sqlite3*);
+#endif
+  void sqlite3BtreeMutexArrayEnter(BtreeMutexArray*);
+  void sqlite3BtreeMutexArrayLeave(BtreeMutexArray*);
+  void sqlite3BtreeMutexArrayInsert(BtreeMutexArray*, Btree*);
+#else
+# define sqlite3BtreeEnter(X)
+# define sqlite3BtreeLeave(X)
+#ifndef NDEBUG
+  /* This routine is used inside assert() statements only. */
+# define sqlite3BtreeHoldsMutex(X) 1
+#endif
+# define sqlite3BtreeEnterCursor(X)
+# define sqlite3BtreeLeaveCursor(X)
+# define sqlite3BtreeEnterAll(X)
+# define sqlite3BtreeLeaveAll(X)
+#ifndef NDEBUG
+  /* This routine is used inside assert() statements only. */
+# define sqlite3BtreeHoldsAllMutexes(X) 1
+#endif
+# define sqlite3BtreeMutexArrayEnter(X)
+# define sqlite3BtreeMutexArrayLeave(X)
+# define sqlite3BtreeMutexArrayInsert(X,Y)
+#endif
+
+
+#endif /* _BTREE_H_ */

src/btreeInt.h

@@ -0,0 +1,627 @@
+/*
+** 2004 April 6
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** $Id: btreeInt.h,v 1.26 2008/07/12 14:52:20 drh Exp $
+**
+** This file implements a external (disk-based) database using BTrees.
+** For a detailed discussion of BTrees, refer to
+**
+**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
+**     "Sorting And Searching", pages 473-480. Addison-Wesley
+**     Publishing Company, Reading, Massachusetts.
+**
+** The basic idea is that each page of the file contains N database
+** entries and N+1 pointers to subpages.
+**
+**   ----------------------------------------------------------------
+**   |  Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) |
+**   ----------------------------------------------------------------
+**
+** All of the keys on the page that Ptr(0) points to have values less
+** than Key(0).  All of the keys on page Ptr(1) and its subpages have
+** values greater than Key(0) and less than Key(1).  All of the keys
+** on Ptr(N) and its subpages have values greater than Key(N-1).  And
+** so forth.
+**
+** Finding a particular key requires reading O(log(M)) pages from the 
+** disk where M is the number of entries in the tree.
+**
+** In this implementation, a single file can hold one or more separate 
+** BTrees.  Each BTree is identified by the index of its root page.  The
+** key and data for any entry are combined to form the "payload".  A
+** fixed amount of payload can be carried directly on the database
+** page.  If the payload is larger than the preset amount then surplus
+** bytes are stored on overflow pages.  The payload for an entry
+** and the preceding pointer are combined to form a "Cell".  Each 
+** page has a small header which contains the Ptr(N) pointer and other
+** information such as the size of key and data.
+**
+** FORMAT DETAILS
+**
+** The file is divided into pages.  The first page is called page 1,
+** the second is page 2, and so forth.  A page number of zero indicates
+** "no such page".  The page size can be anything between 512 and 65536.
+** Each page can be either a btree page, a freelist page or an overflow
+** page.
+**
+** The first page is always a btree page.  The first 100 bytes of the first
+** page contain a special header (the "file header") that describes the file.
+** The format of the file header is as follows:
+**
+**   OFFSET   SIZE    DESCRIPTION
+**      0      16     Header string: "SQLite format 3\000"
+**     16       2     Page size in bytes.  
+**     18       1     File format write version
+**     19       1     File format read version
+**     20       1     Bytes of unused space at the end of each page
+**     21       1     Max embedded payload fraction
+**     22       1     Min embedded payload fraction
+**     23       1     Min leaf payload fraction
+**     24       4     File change counter
+**     28       4     Reserved for future use
+**     32       4     First freelist page
+**     36       4     Number of freelist pages in the file
+**     40      60     15 4-byte meta values passed to higher layers
+**
+** All of the integer values are big-endian (most significant byte first).
+**
+** The file change counter is incremented when the database is changed
+** This counter allows other processes to know when the file has changed
+** and thus when they need to flush their cache.
+**
+** The max embedded payload fraction is the amount of the total usable
+** space in a page that can be consumed by a single cell for standard
+** B-tree (non-LEAFDATA) tables.  A value of 255 means 100%.  The default
+** is to limit the maximum cell size so that at least 4 cells will fit
+** on one page.  Thus the default max embedded payload fraction is 64.
+**
+** If the payload for a cell is larger than the max payload, then extra
+** payload is spilled to overflow pages.  Once an overflow page is allocated,
+** as many bytes as possible are moved into the overflow pages without letting
+** the cell size drop below the min embedded payload fraction.
+**
+** The min leaf payload fraction is like the min embedded payload fraction
+** except that it applies to leaf nodes in a LEAFDATA tree.  The maximum
+** payload fraction for a LEAFDATA tree is always 100% (or 255) and it
+** not specified in the header.
+**
+** Each btree pages is divided into three sections:  The header, the
+** cell pointer array, and the cell content area.  Page 1 also has a 100-byte
+** file header that occurs before the page header.
+**
+**      |----------------|
+**      | file header    |   100 bytes.  Page 1 only.
+**      |----------------|
+**      | page header    |   8 bytes for leaves.  12 bytes for interior nodes
+**      |----------------|
+**      | cell pointer   |   |  2 bytes per cell.  Sorted order.
+**      | array          |   |  Grows downward
+**      |                |   v
+**      |----------------|
+**      | unallocated    |
+**      | space          |
+**      |----------------|   ^  Grows upwards
+**      | cell content   |   |  Arbitrary order interspersed with freeblocks.
+**      | area           |   |  and free space fragments.
+**      |----------------|
+**
+** The page headers looks like this:
+**
+**   OFFSET   SIZE     DESCRIPTION
+**      0       1      Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf
+**      1       2      byte offset to the first freeblock
+**      3       2      number of cells on this page
+**      5       2      first byte of the cell content area
+**      7       1      number of fragmented free bytes
+**      8       4      Right child (the Ptr(N) value).  Omitted on leaves.
+**
+** The flags define the format of this btree page.  The leaf flag means that
+** this page has no children.  The zerodata flag means that this page carries
+** only keys and no data.  The intkey flag means that the key is a integer
+** which is stored in the key size entry of the cell header rather than in
+** the payload area.
+**
+** The cell pointer array begins on the first byte after the page header.
+** The cell pointer array contains zero or more 2-byte numbers which are
+** offsets from the beginning of the page to the cell content in the cell
+** content area.  The cell pointers occur in sorted order.  The system strives
+** to keep free space after the last cell pointer so that new cells can
+** be easily added without having to defragment the page.
+**
+** Cell content is stored at the very end of the page and grows toward the
+** beginning of the page.
+**
+** Unused space within the cell content area is collected into a linked list of
+** freeblocks.  Each freeblock is at least 4 bytes in size.  The byte offset
+** to the first freeblock is given in the header.  Freeblocks occur in
+** increasing order.  Because a freeblock must be at least 4 bytes in size,
+** any group of 3 or fewer unused bytes in the cell content area cannot
+** exist on the freeblock chain.  A group of 3 or fewer free bytes is called
+** a fragment.  The total number of bytes in all fragments is recorded.
+** in the page header at offset 7.
+**
+**    SIZE    DESCRIPTION
+**      2     Byte offset of the next freeblock
+**      2     Bytes in this freeblock
+**
+** Cells are of variable length.  Cells are stored in the cell content area at
+** the end of the page.  Pointers to the cells are in the cell pointer array
+** that immediately follows the page header.  Cells is not necessarily
+** contiguous or in order, but cell pointers are contiguous and in order.
+**
+** Cell content makes use of variable length integers.  A variable
+** length integer is 1 to 9 bytes where the lower 7 bits of each 
+** byte are used.  The integer consists of all bytes that have bit 8 set and
+** the first byte with bit 8 clear.  The most significant byte of the integer
+** appears first.  A variable-length integer may not be more than 9 bytes long.
+** As a special case, all 8 bytes of the 9th byte are used as data.  This
+** allows a 64-bit integer to be encoded in 9 bytes.
+**
+**    0x00                      becomes  0x00000000
+**    0x7f                      becomes  0x0000007f
+**    0x81 0x00                 becomes  0x00000080
+**    0x82 0x00                 becomes  0x00000100
+**    0x80 0x7f                 becomes  0x0000007f
+**    0x8a 0x91 0xd1 0xac 0x78  becomes  0x12345678
+**    0x81 0x81 0x81 0x81 0x01  becomes  0x10204081
+**
+** Variable length integers are used for rowids and to hold the number of
+** bytes of key and data in a btree cell.
+**
+** The content of a cell looks like this:
+**
+**    SIZE    DESCRIPTION
+**      4     Page number of the left child. Omitted if leaf flag is set.
+**     var    Number of bytes of data. Omitted if the zerodata flag is set.
+**     var    Number of bytes of key. Or the key itself if intkey flag is set.
+**      *     Payload
+**      4     First page of the overflow chain.  Omitted if no overflow
+**
+** Overflow pages form a linked list.  Each page except the last is completely
+** filled with data (pagesize - 4 bytes).  The last page can have as little
+** as 1 byte of data.
+**
+**    SIZE    DESCRIPTION
+**      4     Page number of next overflow page
+**      *     Data
+**
+** Freelist pages come in two subtypes: trunk pages and leaf pages.  The
+** file header points to the first in a linked list of trunk page.  Each trunk
+** page points to multiple leaf pages.  The content of a leaf page is
+** unspecified.  A trunk page looks like this:
+**
+**    SIZE    DESCRIPTION
+**      4     Page number of next trunk page
+**      4     Number of leaf pointers on this page
+**      *     zero or more pages numbers of leaves
+*/
+#include "sqliteInt.h"
+#include "pager.h"
+#include "btree.h"
+#include "os.h"
+#include <assert.h>
+
+/* Round up a number to the next larger multiple of 8.  This is used
+** to force 8-byte alignment on 64-bit architectures.
+*/
+#define ROUND8(x)   ((x+7)&~7)
+
+
+/* The following value is the maximum cell size assuming a maximum page
+** size give above.
+*/
+#define MX_CELL_SIZE(pBt)  (pBt->pageSize-8)
+
+/* The maximum number of cells on a single page of the database.  This
+** assumes a minimum cell size of 6 bytes  (4 bytes for the cell itself
+** plus 2 bytes for the index to the cell in the page header).  Such
+** small cells will be rare, but they are possible.
+*/
+#define MX_CELL(pBt) ((pBt->pageSize-8)/6)
+
+/* Forward declarations */
+typedef struct MemPage MemPage;
+typedef struct BtLock BtLock;
+
+/*
+** This is a magic string that appears at the beginning of every
+** SQLite database in order to identify the file as a real database.
+**
+** You can change this value at compile-time by specifying a
+** -DSQLITE_FILE_HEADER="..." on the compiler command-line.  The
+** header must be exactly 16 bytes including the zero-terminator so
+** the string itself should be 15 characters long.  If you change
+** the header, then your custom library will not be able to read 
+** databases generated by the standard tools and the standard tools
+** will not be able to read databases created by your custom library.
+*/
+#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */
+#  define SQLITE_FILE_HEADER "SQLite format 3"
+#endif
+
+/*
+** Page type flags.  An ORed combination of these flags appear as the
+** first byte of on-disk image of every BTree page.
+*/
+#define PTF_INTKEY    0x01
+#define PTF_ZERODATA  0x02
+#define PTF_LEAFDATA  0x04
+#define PTF_LEAF      0x08
+
+/*
+** As each page of the file is loaded into memory, an instance of the following
+** structure is appended and initialized to zero.  This structure stores
+** information about the page that is decoded from the raw file page.
+**
+** The pParent field points back to the parent page.  This allows us to
+** walk up the BTree from any leaf to the root.  Care must be taken to
+** unref() the parent page pointer when this page is no longer referenced.
+** The pageDestructor() routine handles that chore.
+**
+** Access to all fields of this structure is controlled by the mutex
+** stored in MemPage.pBt->mutex.
+*/
+struct MemPage {
+  u8 isInit;           /* True if previously initialized. MUST BE FIRST! */
+  u8 idxShift;         /* True if Cell indices have changed */
+  u8 nOverflow;        /* Number of overflow cell bodies in aCell[] */
+  u8 intKey;           /* True if intkey flag is set */
+  u8 leaf;             /* True if leaf flag is set */
+  u8 zeroData;         /* True if table stores keys only */
+  u8 leafData;         /* True if tables stores data on leaves only */
+  u8 hasData;          /* True if this page stores data */
+  u8 hdrOffset;        /* 100 for page 1.  0 otherwise */
+  u8 childPtrSize;     /* 0 if leaf==1.  4 if leaf==0 */
+  u16 maxLocal;        /* Copy of BtShared.maxLocal or BtShared.maxLeaf */
+  u16 minLocal;        /* Copy of BtShared.minLocal or BtShared.minLeaf */
+  u16 cellOffset;      /* Index in aData of first cell pointer */
+  u16 idxParent;       /* Index in parent of this node */
+  u16 nFree;           /* Number of free bytes on the page */
+  u16 nCell;           /* Number of cells on this page, local and ovfl */
+  struct _OvflCell {   /* Cells that will not fit on aData[] */
+    u8 *pCell;          /* Pointers to the body of the overflow cell */
+    u16 idx;            /* Insert this cell before idx-th non-overflow cell */
+  } aOvfl[5];
+  BtShared *pBt;       /* Pointer to BtShared that this page is part of */
+  u8 *aData;           /* Pointer to disk image of the page data */
+  DbPage *pDbPage;     /* Pager page handle */
+  Pgno pgno;           /* Page number for this page */
+  MemPage *pParent;    /* The parent of this page.  NULL for root */
+};
+
+/*
+** The in-memory image of a disk page has the auxiliary information appended
+** to the end.  EXTRA_SIZE is the number of bytes of space needed to hold
+** that extra information.
+*/
+#define EXTRA_SIZE sizeof(MemPage)
+
+/* A Btree handle
+**
+** A database connection contains a pointer to an instance of
+** this object for every database file that it has open.  This structure
+** is opaque to the database connection.  The database connection cannot
+** see the internals of this structure and only deals with pointers to
+** this structure.
+**
+** For some database files, the same underlying database cache might be 
+** shared between multiple connections.  In that case, each contection
+** has it own pointer to this object.  But each instance of this object
+** points to the same BtShared object.  The database cache and the
+** schema associated with the database file are all contained within
+** the BtShared object.
+**
+** All fields in this structure are accessed under sqlite3.mutex.
+** The pBt pointer itself may not be changed while there exists cursors 
+** in the referenced BtShared that point back to this Btree since those
+** cursors have to do go through this Btree to find their BtShared and
+** they often do so without holding sqlite3.mutex.
+*/
+struct Btree {
+  sqlite3 *db;       /* The database connection holding this btree */
+  BtShared *pBt;     /* Sharable content of this btree */
+  u8 inTrans;        /* TRANS_NONE, TRANS_READ or TRANS_WRITE */
+  u8 sharable;       /* True if we can share pBt with another db */
+  u8 locked;         /* True if db currently has pBt locked */
+  int wantToLock;    /* Number of nested calls to sqlite3BtreeEnter() */
+  Btree *pNext;      /* List of other sharable Btrees from the same db */
+  Btree *pPrev;      /* Back pointer of the same list */
+};
+
+/*
+** Btree.inTrans may take one of the following values.
+**
+** If the shared-data extension is enabled, there may be multiple users
+** of the Btree structure. At most one of these may open a write transaction,
+** but any number may have active read transactions.
+*/
+#define TRANS_NONE  0
+#define TRANS_READ  1
+#define TRANS_WRITE 2
+
+/*
+** An instance of this object represents a single database file.
+** 
+** A single database file can be in use as the same time by two
+** or more database connections.  When two or more connections are
+** sharing the same database file, each connection has it own
+** private Btree object for the file and each of those Btrees points
+** to this one BtShared object.  BtShared.nRef is the number of
+** connections currently sharing this database file.
+**
+** Fields in this structure are accessed under the BtShared.mutex
+** mutex, except for nRef and pNext which are accessed under the
+** global SQLITE_MUTEX_STATIC_MASTER mutex.  The pPager field
+** may not be modified once it is initially set as long as nRef>0.
+** The pSchema field may be set once under BtShared.mutex and
+** thereafter is unchanged as long as nRef>0.
+*/
+struct BtShared {
+  Pager *pPager;        /* The page cache */
+  sqlite3 *db;          /* Database connection currently using this Btree */
+  BtCursor *pCursor;    /* A list of all open cursors */
+  MemPage *pPage1;      /* First page of the database */
+  u8 inStmt;            /* True if we are in a statement subtransaction */
+  u8 readOnly;          /* True if the underlying file is readonly */
+  u8 pageSizeFixed;     /* True if the page size can no longer be changed */
+#ifndef SQLITE_OMIT_AUTOVACUUM
+  u8 autoVacuum;        /* True if auto-vacuum is enabled */
+  u8 incrVacuum;        /* True if incr-vacuum is enabled */
+  Pgno nTrunc;          /* Non-zero if the db will be truncated (incr vacuum) */
+#endif
+  u16 pageSize;         /* Total number of bytes on a page */
+  u16 usableSize;       /* Number of usable bytes on each page */
+  int maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
+  int minLocal;         /* Minimum local payload in non-LEAFDATA tables */
+  int maxLeaf;          /* Maximum local payload in a LEAFDATA table */
+  int minLeaf;          /* Minimum local payload in a LEAFDATA table */
+  u8 inTransaction;     /* Transaction state */
+  int nTransaction;     /* Number of open transactions (read + write) */
+  void *pSchema;        /* Pointer to space allocated by sqlite3BtreeSchema() */
+  void (*xFreeSchema)(void*);  /* Destructor for BtShared.pSchema */
+  sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */
+  BusyHandler busyHdr;  /* The busy handler for this btree */
+#ifndef SQLITE_OMIT_SHARED_CACHE
+  int nRef;             /* Number of references to this structure */
+  BtShared *pNext;      /* Next on a list of sharable BtShared structs */
+  BtLock *pLock;        /* List of locks held on this shared-btree struct */
+  Btree *pExclusive;    /* Btree with an EXCLUSIVE lock on the whole db */
+#endif
+  u8 *pTmpSpace;        /* BtShared.pageSize bytes of space for tmp use */
+};
+
+/*
+** An instance of the following structure is used to hold information
+** about a cell.  The parseCellPtr() function fills in this structure
+** based on information extract from the raw disk page.
+*/
+typedef struct CellInfo CellInfo;
+struct CellInfo {
+  u8 *pCell;     /* Pointer to the start of cell content */
+  i64 nKey;      /* The key for INTKEY tables, or number of bytes in key */
+  u32 nData;     /* Number of bytes of data */
+  u32 nPayload;  /* Total amount of payload */
+  u16 nHeader;   /* Size of the cell content header in bytes */
+  u16 nLocal;    /* Amount of payload held locally */
+  u16 iOverflow; /* Offset to overflow page number.  Zero if no overflow */
+  u16 nSize;     /* Size of the cell content on the main b-tree page */
+};
+
+/*
+** A cursor is a pointer to a particular entry within a particular
+** b-tree within a database file.
+**
+** The entry is identified by its MemPage and the index in
+** MemPage.aCell[] of the entry.
+**
+** When a single database file can shared by two more database connections,
+** but cursors cannot be shared.  Each cursor is associated with a
+** particular database connection identified BtCursor.pBtree.db.
+**
+** Fields in this structure are accessed under the BtShared.mutex
+** found at self->pBt->mutex. 
+*/
+struct BtCursor {
+  Btree *pBtree;            /* The Btree to which this cursor belongs */
+  BtShared *pBt;            /* The BtShared this cursor points to */
+  BtCursor *pNext, *pPrev;  /* Forms a linked list of all cursors */
+  struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */
+  Pgno pgnoRoot;            /* The root page of this tree */
+  MemPage *pPage;           /* Page that contains the entry */
+  int idx;                  /* Index of the entry in pPage->aCell[] */
+  CellInfo info;            /* A parse of the cell we are pointing at */
+  u8 wrFlag;                /* True if writable */
+  u8 atLast;                /* Cursor pointing to the last entry */
+  u8 validNKey;             /* True if info.nKey is valid */
+  u8 eState;                /* One of the CURSOR_XXX constants (see below) */
+  void *pKey;      /* Saved key that was cursor's last known position */
+  i64 nKey;        /* Size of pKey, or last integer key */
+  int skip;        /* (skip<0) -> Prev() is a no-op. (skip>0) -> Next() is */
+#ifndef SQLITE_OMIT_INCRBLOB
+  u8 isIncrblobHandle;      /* True if this cursor is an incr. io handle */
+  Pgno *aOverflow;          /* Cache of overflow page locations */
+#endif
+};
+
+/*
+** Potential values for BtCursor.eState.
+**
+** CURSOR_VALID:
+**   Cursor points to a valid entry. getPayload() etc. may be called.
+**
+** CURSOR_INVALID:
+**   Cursor does not point to a valid entry. This can happen (for example) 
+**   because the table is empty or because BtreeCursorFirst() has not been
+**   called.
+**
+** CURSOR_REQUIRESEEK:
+**   The table that this cursor was opened on still exists, but has been 
+**   modified since the cursor was last used. The cursor position is saved
+**   in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in 
+**   this state, restoreCursorPosition() can be called to attempt to
+**   seek the cursor to the saved position.
+**
+** CURSOR_FAULT:
+**   A unrecoverable error (an I/O error or a malloc failure) has occurred
+**   on a different connection that shares the BtShared cache with this
+**   cursor.  The error has left the cache in an inconsistent state.
+**   Do nothing else with this cursor.  Any attempt to use the cursor
+**   should return the error code stored in BtCursor.skip
+*/
+#define CURSOR_INVALID           0
+#define CURSOR_VALID             1
+#define CURSOR_REQUIRESEEK       2
+#define CURSOR_FAULT             3
+
+/* The database page the PENDING_BYTE occupies. This page is never used.
+** TODO: This macro is very similary to PAGER_MJ_PGNO() in pager.c. They
+** should possibly be consolidated (presumably in pager.h).
+**
+** If disk I/O is omitted (meaning that the database is stored purely
+** in memory) then there is no pending byte.
+*/
+#ifdef SQLITE_OMIT_DISKIO
+# define PENDING_BYTE_PAGE(pBt)  0x7fffffff
+#else
+# define PENDING_BYTE_PAGE(pBt) ((PENDING_BYTE/(pBt)->pageSize)+1)
+#endif
+
+/*
+** A linked list of the following structures is stored at BtShared.pLock.
+** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor 
+** is opened on the table with root page BtShared.iTable. Locks are removed
+** from this list when a transaction is committed or rolled back, or when
+** a btree handle is closed.
+*/
+struct BtLock {
+  Btree *pBtree;        /* Btree handle holding this lock */
+  Pgno iTable;          /* Root page of table */
+  u8 eLock;             /* READ_LOCK or WRITE_LOCK */
+  BtLock *pNext;        /* Next in BtShared.pLock list */
+};
+
+/* Candidate values for BtLock.eLock */
+#define READ_LOCK     1
+#define WRITE_LOCK    2
+
+/*
+** These macros define the location of the pointer-map entry for a 
+** database page. The first argument to each is the number of usable
+** bytes on each page of the database (often 1024). The second is the
+** page number to look up in the pointer map.
+**
+** PTRMAP_PAGENO returns the database page number of the pointer-map
+** page that stores the required pointer. PTRMAP_PTROFFSET returns
+** the offset of the requested map entry.
+**
+** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page,
+** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be
+** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements
+** this test.
+*/
+#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno)
+#define PTRMAP_PTROFFSET(pBt, pgno) (5*(pgno-ptrmapPageno(pBt, pgno)-1))
+#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno))
+
+/*
+** The pointer map is a lookup table that identifies the parent page for
+** each child page in the database file.  The parent page is the page that
+** contains a pointer to the child.  Every page in the database contains
+** 0 or 1 parent pages.  (In this context 'database page' refers
+** to any page that is not part of the pointer map itself.)  Each pointer map
+** entry consists of a single byte 'type' and a 4 byte parent page number.
+** The PTRMAP_XXX identifiers below are the valid types.
+**
+** The purpose of the pointer map is to facility moving pages from one
+** position in the file to another as part of autovacuum.  When a page
+** is moved, the pointer in its parent must be updated to point to the
+** new location.  The pointer map is used to locate the parent page quickly.
+**
+** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not
+**                  used in this case.
+**
+** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number 
+**                  is not used in this case.
+**
+** PTRMAP_OVERFLOW1: The database page is the first page in a list of 
+**                   overflow pages. The page number identifies the page that
+**                   contains the cell with a pointer to this overflow page.
+**
+** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of
+**                   overflow pages. The page-number identifies the previous
+**                   page in the overflow page list.
+**
+** PTRMAP_BTREE: The database page is a non-root btree page. The page number
+**               identifies the parent page in the btree.
+*/
+#define PTRMAP_ROOTPAGE 1
+#define PTRMAP_FREEPAGE 2
+#define PTRMAP_OVERFLOW1 3
+#define PTRMAP_OVERFLOW2 4
+#define PTRMAP_BTREE 5
+
+/* A bunch of assert() statements to check the transaction state variables
+** of handle p (type Btree*) are internally consistent.
+*/
+#define btreeIntegrity(p) \
+  assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \
+  assert( p->pBt->inTransaction>=p->inTrans ); 
+
+
+/*
+** The ISAUTOVACUUM macro is used within balance_nonroot() to determine
+** if the database supports auto-vacuum or not. Because it is used
+** within an expression that is an argument to another macro 
+** (sqliteMallocRaw), it is not possible to use conditional compilation.
+** So, this macro is defined instead.
+*/
+#ifndef SQLITE_OMIT_AUTOVACUUM
+#define ISAUTOVACUUM (pBt->autoVacuum)
+#else
+#define ISAUTOVACUUM 0
+#endif
+
+
+/*
+** This structure is passed around through all the sanity checking routines
+** in order to keep track of some global state information.
+*/
+typedef struct IntegrityCk IntegrityCk;
+struct IntegrityCk {
+  BtShared *pBt;    /* The tree being checked out */
+  Pager *pPager;    /* The associated pager.  Also accessible by pBt->pPager */
+  int nPage;        /* Number of pages in the database */
+  int *anRef;       /* Number of times each page is referenced */
+  int mxErr;        /* Stop accumulating errors when this reaches zero */
+  int nErr;         /* Number of messages written to zErrMsg so far */
+  StrAccum errMsg;  /* Accumulate the error message text here */
+};
+
+/*
+** Read or write a two- and four-byte big-endian integer values.
+*/
+#define get2byte(x)   ((x)[0]<<8 | (x)[1])
+#define put2byte(p,v) ((p)[0] = (v)>>8, (p)[1] = (v))
+#define get4byte sqlite3Get4byte
+#define put4byte sqlite3Put4byte
+
+/*
+** Internal routines that should be accessed by the btree layer only.
+*/
+int sqlite3BtreeGetPage(BtShared*, Pgno, MemPage**, int);
+int sqlite3BtreeInitPage(MemPage *pPage, MemPage *pParent);
+void sqlite3BtreeParseCellPtr(MemPage*, u8*, CellInfo*);
+void sqlite3BtreeParseCell(MemPage*, int, CellInfo*);
+int sqlite3BtreeRestoreCursorPosition(BtCursor *pCur);
+void sqlite3BtreeGetTempCursor(BtCursor *pCur, BtCursor *pTempCur);
+void sqlite3BtreeReleaseTempCursor(BtCursor *pCur);
+int sqlite3BtreeIsRootPage(MemPage *pPage);
+void sqlite3BtreeMoveToParent(BtCursor *pCur);

src/callback.c

@@ -0,0 +1,380 @@
+/*
+** 2005 May 23 
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains functions used to access the internal hash tables
+** of user defined functions and collation sequences.
+**
+** $Id: callback.c,v 1.25 2008/07/08 14:52:10 drh Exp $
+*/
+
+#include "sqliteInt.h"
+
+/*
+** Invoke the 'collation needed' callback to request a collation sequence
+** in the database text encoding of name zName, length nName.
+** If the collation sequence
+*/
+static void callCollNeeded(sqlite3 *db, const char *zName, int nName){
+  assert( !db->xCollNeeded || !db->xCollNeeded16 );
+  if( nName<0 ) nName = sqlite3Strlen(db, zName);
+  if( db->xCollNeeded ){
+    char *zExternal = sqlite3DbStrNDup(db, zName, nName);
+    if( !zExternal ) return;
+    db->xCollNeeded(db->pCollNeededArg, db, (int)ENC(db), zExternal);
+    sqlite3_free(zExternal);
+  }
+#ifndef SQLITE_OMIT_UTF16
+  if( db->xCollNeeded16 ){
+    char const *zExternal;
+    sqlite3_value *pTmp = sqlite3ValueNew(db);
+    sqlite3ValueSetStr(pTmp, nName, zName, SQLITE_UTF8, SQLITE_STATIC);
+    zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE);
+    if( zExternal ){
+      db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal);
+    }
+    sqlite3ValueFree(pTmp);
+  }
+#endif
+}
+
+/*
+** This routine is called if the collation factory fails to deliver a
+** collation function in the best encoding but there may be other versions
+** of this collation function (for other text encodings) available. Use one
+** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if
+** possible.
+*/
+static int synthCollSeq(sqlite3 *db, CollSeq *pColl){
+  CollSeq *pColl2;
+  char *z = pColl->zName;
+  int n = strlen(z);
+  int i;
+  static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 };
+  for(i=0; i<3; i++){
+    pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, n, 0);
+    if( pColl2->xCmp!=0 ){
+      memcpy(pColl, pColl2, sizeof(CollSeq));
+      pColl->xDel = 0;         /* Do not copy the destructor */
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_ERROR;
+}
+
+/*
+** This function is responsible for invoking the collation factory callback
+** or substituting a collation sequence of a different encoding when the
+** requested collation sequence is not available in the database native
+** encoding.
+** 
+** If it is not NULL, then pColl must point to the database native encoding 
+** collation sequence with name zName, length nName.
+**
+** The return value is either the collation sequence to be used in database
+** db for collation type name zName, length nName, or NULL, if no collation
+** sequence can be found.
+*/
+CollSeq *sqlite3GetCollSeq(
+  sqlite3* db, 
+  CollSeq *pColl, 
+  const char *zName, 
+  int nName
+){
+  CollSeq *p;
+
+  p = pColl;
+  if( !p ){
+    p = sqlite3FindCollSeq(db, ENC(db), zName, nName, 0);
+  }
+  if( !p || !p->xCmp ){
+    /* No collation sequence of this type for this encoding is registered.
+    ** Call the collation factory to see if it can supply us with one.
+    */
+    callCollNeeded(db, zName, nName);
+    p = sqlite3FindCollSeq(db, ENC(db), zName, nName, 0);
+  }
+  if( p && !p->xCmp && synthCollSeq(db, p) ){
+    p = 0;
+  }
+  assert( !p || p->xCmp );
+  return p;
+}
+
+/*
+** This routine is called on a collation sequence before it is used to
+** check that it is defined. An undefined collation sequence exists when
+** a database is loaded that contains references to collation sequences
+** that have not been defined by sqlite3_create_collation() etc.
+**
+** If required, this routine calls the 'collation needed' callback to
+** request a definition of the collating sequence. If this doesn't work, 
+** an equivalent collating sequence that uses a text encoding different
+** from the main database is substituted, if one is available.
+*/
+int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){
+  if( pColl ){
+    const char *zName = pColl->zName;
+    CollSeq *p = sqlite3GetCollSeq(pParse->db, pColl, zName, -1);
+    if( !p ){
+      if( pParse->nErr==0 ){
+        sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName);
+      }
+      pParse->nErr++;
+      return SQLITE_ERROR;
+    }
+    assert( p==pColl );
+  }
+  return SQLITE_OK;
+}
+
+
+
+/*
+** Locate and return an entry from the db.aCollSeq hash table. If the entry
+** specified by zName and nName is not found and parameter 'create' is
+** true, then create a new entry. Otherwise return NULL.
+**
+** Each pointer stored in the sqlite3.aCollSeq hash table contains an
+** array of three CollSeq structures. The first is the collation sequence
+** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be.
+**
+** Stored immediately after the three collation sequences is a copy of
+** the collation sequence name. A pointer to this string is stored in
+** each collation sequence structure.
+*/
+static CollSeq *findCollSeqEntry(
+  sqlite3 *db,
+  const char *zName,
+  int nName,
+  int create
+){
+  CollSeq *pColl;
+  if( nName<0 ) nName = sqlite3Strlen(db, zName);
+  pColl = sqlite3HashFind(&db->aCollSeq, zName, nName);
+
+  if( 0==pColl && create ){
+    pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 );
+    if( pColl ){
+      CollSeq *pDel = 0;
+      pColl[0].zName = (char*)&pColl[3];
+      pColl[0].enc = SQLITE_UTF8;
+      pColl[1].zName = (char*)&pColl[3];
+      pColl[1].enc = SQLITE_UTF16LE;
+      pColl[2].zName = (char*)&pColl[3];
+      pColl[2].enc = SQLITE_UTF16BE;
+      memcpy(pColl[0].zName, zName, nName);
+      pColl[0].zName[nName] = 0;
+      pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl);
+
+      /* If a malloc() failure occured in sqlite3HashInsert(), it will 
+      ** return the pColl pointer to be deleted (because it wasn't added
+      ** to the hash table).
+      */
+      assert( pDel==0 || pDel==pColl );
+      if( pDel!=0 ){
+        db->mallocFailed = 1;
+        sqlite3_free(pDel);
+        pColl = 0;
+      }
+    }
+  }
+  return pColl;
+}
+
+/*
+** Parameter zName points to a UTF-8 encoded string nName bytes long.
+** Return the CollSeq* pointer for the collation sequence named zName
+** for the encoding 'enc' from the database 'db'.
+**
+** If the entry specified is not found and 'create' is true, then create a
+** new entry.  Otherwise return NULL.
+**
+** A separate function sqlite3LocateCollSeq() is a wrapper around
+** this routine.  sqlite3LocateCollSeq() invokes the collation factory
+** if necessary and generates an error message if the collating sequence
+** cannot be found.
+*/
+CollSeq *sqlite3FindCollSeq(
+  sqlite3 *db,
+  u8 enc,
+  const char *zName,
+  int nName,
+  int create
+){
+  CollSeq *pColl;
+  if( zName ){
+    pColl = findCollSeqEntry(db, zName, nName, create);
+  }else{
+    pColl = db->pDfltColl;
+  }
+  assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
+  assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE );
+  if( pColl ) pColl += enc-1;
+  return pColl;
+}
+
+/*
+** Locate a user function given a name, a number of arguments and a flag
+** indicating whether the function prefers UTF-16 over UTF-8.  Return a
+** pointer to the FuncDef structure that defines that function, or return
+** NULL if the function does not exist.
+**
+** If the createFlag argument is true, then a new (blank) FuncDef
+** structure is created and liked into the "db" structure if a
+** no matching function previously existed.  When createFlag is true
+** and the nArg parameter is -1, then only a function that accepts
+** any number of arguments will be returned.
+**
+** If createFlag is false and nArg is -1, then the first valid
+** function found is returned.  A function is valid if either xFunc
+** or xStep is non-zero.
+**
+** If createFlag is false, then a function with the required name and
+** number of arguments may be returned even if the eTextRep flag does not
+** match that requested.
+*/
+FuncDef *sqlite3FindFunction(
+  sqlite3 *db,       /* An open database */
+  const char *zName, /* Name of the function.  Not null-terminated */
+  int nName,         /* Number of characters in the name */
+  int nArg,          /* Number of arguments.  -1 means any number */
+  u8 enc,            /* Preferred text encoding */
+  int createFlag     /* Create new entry if true and does not otherwise exist */
+){
+  FuncDef *p;         /* Iterator variable */
+  FuncDef *pFirst;    /* First function with this name */
+  FuncDef *pBest = 0; /* Best match found so far */
+  int bestmatch = 0;  
+
+
+  assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
+  if( nArg<-1 ) nArg = -1;
+
+  pFirst = (FuncDef*)sqlite3HashFind(&db->aFunc, zName, nName);
+  for(p=pFirst; p; p=p->pNext){
+    /* During the search for the best function definition, bestmatch is set
+    ** as follows to indicate the quality of the match with the definition
+    ** pointed to by pBest:
+    **
+    ** 0: pBest is NULL. No match has been found.
+    ** 1: A variable arguments function that prefers UTF-8 when a UTF-16
+    **    encoding is requested, or vice versa.
+    ** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is
+    **    requested, or vice versa.
+    ** 3: A variable arguments function using the same text encoding.
+    ** 4: A function with the exact number of arguments requested that
+    **    prefers UTF-8 when a UTF-16 encoding is requested, or vice versa.
+    ** 5: A function with the exact number of arguments requested that
+    **    prefers UTF-16LE when UTF-16BE is requested, or vice versa.
+    ** 6: An exact match.
+    **
+    ** A larger value of 'matchqual' indicates a more desirable match.
+    */
+    if( p->nArg==-1 || p->nArg==nArg || nArg==-1 ){
+      int match = 1;          /* Quality of this match */
+      if( p->nArg==nArg || nArg==-1 ){
+        match = 4;
+      }
+      if( enc==p->iPrefEnc ){
+        match += 2;
+      }
+      else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) ||
+               (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){
+        match += 1;
+      }
+
+      if( match>bestmatch ){
+        pBest = p;
+        bestmatch = match;
+      }
+    }
+  }
+
+  /* If the createFlag parameter is true, and the seach did not reveal an
+  ** exact match for the name, number of arguments and encoding, then add a
+  ** new entry to the hash table and return it.
+  */
+  if( createFlag && bestmatch<6 && 
+      (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName))!=0 ){
+    pBest->nArg = nArg;
+    pBest->pNext = pFirst;
+    pBest->iPrefEnc = enc;
+    memcpy(pBest->zName, zName, nName);
+    pBest->zName[nName] = 0;
+    if( pBest==sqlite3HashInsert(&db->aFunc,pBest->zName,nName,(void*)pBest) ){
+      db->mallocFailed = 1;
+      sqlite3_free(pBest);
+      return 0;
+    }
+  }
+
+  if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){
+    return pBest;
+  }
+  return 0;
+}
+
+/*
+** Free all resources held by the schema structure. The void* argument points
+** at a Schema struct. This function does not call sqlite3_free() on the 
+** pointer itself, it just cleans up subsiduary resources (i.e. the contents
+** of the schema hash tables).
+**
+** The Schema.cache_size variable is not cleared.
+*/
+void sqlite3SchemaFree(void *p){
+  Hash temp1;
+  Hash temp2;
+  HashElem *pElem;
+  Schema *pSchema = (Schema *)p;
+
+  temp1 = pSchema->tblHash;
+  temp2 = pSchema->trigHash;
+  sqlite3HashInit(&pSchema->trigHash, SQLITE_HASH_STRING, 0);
+  sqlite3HashClear(&pSchema->aFKey);
+  sqlite3HashClear(&pSchema->idxHash);
+  for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){
+    sqlite3DeleteTrigger((Trigger*)sqliteHashData(pElem));
+  }
+  sqlite3HashClear(&temp2);
+  sqlite3HashInit(&pSchema->tblHash, SQLITE_HASH_STRING, 0);
+  for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){
+    Table *pTab = sqliteHashData(pElem);
+    sqlite3DeleteTable(pTab);
+  }
+  sqlite3HashClear(&temp1);
+  pSchema->pSeqTab = 0;
+  pSchema->flags &= ~DB_SchemaLoaded;
+}
+
+/*
+** Find and return the schema associated with a BTree.  Create
+** a new one if necessary.
+*/
+Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){
+  Schema * p;
+  if( pBt ){
+    p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree);
+  }else{
+    p = (Schema *)sqlite3MallocZero(sizeof(Schema));
+  }
+  if( !p ){
+    db->mallocFailed = 1;
+  }else if ( 0==p->file_format ){
+    sqlite3HashInit(&p->tblHash, SQLITE_HASH_STRING, 0);
+    sqlite3HashInit(&p->idxHash, SQLITE_HASH_STRING, 0);
+    sqlite3HashInit(&p->trigHash, SQLITE_HASH_STRING, 0);
+    sqlite3HashInit(&p->aFKey, SQLITE_HASH_STRING, 1);
+    p->enc = SQLITE_UTF8;
+  }
+  return p;
+}

src/complete.c

@@ -0,0 +1,277 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** An tokenizer for SQL
+**
+** This file contains C code that implements the sqlite3_complete() API.
+** This code used to be part of the tokenizer.c source file.  But by
+** separating it out, the code will be automatically omitted from
+** static links that do not use it.
+**
+** $Id: complete.c,v 1.7 2008/06/13 18:24:27 drh Exp $
+*/
+#include "sqliteInt.h"
+#ifndef SQLITE_OMIT_COMPLETE
+
+/*
+** This is defined in tokenize.c.  We just have to import the definition.
+*/
+#ifndef SQLITE_AMALGAMATION
+#ifdef SQLITE_ASCII
+extern const char sqlite3IsAsciiIdChar[];
+#define IdChar(C)  (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
+#endif
+#ifdef SQLITE_EBCDIC
+extern const char sqlite3IsEbcdicIdChar[];
+#define IdChar(C)  (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
+#endif
+#endif /* SQLITE_AMALGAMATION */
+
+
+/*
+** Token types used by the sqlite3_complete() routine.  See the header
+** comments on that procedure for additional information.
+*/
+#define tkSEMI    0
+#define tkWS      1
+#define tkOTHER   2
+#define tkEXPLAIN 3
+#define tkCREATE  4
+#define tkTEMP    5
+#define tkTRIGGER 6
+#define tkEND     7
+
+/*
+** Return TRUE if the given SQL string ends in a semicolon.
+**
+** Special handling is require for CREATE TRIGGER statements.
+** Whenever the CREATE TRIGGER keywords are seen, the statement
+** must end with ";END;".
+**
+** This implementation uses a state machine with 7 states:
+**
+**   (0) START     At the beginning or end of an SQL statement.  This routine
+**                 returns 1 if it ends in the START state and 0 if it ends
+**                 in any other state.
+**
+**   (1) NORMAL    We are in the middle of statement which ends with a single
+**                 semicolon.
+**
+**   (2) EXPLAIN   The keyword EXPLAIN has been seen at the beginning of 
+**                 a statement.
+**
+**   (3) CREATE    The keyword CREATE has been seen at the beginning of a
+**                 statement, possibly preceeded by EXPLAIN and/or followed by
+**                 TEMP or TEMPORARY
+**
+**   (4) TRIGGER   We are in the middle of a trigger definition that must be
+**                 ended by a semicolon, the keyword END, and another semicolon.
+**
+**   (5) SEMI      We've seen the first semicolon in the ";END;" that occurs at
+**                 the end of a trigger definition.
+**
+**   (6) END       We've seen the ";END" of the ";END;" that occurs at the end
+**                 of a trigger difinition.
+**
+** Transitions between states above are determined by tokens extracted
+** from the input.  The following tokens are significant:
+**
+**   (0) tkSEMI      A semicolon.
+**   (1) tkWS        Whitespace
+**   (2) tkOTHER     Any other SQL token.
+**   (3) tkEXPLAIN   The "explain" keyword.
+**   (4) tkCREATE    The "create" keyword.
+**   (5) tkTEMP      The "temp" or "temporary" keyword.
+**   (6) tkTRIGGER   The "trigger" keyword.
+**   (7) tkEND       The "end" keyword.
+**
+** Whitespace never causes a state transition and is always ignored.
+**
+** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed
+** to recognize the end of a trigger can be omitted.  All we have to do
+** is look for a semicolon that is not part of an string or comment.
+*/
+int sqlite3_complete(const char *zSql){
+  u8 state = 0;   /* Current state, using numbers defined in header comment */
+  u8 token;       /* Value of the next token */
+
+#ifndef SQLITE_OMIT_TRIGGER
+  /* A complex statement machine used to detect the end of a CREATE TRIGGER
+  ** statement.  This is the normal case.
+  */
+  static const u8 trans[7][8] = {
+                     /* Token:                                                */
+     /* State:       **  SEMI  WS  OTHER EXPLAIN  CREATE  TEMP  TRIGGER  END  */
+     /* 0   START: */ {    0,  0,     1,      2,      3,    1,       1,   1,  },
+     /* 1  NORMAL: */ {    0,  1,     1,      1,      1,    1,       1,   1,  },
+     /* 2 EXPLAIN: */ {    0,  2,     1,      1,      3,    1,       1,   1,  },
+     /* 3  CREATE: */ {    0,  3,     1,      1,      1,    3,       4,   1,  },
+     /* 4 TRIGGER: */ {    5,  4,     4,      4,      4,    4,       4,   4,  },
+     /* 5    SEMI: */ {    5,  5,     4,      4,      4,    4,       4,   6,  },
+     /* 6     END: */ {    0,  6,     4,      4,      4,    4,       4,   4,  },
+  };
+#else
+  /* If triggers are not suppored by this compile then the statement machine
+  ** used to detect the end of a statement is much simplier
+  */
+  static const u8 trans[2][3] = {
+                     /* Token:           */
+     /* State:       **  SEMI  WS  OTHER */
+     /* 0   START: */ {    0,  0,     1, },
+     /* 1  NORMAL: */ {    0,  1,     1, },
+  };
+#endif /* SQLITE_OMIT_TRIGGER */
+
+  while( *zSql ){
+    switch( *zSql ){
+      case ';': {  /* A semicolon */
+        token = tkSEMI;
+        break;
+      }
+      case ' ':
+      case '\r':
+      case '\t':
+      case '\n':
+      case '\f': {  /* White space is ignored */
+        token = tkWS;
+        break;
+      }
+      case '/': {   /* C-style comments */
+        if( zSql[1]!='*' ){
+          token = tkOTHER;
+          break;
+        }
+        zSql += 2;
+        while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; }
+        if( zSql[0]==0 ) return 0;
+        zSql++;
+        token = tkWS;
+        break;
+      }
+      case '-': {   /* SQL-style comments from "--" to end of line */
+        if( zSql[1]!='-' ){
+          token = tkOTHER;
+          break;
+        }
+        while( *zSql && *zSql!='\n' ){ zSql++; }
+        if( *zSql==0 ) return state==0;
+        token = tkWS;
+        break;
+      }
+      case '[': {   /* Microsoft-style identifiers in [...] */
+        zSql++;
+        while( *zSql && *zSql!=']' ){ zSql++; }
+        if( *zSql==0 ) return 0;
+        token = tkOTHER;
+        break;
+      }
+      case '`':     /* Grave-accent quoted symbols used by MySQL */
+      case '"':     /* single- and double-quoted strings */
+      case '\'': {
+        int c = *zSql;
+        zSql++;
+        while( *zSql && *zSql!=c ){ zSql++; }
+        if( *zSql==0 ) return 0;
+        token = tkOTHER;
+        break;
+      }
+      default: {
+        int c;
+        if( IdChar((u8)*zSql) ){
+          /* Keywords and unquoted identifiers */
+          int nId;
+          for(nId=1; IdChar(zSql[nId]); nId++){}
+#ifdef SQLITE_OMIT_TRIGGER
+          token = tkOTHER;
+#else
+          switch( *zSql ){
+            case 'c': case 'C': {
+              if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){
+                token = tkCREATE;
+              }else{
+                token = tkOTHER;
+              }
+              break;
+            }
+            case 't': case 'T': {
+              if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){
+                token = tkTRIGGER;
+              }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){
+                token = tkTEMP;
+              }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){
+                token = tkTEMP;
+              }else{
+                token = tkOTHER;
+              }
+              break;
+            }
+            case 'e':  case 'E': {
+              if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){
+                token = tkEND;
+              }else
+#ifndef SQLITE_OMIT_EXPLAIN
+              if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){
+                token = tkEXPLAIN;
+              }else
+#endif
+              {
+                token = tkOTHER;
+              }
+              break;
+            }
+            default: {
+              token = tkOTHER;
+              break;
+            }
+          }
+#endif /* SQLITE_OMIT_TRIGGER */
+          zSql += nId-1;
+        }else{
+          /* Operators and special symbols */
+          token = tkOTHER;
+        }
+        break;
+      }
+    }
+    state = trans[state][token];
+    zSql++;
+  }
+  return state==0;
+}
+
+#ifndef SQLITE_OMIT_UTF16
+/*
+** This routine is the same as the sqlite3_complete() routine described
+** above, except that the parameter is required to be UTF-16 encoded, not
+** UTF-8.
+*/
+int sqlite3_complete16(const void *zSql){
+  sqlite3_value *pVal;
+  char const *zSql8;
+  int rc = SQLITE_NOMEM;
+
+#ifndef SQLITE_OMIT_AUTOINIT
+  rc = sqlite3_initialize();
+  if( rc ) return rc;
+#endif
+  pVal = sqlite3ValueNew(0);
+  sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC);
+  zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8);
+  if( zSql8 ){
+    rc = sqlite3_complete(zSql8);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+  sqlite3ValueFree(pVal);
+  return sqlite3ApiExit(0, rc);
+}
+#endif /* SQLITE_OMIT_UTF16 */
+#endif /* SQLITE_OMIT_COMPLETE */

src/delete.c

@@ -0,0 +1,544 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains C code routines that are called by the parser
+** in order to generate code for DELETE FROM statements.
+**
+** $Id: delete.c,v 1.170 2008/07/08 23:40:20 drh Exp $
+*/
+#include "sqliteInt.h"
+
+/*
+** Look up every table that is named in pSrc.  If any table is not found,
+** add an error message to pParse->zErrMsg and return NULL.  If all tables
+** are found, return a pointer to the last table.
+*/
+Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){
+  Table *pTab = 0;
+  int i;
+  struct SrcList_item *pItem;
+  for(i=0, pItem=pSrc->a; i<pSrc->nSrc; i++, pItem++){
+    pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase);
+    sqlite3DeleteTable(pItem->pTab);
+    pItem->pTab = pTab;
+    if( pTab ){
+      pTab->nRef++;
+    }
+  }
+  return pTab;
+}
+
+/*
+** Check to make sure the given table is writable.  If it is not
+** writable, generate an error message and return 1.  If it is
+** writable return 0;
+*/
+int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){
+  if( (pTab->readOnly && (pParse->db->flags & SQLITE_WriteSchema)==0
+        && pParse->nested==0) 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+      || (pTab->pMod && pTab->pMod->pModule->xUpdate==0)
+#endif
+  ){
+    sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName);
+    return 1;
+  }
+#ifndef SQLITE_OMIT_VIEW
+  if( !viewOk && pTab->pSelect ){
+    sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName);
+    return 1;
+  }
+#endif
+  return 0;
+}
+
+/*
+** Generate code that will open a table for reading.
+*/
+void sqlite3OpenTable(
+  Parse *p,       /* Generate code into this VDBE */
+  int iCur,       /* The cursor number of the table */
+  int iDb,        /* The database index in sqlite3.aDb[] */
+  Table *pTab,    /* The table to be opened */
+  int opcode      /* OP_OpenRead or OP_OpenWrite */
+){
+  Vdbe *v;
+  if( IsVirtual(pTab) ) return;
+  v = sqlite3GetVdbe(p);
+  assert( opcode==OP_OpenWrite || opcode==OP_OpenRead );
+  sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite), pTab->zName);
+  sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol);
+  sqlite3VdbeAddOp3(v, opcode, iCur, pTab->tnum, iDb);
+  VdbeComment((v, "%s", pTab->zName));
+}
+
+
+#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
+/*
+** Evaluate a view and store its result in an ephemeral table.  The
+** pWhere argument is an optional WHERE clause that restricts the
+** set of rows in the view that are to be added to the ephemeral table.
+*/
+void sqlite3MaterializeView(
+  Parse *pParse,       /* Parsing context */
+  Select *pView,       /* View definition */
+  Expr *pWhere,        /* Optional WHERE clause to be added */
+  int iCur             /* Cursor number for ephemerial table */
+){
+  SelectDest dest;
+  Select *pDup;
+  sqlite3 *db = pParse->db;
+
+  pDup = sqlite3SelectDup(db, pView);
+  if( pWhere ){
+    SrcList *pFrom;
+    
+    pWhere = sqlite3ExprDup(db, pWhere);
+    pFrom = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, 0, pDup, 0, 0);
+    pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
+  }
+  sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
+  sqlite3Select(pParse, pDup, &dest, 0, 0, 0);
+  sqlite3SelectDelete(pDup);
+}
+#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */
+
+
+/*
+** Generate code for a DELETE FROM statement.
+**
+**     DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL;
+**                 \________/       \________________/
+**                  pTabList              pWhere
+*/
+void sqlite3DeleteFrom(
+  Parse *pParse,         /* The parser context */
+  SrcList *pTabList,     /* The table from which we should delete things */
+  Expr *pWhere           /* The WHERE clause.  May be null */
+){
+  Vdbe *v;               /* The virtual database engine */
+  Table *pTab;           /* The table from which records will be deleted */
+  const char *zDb;       /* Name of database holding pTab */
+  int end, addr = 0;     /* A couple addresses of generated code */
+  int i;                 /* Loop counter */
+  WhereInfo *pWInfo;     /* Information about the WHERE clause */
+  Index *pIdx;           /* For looping over indices of the table */
+  int iCur;              /* VDBE Cursor number for pTab */
+  sqlite3 *db;           /* Main database structure */
+  AuthContext sContext;  /* Authorization context */
+  int oldIdx = -1;       /* Cursor for the OLD table of AFTER triggers */
+  NameContext sNC;       /* Name context to resolve expressions in */
+  int iDb;               /* Database number */
+  int memCnt = 0;        /* Memory cell used for change counting */
+
+#ifndef SQLITE_OMIT_TRIGGER
+  int isView;                  /* True if attempting to delete from a view */
+  int triggers_exist = 0;      /* True if any triggers exist */
+#endif
+  int iBeginAfterTrigger;      /* Address of after trigger program */
+  int iEndAfterTrigger;        /* Exit of after trigger program */
+  int iBeginBeforeTrigger;     /* Address of before trigger program */
+  int iEndBeforeTrigger;       /* Exit of before trigger program */
+  u32 old_col_mask = 0;        /* Mask of OLD.* columns in use */
+
+  sContext.pParse = 0;
+  db = pParse->db;
+  if( pParse->nErr || db->mallocFailed ){
+    goto delete_from_cleanup;
+  }
+  assert( pTabList->nSrc==1 );
+
+  /* Locate the table which we want to delete.  This table has to be
+  ** put in an SrcList structure because some of the subroutines we
+  ** will be calling are designed to work with multiple tables and expect
+  ** an SrcList* parameter instead of just a Table* parameter.
+  */
+  pTab = sqlite3SrcListLookup(pParse, pTabList);
+  if( pTab==0 )  goto delete_from_cleanup;
+
+  /* Figure out if we have any triggers and if the table being
+  ** deleted from is a view
+  */
+#ifndef SQLITE_OMIT_TRIGGER
+  triggers_exist = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0);
+  isView = pTab->pSelect!=0;
+#else
+# define triggers_exist 0
+# define isView 0
+#endif
+#ifdef SQLITE_OMIT_VIEW
+# undef isView
+# define isView 0
+#endif
+
+  if( sqlite3IsReadOnly(pParse, pTab, triggers_exist) ){
+    goto delete_from_cleanup;
+  }
+  iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+  assert( iDb<db->nDb );
+  zDb = db->aDb[iDb].zName;
+  if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){
+    goto delete_from_cleanup;
+  }
+
+  /* If pTab is really a view, make sure it has been initialized.
+  */
+  if( sqlite3ViewGetColumnNames(pParse, pTab) ){
+    goto delete_from_cleanup;
+  }
+
+  /* Allocate a cursor used to store the old.* data for a trigger.
+  */
+  if( triggers_exist ){ 
+    oldIdx = pParse->nTab++;
+  }
+
+  /* Assign  cursor number to the table and all its indices.
+  */
+  assert( pTabList->nSrc==1 );
+  iCur = pTabList->a[0].iCursor = pParse->nTab++;
+  for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    pParse->nTab++;
+  }
+
+  /* Start the view context
+  */
+  if( isView ){
+    sqlite3AuthContextPush(pParse, &sContext, pTab->zName);
+  }
+
+  /* Begin generating code.
+  */
+  v = sqlite3GetVdbe(pParse);
+  if( v==0 ){
+    goto delete_from_cleanup;
+  }
+  if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
+  sqlite3BeginWriteOperation(pParse, triggers_exist, iDb);
+
+  if( triggers_exist ){
+    int orconf = ((pParse->trigStack)?pParse->trigStack->orconf:OE_Default);
+    int iGoto = sqlite3VdbeAddOp0(v, OP_Goto);
+    addr = sqlite3VdbeMakeLabel(v);
+
+    iBeginBeforeTrigger = sqlite3VdbeCurrentAddr(v);
+    (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_BEFORE, pTab,
+        -1, oldIdx, orconf, addr, &old_col_mask, 0);
+    iEndBeforeTrigger = sqlite3VdbeAddOp0(v, OP_Goto);
+
+    iBeginAfterTrigger = sqlite3VdbeCurrentAddr(v);
+    (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_AFTER, pTab, -1,
+        oldIdx, orconf, addr, &old_col_mask, 0);
+    iEndAfterTrigger = sqlite3VdbeAddOp0(v, OP_Goto);
+
+    sqlite3VdbeJumpHere(v, iGoto);
+  }
+
+  /* If we are trying to delete from a view, realize that view into
+  ** a ephemeral table.
+  */
+  if( isView ){
+    sqlite3MaterializeView(pParse, pTab->pSelect, pWhere, iCur);
+  }
+
+  /* Resolve the column names in the WHERE clause.
+  */
+  memset(&sNC, 0, sizeof(sNC));
+  sNC.pParse = pParse;
+  sNC.pSrcList = pTabList;
+  if( sqlite3ExprResolveNames(&sNC, pWhere) ){
+    goto delete_from_cleanup;
+  }
+
+  /* Initialize the counter of the number of rows deleted, if
+  ** we are counting rows.
+  */
+  if( db->flags & SQLITE_CountRows ){
+    memCnt = ++pParse->nMem;
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt);
+  }
+
+  /* Special case: A DELETE without a WHERE clause deletes everything.
+  ** It is easier just to erase the whole table.  Note, however, that
+  ** this means that the row change count will be incorrect.
+  */
+  if( pWhere==0 && !triggers_exist && !IsVirtual(pTab) ){
+    if( db->flags & SQLITE_CountRows ){
+      /* If counting rows deleted, just count the total number of
+      ** entries in the table. */
+      int addr2;
+      if( !isView ){
+        sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead);
+      }
+      sqlite3VdbeAddOp2(v, OP_Rewind, iCur, sqlite3VdbeCurrentAddr(v)+2);
+      addr2 = sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
+      sqlite3VdbeAddOp2(v, OP_Next, iCur, addr2);
+      sqlite3VdbeAddOp1(v, OP_Close, iCur);
+    }
+    if( !isView ){
+      sqlite3VdbeAddOp2(v, OP_Clear, pTab->tnum, iDb);
+      if( !pParse->nested ){
+        sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC);
+      }
+      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+        assert( pIdx->pSchema==pTab->pSchema );
+        sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb);
+      }
+    }
+  } 
+  /* The usual case: There is a WHERE clause so we have to scan through
+  ** the table and pick which records to delete.
+  */
+  else{
+    int iRowid = ++pParse->nMem;    /* Used for storing rowid values. */
+
+    /* Begin the database scan
+    */
+    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0);
+    if( pWInfo==0 ) goto delete_from_cleanup;
+
+    /* Remember the rowid of every item to be deleted.
+    */
+    sqlite3VdbeAddOp2(v, IsVirtual(pTab) ? OP_VRowid : OP_Rowid, iCur, iRowid);
+    sqlite3VdbeAddOp1(v, OP_FifoWrite, iRowid);
+    if( db->flags & SQLITE_CountRows ){
+      sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
+    }
+
+    /* End the database scan loop.
+    */
+    sqlite3WhereEnd(pWInfo);
+
+    /* Open the pseudo-table used to store OLD if there are triggers.
+    */
+    if( triggers_exist ){
+      sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol);
+      sqlite3VdbeAddOp1(v, OP_OpenPseudo, oldIdx);
+    }
+
+    /* Delete every item whose key was written to the list during the
+    ** database scan.  We have to delete items after the scan is complete
+    ** because deleting an item can change the scan order.
+    */
+    end = sqlite3VdbeMakeLabel(v);
+
+    if( !isView ){
+      /* Open cursors for the table we are deleting from and 
+      ** all its indices.
+      */
+      sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite);
+    }
+
+    /* This is the beginning of the delete loop. If a trigger encounters
+    ** an IGNORE constraint, it jumps back to here.
+    */
+    if( triggers_exist ){
+      sqlite3VdbeResolveLabel(v, addr);
+    }
+    addr = sqlite3VdbeAddOp2(v, OP_FifoRead, iRowid, end);
+
+    if( triggers_exist ){
+      int iData = ++pParse->nMem;   /* For storing row data of OLD table */
+
+      /* If the record is no longer present in the table, jump to the
+      ** next iteration of the loop through the contents of the fifo.
+      */
+      sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, iRowid);
+
+      /* Populate the OLD.* pseudo-table */
+      if( old_col_mask ){
+        sqlite3VdbeAddOp2(v, OP_RowData, iCur, iData);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Null, 0, iData);
+      }
+      sqlite3VdbeAddOp3(v, OP_Insert, oldIdx, iData, iRowid);
+
+      /* Jump back and run the BEFORE triggers */
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, iBeginBeforeTrigger);
+      sqlite3VdbeJumpHere(v, iEndBeforeTrigger);
+    }
+
+    if( !isView ){
+      /* Delete the row */
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+      if( IsVirtual(pTab) ){
+        const char *pVtab = (const char *)pTab->pVtab;
+        sqlite3VtabMakeWritable(pParse, pTab);
+        sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVtab, P4_VTAB);
+      }else
+#endif
+      {
+        sqlite3GenerateRowDelete(pParse, pTab, iCur, iRowid, pParse->nested==0);
+      }
+    }
+
+    /* If there are row triggers, close all cursors then invoke
+    ** the AFTER triggers
+    */
+    if( triggers_exist ){
+      /* Jump back and run the AFTER triggers */
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, iBeginAfterTrigger);
+      sqlite3VdbeJumpHere(v, iEndAfterTrigger);
+    }
+
+    /* End of the delete loop */
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, addr);
+    sqlite3VdbeResolveLabel(v, end);
+
+    /* Close the cursors after the loop if there are no row triggers */
+    if( !isView  && !IsVirtual(pTab) ){
+      for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){
+        sqlite3VdbeAddOp2(v, OP_Close, iCur + i, pIdx->tnum);
+      }
+      sqlite3VdbeAddOp1(v, OP_Close, iCur);
+    }
+  }
+
+  /*
+  ** Return the number of rows that were deleted. If this routine is 
+  ** generating code because of a call to sqlite3NestedParse(), do not
+  ** invoke the callback function.
+  */
+  if( db->flags & SQLITE_CountRows && pParse->nested==0 && !pParse->trigStack ){
+    sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1);
+    sqlite3VdbeSetNumCols(v, 1);
+    sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", P4_STATIC);
+  }
+
+delete_from_cleanup:
+  sqlite3AuthContextPop(&sContext);
+  sqlite3SrcListDelete(pTabList);
+  sqlite3ExprDelete(pWhere);
+  return;
+}
+
+/*
+** This routine generates VDBE code that causes a single row of a
+** single table to be deleted.
+**
+** The VDBE must be in a particular state when this routine is called.
+** These are the requirements:
+**
+**   1.  A read/write cursor pointing to pTab, the table containing the row
+**       to be deleted, must be opened as cursor number "base".
+**
+**   2.  Read/write cursors for all indices of pTab must be open as
+**       cursor number base+i for the i-th index.
+**
+**   3.  The record number of the row to be deleted must be stored in
+**       memory cell iRowid.
+**
+** This routine pops the top of the stack to remove the record number
+** and then generates code to remove both the table record and all index
+** entries that point to that record.
+*/
+void sqlite3GenerateRowDelete(
+  Parse *pParse,     /* Parsing context */
+  Table *pTab,       /* Table containing the row to be deleted */
+  int iCur,          /* Cursor number for the table */
+  int iRowid,        /* Memory cell that contains the rowid to delete */
+  int count          /* Increment the row change counter */
+){
+  int addr;
+  Vdbe *v;
+
+  v = pParse->pVdbe;
+  addr = sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowid);
+  sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, 0);
+  sqlite3VdbeAddOp2(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0));
+  if( count ){
+    sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC);
+  }
+  sqlite3VdbeJumpHere(v, addr);
+}
+
+/*
+** This routine generates VDBE code that causes the deletion of all
+** index entries associated with a single row of a single table.
+**
+** The VDBE must be in a particular state when this routine is called.
+** These are the requirements:
+**
+**   1.  A read/write cursor pointing to pTab, the table containing the row
+**       to be deleted, must be opened as cursor number "iCur".
+**
+**   2.  Read/write cursors for all indices of pTab must be open as
+**       cursor number iCur+i for the i-th index.
+**
+**   3.  The "iCur" cursor must be pointing to the row that is to be
+**       deleted.
+*/
+void sqlite3GenerateRowIndexDelete(
+  Parse *pParse,     /* Parsing and code generating context */
+  Table *pTab,       /* Table containing the row to be deleted */
+  int iCur,          /* Cursor number for the table */
+  int *aRegIdx       /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */
+){
+  int i;
+  Index *pIdx;
+  int r1;
+
+  for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){
+    if( aRegIdx!=0 && aRegIdx[i-1]==0 ) continue;
+    r1 = sqlite3GenerateIndexKey(pParse, pIdx, iCur, 0, 0);
+    sqlite3VdbeAddOp3(pParse->pVdbe, OP_IdxDelete, iCur+i, r1,pIdx->nColumn+1);
+  }
+}
+
+/*
+** Generate code that will assemble an index key and put it in register
+** regOut.  The key with be for index pIdx which is an index on pTab.
+** iCur is the index of a cursor open on the pTab table and pointing to
+** the entry that needs indexing.
+**
+** Return a register number which is the first in a block of
+** registers that holds the elements of the index key.  The
+** block of registers has already been deallocated by the time
+** this routine returns.
+*/
+int sqlite3GenerateIndexKey(
+  Parse *pParse,     /* Parsing context */
+  Index *pIdx,       /* The index for which to generate a key */
+  int iCur,          /* Cursor number for the pIdx->pTable table */
+  int regOut,        /* Write the new index key to this register */
+  int doMakeRec      /* Run the OP_MakeRecord instruction if true */
+){
+  Vdbe *v = pParse->pVdbe;
+  int j;
+  Table *pTab = pIdx->pTable;
+  int regBase;
+  int nCol;
+
+  nCol = pIdx->nColumn;
+  regBase = sqlite3GetTempRange(pParse, nCol+1);
+  sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regBase+nCol);
+  for(j=0; j<nCol; j++){
+    int idx = pIdx->aiColumn[j];
+    if( idx==pTab->iPKey ){
+      sqlite3VdbeAddOp2(v, OP_SCopy, regBase+nCol, regBase+j);
+    }else{
+      sqlite3VdbeAddOp3(v, OP_Column, iCur, idx, regBase+j);
+      sqlite3ColumnDefault(v, pTab, idx);
+    }
+  }
+  if( doMakeRec ){
+    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut);
+    sqlite3IndexAffinityStr(v, pIdx);
+    sqlite3ExprCacheAffinityChange(pParse, regBase, nCol+1);
+  }
+  sqlite3ReleaseTempRange(pParse, regBase, nCol+1);
+  return regBase;
+}
+
+/* Make sure "isView" gets undefined in case this file becomes part of
+** the amalgamation - so that subsequent files do not see isView as a
+** macro. */
+#undef isView

src/fault.c

@@ -0,0 +1,71 @@
+/*
+** 2008 Jan 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** $Id: fault.c,v 1.10 2008/06/22 12:37:58 drh Exp $
+*/
+
+/*
+** This file contains code to support the concept of "benign" 
+** malloc failures (when the xMalloc() or xRealloc() method of the
+** sqlite3_mem_methods structure fails to allocate a block of memory
+** and returns 0). 
+**
+** Most malloc failures are non-benign. After they occur, SQLite
+** abandons the current operation and returns an error code (usually
+** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily
+** fatal. For example, if a malloc fails while resizing a hash table, this 
+** is completely recoverable simply by not carrying out the resize. The 
+** hash table will continue to function normally.  So a malloc failure 
+** during a hash table resize is a benign fault.
+*/
+
+#include "sqliteInt.h"
+
+#ifndef SQLITE_OMIT_BUILTIN_TEST
+
+/*
+** Global variables.
+*/
+static struct BenignMallocHooks {
+  void (*xBenignBegin)(void);
+  void (*xBenignEnd)(void);
+} hooks;
+
+/*
+** Register hooks to call when sqlite3BeginBenignMalloc() and
+** sqlite3EndBenignMalloc() are called, respectively.
+*/
+void sqlite3BenignMallocHooks(
+  void (*xBenignBegin)(void),
+  void (*xBenignEnd)(void)
+){
+  hooks.xBenignBegin = xBenignBegin;
+  hooks.xBenignEnd = xBenignEnd;
+}
+
+/*
+** This (sqlite3EndBenignMalloc()) is called by SQLite code to indicate that
+** subsequent malloc failures are benign. A call to sqlite3EndBenignMalloc()
+** indicates that subsequent malloc failures are non-benign.
+*/
+void sqlite3BeginBenignMalloc(void){
+  if( hooks.xBenignBegin ){
+    hooks.xBenignBegin();
+  }
+}
+void sqlite3EndBenignMalloc(void){
+  if( hooks.xBenignEnd ){
+    hooks.xBenignEnd();
+  }
+}
+
+#endif   /* #ifndef SQLITE_OMIT_BUILTIN_TEST */

src/global.c

@@ -0,0 +1,69 @@
+/*
+** 2008 June 13
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains definitions of global variables and contants.
+**
+** $Id: global.c,v 1.3 2008/07/08 14:52:10 drh Exp $
+*/
+#include "sqliteInt.h"
+
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character. 
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters.  We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+const unsigned char sqlite3UpperToLower[] = {
+#ifdef SQLITE_ASCII
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+    252,253,254,255
+#endif
+#ifdef SQLITE_EBCDIC
+      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, /* 0x */
+     16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, /* 1x */
+     32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, /* 2x */
+     48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, /* 3x */
+     64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, /* 4x */
+     80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, /* 5x */
+     96, 97, 66, 67, 68, 69, 70, 71, 72, 73,106,107,108,109,110,111, /* 6x */
+    112, 81, 82, 83, 84, 85, 86, 87, 88, 89,122,123,124,125,126,127, /* 7x */
+    128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, /* 8x */
+    144,145,146,147,148,149,150,151,152,153,154,155,156,157,156,159, /* 9x */
+    160,161,162,163,164,165,166,167,168,169,170,171,140,141,142,175, /* Ax */
+    176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, /* Bx */
+    192,129,130,131,132,133,134,135,136,137,202,203,204,205,206,207, /* Cx */
+    208,145,146,147,148,149,150,151,152,153,218,219,220,221,222,223, /* Dx */
+    224,225,162,163,164,165,166,167,168,169,232,203,204,205,206,207, /* Ex */
+    239,240,241,242,243,244,245,246,247,248,249,219,220,221,222,255, /* Fx */
+#endif
+};
+
+/*
+** The following singleton contains the global configuration for
+** the SQLite library.
+*/
+struct Sqlite3Config sqlite3Config = { 1, 1, 1, 0x7ffffffe };

src/hash.c

@@ -0,0 +1,423 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables
+** used in SQLite.
+**
+** $Id: hash.c,v 1.30 2008/06/20 14:59:51 danielk1977 Exp $
+*/
+#include "sqliteInt.h"
+#include <assert.h>
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants SQLITE_HASH_INT, SQLITE_HASH_POINTER,
+** SQLITE_HASH_BINARY, or SQLITE_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.  CopyKey only makes
+** sense for SQLITE_HASH_STRING and SQLITE_HASH_BINARY and is ignored
+** for other key classes.
+*/
+void sqlite3HashInit(Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=SQLITE_HASH_STRING && keyClass<=SQLITE_HASH_BINARY );
+  pNew->keyClass = keyClass;
+#if 0
+  if( keyClass==SQLITE_HASH_POINTER || keyClass==SQLITE_HASH_INT ) copyKey = 0;
+#endif
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3HashClear(Hash *pH){
+  HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  sqlite3_free(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      sqlite3_free(elem->pKey);
+    }
+    sqlite3_free(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is SQLITE_HASH_INT
+*/
+static int intHash(const void *pKey, int nKey){
+  return nKey ^ (nKey<<8) ^ (nKey>>8);
+}
+static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  return n2 - n1;
+}
+#endif
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is SQLITE_HASH_POINTER
+*/
+static int ptrHash(const void *pKey, int nKey){
+  uptr x = Addr(pKey);
+  return x ^ (x<<8) ^ (x>>8);
+}
+static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( pKey1==pKey2 ) return 0;
+  if( pKey1<pKey2 ) return -1;
+  return 1;
+}
+#endif
+
+/*
+** Hash and comparison functions when the mode is SQLITE_HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++];
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return sqlite3StrNICmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is SQLITE_HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+#if 0  /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case SQLITE_HASH_INT:     return &intHash;
+    case SQLITE_HASH_POINTER: return &ptrHash;
+    case SQLITE_HASH_STRING:  return &strHash;
+    case SQLITE_HASH_BINARY:  return &binHash;;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==SQLITE_HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==SQLITE_HASH_BINARY );
+    return &binHash;
+  }
+#endif
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+#if 0 /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case SQLITE_HASH_INT:     return &intCompare;
+    case SQLITE_HASH_POINTER: return &ptrCompare;
+    case SQLITE_HASH_STRING:  return &strCompare;
+    case SQLITE_HASH_BINARY:  return &binCompare;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==SQLITE_HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==SQLITE_HASH_BINARY );
+    return &binCompare;
+  }
+#endif
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  Hash *pH,              /* The complete hash table */
+  struct _ht *pEntry,    /* The entry into which pNew is inserted */
+  HashElem *pNew         /* The element to be inserted */
+){
+  HashElem *pHead;       /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqlite3_malloc() fails.
+*/
+static void rehash(Hash *pH, int new_size){
+  struct _ht *new_ht;            /* The new hash table */
+  HashElem *elem, *next_elem;    /* For looping over existing elements */
+  int (*xHash)(const void*,int); /* The hash function */
+
+#ifdef SQLITE_MALLOC_SOFT_LIMIT
+  if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){
+    new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht);
+  }
+  if( new_size==pH->htsize ) return;
+#endif
+
+  /* There is a call to sqlite3_malloc() inside rehash(). If there is
+  ** already an allocation at pH->ht, then if this malloc() fails it
+  ** is benign (since failing to resize a hash table is a performance
+  ** hit only, not a fatal error).
+  */
+  if( pH->htsize>0 ) sqlite3BeginBenignMalloc();
+  new_ht = (struct _ht *)sqlite3MallocZero( new_size*sizeof(struct _ht) );
+  if( pH->htsize>0 ) sqlite3EndBenignMalloc();
+
+  if( new_ht==0 ) return;
+  sqlite3_free(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static HashElem *findElementGivenHash(
+  const Hash *pH,     /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  HashElem *elem;                /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  Hash *pH,         /* The pH containing "elem" */
+  HashElem* elem,   /* The element to be removed from the pH */
+  int h             /* Hash value for the element */
+){
+  struct _ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey ){
+    sqlite3_free(elem->pKey);
+  }
+  sqlite3_free( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    sqlite3HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return a pointer to the corresponding 
+** HashElem structure for this element if it is found, or NULL
+** otherwise.
+*/
+HashElem *sqlite3HashFindElem(const Hash *pH, const void *pKey, int nKey){
+  int h;             /* A hash on key */
+  HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  elem = findElementGivenHash(pH,pKey,nKey, h % pH->htsize);
+  return elem;
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3HashFind(const Hash *pH, const void *pKey, int nKey){
+  HashElem *elem;    /* The element that matches key */
+  elem = sqlite3HashFindElem(pH, pKey, nKey);
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
+  int hraw;             /* Raw hash value of the key */
+  int h;                /* the hash of the key modulo hash table size */
+  HashElem *elem;       /* Used to loop thru the element list */
+  HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  if( pH->htsize ){
+    h = hraw % pH->htsize;
+    elem = findElementGivenHash(pH,pKey,nKey,h);
+    if( elem ){
+      void *old_data = elem->data;
+      if( data==0 ){
+        removeElementGivenHash(pH,elem,h);
+      }else{
+        elem->data = data;
+        if( !pH->copyKey ){
+          elem->pKey = (void *)pKey;
+        }
+        assert(nKey==elem->nKey);
+      }
+      return old_data;
+    }
+  }
+  if( data==0 ) return 0;
+  new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = sqlite3Malloc( nKey );
+    if( new_elem->pKey==0 ){
+      sqlite3_free(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH, 128/sizeof(pH->ht[0]));
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      if( pH->copyKey ){
+        sqlite3_free(new_elem->pKey);
+      }
+      sqlite3_free(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  h = hraw % pH->htsize;
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}

src/hash.h

@@ -0,0 +1,110 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite.
+**
+** $Id: hash.h,v 1.11 2007/09/04 14:31:47 danielk1977 Exp $
+*/
+#ifndef _SQLITE_HASH_H_
+#define _SQLITE_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct Hash Hash;
+typedef struct HashElem HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct Hash {
+  char keyClass;          /* SQLITE_HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  int htsize;             /* Number of buckets in the hash table */
+  HashElem *first;        /* The first element of the array */
+  struct _ht {            /* the hash table */
+    int count;               /* Number of entries with this hash */
+    HashElem *chain;         /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct HashElem {
+  HashElem *next, *prev;   /* Next and previous elements in the table */
+  void *data;              /* Data associated with this element */
+  void *pKey; int nKey;    /* Key associated with this element */
+};
+
+/*
+** There are 4 different modes of operation for a hash table:
+**
+**   SQLITE_HASH_INT         nKey is used as the key and pKey is ignored.
+**
+**   SQLITE_HASH_POINTER     pKey is used as the key and nKey is ignored.
+**
+**   SQLITE_HASH_STRING      pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is ignored in comparisons.
+**
+**   SQLITE_HASH_BINARY      pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made for SQLITE_HASH_STRING and SQLITE_HASH_BINARY
+** if the copyKey parameter to HashInit is 1.  
+*/
+/* #define SQLITE_HASH_INT       1 // NOT USED */
+/* #define SQLITE_HASH_POINTER   2 // NOT USED */
+#define SQLITE_HASH_STRING    3
+#define SQLITE_HASH_BINARY    4
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3HashInit(Hash*, int keytype, int copyKey);
+void *sqlite3HashInsert(Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3HashFind(const Hash*, const void *pKey, int nKey);
+HashElem *sqlite3HashFindElem(const Hash*, const void *pKey, int nKey);
+void sqlite3HashClear(Hash*);
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   Hash h;
+**   HashElem *p;
+**   ...
+**   for(p=sqliteHashFirst(&h); p; p=sqliteHashNext(p)){
+**     SomeStructure *pData = sqliteHashData(p);
+**     // do something with pData
+**   }
+*/
+#define sqliteHashFirst(H)  ((H)->first)
+#define sqliteHashNext(E)   ((E)->next)
+#define sqliteHashData(E)   ((E)->data)
+#define sqliteHashKey(E)    ((E)->pKey)
+#define sqliteHashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define sqliteHashCount(H)  ((H)->count)
+
+#endif /* _SQLITE_HASH_H_ */

src/hwtime.h

@@ -0,0 +1,72 @@
+/*
+** 2008 May 27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains inline asm code for retrieving "high-performance"
+** counters for x86 class CPUs.
+**
+** $Id: hwtime.h,v 1.2 2008/06/12 02:24:39 shane Exp $
+*/
+#ifndef _HWTIME_H_
+#define _HWTIME_H_
+
+/*
+** The following routine only works on pentium-class (or newer) processors.
+** It uses the RDTSC opcode to read the cycle count value out of the
+** processor and returns that value.  This can be used for high-res
+** profiling.
+*/
+#if (defined(__GNUC__) || defined(_MSC_VER)) && \
+      (defined(i386) || defined(__i386__) || defined(_M_IX86))
+
+  #if defined(__GNUC__)
+
+  __inline__ sqlite_uint64 sqlite3Hwtime(void){
+     unsigned int lo, hi;
+     __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi));
+     return (sqlite_uint64)hi << 32 | lo;
+  }
+
+  #elif defined(_MSC_VER)
+
+  __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){
+     __asm {
+        rdtsc
+        ret       ; return value at EDX:EAX
+     }
+  }
+
+  #endif
+
+#elif (defined(__GNUC__) && defined(__x86_64__))
+
+  __inline__ sqlite_uint64 sqlite3Hwtime(void){
+      unsigned long val;
+      __asm__ __volatile__ ("rdtsc" : "=A" (val));
+      return val;
+  }
+ 
+#else
+
+  #error Need implementation of sqlite3Hwtime() for your platform.
+
+  /*
+  ** To compile without implementing sqlite3Hwtime() for your platform,
+  ** you can remove the above #error and use the following
+  ** stub function.  You will lose timing support for many
+  ** of the debugging and testing utilities, but it should at
+  ** least compile and run.
+  */
+  sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); }
+
+#endif
+
+#endif /* !defined(_HWTIME_H_) */