Merge sqlite-release(3.34.1) into prerelease-integration

2026-08-31 14:31:11 +00:00 · 2021-02-10 11:46:38 -05:00
parent 804dfb3a38 718beb4d42
commit 4a459028c5
152 changed files with 4642 additions and 1277 deletions
@@ -1196,13 +1196,21 @@ static int renameResolveTrigger(Parse *pParse){
        int i;
        for(i=0; i<pSrc->nSrc && rc==SQLITE_OK; i++){
          struct SrcList_item *p = &pSrc->a[i];
-          p->pTab = sqlite3LocateTableItem(pParse, 0, p);
          p->iCursor = pParse->nTab++;
-          if( p->pTab==0 ){
-            rc = SQLITE_ERROR;
+          if( p->pSelect ){
+            sqlite3SelectPrep(pParse, p->pSelect, 0);
+            sqlite3ExpandSubquery(pParse, p);
+            assert( i>0 );
+            assert( pStep->pFrom->a[i-1].pSelect );
+            sqlite3SelectPrep(pParse, pStep->pFrom->a[i-1].pSelect, 0);
          }else{
-            p->pTab->nTabRef++;
-            rc = sqlite3ViewGetColumnNames(pParse, p->pTab);
+            p->pTab = sqlite3LocateTableItem(pParse, 0, p);
+            if( p->pTab==0 ){
+              rc = SQLITE_ERROR;
+            }else{
+              p->pTab->nTabRef++;
+              rc = sqlite3ViewGetColumnNames(pParse, p->pTab);
+            }
          }
        }
        sNC.pSrcList = pSrc;
@@ -1264,6 +1272,12 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){
      sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere);
      sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere);
    }
+    if( pStep->pFrom ){
+      int i;
+      for(i=0; i<pStep->pFrom->nSrc; i++){
+        sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect);
+      }
+    }
  }
 }

@@ -1772,6 +1772,7 @@ static int loadStatTbl(
    }
    pSpace = (tRowcnt*)&pIdx->aSample[nSample];
    pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
+    pIdx->pTable->tabFlags |= TF_HasStat4;
    for(i=0; i<nSample; i++){
      pIdx->aSample[i].anEq = pSpace; pSpace += nIdxCol;
      pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol;
@@ -330,7 +330,9 @@ static void detachFunc(
    sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName);
    goto detach_error;
  }
-  if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){
+  if( sqlite3BtreeTxnState(pDb->pBt)!=SQLITE_TXN_NONE
+   || sqlite3BtreeIsInBackup(pDb->pBt)
+  ){
    sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName);
    goto detach_error;
  }
@@ -123,7 +123,7 @@ static int setDestPgsz(sqlite3_backup *p){
 ** message in database handle db.
 */
 static int checkReadTransaction(sqlite3 *db, Btree *p){
-  if( sqlite3BtreeIsInReadTrans(p) ){
+  if( sqlite3BtreeTxnState(p)!=SQLITE_TXN_NONE ){
    sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use");
    return SQLITE_ERROR;
  }
@@ -406,7 +406,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
    ** one now. If a transaction is opened here, then it will be closed
    ** before this function exits.
    */
-    if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){
+    if( rc==SQLITE_OK && SQLITE_TXN_NONE==sqlite3BtreeTxnState(p->pSrc) ){
      rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0);
      bCloseTrans = 1;
    }
@@ -778,7 +778,7 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){
  sqlite3BtreeEnter(pTo);
  sqlite3BtreeEnter(pFrom);

-  assert( sqlite3BtreeIsInTrans(pTo) );
+  assert( sqlite3BtreeTxnState(pTo)==SQLITE_TXN_WRITE );
  pFd = sqlite3PagerFile(sqlite3BtreePager(pTo));
  if( pFd->pMethods ){
    i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom);
@@ -820,7 +820,7 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){
    sqlite3PagerClearCache(sqlite3BtreePager(b.pDest));
  }

-  assert( sqlite3BtreeIsInTrans(pTo)==0 );
+  assert( sqlite3BtreeTxnState(pTo)!=SQLITE_TXN_WRITE );
 copy_finished:
  sqlite3BtreeLeave(pFrom);
  sqlite3BtreeLeave(pTo);
@@ -112,6 +112,17 @@ int sqlite3_enable_shared_cache(int enable){
  #define hasReadConflicts(a, b) 0
 #endif

+#ifdef SQLITE_DEBUG
+/*
+** Return and reset the seek counter for a Btree object.
+*/
+sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){
+  u64 n =  pBt->nSeek;
+  pBt->nSeek = 0;
+  return n;
+}
+#endif
+
 /*
 ** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single
 ** (MemPage*) as an argument. The (MemPage*) must not be NULL.
@@ -2605,7 +2616,7 @@ btree_open_out:
    ** do not change the pager-cache size.
    */
    if( sqlite3BtreeSchema(p, 0, 0)==0 ){
-      sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE);
+      sqlite3BtreeSetCacheSize(p, SQLITE_DEFAULT_CACHE_SIZE);
    }

    pFile = sqlite3PagerFile(pBt->pPager);
@@ -5459,6 +5470,10 @@ int sqlite3BtreeMovetoUnpacked(
    }
  }

+#ifdef SQLITE_DEBUG
+  pCur->pBtree->nSeek++;   /* Performance measurement during testing */
+#endif
+
  if( pIdxKey ){
    xRecordCompare = sqlite3VdbeFindCompare(pIdxKey);
    pIdxKey->errCode = 0;
@@ -5735,7 +5750,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){

  pPage = pCur->pPage;
  idx = ++pCur->ix;
-  if( !pPage->isInit ){
+  if( !pPage->isInit || sqlite3FaultSim(412) ){
    /* The only known way for this to happen is for there to be a
    ** recursive SQL function that does a DELETE operation as part of a
    ** SELECT which deletes content out from under an active cursor
@@ -10310,11 +10325,12 @@ const char *sqlite3BtreeGetJournalname(Btree *p){
 }

 /*
-** Return non-zero if a transaction is active.
+** Return one of SQLITE_TXN_NONE, SQLITE_TXN_READ, or SQLITE_TXN_WRITE
+** to describe the current transaction state of Btree p.
 */
-int sqlite3BtreeIsInTrans(Btree *p){
+int sqlite3BtreeTxnState(Btree *p){
  assert( p==0 || sqlite3_mutex_held(p->db->mutex) );
-  return (p && (p->inTrans==TRANS_WRITE));
+  return p ? p->inTrans : 0;
 }

 #ifndef SQLITE_OMIT_WAL
@@ -10343,14 +10359,8 @@ int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){
 #endif

 /*
-** Return non-zero if a read (or write) transaction is active.
+** Return true if there is currently a backup running on Btree p.
 */
-int sqlite3BtreeIsInReadTrans(Btree *p){
-  assert( p );
-  assert( sqlite3_mutex_held(p->db->mutex) );
-  return p->inTrans!=TRANS_NONE;
-}
-
 int sqlite3BtreeIsInBackup(Btree *p){
  assert( p );
  assert( sqlite3_mutex_held(p->db->mutex) );
@@ -85,16 +85,24 @@ int sqlite3BtreeCommit(Btree*);
 int sqlite3BtreeRollback(Btree*,int,int);
 int sqlite3BtreeBeginStmt(Btree*,int);
 int sqlite3BtreeCreateTable(Btree*, Pgno*, int flags);
-int sqlite3BtreeIsInTrans(Btree*);
-int sqlite3BtreeIsInReadTrans(Btree*);
+int sqlite3BtreeTxnState(Btree*);
 int sqlite3BtreeIsInBackup(Btree*);
+
 void *sqlite3BtreeSchema(Btree *, int, void(*)(void *));
 int sqlite3BtreeSchemaLocked(Btree *pBtree);
 #ifndef SQLITE_OMIT_SHARED_CACHE
 int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock);
 #endif
+
+/* Savepoints are named, nestable SQL transactions mostly implemented */ 
+/* in vdbe.c and pager.c See https://sqlite.org/lang_savepoint.html */
 int sqlite3BtreeSavepoint(Btree *, int, int);

+/* "Checkpoint" only refers to WAL. See https://sqlite.org/wal.html#ckpt */
+#ifndef SQLITE_OMIT_WAL
+  int sqlite3BtreeCheckpoint(Btree*, int, int *, int *);  
+#endif
+
 const char *sqlite3BtreeGetFilename(Btree *);
 const char *sqlite3BtreeGetJournalname(Btree *);
 int sqlite3BtreeCopyFile(Btree *, Btree *);
@@ -331,6 +339,12 @@ int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask);
 int sqlite3BtreeIsReadonly(Btree *pBt);
 int sqlite3HeaderSizeBtree(void);

+#ifdef SQLITE_DEBUG
+sqlite3_uint64 sqlite3BtreeSeekCount(Btree*);
+#else
+# define sqlite3BtreeSeekCount(X) 0
+#endif
+
 #ifndef NDEBUG
 int sqlite3BtreeCursorIsValid(BtCursor*);
 #endif
@@ -353,6 +353,9 @@ struct Btree {
  u32 iDataVersion;  /* Combines with pBt->pPager->iDataVersion */
  Btree *pNext;      /* List of other sharable Btrees from the same db */
  Btree *pPrev;      /* Back pointer of the same list */
+#ifdef SQLITE_DEBUG
+  u64 nSeek;         /* Calls to sqlite3BtreeMovetoUnpacked() */
+#endif
 #ifndef SQLITE_OMIT_SHARED_CACHE
  BtLock lock;       /* Object used to lock page 1 */
 #endif
@@ -364,11 +367,25 @@ struct Btree {
 ** If the shared-data extension is enabled, there may be multiple users
 ** of the Btree structure. At most one of these may open a write transaction,
 ** but any number may have active read transactions.
+**
+** These values must match SQLITE_TXN_NONE, SQLITE_TXN_READ, and
+** SQLITE_TXN_WRITE
 */
 #define TRANS_NONE  0
 #define TRANS_READ  1
 #define TRANS_WRITE 2

+#if TRANS_NONE!=SQLITE_TXN_NONE
+# error wrong numeric code for no-transaction
+#endif
+#if TRANS_READ!=SQLITE_TXN_READ
+# error wrong numeric code for read-transaction
+#endif
+#if TRANS_WRITE!=SQLITE_TXN_WRITE
+# error wrong numeric code for write-transaction
+#endif
+
+
 /*
 ** An instance of this object represents a single database file.
 ** 
@@ -53,7 +53,7 @@ void sqlite3TableLock(
  u8 isWriteLock,    /* True for a write lock */
  const char *zName  /* Name of the table to be locked */
 ){
-  Parse *pToplevel = sqlite3ParseToplevel(pParse);
+  Parse *pToplevel;
  int i;
  int nBytes;
  TableLock *p;
@@ -61,6 +61,7 @@ void sqlite3TableLock(

  if( iDb==1 ) return;
  if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return;
+  pToplevel = sqlite3ParseToplevel(pParse);
  for(i=0; i<pToplevel->nTableLock; i++){
    p = &pToplevel->aTableLock[i];
    if( p->iDb==iDb && p->iTab==iTab ){
@@ -90,10 +91,8 @@ void sqlite3TableLock(
 */
 static void codeTableLocks(Parse *pParse){
  int i;
-  Vdbe *pVdbe; 
-
-  pVdbe = sqlite3GetVdbe(pParse);
-  assert( pVdbe!=0 ); /* sqlite3GetVdbe cannot fail: VDBE already allocated */
+  Vdbe *pVdbe = pParse->pVdbe; 
+  assert( pVdbe!=0 );

  for(i=0; i<pParse->nTableLock; i++){
    TableLock *p = &pParse->aTableLock[i];
@@ -1618,8 +1617,10 @@ primary_key_exit:
 ** Add a new CHECK constraint to the table currently under construction.
 */
 void sqlite3AddCheckConstraint(
-  Parse *pParse,    /* Parsing context */
-  Expr *pCheckExpr  /* The check expression */
+  Parse *pParse,      /* Parsing context */
+  Expr *pCheckExpr,   /* The check expression */
+  const char *zStart, /* Opening "(" */
+  const char *zEnd    /* Closing ")" */
 ){
 #ifndef SQLITE_OMIT_CHECK
  Table *pTab = pParse->pNewTable;
@@ -1630,6 +1631,13 @@ void sqlite3AddCheckConstraint(
    pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr);
    if( pParse->constraintName.n ){
      sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1);
+    }else{
+      Token t;
+      for(zStart++; sqlite3Isspace(zStart[0]); zStart++){}
+      while( sqlite3Isspace(zEnd[-1]) ){ zEnd--; }
+      t.z = zStart;
+      t.n = (int)(zEnd - t.z);
+      sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1);    
    }
  }else
 #endif
@@ -1648,7 +1656,7 @@ void sqlite3AddCollateType(Parse *pParse, Token *pToken){
  char *zColl;              /* Dequoted name of collation sequence */
  sqlite3 *db;

-  if( (p = pParse->pNewTable)==0 ) return;
+  if( (p = pParse->pNewTable)==0 || IN_RENAME_OBJECT ) return;
  i = p->nCol-1;
  db = pParse->db;
  zColl = sqlite3NameFromToken(db, pToken);
@@ -1883,12 +1891,15 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){
  int nByte;
  if( pIdx->nColumn>=N ) return SQLITE_OK;
  assert( pIdx->isResized==0 );
-  nByte = (sizeof(char*) + sizeof(i16) + 1)*N;
+  nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N;
  zExtra = sqlite3DbMallocZero(db, nByte);
  if( zExtra==0 ) return SQLITE_NOMEM_BKPT;
  memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn);
  pIdx->azColl = (const char**)zExtra;
  zExtra += sizeof(char*)*N;
+  memcpy(zExtra, pIdx->aiRowLogEst, sizeof(LogEst)*(pIdx->nKeyCol+1));
+  pIdx->aiRowLogEst = (LogEst*)zExtra;
+  zExtra += sizeof(LogEst)*N;
  memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn);
  pIdx->aiColumn = (i16*)zExtra;
  zExtra += sizeof(i16)*N;
@@ -4449,7 +4460,7 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){
  assert(pList || pParse->db->mallocFailed );
  if( pList ){
    for(i=0, pItem=pList->a; i<pList->nSrc; i++, pItem++){
-      if( pItem->iCursor>=0 ) break;
+      if( pItem->iCursor>=0 ) continue;
      pItem->iCursor = pParse->nTab++;
      if( pItem->pSelect ){
        sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc);
@@ -4466,15 +4477,15 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){
  struct SrcList_item *pItem;
  if( pList==0 ) return;
  for(pItem=pList->a, i=0; i<pList->nSrc; i++, pItem++){
-    sqlite3DbFree(db, pItem->zDatabase);
+    if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase);
    sqlite3DbFree(db, pItem->zName);
-    sqlite3DbFree(db, pItem->zAlias);
+    if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias);
    if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy);
    if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg);
    sqlite3DeleteTable(db, pItem->pTab);
-    sqlite3SelectDelete(db, pItem->pSelect);
-    sqlite3ExprDelete(db, pItem->pOn);
-    sqlite3IdListDelete(db, pItem->pUsing);
+    if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect);
+    if( pItem->pOn ) sqlite3ExprDelete(db, pItem->pOn);
+    if( pItem->pUsing ) sqlite3IdListDelete(db, pItem->pUsing);
  }
  sqlite3DbFreeNN(db, pList);
 }
@@ -4646,7 +4657,16 @@ void sqlite3BeginTransaction(Parse *pParse, int type){
  if( !v ) return;
  if( type!=TK_DEFERRED ){
    for(i=0; i<db->nDb; i++){
-      sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1);
+      int eTxnType;
+      Btree *pBt = db->aDb[i].pBt;
+      if( pBt && sqlite3BtreeIsReadonly(pBt) ){
+        eTxnType = 0;  /* Read txn */
+      }else if( type==TK_EXCLUSIVE ){
+        eTxnType = 2;  /* Exclusive txn */
+      }else{
+        eTxnType = 1;  /* Write txn */
+      }
+      sqlite3VdbeAddOp2(v, OP_Transaction, i, eTxnType);
      sqlite3VdbeUsesBtree(v, i);
    }
  }
@@ -4735,13 +4755,11 @@ int sqlite3OpenTempDatabase(Parse *pParse){
 ** will occur at the end of the top-level VDBE and will be generated
 ** later, by sqlite3FinishCoding().
 */
-void sqlite3CodeVerifySchema(Parse *pParse, int iDb){
-  Parse *pToplevel = sqlite3ParseToplevel(pParse);
-
-  assert( iDb>=0 && iDb<pParse->db->nDb );
-  assert( pParse->db->aDb[iDb].pBt!=0 || iDb==1 );
+static void sqlite3CodeVerifySchemaAtToplevel(Parse *pToplevel, int iDb){
+  assert( iDb>=0 && iDb<pToplevel->db->nDb );
+  assert( pToplevel->db->aDb[iDb].pBt!=0 || iDb==1 );
  assert( iDb<SQLITE_MAX_ATTACHED+2 );
-  assert( sqlite3SchemaMutexHeld(pParse->db, iDb, 0) );
+  assert( sqlite3SchemaMutexHeld(pToplevel->db, iDb, 0) );
  if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){
    DbMaskSet(pToplevel->cookieMask, iDb);
    if( !OMIT_TEMPDB && iDb==1 ){
@@ -4749,6 +4767,10 @@ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){
    }
  }
 }
+void sqlite3CodeVerifySchema(Parse *pParse, int iDb){
+  sqlite3CodeVerifySchemaAtToplevel(sqlite3ParseToplevel(pParse), iDb);
+}
+

 /*
 ** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each 
@@ -4780,7 +4802,7 @@ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){
 */
 void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){
  Parse *pToplevel = sqlite3ParseToplevel(pParse);
-  sqlite3CodeVerifySchema(pParse, iDb);
+  sqlite3CodeVerifySchemaAtToplevel(pToplevel, iDb);
  DbMaskSet(pToplevel->writeMask, iDb);
  pToplevel->isMultiWrite |= setStatement;
 }
@@ -4831,7 +4853,9 @@ void sqlite3HaltConstraint(
  i8 p4type,        /* P4_STATIC or P4_TRANSIENT */
  u8 p5Errmsg       /* P5_ErrMsg type */
 ){
-  Vdbe *v = sqlite3GetVdbe(pParse);
+  Vdbe *v;
+  assert( pParse->pVdbe!=0 );
+  v = sqlite3GetVdbe(pParse);
  assert( (errCode&0xff)==SQLITE_CONSTRAINT || pParse->nested );
  if( onError==OE_Abort ){
    sqlite3MayAbort(pParse);
@@ -425,7 +425,7 @@ void sqlite3DeleteFrom(
  }else
 #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */
  {
-    u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE;
+    u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK;
    if( sNC.ncFlags & NC_VarSelect ) bComplex = 1;
    wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW);
    if( HasRowid(pTab) ){
@@ -461,6 +461,9 @@ void sqlite3DeleteFrom(
    assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );
    assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF );
    if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse);
+    if( sqlite3WhereUsesDeferredSeek(pWInfo) ){
+      sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur);
+    }
  
    /* Keep track of the number of rows to be deleted */
    if( memCnt ){
@@ -495,6 +498,7 @@ void sqlite3DeleteFrom(
      if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0;
      if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0;
      if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen);
+      addrBypass = sqlite3VdbeMakeLabel(pParse);
    }else{
      if( pPk ){
        /* Add the PK key for this row to the temporary table */
@@ -508,13 +512,6 @@ void sqlite3DeleteFrom(
        nKey = 1;  /* OP_DeferredSeek always uses a single rowid */
        sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey);
      }
-    }
-  
-    /* If this DELETE cannot use the ONEPASS strategy, this is the 
-    ** end of the WHERE loop */
-    if( eOnePass!=ONEPASS_OFF ){
-      addrBypass = sqlite3VdbeMakeLabel(pParse);
-    }else{
      sqlite3WhereEnd(pWInfo);
    }
  
@@ -945,10 +942,6 @@ int sqlite3GenerateIndexKey(
  }
  if( regOut ){
    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut);
-    if( pIdx->pTable->pSelect ){
-      const char *zAff = sqlite3IndexAffinityStr(pParse->db, pIdx);
-      sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT);
-    }
  }
  sqlite3ReleaseTempRange(pParse, regBase, nCol);
  return regBase;
@@ -44,8 +44,10 @@ char sqlite3TableColumnAffinity(Table *pTab, int iCol){
 */
 char sqlite3ExprAffinity(const Expr *pExpr){
  int op;
-  while( ExprHasProperty(pExpr, EP_Skip) ){
-    assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
+  while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){
+    assert( pExpr->op==TK_COLLATE
+         || pExpr->op==TK_IF_NULL_ROW
+         || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) );
    pExpr = pExpr->pLeft;
    assert( pExpr!=0 );
  }
@@ -115,7 +117,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){
 */
 Expr *sqlite3ExprSkipCollate(Expr *pExpr){
  while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){
-    assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
+    assert( pExpr->op==TK_COLLATE );
    pExpr = pExpr->pLeft;
  }   
  return pExpr;
@@ -134,7 +136,7 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){
      assert( pExpr->op==TK_FUNCTION );
      pExpr = pExpr->x.pList->a[0].pExpr;
    }else{
-      assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW );
+      assert( pExpr->op==TK_COLLATE );
      pExpr = pExpr->pLeft;
    }
  }   
@@ -768,6 +770,7 @@ int sqlite3SelectExprHeight(Select *p){
 ** Expr.flags. 
 */
 void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){
+  if( pParse->nErr ) return;
  if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){
    p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList);
  }
@@ -3620,6 +3623,7 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){
 */
 static void exprToRegister(Expr *pExpr, int iReg){
  Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr);
+  if( NEVER(p==0) ) return;
  p->op2 = p->op;
  p->op = TK_REGISTER;
  p->iTable = iReg;
@@ -4607,6 +4611,7 @@ int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){
  int r2;
  pExpr = sqlite3ExprSkipCollateAndLikely(pExpr);
  if( ConstFactorOk(pParse)
+   && ALWAYS(pExpr!=0)
   && pExpr->op!=TK_REGISTER
   && sqlite3ExprIsConstantNotJoin(pExpr)
  ){
@@ -2006,6 +2006,8 @@ void sqlite3RegisterBuiltinFunctions(void){
    FUNCTION(zeroblob,           1, 0, 0, zeroblobFunc     ),
    FUNCTION(substr,             2, 0, 0, substrFunc       ),
    FUNCTION(substr,             3, 0, 0, substrFunc       ),
+    FUNCTION(substring,          2, 0, 0, substrFunc       ),
+    FUNCTION(substring,          3, 0, 0, substrFunc       ),
    WAGGREGATE(sum,   1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0),
    WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0),
    WAGGREGATE(avg,   1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0),
@@ -32,7 +32,8 @@ void sqlite3OpenTable(
 ){
  Vdbe *v;
  assert( !IsVirtual(pTab) );
-  v = sqlite3GetVdbe(pParse);
+  assert( pParse->pVdbe!=0 );
+  v = pParse->pVdbe;
  assert( opcode==OP_OpenWrite || opcode==OP_OpenRead );
  sqlite3TableLock(pParse, iDb, pTab->tnum, 
                   (opcode==OP_OpenWrite)?1:0, pTab->zName);
@@ -1531,7 +1532,7 @@ void sqlite3GenerateConstraintChecks(

  isUpdate = regOldData!=0;
  db = pParse->db;
-  v = sqlite3GetVdbe(pParse);
+  v = pParse->pVdbe;
  assert( v!=0 );
  assert( pTab->pSelect==0 );  /* This table is not a VIEW */
  nCol = pTab->nCol;
@@ -1685,7 +1686,7 @@ void sqlite3GenerateConstraintChecks(
        sqlite3VdbeGoto(v, ignoreDest);
      }else{
        char *zName = pCheck->a[i].zEName;
-        if( zName==0 ) zName = pTab->zName;
+        assert( zName!=0 || pParse->db->mallocFailed );
        if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */
        sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK,
                              onError, zName, P4_TRANSIENT,
@@ -2304,7 +2305,7 @@ void sqlite3CompleteInsertion(
       || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION)
  );

-  v = sqlite3GetVdbe(pParse);
+  v = pParse->pVdbe;
  assert( v!=0 );
  assert( pTab->pSelect==0 );  /* This table is not a VIEW */
  for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){
@@ -2405,7 +2406,7 @@ int sqlite3OpenTableAndIndices(
    return 0;
  }
  iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
-  v = sqlite3GetVdbe(pParse);
+  v = pParse->pVdbe;
  assert( v!=0 );
  if( iBase<0 ) iBase = pParse->nTab;
  iDataCur = iBase++;
@@ -478,6 +478,8 @@ static const sqlite3_api_routines sqlite3Apis = {
  sqlite3_create_filename,
  sqlite3_free_filename,
  sqlite3_database_file_object,
+  /* Version 3.34.0 and later */
+  sqlite3_txn_state,
 };

 /* True if x is the directory separator character
@@ -897,7 +897,7 @@ int sqlite3_db_cacheflush(sqlite3 *db){
  sqlite3BtreeEnterAll(db);
  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
    Btree *pBt = db->aDb[i].pBt;
-    if( pBt && sqlite3BtreeIsInTrans(pBt) ){
+    if( pBt && sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
      Pager *pPager = sqlite3BtreePager(pBt);
      rc = sqlite3PagerFlush(pPager);
      if( rc==SQLITE_BUSY ){
@@ -1241,6 +1241,36 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){
  return SQLITE_OK;
 }

+/*
+** Return the transaction state for a single databse, or the maximum
+** transaction state over all attached databases if zSchema is null.
+*/
+int sqlite3_txn_state(sqlite3 *db, const char *zSchema){
+  int iDb, nDb;
+  int iTxn = -1;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    (void)SQLITE_MISUSE_BKPT;
+    return -1;
+  }
+#endif
+  sqlite3_mutex_enter(db->mutex);
+  if( zSchema ){
+    nDb = iDb = sqlite3FindDbName(db, zSchema);
+    if( iDb<0 ) nDb--;
+  }else{
+    iDb = 0;
+    nDb = db->nDb-1;
+  }
+  for(; iDb<=nDb; iDb++){
+    Btree *pBt = db->aDb[iDb].pBt;
+    int x = pBt!=0 ? sqlite3BtreeTxnState(pBt) : SQLITE_TXN_NONE;
+    if( x>iTxn ) iTxn = x;
+  }
+  sqlite3_mutex_leave(db->mutex);
+  return iTxn;
+}
+
 /*
 ** Two variations on the public interface for closing a database
 ** connection. The sqlite3_close() version returns SQLITE_BUSY and
@@ -1401,7 +1431,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
  for(i=0; i<db->nDb; i++){
    Btree *p = db->aDb[i].pBt;
    if( p ){
-      if( sqlite3BtreeIsInTrans(p) ){
+      if( sqlite3BtreeTxnState(p)==SQLITE_TXN_WRITE ){
        inTrans = 1;
      }
      sqlite3BtreeRollback(p, tripCode, !schemaChange);
@@ -3871,7 +3901,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){
      }
      rc = SQLITE_OK;
    }else{
+      int nSave = db->busyHandler.nBusy;
      rc = sqlite3OsFileControl(fd, op, pArg);
+      db->busyHandler.nBusy = nSave;
    }
    sqlite3BtreeLeave(pBtree);
  }
@@ -4254,6 +4286,25 @@ int sqlite3_test_control(int op, ...){
      sqlite3ResultIntReal(pCtx);
      break;
    }
+
+    /*  sqlite3_test_control(SQLITE_TESTCTRL_SEEK_COUNT,
+    **    sqlite3 *db,    // Database connection
+    **    u64 *pnSeek     // Write seek count here
+    **  );
+    **
+    ** This test-control queries the seek-counter on the "main" database
+    ** file.  The seek-counter is written into *pnSeek and is then reset.
+    ** The seek-count is only available if compiled with SQLITE_DEBUG.
+    */
+    case SQLITE_TESTCTRL_SEEK_COUNT: {
+      sqlite3 *db = va_arg(ap, sqlite3*);
+      u64 *pn = va_arg(ap, sqlite3_uint64*);
+      *pn = sqlite3BtreeSeekCount(db->aDb->pBt);
+      (void)db;  /* Silence harmless unused variable warning */
+      break;
+    }
+
+
  }
  va_end(ap);
 #endif /* SQLITE_UNTESTABLE */
@@ -4489,7 +4540,7 @@ int sqlite3_snapshot_get(
    int iDb = sqlite3FindDbName(db, zDb);
    if( iDb==0 || iDb>1 ){
      Btree *pBt = db->aDb[iDb].pBt;
-      if( 0==sqlite3BtreeIsInTrans(pBt) ){
+      if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){
        rc = sqlite3BtreeBeginTrans(pBt, 0, 0);
        if( rc==SQLITE_OK ){
          rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot);
@@ -4525,10 +4576,10 @@ int sqlite3_snapshot_open(
    iDb = sqlite3FindDbName(db, zDb);
    if( iDb==0 || iDb>1 ){
      Btree *pBt = db->aDb[iDb].pBt;
-      if( sqlite3BtreeIsInTrans(pBt)==0 ){
+      if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ){
        Pager *pPager = sqlite3BtreePager(pBt);
        int bUnlock = 0;
-        if( sqlite3BtreeIsInReadTrans(pBt) ){
+        if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_NONE ){
          if( db->nVdbeActive==0 ){
            rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot);
            if( rc==SQLITE_OK ){
@@ -4577,7 +4628,7 @@ int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){
  iDb = sqlite3FindDbName(db, zDb);
  if( iDb==0 || iDb>1 ){
    Btree *pBt = db->aDb[iDb].pBt;
-    if( 0==sqlite3BtreeIsInReadTrans(pBt) ){
+    if( SQLITE_TXN_NONE==sqlite3BtreeTxnState(pBt) ){
      rc = sqlite3BtreeBeginTrans(pBt, 0, 0);
      if( rc==SQLITE_OK ){
        rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt));
@@ -485,12 +485,17 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){
  if( nOld==nNew ){
    pNew = pOld;
  }else if( sqlite3GlobalConfig.bMemstat ){
+    sqlite3_int64 nUsed;
    sqlite3_mutex_enter(mem0.mutex);
    sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes);
    nDiff = nNew - nOld;
-    if( nDiff>0 && sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= 
+    if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= 
          mem0.alarmThreshold-nDiff ){
      sqlite3MallocAlarm(nDiff);
+      if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){
+        sqlite3_mutex_leave(mem0.mutex);
+        return 0;
+      }
    }
    pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
@@ -797,12 +802,15 @@ void sqlite3OomClear(sqlite3 *db){
 }

 /*
-** Take actions at the end of an API call to indicate an OOM error
+** Take actions at the end of an API call to deal with error codes.
 */
-static SQLITE_NOINLINE int apiOomError(sqlite3 *db){
-  sqlite3OomClear(db);
-  sqlite3Error(db, SQLITE_NOMEM);
-  return SQLITE_NOMEM_BKPT;
+static SQLITE_NOINLINE int apiHandleError(sqlite3 *db, int rc){
+  if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){
+    sqlite3OomClear(db);
+    sqlite3Error(db, SQLITE_NOMEM);
+    return SQLITE_NOMEM_BKPT;
+  }
+  return rc & db->errMask;
 }

 /*
@@ -824,8 +832,8 @@ int sqlite3ApiExit(sqlite3* db, int rc){
  */
  assert( db!=0 );
  assert( sqlite3_mutex_held(db->mutex) );
-  if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){
-    return apiOomError(db);
+  if( db->mallocFailed || rc ){
+    return apiHandleError(db, rc);
  }
  return rc & db->errMask;
 }
@@ -126,11 +126,14 @@ static const sqlite3_io_methods memdb_io_methods = {
 ** Close an memdb-file.
 **
 ** The pData pointer is owned by the application, so there is nothing
-** to free.
+** to free.  Unless the SQLITE_DESERIALIZE_FREEONCLOSE flag is set,
+** in which case we own the pData pointer and need to free it.
 */
 static int memdbClose(sqlite3_file *pFile){
  MemFile *p = (MemFile *)pFile;
-  if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ) sqlite3_free(p->aData);
+  if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){
+    sqlite3_free(p->aData);
+  }
  return SQLITE_OK;
 }

@@ -573,8 +576,12 @@ int sqlite3_deserialize(
    goto end_deserialize;
  }    
  zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+  }
  if( rc ) goto end_deserialize;
  db->init.iDb = (u8)iDb;
  db->init.reopenMemdb = 1;
@@ -589,6 +596,7 @@ int sqlite3_deserialize(
    rc = SQLITE_ERROR;
  }else{
    p->aData = pData;
+    pData = 0;
    p->sz = szDb;
    p->szAlloc = szBuf;
    p->szMax = szBuf;
@@ -601,6 +609,9 @@ int sqlite3_deserialize(

 end_deserialize:
  sqlite3_finalize(pStmt);
+  if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){
+    sqlite3_free(pData);
+  }
  sqlite3_mutex_leave(db->mutex);
  return rc;
 }
@@ -122,7 +122,8 @@
 # if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \
                            (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000))
 #    if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \
-         && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0))
+        && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0))\
+        && (!defined(TARGET_OS_MACCATALYST) || (TARGET_OS_MACCATALYST==0))
 #      undef HAVE_GETHOSTUUID
 #      define HAVE_GETHOSTUUID 1
 #    else
@@ -1544,6 +1545,9 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){
  return rc;
 }

+/* Forward declaration*/
+static int unixSleep(sqlite3_vfs*,int);
+
 /*
 ** Set a posix-advisory-lock.
 **
@@ -1573,7 +1577,7 @@ static int osSetPosixAdvisoryLock(
    ** generic posix, however, there is no such API.  So we simply try the
    ** lock once every millisecond until either the timeout expires, or until
    ** the lock is obtained. */
-    usleep(1000);
+    unixSleep(0,1000);
    rc = osFcntl(h,F_SETLK,pLock);
    tm--;
  }
@@ -2144,6 +2148,7 @@ static int unixClose(sqlite3_file *id){
  }
  sqlite3_mutex_leave(pInode->pLockMutex);
  releaseInodeInfo(pFile);
+  assert( pFile->pShm==0 );
  rc = closeUnixFile(id);
  unixLeaveMutex();
  return rc;
@@ -3370,7 +3375,24 @@ static int unixRead(
  if( got==amt ){
    return SQLITE_OK;
  }else if( got<0 ){
-    /* lastErrno set by seekAndRead */
+    /* pFile->lastErrno has been set by seekAndRead().
+    ** Usually we return SQLITE_IOERR_READ here, though for some
+    ** kinds of errors we return SQLITE_IOERR_CORRUPTFS.  The
+    ** SQLITE_IOERR_CORRUPTFS will be converted into SQLITE_CORRUPT
+    ** prior to returning to the application by the sqlite3ApiExit()
+    ** routine.
+    */
+    switch( pFile->lastErrno ){
+      case ERANGE:
+      case EIO:
+#ifdef ENXIO
+      case ENXIO:
+#endif
+#ifdef EDEVERR
+      case EDEVERR:
+#endif
+        return SQLITE_IOERR_CORRUPTFS;
+    }
    return SQLITE_IOERR_READ;
  }else{
    storeLastErrno(pFile, 0);   /* not a system error */
@@ -4254,6 +4276,7 @@ struct unixShmNode {
  char **apRegion;           /* Array of mapped shared-memory regions */
  int nRef;                  /* Number of unixShm objects pointing to this */
  unixShm *pFirst;           /* All unixShm objects pointing to this */
+  int aLock[SQLITE_SHM_NLOCK];  /* # shared locks on slot, -1==excl lock */
 #ifdef SQLITE_DEBUG
  u8 exclMask;               /* Mask of exclusive locks held */
  u8 sharedMask;             /* Mask of shared locks held */
@@ -4794,6 +4817,38 @@ shmpage_out:
  return rc;
 }

+/*
+** Check that the pShmNode->aLock[] array comports with the locking bitmasks
+** held by each client. Return true if it does, or false otherwise. This
+** is to be used in an assert(). e.g.
+**
+**     assert( assertLockingArrayOk(pShmNode) );
+*/
+#ifdef SQLITE_DEBUG
+static int assertLockingArrayOk(unixShmNode *pShmNode){
+  unixShm *pX;
+  int aLock[SQLITE_SHM_NLOCK];
+  assert( sqlite3_mutex_held(pShmNode->pShmMutex) );
+
+  memset(aLock, 0, sizeof(aLock));
+  for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
+    int i;
+    for(i=0; i<SQLITE_SHM_NLOCK; i++){
+      if( pX->exclMask & (1<<i) ){
+        assert( aLock[i]==0 );
+        aLock[i] = -1;
+      }else if( pX->sharedMask & (1<<i) ){
+        assert( aLock[i]>=0 );
+        aLock[i]++;
+      }
+    }
+  }
+
+  assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) );
+  return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0);
+}
+#endif
+
 /*
 ** Change the lock state for a shared-memory segment.
 **
@@ -4810,10 +4865,10 @@ static int unixShmLock(
 ){
  unixFile *pDbFd = (unixFile*)fd;      /* Connection holding shared memory */
  unixShm *p = pDbFd->pShm;             /* The shared memory being locked */
-  unixShm *pX;                          /* For looping over all siblings */
  unixShmNode *pShmNode = p->pShmNode;  /* The underlying file iNode */
  int rc = SQLITE_OK;                   /* Result code */
  u16 mask;                             /* Mask of locks to take or release */
+  int *aLock = pShmNode->aLock;

  assert( pShmNode==pDbFd->pInode->pShmNode );
  assert( pShmNode->pInode==pDbFd->pInode );
@@ -4852,78 +4907,76 @@ static int unixShmLock(
  mask = (1<<(ofst+n)) - (1<<ofst);
  assert( n>1 || mask==(1<<ofst) );
  sqlite3_mutex_enter(pShmNode->pShmMutex);
+  assert( assertLockingArrayOk(pShmNode) );
  if( flags & SQLITE_SHM_UNLOCK ){
-    u16 allMask = 0; /* Mask of locks held by siblings */
+    if( (p->exclMask|p->sharedMask) & mask ){
+      int ii;
+      int bUnlock = 1;

-    /* See if any siblings hold this same lock */
-    for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
-      if( pX==p ) continue;
-      assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 );
-      allMask |= pX->sharedMask;
+      for(ii=ofst; ii<ofst+n; ii++){
+        if( aLock[ii]>((p->sharedMask & (1<<ii)) ? 1 : 0) ){
+          bUnlock = 0;
+        }
+      }
+
+      if( bUnlock ){
+        rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
+        if( rc==SQLITE_OK ){
+          memset(&aLock[ofst], 0, sizeof(int)*n);
+        }
+      }else if( ALWAYS(p->sharedMask & (1<<ofst)) ){
+        assert( n==1 && aLock[ofst]>1 );
+        aLock[ofst]--;
+      }
+
+      /* Undo the local locks */
+      if( rc==SQLITE_OK ){
+        p->exclMask &= ~mask;
+        p->sharedMask &= ~mask;
+      } 
    }
-
-    /* Unlock the system-level locks */
-    if( (mask & allMask)==0 ){
-      rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n);
-    }else{
-      rc = SQLITE_OK;
-    }
-
-    /* Undo the local locks */
-    if( rc==SQLITE_OK ){
-      p->exclMask &= ~mask;
-      p->sharedMask &= ~mask;
-    } 
  }else if( flags & SQLITE_SHM_SHARED ){
-    u16 allShared = 0;  /* Union of locks held by connections other than "p" */
-
-    /* Find out which shared locks are already held by sibling connections.
-    ** If any sibling already holds an exclusive lock, go ahead and return
-    ** SQLITE_BUSY.
-    */
-    for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
-      if( (pX->exclMask & mask)!=0 ){
+    assert( n==1 );
+    assert( (p->exclMask & (1<<ofst))==0 );
+    if( (p->sharedMask & mask)==0 ){
+      if( aLock[ofst]<0 ){
        rc = SQLITE_BUSY;
-        break;
-      }
-      allShared |= pX->sharedMask;
-    }
-
-    /* Get shared locks at the system level, if necessary */
-    if( rc==SQLITE_OK ){
-      if( (allShared & mask)==0 ){
+      }else if( aLock[ofst]==0 ){
        rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n);
-      }else{
-        rc = SQLITE_OK;
      }
-    }

-    /* Get the local shared locks */
-    if( rc==SQLITE_OK ){
-      p->sharedMask |= mask;
+      /* Get the local shared locks */
+      if( rc==SQLITE_OK ){
+        p->sharedMask |= mask;
+        aLock[ofst]++;
+      }
    }
  }else{
    /* Make sure no sibling connections hold locks that will block this
-    ** lock.  If any do, return SQLITE_BUSY right away.
-    */
-    for(pX=pShmNode->pFirst; pX; pX=pX->pNext){
-      if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){
+    ** lock.  If any do, return SQLITE_BUSY right away.  */
+    int ii;
+    for(ii=ofst; ii<ofst+n; ii++){
+      assert( (p->sharedMask & mask)==0 );
+      if( ALWAYS((p->exclMask & (1<<ii))==0) && aLock[ii] ){
        rc = SQLITE_BUSY;
        break;
      }
    }
-  
-    /* Get the exclusive locks at the system level.  Then if successful
-    ** also mark the local connection as being locked.
-    */
+
+    /* Get the exclusive locks at the system level. Then if successful
+    ** also update the in-memory values. */
    if( rc==SQLITE_OK ){
      rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n);
      if( rc==SQLITE_OK ){
        assert( (p->sharedMask & mask)==0 );
        p->exclMask |= mask;
+        for(ii=ofst; ii<ofst+n; ii++){
+          aLock[ii] = -1;
+        }
      }
    }
  }
+  assert( assertLockingArrayOk(pShmNode) );
  sqlite3_mutex_leave(pShmNode->pShmMutex);
  OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n",
           p->id, osGetpid(0), p->sharedMask, p->exclMask));
@@ -6300,7 +6353,26 @@ static int unixAccess(
 }

 /*
+** If the last component of the pathname in z[0]..z[j-1] is something
+** other than ".." then back it out and return true.  If the last
+** component is empty or if it is ".." then return false.
+*/
+static int unixBackupDir(const char *z, int *pJ){
+  int j = *pJ;
+  int i;
+  if( j<=0 ) return 0;
+  for(i=j-1; ALWAYS(i>0) && z[i-1]!='/'; i--){}
+  if( z[i]=='.' && i==j-2 && z[i+1]=='.' ) return 0;
+  *pJ = i-1;
+  return 1;
+}
+
+/*
+** Convert a relative pathname into a full pathname.  Also
+** simplify the pathname as follows:
 **
+**    Remove all instances of /./
+**    Remove all isntances of /X/../ for any X
 */
 static int mkFullPathname(
  const char *zPath,              /* Input path */
@@ -6309,6 +6381,7 @@ static int mkFullPathname(
 ){
  int nPath = sqlite3Strlen30(zPath);
  int iOff = 0;
+  int i, j;
  if( zPath[0]!='/' ){
    if( osGetcwd(zOut, nOut-2)==0 ){
      return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath);
@@ -6323,6 +6396,41 @@ static int mkFullPathname(
    return SQLITE_CANTOPEN_BKPT;
  }
  sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath);
+
+  /* Remove duplicate '/' characters.  Except, two // at the beginning
+  ** of a pathname is allowed since this is important on windows. */
+  for(i=j=1; zOut[i]; i++){
+    zOut[j++] = zOut[i];
+    while( zOut[i]=='/' && zOut[i+1]=='/' ) i++;
+  }
+  zOut[j] = 0;
+
+  assert( zOut[0]=='/' );
+  for(i=j=0; zOut[i]; i++){
+    if( zOut[i]=='/' ){
+      /* Skip over internal "/." directory components */
+      if( zOut[i+1]=='.' && zOut[i+2]=='/' ){
+        i += 1;
+        continue;
+      }
+
+      /* If this is a "/.." directory component then back out the
+      ** previous term of the directory if it is something other than "..".
+      */
+      if( zOut[i+1]=='.'
+       && zOut[i+2]=='.'
+       && zOut[i+3]=='/'
+       && unixBackupDir(zOut, &j)
+      ){
+        i += 2;
+        continue;
+      }
+    }
+    if( ALWAYS(j>=0) ) zOut[j] = zOut[i];
+    j++;
+  }
+  if( NEVER(j==0) ) zOut[j++] = '/';
+  zOut[j] = 0;
  return SQLITE_OK;
 }

@@ -6543,7 +6651,8 @@ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){
  UNUSED_PARAMETER(NotUsed);
  return microseconds;
 #elif defined(HAVE_USLEEP) && HAVE_USLEEP
-  usleep(microseconds);
+  if( microseconds>=1000000 ) sleep(microseconds/1000000);
+  if( microseconds%1000000 ) usleep(microseconds%1000000);
  UNUSED_PARAMETER(NotUsed);
  return microseconds;
 #else
@@ -7116,7 +7225,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){
      
      if( nTries==1 ){
        conchModTime = buf.st_mtimespec;
-        usleep(500000); /* wait 0.5 sec and try the lock again*/
+        unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/
        continue;  
      }

@@ -7142,7 +7251,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){
          /* don't break the lock on short read or a version mismatch */
          return SQLITE_BUSY;
        }
-        usleep(10000000); /* wait 10 sec and try the lock again */
+        unixSleep(0,10000000); /* wait 10 sec and try the lock again */
        continue; 
      }
      
@@ -5126,7 +5126,11 @@ static int winOpen(
    dwCreationDisposition = OPEN_EXISTING;
  }

-  dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
+  if( 0==sqlite3_uri_boolean(zName, "exclusive", 0) ){
+    dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
+  }else{
+    dwShareMode = 0;
+  }

  if( isDelete ){
 #if SQLITE_OS_WINCE
@@ -2587,6 +2587,7 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){
  i64 nSuperJournal;        /* Size of super-journal file */
  char *zJournal;           /* Pointer to one journal within MJ file */
  char *zSuperPtr;          /* Space to hold super-journal filename */
+  char *zFree = 0;          /* Free this buffer */
  int nSuperPtr;            /* Amount of space allocated to zSuperPtr[] */

  /* Allocate space for both the pJournal and pSuper file descriptors.
@@ -2611,11 +2612,13 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){
  rc = sqlite3OsFileSize(pSuper, &nSuperJournal);
  if( rc!=SQLITE_OK ) goto delsuper_out;
  nSuperPtr = pVfs->mxPathname+1;
-  zSuperJournal = sqlite3Malloc(nSuperJournal + nSuperPtr + 2);
-  if( !zSuperJournal ){
+  zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2);
+  if( !zFree ){
    rc = SQLITE_NOMEM_BKPT;
    goto delsuper_out;
  }
+  zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0;
+  zSuperJournal = &zFree[4];
  zSuperPtr = &zSuperJournal[nSuperJournal+2];
  rc = sqlite3OsRead(pSuper, zSuperJournal, (int)nSuperJournal, 0);
  if( rc!=SQLITE_OK ) goto delsuper_out;
@@ -2663,7 +2666,7 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){
  rc = sqlite3OsDelete(pVfs, zSuper, 0);

 delsuper_out:
-  sqlite3_free(zSuperJournal);
+  sqlite3_free(zFree);
  if( pSuper ){
    sqlite3OsClose(pSuper);
    assert( !isOpen(pJournal) );
@@ -3001,7 +3004,11 @@ end_playback:
  pPager->changeCountDone = pPager->tempFile;

  if( rc==SQLITE_OK ){
-    zSuper = pPager->pTmpSpace;
+    /* Leave 4 bytes of space before the super-journal filename in memory.
+    ** This is because it may end up being passed to sqlite3OsOpen(), in
+    ** which case it requires 4 0x00 bytes in memory immediately before
+    ** the filename. */
+    zSuper = &pPager->pTmpSpace[4];
    rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1);
    testcase( rc!=SQLITE_OK );
  }
@@ -3018,6 +3025,8 @@ end_playback:
    /* If there was a super-journal and this routine will return success,
    ** see if it is possible to delete the super-journal.
    */
+    assert( zSuper==&pPager->pTmpSpace[4] );
+    memset(&zSuper[-4], 0, 4);
    rc = pager_delsuper(pPager, zSuper);
    testcase( rc!=SQLITE_OK );
  }
@@ -1,5 +1,6 @@
+%include {
 /*
-** 2001 September 15
+** 2001-09-15
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -9,11 +10,16 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** This file contains SQLite's grammar for SQL.  Process this file
-** using the lemon parser generator to generate C code that runs
-** the parser.  Lemon will also generate a header file containing
-** numeric codes for all of the tokens.
+** This file contains SQLite's SQL parser.
+**
+** The canonical source code to this file ("parse.y") is a Lemon grammar 
+** file that specifies the input grammar and actions to take while parsing.
+** That input file is processed by Lemon to generate a C-language 
+** implementation of a parser for the given grammer.  You might be reading
+** this comment as part of the translated C-code.  Edits should be made
+** to the original parse.y sources.
 */
+}

 // All token codes are small integers with #defines that begin with "TK_"
 %token_prefix TK_
@@ -366,7 +372,7 @@ ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I).
                                 {sqlite3AddPrimaryKey(pParse,0,R,I,Z);}
 ccons ::= UNIQUE onconf(R).      {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0,
                                   SQLITE_IDXTYPE_UNIQUE);}
-ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X);}
+ccons ::= CHECK LP(A) expr(X) RP(B).  {sqlite3AddCheckConstraint(pParse,X,A.z,B.z);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                 {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
 ccons ::= defer_subclause(D).    {sqlite3DeferForeignKey(pParse,D);}
@@ -420,8 +426,8 @@ tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R).
 tcons ::= UNIQUE LP sortlist(X) RP onconf(R).
                                 {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0,
                                       SQLITE_IDXTYPE_UNIQUE);}
-tcons ::= CHECK LP expr(E) RP onconf.
-                                 {sqlite3AddCheckConstraint(pParse,E);}
+tcons ::= CHECK LP(A) expr(E) RP(B) onconf.
+                                 {sqlite3AddCheckConstraint(pParse,E,A.z,B.z);}
 tcons ::= FOREIGN KEY LP eidlist(FA) RP
          REFERENCES nm(T) eidlist_opt(TA) refargs(R) defer_subclause_opt(D). {
    sqlite3CreateForeignKey(pParse, FA, &T, TA, R);
@@ -131,7 +131,9 @@ static int getTempStore(const char *z){
 static int invalidateTempStorage(Parse *pParse){
  sqlite3 *db = pParse->db;
  if( db->aDb[1].pBt!=0 ){
-    if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){
+    if( !db->autoCommit
+     || sqlite3BtreeTxnState(db->aDb[1].pBt)!=SQLITE_TXN_NONE
+    ){
      sqlite3ErrorMsg(pParse, "temporary storage cannot be changed "
        "from within a transaction");
      return SQLITE_ERROR;
@@ -1466,7 +1468,7 @@ void sqlite3Pragma(
        aiCols = 0;
        if( pParent ){
          x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols);
-          assert( x==0 );
+          assert( x==0 || db->mallocFailed );
        }
        addrOk = sqlite3VdbeMakeLabel(pParse);

@@ -1491,7 +1493,7 @@ void sqlite3Pragma(
          int jmp = sqlite3VdbeCurrentAddr(v)+2;
          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v);
          sqlite3VdbeGoto(v, addrOk);
-          assert( pFK->nCol==1 );
+          assert( pFK->nCol==1 || db->mallocFailed );
        }

        /* Generate code to report an FK violation to the caller. */
@@ -240,7 +240,7 @@ int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){
  ** on the b-tree database, open one now. If a transaction is opened, it 
  ** will be closed before this function returns.  */
  sqlite3BtreeEnter(pDb->pBt);
-  if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){
+  if( sqlite3BtreeTxnState(pDb->pBt)==SQLITE_TXN_NONE ){
    rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0);
    if( rc!=SQLITE_OK ){
      sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc));
@@ -483,7 +483,7 @@ static void schemaIsValid(Parse *pParse){
    /* If there is not already a read-only (or read-write) transaction opened
    ** on the b-tree database, open one now. If a transaction is opened, it 
    ** will be closed immediately after reading the meta-value. */
-    if( !sqlite3BtreeIsInReadTrans(pBt) ){
+    if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_NONE ){
      rc = sqlite3BtreeBeginTrans(pBt, 0, 0);
      if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
        sqlite3OomFault(db);
@@ -746,6 +746,7 @@ static int sqlite3LockAndPrepare(
  sqlite3BtreeLeaveAll(db);
  rc = sqlite3ApiExit(db, rc);
  assert( (rc&db->errMask)==rc );
+  db->busyHandler.nBusy = 0;
  sqlite3_mutex_leave(db->mutex);
  return rc;
 }
@@ -427,11 +427,10 @@ void sqlite3_str_vappendf(
            v = va_arg(ap,int);
          }
          if( v<0 ){
-            if( v==SMALLEST_INT64 ){
-              longvalue = ((u64)1)<<63;
-            }else{
-              longvalue = -v;
-            }
+            testcase( v==SMALLEST_INT64 );
+            testcase( v==(-1) );
+            longvalue = ~v;
+            longvalue++;
            prefix = '-';
          }else{
            longvalue = v;
@@ -1075,7 +1075,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
      assert( !ExprHasProperty(pExpr, EP_Reduced) );
      /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE",
      ** and "x IS NOT FALSE". */
-      if( pRight && pRight->op==TK_ID ){
+      if( pRight && (pRight->op==TK_ID || pRight->op==TK_TRUEFALSE) ){
        int rc = resolveExprStep(pWalker, pRight);
        if( rc==WRC_Abort ) return WRC_Abort;
        if( pRight->op==TK_TRUEFALSE ){
@@ -1284,6 +1284,7 @@ static int resolveCompoundOrderBy(
      Expr *pE, *pDup;
      if( pItem->done ) continue;
      pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr);
+      if( NEVER(pE==0) ) continue;
      if( sqlite3ExprIsInteger(pE, &iCol) ){
        if( iCol<=0 || iCol>pEList->nExpr ){
          resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr);
@@ -1463,6 +1464,7 @@ static int resolveOrderGroupBy(
  for(i=0, pItem=pOrderBy->a; i<pOrderBy->nExpr; i++, pItem++){
    Expr *pE = pItem->pExpr;
    Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE);
+    if( NEVER(pE2==0) ) continue;
    if( zType[0]!='G' ){
      iCol = resolveAsName(pParse, pSelect->pEList, pE2);
      if( iCol>0 ){
@@ -1972,6 +1972,7 @@ int sqlite3ColumnsFromExprList(
  char *zName;                /* Column name */
  int nName;                  /* Size of name in zName[] */
  Hash ht;                    /* Hash table of column names */
+  Table *pTab;

  sqlite3HashInit(&ht);
  if( pEList ){
@@ -1994,15 +1995,13 @@ int sqlite3ColumnsFromExprList(
      /* If the column contains an "AS <name>" phrase, use <name> as the name */
    }else{
      Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr);
-      while( pColExpr->op==TK_DOT ){
+      while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){
        pColExpr = pColExpr->pRight;
        assert( pColExpr!=0 );
      }
-      if( pColExpr->op==TK_COLUMN ){
+      if( pColExpr->op==TK_COLUMN && (pTab = pColExpr->y.pTab)!=0 ){
        /* For columns use the column name name */
        int iCol = pColExpr->iColumn;
-        Table *pTab = pColExpr->y.pTab;
-        assert( pTab!=0 );
        if( iCol<0 ) iCol = pTab->iPKey;
        zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
      }else if( pColExpr->op==TK_ID ){
@@ -2340,6 +2339,7 @@ static void generateWithRecursiveQuery(
  int nCol = p->pEList->nExpr;  /* Number of columns in the recursive table */
  Vdbe *v = pParse->pVdbe;      /* The prepared statement under construction */
  Select *pSetup = p->pPrior;   /* The setup query */
+  Select *pFirstRec;            /* Left-most recursive term */
  int addrTop;                  /* Top of the loop */
  int addrCont, addrBreak;      /* CONTINUE and BREAK addresses */
  int iCurrent = 0;             /* The Current table */
@@ -2415,7 +2415,25 @@ static void generateWithRecursiveQuery(
  /* Detach the ORDER BY clause from the compound SELECT */
  p->pOrderBy = 0;

+  /* Figure out how many elements of the compound SELECT are part of the
+  ** recursive query.  Make sure no recursive elements use aggregate
+  ** functions.  Mark the recursive elements as UNION ALL even if they
+  ** are really UNION because the distinctness will be enforced by the
+  ** iDistinct table.  pFirstRec is left pointing to the left-most
+  ** recursive term of the CTE.
+  */
+  pFirstRec = p;
+  for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){
+    if( pFirstRec->selFlags & SF_Aggregate ){
+      sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
+      goto end_of_recursive_query;
+    }
+    pFirstRec->op = TK_ALL;
+    if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break;
+  }
+
  /* Store the results of the setup-query in Queue. */
+  pSetup = pFirstRec->pPrior;
  pSetup->pNext = 0;
  ExplainQueryPlan((pParse, 1, "SETUP"));
  rc = sqlite3Select(pParse, pSetup, &destQueue);
@@ -2448,15 +2466,11 @@ static void generateWithRecursiveQuery(
  /* Execute the recursive SELECT taking the single row in Current as
  ** the value for the recursive-table. Store the results in the Queue.
  */
-  if( p->selFlags & SF_Aggregate ){
-    sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
-  }else{
-    p->pPrior = 0;
-    ExplainQueryPlan((pParse, 1, "RECURSIVE STEP"));
-    sqlite3Select(pParse, p, &destQueue);
-    assert( p->pPrior==0 );
-    p->pPrior = pSetup;
-  }
+  pFirstRec->pPrior = 0;
+  ExplainQueryPlan((pParse, 1, "RECURSIVE STEP"));
+  sqlite3Select(pParse, p, &destQueue);
+  assert( pFirstRec->pPrior==0 );
+  pFirstRec->pPrior = pSetup;

  /* Keep running the loop until the Queue is empty */
  sqlite3VdbeGoto(v, addrTop);
@@ -2525,6 +2539,16 @@ static int multiSelectValues(
  return rc;
 }

+/*
+** Return true if the SELECT statement which is known to be the recursive
+** part of a recursive CTE still has its anchor terms attached.  If the
+** anchor terms have already been removed, then return false.
+*/
+static int hasAnchor(Select *p){
+  while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; }
+  return p!=0;
+}
+
 /*
 ** This routine is called to process a compound query form from
 ** two or more separate queries using UNION, UNION ALL, EXCEPT, or
@@ -2610,7 +2634,7 @@ static int multiSelect(
  assert( p->pEList->nExpr==pPrior->pEList->nExpr );

 #ifndef SQLITE_OMIT_CTE
-  if( p->selFlags & SF_Recursive ){
+  if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){
    generateWithRecursiveQuery(pParse, p, &dest);
  }else
 #endif
@@ -2701,6 +2725,7 @@ static int multiSelect(
          findRightmost(p)->selFlags |= SF_UsesEphemeral;
          assert( p->pEList );
        }
+          
  
        /* Code the SELECT statements to our left
        */
@@ -3506,7 +3531,7 @@ static Expr *substExpr(
          ifNullRow.op = TK_IF_NULL_ROW;
          ifNullRow.pLeft = pCopy;
          ifNullRow.iTable = pSubst->iNewTable;
-          ifNullRow.flags = EP_Skip;
+          ifNullRow.flags = EP_IfNullRow;
          pCopy = &ifNullRow;
        }
        testcase( ExprHasProperty(pCopy, EP_Subquery) );
@@ -3515,8 +3540,7 @@ static Expr *substExpr(
          ExprSetProperty(pNew, EP_CanBeNull);
        }
        if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
-          pNew->iRightJoinTable = pExpr->iRightJoinTable;
-          ExprSetProperty(pNew, EP_FromJoin);
+          sqlite3SetJoinExpr(pNew, pExpr->iRightJoinTable);
        }
        sqlite3ExprDelete(db, pExpr);
        pExpr = pNew;
@@ -4795,8 +4819,10 @@ static int withExpand(
    ExprList *pEList;
    Select *pSel;
    Select *pLeft;                /* Left-most SELECT statement */
+    Select *pRecTerm;             /* Left-most recursive term */
    int bMayRecursive;            /* True if compound joined by UNION [ALL] */
    With *pSavedWith;             /* Initial value of pParse->pWith */
+    int iRecTab = -1;             /* Cursor for recursive table */

    /* If pCte->zCteErr is non-NULL at this point, then this is an illegal
    ** recursive reference to CTE pCte. Leave an error in pParse and return
@@ -4821,44 +4847,48 @@ static int withExpand(
    assert( pFrom->pSelect );

    /* Check if this is a recursive CTE. */
-    pSel = pFrom->pSelect;
+    pRecTerm = pSel = pFrom->pSelect;
    bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION );
-    if( bMayRecursive ){
+    while( bMayRecursive && pRecTerm->op==pSel->op ){
      int i;
-      SrcList *pSrc = pFrom->pSelect->pSrc;
+      SrcList *pSrc = pRecTerm->pSrc;
+      assert( pRecTerm->pPrior!=0 );
      for(i=0; i<pSrc->nSrc; i++){
        struct SrcList_item *pItem = &pSrc->a[i];
        if( pItem->zDatabase==0 
         && pItem->zName!=0 
         && 0==sqlite3StrICmp(pItem->zName, pCte->zName)
-          ){
+        ){
          pItem->pTab = pTab;
-          pItem->fg.isRecursive = 1;
          pTab->nTabRef++;
-          pSel->selFlags |= SF_Recursive;
+          pItem->fg.isRecursive = 1;
+          if( pRecTerm->selFlags & SF_Recursive ){
+            sqlite3ErrorMsg(pParse,
+               "multiple references to recursive table: %s", pCte->zName
+            );
+            return SQLITE_ERROR;
+          }
+          pRecTerm->selFlags |= SF_Recursive;
+          if( iRecTab<0 ) iRecTab = pParse->nTab++;
+          pItem->iCursor = iRecTab;
        }
      }
+      if( (pRecTerm->selFlags & SF_Recursive)==0 ) break;
+      pRecTerm = pRecTerm->pPrior;
    }

-    /* Only one recursive reference is permitted. */ 
-    if( pTab->nTabRef>2 ){
-      sqlite3ErrorMsg(
-          pParse, "multiple references to recursive table: %s", pCte->zName
-      );
-      return SQLITE_ERROR;
-    }
-    assert( pTab->nTabRef==1 || 
-            ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 ));
-
    pCte->zCteErr = "circular reference: %s";
    pSavedWith = pParse->pWith;
    pParse->pWith = pWith;
-    if( bMayRecursive ){
-      Select *pPrior = pSel->pPrior;
-      assert( pPrior->pWith==0 );
-      pPrior->pWith = pSel->pWith;
-      sqlite3WalkSelect(pWalker, pPrior);
-      pPrior->pWith = 0;
+    if( pSel->selFlags & SF_Recursive ){
+      assert( pRecTerm!=0 );
+      assert( (pRecTerm->selFlags & SF_Recursive)==0 );
+      assert( pRecTerm->pNext!=0 );
+      assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 );
+      assert( pRecTerm->pWith==0 );
+      pRecTerm->pWith = pSel->pWith;
+      sqlite3WalkSelect(pWalker, pRecTerm);
+      pRecTerm->pWith = 0;
    }else{
      sqlite3WalkSelect(pWalker, pSel);
    }
@@ -5585,7 +5615,9 @@ static void explainSimpleCount(
 static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){
  if( pExpr->op!=TK_AND ){
    Select *pS = pWalker->u.pSelect;
-    if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) ){
+    if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) 
+     && ExprAlwaysFalse(pExpr)==0
+    ){
      sqlite3 *db = pWalker->pParse->db;
      Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1");
      if( pNew ){
@@ -5808,13 +5840,11 @@ int sqlite3Select(
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue );
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue );
-  if( IgnorableOrderby(pDest) ){
-    assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || 
-           pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard ||
-           pDest->eDest==SRT_Queue  || pDest->eDest==SRT_DistFifo ||
-           pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo);
-    /* If ORDER BY makes no difference in the output then neither does
-    ** DISTINCT so it can be removed too. */
+  if( IgnorableDistinct(pDest) ){
+    assert(pDest->eDest==SRT_Exists     || pDest->eDest==SRT_Union || 
+           pDest->eDest==SRT_Except     || pDest->eDest==SRT_Discard ||
+           pDest->eDest==SRT_DistQueue  || pDest->eDest==SRT_DistFifo );
+    /* All of these destinations are also able to ignore the ORDER BY clause */
    sqlite3ExprListDelete(db, p->pOrderBy);
    p->pOrderBy = 0;
    p->selFlags &= ~SF_Distinct;
@@ -553,8 +553,6 @@ static void utf8_width_print(FILE *pOut, int w, const char *zUtf){
  int i;
  int n;
  int aw = w<0 ? -w : w;
-  char zBuf[1000];
-  if( aw>(int)sizeof(zBuf)/3 ) aw = (int)sizeof(zBuf)/3;
  for(i=n=0; zUtf[i]; i++){
    if( (zUtf[i]&0xc0)!=0x80 ){
      n++;
@@ -1041,6 +1039,7 @@ INCLUDE ../ext/misc/memtrace.c
 INCLUDE ../ext/misc/uint.c
 INCLUDE ../ext/misc/decimal.c
 INCLUDE ../ext/misc/ieee754.c
+INCLUDE ../ext/misc/series.c
 #ifdef SQLITE_HAVE_ZLIB
 INCLUDE ../ext/misc/zipfile.c
 INCLUDE ../ext/misc/sqlar.c
@@ -1198,6 +1197,8 @@ struct ShellState {
 #define SHFLG_CountChanges   0x00000020 /* .changes setting */
 #define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
 #define SHFLG_HeaderSet      0x00000080 /* .header has been used */
+#define SHFLG_DumpDataOnly   0x00000100 /* .dump show data only */
+#define SHFLG_DumpNoSys      0x00000200 /* .dump omits system tables */

 /*
 ** Macros for testing and setting shellFlgs
@@ -3718,19 +3719,25 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
  const char *zType;
  const char *zSql;
  ShellState *p = (ShellState *)pArg;
+  int dataOnly;
+  int noSys;

  UNUSED_PARAMETER(azNotUsed);
  if( nArg!=3 || azArg==0 ) return 0;
  zTable = azArg[0];
  zType = azArg[1];
  zSql = azArg[2];
+  dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0;
+  noSys    = (p->shellFlgs & SHFLG_DumpNoSys)!=0;

-  if( strcmp(zTable, "sqlite_sequence")==0 ){
-    raw_printf(p->out, "DELETE FROM sqlite_sequence;\n");
-  }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){
-    raw_printf(p->out, "ANALYZE sqlite_schema;\n");
+  if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
+    if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n");
+  }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
+    if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n");
  }else if( strncmp(zTable, "sqlite_", 7)==0 ){
    return 0;
+  }else if( dataOnly ){
+    /* no-op */
  }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
    char *zIns;
    if( !p->writableSchema ){
@@ -3903,8 +3910,10 @@ static const char *(azHelp[]) = {
  ".dbinfo ?DB?             Show status information about the database",
  ".dump ?TABLE?            Render database content as SQL",
  "   Options:",
-  "     --preserve-rowids      Include ROWID values in the output",
+  "     --data-only            Output only INSERT statements",
  "     --newlines             Allow unescaped newline characters in output",
+  "     --nosys                Omit system tables (ex: \"sqlite_stat1\")",
+  "     --preserve-rowids      Include ROWID values in the output",
  "   TABLE is a LIKE pattern for the tables to dump",
  "   Additional LIKE patterns can be given in subsequent arguments",
  ".echo on|off             Turn command echo on or off",
@@ -4029,8 +4038,9 @@ static const char *(azHelp[]) = {
  ".save FILE               Write in-memory database into FILE",
  ".scanstats on|off        Turn sqlite3_stmt_scanstatus() metrics on or off",
  ".schema ?PATTERN?        Show the CREATE statements matching PATTERN",
-  "     Options:",
-  "         --indent            Try to pretty-print the schema",
+  "   Options:",
+  "      --indent             Try to pretty-print the schema",
+  "      --nosys              Omit objects whose names start with \"sqlite_\"",
  ".selftest ?OPTIONS?      Run tests defined in the SELFTEST table",
  "    Options:",
  "       --init               Create a new SELFTEST table",
@@ -4439,6 +4449,20 @@ static void shellIdQuote(
  }
 }

+/*
+** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X.
+*/
+static void shellUSleepFunc(
+  sqlite3_context *context, 
+  int argcUnused, 
+  sqlite3_value **argv
+){
+  int sleep = sqlite3_value_int(argv[0]);
+  (void)argcUnused;
+  sqlite3_sleep(sleep/1000);
+  sqlite3_result_int(context, sleep);
+}
+
 /*
 ** Scalar function "shell_escape_crnl" used by the .recover command.
 ** The argument passed to this function is the output of built-in
@@ -4603,6 +4627,7 @@ static void open_db(ShellState *p, int openFlags){
    sqlite3_uint_init(p->db, 0, 0);
    sqlite3_decimal_init(p->db, 0, 0);
    sqlite3_ieee_init(p->db, 0, 0);
+    sqlite3_series_init(p->db, 0, 0);
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB)
    sqlite3_dbdata_init(p->db, 0, 0);
 #endif
@@ -4622,6 +4647,8 @@ static void open_db(ShellState *p, int openFlags){
                            shellInt32, 0, 0);
    sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0,
                            shellIdQuote, 0, 0);
+    sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0,
+                            shellUSleepFunc, 0, 0);
 #ifndef SQLITE_NOHAVE_SYSTEM
    sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0,
                            editFunc, 0, 0);
@@ -7596,21 +7623,41 @@ static int do_meta_command(char *zLine, ShellState *p){
  }else

  if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){
-    ShellState data;
-    char *zErrMsg = 0;
+    char **azName = 0;
+    int nName = 0;
+    sqlite3_stmt *pStmt;
+    int i;
    open_db(p, 0);
-    memcpy(&data, p, sizeof(data));
-    data.showHeader = 0;
-    data.cMode = data.mode = MODE_List;
-    sqlite3_snprintf(sizeof(data.colSeparator),data.colSeparator,": ");
-    data.cnt = 0;
-    sqlite3_exec(p->db, "SELECT name, file FROM pragma_database_list",
-                 callback, &data, &zErrMsg);
-    if( zErrMsg ){
-      utf8_printf(stderr,"Error: %s\n", zErrMsg);
-      sqlite3_free(zErrMsg);
+    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
+    if( rc ){
+      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
      rc = 1;
+    }else{
+      while( sqlite3_step(pStmt)==SQLITE_ROW ){
+        const char *zSchema = (const char *)sqlite3_column_text(pStmt,1);
+        const char *zFile = (const char*)sqlite3_column_text(pStmt,2);
+        azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*));
+        if( azName==0 ){ shell_out_of_memory();  /* Does not return */ }
+        azName[nName*2] = strdup(zSchema);
+        azName[nName*2+1] = strdup(zFile);
+        nName++;
+      }
    }
+    sqlite3_finalize(pStmt);
+    for(i=0; i<nName; i++){
+      int eTxn = sqlite3_txn_state(p->db, azName[i*2]);
+      int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]);
+      const char *z = azName[i*2+1];
+      utf8_printf(p->out, "%s: %s %s%s\n",
+         azName[i*2],
+         z && z[0] ? z : "\"\"",
+         bRdonly ? "r/o" : "r/w",
+         eTxn==SQLITE_TXN_NONE ? "" :
+            eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn");
+      free(azName[i*2]);
+      free(azName[i*2+1]);
+    }
+    sqlite3_free(azName);
  }else

  if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){
@@ -7669,7 +7716,9 @@ static int do_meta_command(char *zLine, ShellState *p){
    int i;
    int savedShowHeader = p->showHeader;
    int savedShellFlags = p->shellFlgs;
-    ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo);
+    ShellClearFlag(p, 
+       SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo
+       |SHFLG_DumpDataOnly|SHFLG_DumpNoSys);
    for(i=1; i<nArg; i++){
      if( azArg[i][0]=='-' ){
        const char *z = azArg[i]+1;
@@ -7688,6 +7737,12 @@ static int do_meta_command(char *zLine, ShellState *p){
        if( strcmp(z,"newlines")==0 ){
          ShellSetFlag(p, SHFLG_Newlines);
        }else
+        if( strcmp(z,"data-only")==0 ){
+          ShellSetFlag(p, SHFLG_DumpDataOnly);
+        }else
+        if( strcmp(z,"nosys")==0 ){
+          ShellSetFlag(p, SHFLG_DumpNoSys);
+        }else
        {
          raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
          rc = 1;
@@ -7704,11 +7759,13 @@ static int do_meta_command(char *zLine, ShellState *p){

    open_db(p, 0);

-    /* When playing back a "dump", the content might appear in an order
-    ** which causes immediate foreign key constraints to be violated.
-    ** So disable foreign-key constraint enforcement to prevent problems. */
-    raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
-    raw_printf(p->out, "BEGIN TRANSACTION;\n");
+    if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
+      /* When playing back a "dump", the content might appear in an order
+      ** which causes immediate foreign key constraints to be violated.
+      ** So disable foreign-key constraint enforcement to prevent problems. */
+      raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
+      raw_printf(p->out, "BEGIN TRANSACTION;\n");
+    }
    p->writableSchema = 0;
    p->showHeader = 0;
    /* Set writable_schema=ON since doing so forces SQLite to initialize
@@ -7726,14 +7783,16 @@ static int do_meta_command(char *zLine, ShellState *p){
    );
    run_schema_dump_query(p,zSql);
    sqlite3_free(zSql);
-    zSql = sqlite3_mprintf(
-      "SELECT sql FROM sqlite_schema "
-      "WHERE (%s) AND sql NOT NULL"
-      "  AND type IN ('index','trigger','view')",
-      zLike
-    );
-    run_table_dump_query(p, zSql);
-    sqlite3_free(zSql);
+    if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
+      zSql = sqlite3_mprintf(
+        "SELECT sql FROM sqlite_schema "
+        "WHERE (%s) AND sql NOT NULL"
+        "  AND type IN ('index','trigger','view')",
+        zLike
+      );
+      run_table_dump_query(p, zSql);
+      sqlite3_free(zSql);
+    }
    sqlite3_free(zLike);
    if( p->writableSchema ){
      raw_printf(p->out, "PRAGMA writable_schema=OFF;\n");
@@ -7741,7 +7800,9 @@ static int do_meta_command(char *zLine, ShellState *p){
    }
    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
    sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
-    raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n");
+    if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
+      raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n");
+    }
    p->showHeader = savedShowHeader;
    p->shellFlgs = savedShellFlags;
  }else
@@ -8180,7 +8241,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    while( (nSkip--)>0 ){
      while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){}
    }
-    zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
+    zSql = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable);
    if( zSql==0 ){
      import_cleanup(&sCtx);
      shell_out_of_memory();
@@ -8189,7 +8250,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
    import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
    if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){
-      char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
+      char *zCreate = sqlite3_mprintf("CREATE TABLE \"%w\"", zTable);
      char cSep = '(';
      while( xRead(&sCtx) ){
        zCreate = sqlite3_mprintf("%z%c\n  \"%w\" TEXT", zCreate, cSep, sCtx.z);
@@ -8210,7 +8271,7 @@ static int do_meta_command(char *zLine, ShellState *p){
      rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
      sqlite3_free(zCreate);
      if( rc ){
-        utf8_printf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable,
+        utf8_printf(stderr, "CREATE TABLE \"%s\"(...) failed: %s\n", zTable,
                sqlite3_errmsg(p->db));
        import_cleanup(&sCtx);
        rc = 1;
@@ -8975,9 +9036,16 @@ static int do_meta_command(char *zLine, ShellState *p){
      rc = 1;
      goto meta_command_exit;
    }
-    if( notNormalFile(azArg[1])
-     || (p->in = fopen(azArg[1], "rb"))==0
-    ){
+    if( azArg[1][0]=='|' ){
+      p->in = popen(azArg[1]+1, "r");
+      if( p->in==0 ){
+        utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]);
+        rc = 1;
+      }else{
+        rc = process_input(p);
+        pclose(p->in);
+      }
+    }else if( notNormalFile(azArg[1]) || (p->in = fopen(azArg[1], "rb"))==0 ){
      utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
      rc = 1;
    }else{
@@ -9059,6 +9127,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    const char *zName = 0;
    int iSchema = 0;
    int bDebug = 0;
+    int bNoSystemTabs = 0;
    int ii;

    open_db(p, 0);
@@ -9071,10 +9140,16 @@ static int do_meta_command(char *zLine, ShellState *p){
        data.cMode = data.mode = MODE_Pretty;
      }else if( optionMatch(azArg[ii],"debug") ){
        bDebug = 1;
+      }else if( optionMatch(azArg[ii],"nosys") ){
+        bNoSystemTabs = 1;
+      }else if( azArg[ii][0]=='-' ){
+        utf8_printf(stderr, "Unknown option: \"%s\"\n", azArg[ii]);
+        rc = 1;
+        goto meta_command_exit;
      }else if( zName==0 ){
        zName = azArg[ii];
      }else{
-        raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
+        raw_printf(stderr, "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n");
        rc = 1;
        goto meta_command_exit;
      }
@@ -9160,7 +9235,10 @@ static int do_meta_command(char *zLine, ShellState *p){
        appendText(&sSelect, " AND ", 0);
        sqlite3_free(zQarg);
      }
-      appendText(&sSelect, "type!='meta' AND sql IS NOT NULL"
+      if( bNoSystemTabs ){
+        appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0);
+      }
+      appendText(&sSelect, "sql IS NOT NULL"
                           " ORDER BY snum, rowid", 0);
      if( bDebug ){
        utf8_printf(p->out, "SQL: %s;\n", sSelect.z);
@@ -9857,6 +9935,7 @@ static int do_meta_command(char *zLine, ShellState *p){
      { "prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,  ""               },
      { "prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,     ""               },
      { "prng_seed",          SQLITE_TESTCTRL_PRNG_SEED,     "SEED ?db?"      },
+      { "seek_count",         SQLITE_TESTCTRL_SEEK_COUNT,    ""               },
    };
    int testctrl = -1;
    int iCtrl = -1;
@@ -9919,7 +9998,6 @@ static int do_meta_command(char *zLine, ShellState *p){
        /* sqlite3_test_control(int) */
        case SQLITE_TESTCTRL_PRNG_SAVE:
        case SQLITE_TESTCTRL_PRNG_RESTORE:
-        case SQLITE_TESTCTRL_PRNG_RESET:
        case SQLITE_TESTCTRL_BYTEORDER:
          if( nArg==2 ){
            rc2 = sqlite3_test_control(testctrl);
@@ -9993,6 +10071,14 @@ static int do_meta_command(char *zLine, ShellState *p){
          }
          break;

+        case SQLITE_TESTCTRL_SEEK_COUNT: {
+          u64 x = 0;
+          rc2 = sqlite3_test_control(testctrl, p->db, &x);
+          utf8_printf(p->out, "%llu\n", x);
+          isOk = 3;
+          break;
+        }
+
 #ifdef YYCOVERAGE
        case SQLITE_TESTCTRL_PARSER_COVERAGE:
          if( nArg==2 ){
@@ -10581,8 +10667,11 @@ static void process_sqliterc(
    if( stdin_is_interactive ){
      utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
    }
-    process_input(p);
+    if( process_input(p) && bail_on_error ) exit(1);
    fclose(p->in);
+  }else if( sqliterc_override!=0 ){
+    utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc);
+    if( bail_on_error ) exit(1);
  }
  p->in = inSaved;
  p->lineno = savedLineno;
@@ -10641,6 +10730,7 @@ static const char zOptions[] =
 #endif
  "   -stats               print memory stats before each finalize\n"
  "   -table               set output mode to 'table'\n"
+  "   -tabs                set output mode to 'tabs'\n"
  "   -version             show SQLite version\n"
  "   -vfs NAME            use NAME as the default VFS\n"
 #ifdef SQLITE_ENABLE_VFSTRACE
@@ -10898,11 +10988,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
      (void)cmdline_option_value(argc, argv, ++i);
 #endif
    }else if( strcmp(z,"-pagecache")==0 ){
-      int n, sz;
-      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
+      sqlite3_int64 n, sz;
+      sz = integerValue(cmdline_option_value(argc,argv,++i));
      if( sz>70000 ) sz = 70000;
      if( sz<0 ) sz = 0;
-      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
+      n = integerValue(cmdline_option_value(argc,argv,++i));
+      if( sz>0 && n>0 && 0xffffffffffffLL/sz<n ){
+        n = 0xffffffffffffLL/sz;
+      }
      sqlite3_config(SQLITE_CONFIG_PAGECACHE,
                    (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
      data.shellFlgs |= SHFLG_Pagecache;
@@ -10964,6 +11057,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
    }else if( strcmp(z, "-memtrace")==0 ){
      sqlite3MemTraceActivate(stderr);
+    }else if( strcmp(z,"-bail")==0 ){
+      bail_on_error = 1;
    }
  }
  verify_uninitialized();
@@ -11038,6 +11133,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
      data.mode = MODE_List;
    }else if( strcmp(z,"-quote")==0 ){
      data.mode = MODE_Quote;
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row);
    }else if( strcmp(z,"-line")==0 ){
      data.mode = MODE_Line;
    }else if( strcmp(z,"-column")==0 ){
@@ -11071,10 +11168,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
      data.openFlags |= SQLITE_OPEN_NOFOLLOW;
    }else if( strcmp(z,"-ascii")==0 ){
      data.mode = MODE_Ascii;
-      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
-                       SEP_Unit);
-      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,
-                       SEP_Record);
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Unit);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Record);
+    }else if( strcmp(z,"-tabs")==0 ){
+      data.mode = MODE_List;
+      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Tab);
+      sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row);
    }else if( strcmp(z,"-separator")==0 ){
      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
                       "%s",cmdline_option_value(argc,argv,++i));
@@ -11106,7 +11205,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
      */
      ShellSetFlag(&data, SHFLG_Backslash);
    }else if( strcmp(z,"-bail")==0 ){
-      bail_on_error = 1;
+      /* No-op.  The bail_on_error flag should already be set. */
    }else if( strcmp(z,"-version")==0 ){
 /* BEGIN SQLCIPHER */
 #ifdef SQLITE_HAS_CODEC
@@ -11204,20 +11303,25 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
    for(i=0; i<nCmd; i++){
      if( azCmd[i][0]=='.' ){
        rc = do_meta_command(azCmd[i], &data);
-        if( rc ) return rc==2 ? 0 : rc;
+        if( rc ){
+          free(azCmd);
+          return rc==2 ? 0 : rc;
+        }
      }else{
        open_db(&data, 0);
        rc = shell_exec(&data, azCmd[i], &zErrMsg);
-        if( zErrMsg!=0 ){
-          utf8_printf(stderr,"Error: %s\n", zErrMsg);
+        if( zErrMsg || rc ){
+          if( zErrMsg!=0 ){
+            utf8_printf(stderr,"Error: %s\n", zErrMsg);
+          }else{
+            utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]);
+          }
+          sqlite3_free(zErrMsg);
+          free(azCmd);
          return rc!=0 ? rc : 1;
-        }else if( rc!=0 ){
-          utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]);
-          return rc;
        }
      }
    }
-    free(azCmd);
  }else{
    /* Run commands received from standard input
    */
@@ -11277,6 +11381,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
      rc = process_input(&data);
    }
  }
+  free(azCmd);
  set_table_name(&data, 0);
  if( data.db ){
    session_close_all(&data);
@@ -504,6 +504,7 @@ int sqlite3_exec(
 #define SQLITE_IOERR_COMMIT_ATOMIC     (SQLITE_IOERR | (30<<8))
 #define SQLITE_IOERR_ROLLBACK_ATOMIC   (SQLITE_IOERR | (31<<8))
 #define SQLITE_IOERR_DATA              (SQLITE_IOERR | (32<<8))
+#define SQLITE_IOERR_CORRUPTFS         (SQLITE_IOERR | (33<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_LOCKED_VTAB             (SQLITE_LOCKED |  (2<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
@@ -3696,7 +3697,7 @@ sqlite3_file *sqlite3_database_file_object(const char*);
 ** If the Y parameter to sqlite3_free_filename(Y) is anything other
 ** than a NULL pointer or a pointer previously acquired from
 ** sqlite3_create_filename(), then bad things such as heap
-** corruption or segfaults may occur. The value Y should be 
+** corruption or segfaults may occur. The value Y should not be 
 ** used again after sqlite3_free_filename(Y) has been called.  This means
 ** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y,
 ** then the corresponding [sqlite3_module.xClose() method should also be
@@ -6233,6 +6234,57 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName);
 */
 int sqlite3_db_readonly(sqlite3 *db, const char *zDbName);

+/*
+** CAPI3REF: Determine the transaction state of a database
+** METHOD: sqlite3
+**
+** ^The sqlite3_txn_state(D,S) interface returns the current
+** [transaction state] of schema S in database connection D.  ^If S is NULL,
+** then the highest transaction state of any schema on database connection D
+** is returned.  Transaction states are (in order of lowest to highest):
+** <ol>
+** <li value="0"> SQLITE_TXN_NONE
+** <li value="1"> SQLITE_TXN_READ
+** <li value="2"> SQLITE_TXN_WRITE
+** </ol>
+** ^If the S argument to sqlite3_txn_state(D,S) is not the name of
+** a valid schema, then -1 is returned.
+*/
+int sqlite3_txn_state(sqlite3*,const char *zSchema);
+
+/*
+** CAPI3REF: Allowed return values from [sqlite3_txn_state()]
+** KEYWORDS: {transaction state}
+**
+** These constants define the current transaction state of a database file.
+** ^The [sqlite3_txn_state(D,S)] interface returns one of these
+** constants in order to describe the transaction state of schema S
+** in [database connection] D.
+**
+** <dl>
+** [[SQLITE_TXN_NONE]] <dt>SQLITE_TXN_NONE</dt>
+** <dd>The SQLITE_TXN_NONE state means that no transaction is currently
+** pending.</dd>
+**
+** [[SQLITE_TXN_READ]] <dt>SQLITE_TXN_READ</dt>
+** <dd>The SQLITE_TXN_READ state means that the database is currently
+** in a read transaction.  Content has been read from the database file
+** but nothing in the database file has changed.  The transaction state
+** will advanced to SQLITE_TXN_WRITE if any changes occur and there are
+** no other conflicting concurrent write transactions.  The transaction
+** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or
+** [COMMIT].</dd>
+**
+** [[SQLITE_TXN_WRITE]] <dt>SQLITE_TXN_WRITE</dt>
+** <dd>The SQLITE_TXN_WRITE state means that the database is currently
+** in a write transaction.  Content has been written to the database file
+** but has not yet committed.  The transaction state will change to
+** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].</dd>
+*/
+#define SQLITE_TXN_NONE  0
+#define SQLITE_TXN_READ  1
+#define SQLITE_TXN_WRITE 2
+
 /*
 ** CAPI3REF: Find the next prepared statement
 ** METHOD: sqlite3
@@ -7759,7 +7811,8 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_RESULT_INTREAL          27
 #define SQLITE_TESTCTRL_PRNG_SEED               28
 #define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS     29
-#define SQLITE_TESTCTRL_LAST                    29  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_SEEK_COUNT              30
+#define SQLITE_TESTCTRL_LAST                    30  /* Largest TESTCTRL */

 /*
 ** CAPI3REF: SQL Keyword Checking
@@ -9239,10 +9292,11 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 ** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE
 **
 ** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn]
-** method of a [virtual table], then it returns true if and only if the
+** method of a [virtual table], then it might return true if the
 ** column is being fetched as part of an UPDATE operation during which the
-** column value will not change.  Applications might use this to substitute
-** a return value that is less expensive to compute and that the corresponding
+** column value will not change.  The virtual table implementation can use
+** this hint as permission to substitute a return value that is less
+** expensive to compute and that the corresponding
 ** [xUpdate] method understands as a "no-change" value.
 **
 ** If the [xColumn] method calls sqlite3_vtab_nochange() and finds that
@@ -9251,6 +9305,12 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 ** any of the [sqlite3_result_int|sqlite3_result_xxxxx() interfaces].
 ** In that case, [sqlite3_value_nochange(X)] will return true for the
 ** same column in the [xUpdate] method.
+**
+** The sqlite3_vtab_nochange() routine is an optimization.  Virtual table
+** implementations should continue to give a correct answer even if the
+** sqlite3_vtab_nochange() interface were to always return false.  In the
+** current implementation, the sqlite3_vtab_nochange() interface does always
+** returns false for the enhanced [UPDATE FROM] statement.
 */
 int sqlite3_vtab_nochange(sqlite3_context*);

@@ -9392,6 +9452,7 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*);

 /*
 ** CAPI3REF: Flush caches to disk mid-transaction
+** METHOD: sqlite3
 **
 ** ^If a write-transaction is open on [database connection] D when the
 ** [sqlite3_db_cacheflush(D)] interface invoked, any dirty
@@ -9424,6 +9485,7 @@ int sqlite3_db_cacheflush(sqlite3*);

 /*
 ** CAPI3REF: The pre-update hook.
+** METHOD: sqlite3
 **
 ** ^These interfaces are only available if SQLite is compiled using the
 ** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option.
@@ -9464,7 +9526,7 @@ int sqlite3_db_cacheflush(sqlite3*);
 ** seventh parameter is the final rowid value of the row being inserted
 ** or updated. The value of the seventh parameter passed to the callback
 ** function is not defined for operations on WITHOUT ROWID tables, or for
-** INSERT operations on rowid tables.
+** DELETE operations on rowid tables.
 **
 ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()],
 ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces
@@ -9526,6 +9588,7 @@ int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **);

 /*
 ** CAPI3REF: Low-level system error code
+** METHOD: sqlite3
 **
 ** ^Attempt to return the underlying operating system error code or error
 ** number that caused the most recent I/O error or failure to open a file.
@@ -335,6 +335,8 @@ struct sqlite3_api_routines {
                           int,const char**);
  void (*free_filename)(char*);
  sqlite3_file *(*database_file_object)(const char*);
+  /* Version 3.34.0 and later */
+  int (*txn_state)(sqlite3*,const char*);
 };

 /*
@@ -639,6 +641,8 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_create_filename        sqlite3_api->create_filename
 #define sqlite3_free_filename          sqlite3_api->free_filename
 #define sqlite3_database_file_object   sqlite3_api->database_file_object
+/* Version 3.34.0 and later */
+#define sqlite3_txn_state              sqlite3_api->txn_state
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */

 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
@@ -2187,6 +2187,7 @@ struct Table {
 #define TF_OOOHidden       0x0400    /* Out-of-Order hidden columns */
 #define TF_HasNotNull      0x0800    /* Contains NOT NULL constraints */
 #define TF_Shadow          0x1000    /* True for a shadow table */
+#define TF_HasStat4        0x2000    /* STAT4 info available for this table */

 /*
 ** Test to see whether or not a table is a virtual table.  This is
@@ -2715,7 +2716,7 @@ struct Expr {
 #define EP_TokenOnly  0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */
 #define EP_Win        0x008000 /* Contains window functions */
 #define EP_MemToken   0x010000 /* Need to sqlite3DbFree() Expr.zToken */
-                  /*  0x020000 // available for reuse */
+#define EP_IfNullRow  0x020000 /* The TK_IF_NULL_ROW opcode */
 #define EP_Unlikely   0x040000 /* unlikely() or likelihood() function */
 #define EP_ConstFunc  0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */
 #define EP_CanBeNull  0x100000 /* Can be null despite NOT NULL constraint */
@@ -2953,9 +2954,9 @@ struct SrcList {
 #define WHERE_DISTINCTBY       0x0080 /* pOrderby is really a DISTINCT clause */
 #define WHERE_WANT_DISTINCT    0x0100 /* All output needs to be distinct */
 #define WHERE_SORTBYGROUP      0x0200 /* Support sqlite3WhereIsSorted() */
-#define WHERE_SEEK_TABLE       0x0400 /* Do not defer seeks on main table */
+                        /*     0x0400    not currently used */
 #define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
-#define WHERE_SEEK_UNIQ_TABLE  0x1000 /* Do not defer seeks if unique */
+                        /*     0x1000    not currently used */
                        /*     0x2000    not currently used */
 #define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
                        /*     0x8000    not currently used */
@@ -3152,9 +3153,6 @@ struct Select {
 **                     statements within triggers whose only purpose is
 **                     the side-effects of functions.
 **
-** All of the above are free to ignore their ORDER BY clause. Those that
-** follow must honor the ORDER BY clause.
-**
 **     SRT_Output      Generate a row of output (using the OP_ResultRow
 **                     opcode) for each row in the result set.
 **
@@ -3211,13 +3209,18 @@ struct Select {
 #define SRT_Except       2  /* Remove result from a UNION index */
 #define SRT_Exists       3  /* Store 1 if the result is not empty */
 #define SRT_Discard      4  /* Do not save the results anywhere */
-#define SRT_Fifo         5  /* Store result as data with an automatic rowid */
-#define SRT_DistFifo     6  /* Like SRT_Fifo, but unique results only */
+#define SRT_DistFifo     5  /* Like SRT_Fifo, but unique results only */
+#define SRT_DistQueue    6  /* Like SRT_Queue, but unique results only */
+
+/* The DISTINCT clause is ignored for all of the above.  Not that
+** IgnorableDistinct() implies IgnorableOrderby() */
+#define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue)
+
 #define SRT_Queue        7  /* Store result in an queue */
-#define SRT_DistQueue    8  /* Like SRT_Queue, but unique results only */
+#define SRT_Fifo         8  /* Store result as data with an automatic rowid */

 /* The ORDER BY clause is ignored for all of the above */
-#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue)
+#define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo)

 #define SRT_Output       9  /* Output each row of result */
 #define SRT_Mem         10  /* Store result in a memory cell */
@@ -4184,7 +4187,7 @@ void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
 void sqlite3AddColumn(Parse*,Token*,Token*);
 void sqlite3AddNotNull(Parse*, int);
 void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int);
-void sqlite3AddCheckConstraint(Parse*, Expr*);
+void sqlite3AddCheckConstraint(Parse*, Expr*, const char*, const char*);
 void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*);
 void sqlite3AddCollateType(Parse*, Token*);
 void sqlite3AddGenerated(Parse*,Expr*,Token*);
@@ -60,11 +60,7 @@
 ** The maximum depth of an expression tree. This is limited to 
 ** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might 
 ** want to place more severe limits on the complexity of an 
-** expression.
-**
-** A value of 0 used to mean that the limit was not enforced.
-** But that is no longer true.  The limit is now strictly enforced
-** at all times.
+** expression. A value of 0 means that there is no limit.
 */
 #ifndef SQLITE_MAX_EXPR_DEPTH
 # define SQLITE_MAX_EXPR_DEPTH 1000
@@ -3968,7 +3968,7 @@ static int SQLITE_TCLAPI test_bind_blob(
    char zBuf[200];
    sqlite3_snprintf(sizeof(zBuf), zBuf,
                     "cannot use %d blob bytes, have %d", bytes, len);
-    Tcl_AppendResult(interp, zBuf, -1);
+    Tcl_AppendResult(interp, zBuf, (char*)0);
    return TCL_ERROR;
  }

@@ -3981,6 +3981,195 @@ static int SQLITE_TCLAPI test_bind_blob(
  return TCL_OK;
 }

+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+/*
+** sqlite3_carray_bind [options...] STMT NAME VALUE ...
+**
+** Options:
+**    -transient
+**    -static
+**    -int32
+**    -int64
+**    -double
+**    -text
+**
+** Each call clears static data.  Called with no options does nothing
+** but clear static data.
+*/
+static int SQLITE_TCLAPI test_carray_bind(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  int eType = 0;   /* CARRAY_INT32 */
+  int nData = 0;
+  void *aData = 0;
+  int isTransient = 0;
+  int isStatic = 0;
+  int idx;
+  int i, j;
+  int rc;
+  void (*xDel)(void*) = sqlite3_free;
+  static void *aStaticData = 0;
+  static int nStaticData = 0;
+  static int eStaticType = 0;
+  extern int sqlite3_carray_bind(
+    sqlite3_stmt *pStmt,
+    int i,
+    void *aData,
+    int nData,
+    int mFlags,
+    void (*xDestroy)(void*)
+  );
+  
+  if( aStaticData ){
+    /* Always clear preexisting static data on every call */
+    if( eStaticType==3 ){
+      for(i=0; i<nStaticData; i++){
+        sqlite3_free(((char**)aStaticData)[i]);
+      }
+    }
+    sqlite3_free(aStaticData);
+    aStaticData = 0;
+    nStaticData = 0;
+    eStaticType = 0;
+  }
+  if( objc==1 ) return TCL_OK;
+
+  for(i=1; i<objc && Tcl_GetString(objv[i])[0]=='-'; i++){
+    const char *z = Tcl_GetString(objv[i]);
+    if( strcmp(z, "-transient")==0 ){
+      isTransient = 1;
+      xDel = SQLITE_TRANSIENT;
+    }else
+    if( strcmp(z, "-static")==0 ){
+      isStatic = 1;
+      xDel = SQLITE_STATIC;
+    }else
+    if( strcmp(z, "-int32")==0 ){
+      eType = 0;  /* CARRAY_INT32 */
+    }else
+    if( strcmp(z, "-int64")==0 ){
+      eType = 1;  /* CARRAY_INT64 */
+    }else
+    if( strcmp(z, "-double")==0 ){
+      eType = 2;  /* CARRAY_DOUBLE */
+    }else
+    if( strcmp(z, "-text")==0 ){
+      eType = 3;  /* CARRAY_TEXT */
+    }else
+    if( strcmp(z, "--")==0 ){
+      break;
+    }else
+    {
+      Tcl_AppendResult(interp, "unknown option: ", z, (char*)0);
+      return TCL_ERROR;
+    }
+  }
+  if( eType==3 && !isStatic && !isTransient ){
+    Tcl_AppendResult(interp, "text data must be either -static or -transient",
+                     (char*)0);
+    return TCL_ERROR;
+  }
+  if( isStatic && isTransient ){
+    Tcl_AppendResult(interp, "cannot be both -static and -transient",
+                     (char*)0);
+    return TCL_ERROR;
+  }
+  if( objc-i < 2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[OPTIONS] STMT IDX VALUE ...");
+    return TCL_ERROR;
+  }
+  if( getStmtPointer(interp, Tcl_GetString(objv[i]), &pStmt) ) return TCL_ERROR;
+  i++;
+  if( Tcl_GetIntFromObj(interp, objv[i], &idx) ) return TCL_ERROR;
+  i++;
+  nData = objc - i;
+  switch( eType + 4*(nData<=0) ){
+    case 0: { /* INT32 */
+      int *a = sqlite3_malloc( sizeof(int)*nData );
+      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
+      for(j=0; j<nData; j++){
+        int v;
+        if( Tcl_GetIntFromObj(interp, objv[i+j], &v) ){
+          sqlite3_free(a);
+          return TCL_ERROR;
+        }
+        a[j] = v;
+      }
+      aData = a;
+      break;
+    }
+    case 1: { /* INT64 */
+      sqlite3_int64 *a = sqlite3_malloc( sizeof(sqlite3_int64)*nData );
+      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
+      for(j=0; j<nData; j++){
+        Tcl_WideInt v;
+        if( Tcl_GetWideIntFromObj(interp, objv[i+j], &v) ){
+          sqlite3_free(a);
+          return TCL_ERROR;
+        }
+        a[j] = v;
+      }
+      aData = a;
+      break;
+    }
+    case 2: { /* DOUBLE */
+      double *a = sqlite3_malloc( sizeof(double)*nData );
+      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
+      for(j=0; j<nData; j++){
+        double v;
+        if( Tcl_GetDoubleFromObj(interp, objv[i+j], &v) ){
+          sqlite3_free(a);
+          return TCL_ERROR;
+        }
+        a[j] = v;
+      }
+      aData = a;
+      break;
+    }
+    case 3: { /* TEXT */
+      char **a = sqlite3_malloc( sizeof(char*)*nData );
+      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
+      for(j=0; j<nData; j++){
+        const char *v = Tcl_GetString(objv[i+j]);
+        a[j] = sqlite3_mprintf("%s", v);
+      }
+      aData = a;
+      break;
+    }
+    case 4: { /* nData==0 */
+      aData = "";
+      xDel = SQLITE_STATIC;
+      isTransient = 0;
+      isStatic = 0;
+      break;
+    }
+  }
+  if( isStatic ){
+    aStaticData = aData;
+    nStaticData = nData;
+    eStaticType = eType;
+  }
+  rc = sqlite3_carray_bind(pStmt, idx, aData, nData, eType, xDel);
+  if( isTransient ){
+    if( eType==3 ){
+      for(i=0; i<nData; i++) sqlite3_free(((char**)aData)[i]);
+    }
+    sqlite3_free(aData);
+  }
+carray_bind_done:
+  if( rc ){
+    Tcl_AppendResult(interp, sqlite3_errstr(rc), (char*)0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
 /*
 ** Usage:   sqlite3_bind_parameter_count  STMT
 **
@@ -6267,6 +6456,36 @@ static int SQLITE_TCLAPI file_control_vfsname(
  return TCL_OK;  
 }

+/*
+** tclcmd:   file_control_reservebytes DB N
+*/
+static int SQLITE_TCLAPI file_control_reservebytes(
+  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int objc,              /* Number of arguments */
+  Tcl_Obj *CONST objv[]  /* Command arguments */
+){
+  sqlite3 *db;
+  const char *zDbName = "main";
+  int n = 0;
+  int rc;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB N");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db)
+   || Tcl_GetIntFromObj(interp, objv[2], &n) 
+  ){
+    return TCL_ERROR;
+  }
+
+  rc = sqlite3_file_control(db, zDbName, SQLITE_FCNTL_RESERVE_BYTES, (void*)&n);
+  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+  return TCL_OK;  
+}
+
+
 /*
 ** tclcmd:   file_control_tempfilename DB ?AUXDB?
 **
@@ -7777,6 +7996,33 @@ static int SQLITE_TCLAPI test_sqlite3_db_config(
  Tcl_SetObjResult(interp, Tcl_NewIntObj(v));
  return TCL_OK;
 }
+/*
+** tclcmd:   sqlite3_txn_state DB ?SCHEMA?
+**
+** Invoke sqlite3_txn_state(DB,SCHEMA) and return the
+** numeric value that results.  Use NULL for SCHEMA if the 3 argument
+** is omitted.
+*/
+static int SQLITE_TCLAPI test_sqlite3_txn_state(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  const char *zSchema;
+  int iTxn;
+
+  if( objc!=2 && objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB ?SCHEMA?");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zSchema = objc==3 ? Tcl_GetString(objv[2]) : 0;
+  iTxn = sqlite3_txn_state(db, zSchema);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(iTxn));
+  return TCL_OK;
+}

 /*
 ** Change the name of the main database schema from "main" to "icecube".
@@ -7865,6 +8111,48 @@ static int SQLITE_TCLAPI test_write_db(
  return TCL_OK;
 }

+/*
+** Usage:  sqlite3_register_cksumvfs
+**
+*/
+static int SQLITE_TCLAPI test_register_cksumvfs(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }else{
+    extern int sqlite3_register_cksumvfs(const char*);
+    int rc = sqlite3_register_cksumvfs(0);
+    Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
+  }
+  return TCL_OK;
+}
+
+/*
+** Usage:  sqlite3_unregister_cksumvfs
+**
+*/
+static int SQLITE_TCLAPI test_unregister_cksumvfs(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }else{
+    extern int sqlite3_unregister_cksumvfs(void);
+    int rc = sqlite3_unregister_cksumvfs();
+    Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage:  decode_hexdb TEXT
 **
@@ -8013,6 +8301,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
     void *clientData;
  } aObjCmd[] = {
     { "sqlite3_db_config",             test_sqlite3_db_config, 0 },
+     { "sqlite3_txn_state",             test_sqlite3_txn_state, 0 },
     { "bad_behavior",                  test_bad_behavior,  (void*)&iZero },
     { "register_dbstat_vtab",          test_register_dbstat_vtab  },
     { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
@@ -8029,6 +8318,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
     { "sqlite3_bind_text",             test_bind_text     ,0 },
     { "sqlite3_bind_text16",           test_bind_text16   ,0 },
     { "sqlite3_bind_blob",             test_bind_blob     ,0 },
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+     { "sqlite3_carray_bind",           test_carray_bind   ,0 },
+#endif
     { "sqlite3_bind_parameter_count",  test_bind_parameter_count, 0},
     { "sqlite3_bind_parameter_name",   test_bind_parameter_name,  0},
     { "sqlite3_bind_parameter_index",  test_bind_parameter_index, 0},
@@ -8160,6 +8452,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
     { "file_control_persist_wal",    file_control_persist_wal,     0   },
     { "file_control_powersafe_overwrite",file_control_powersafe_overwrite,0},
     { "file_control_vfsname",        file_control_vfsname,         0   },
+     { "file_control_reservebytes",   file_control_reservebytes,    0   },
     { "file_control_tempfilename",   file_control_tempfilename,    0   },
     { "sqlite3_vfs_list",           vfs_list,     0   },
     { "sqlite3_create_function_v2", test_create_function_v2, 0 },
@@ -8230,6 +8523,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
     { "sqlite3_config_sorterref", test_config_sorterref,   0 },
     { "decode_hexdb",             test_decode_hexdb,       0 },
     { "test_write_db",            test_write_db,           0 },
+     { "sqlite3_register_cksumvfs", test_register_cksumvfs,  0 },
+     { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs,  0 },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
@@ -550,13 +550,16 @@ static int cfDeviceCharacteristics(sqlite3_file *pFile){
 ** Pass-throughs for WAL support.
 */
 static int cfShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
-  return sqlite3OsShmLock(((CrashFile*)pFile)->pRealFile, ofst, n, flags);
+  sqlite3_file *pReal = ((CrashFile*)pFile)->pRealFile;
+  return pReal->pMethods->xShmLock(pReal, ofst, n, flags);
 }
 static void cfShmBarrier(sqlite3_file *pFile){
-  sqlite3OsShmBarrier(((CrashFile*)pFile)->pRealFile);
+  sqlite3_file *pReal = ((CrashFile*)pFile)->pRealFile;
+  pReal->pMethods->xShmBarrier(pReal);
 }
 static int cfShmUnmap(sqlite3_file *pFile, int delFlag){
-  return sqlite3OsShmUnmap(((CrashFile*)pFile)->pRealFile, delFlag);
+  sqlite3_file *pReal = ((CrashFile*)pFile)->pRealFile;
+  return pReal->pMethods->xShmUnmap(pReal, delFlag);
 }
 static int cfShmMap(
  sqlite3_file *pFile,            /* Handle open on database file */
@@ -565,7 +568,8 @@ static int cfShmMap(
  int w,                          /* True to extend file if necessary */
  void volatile **pp              /* OUT: Mapped memory */
 ){
-  return sqlite3OsShmMap(((CrashFile*)pFile)->pRealFile, iRegion, sz, w, pp);
+  sqlite3_file *pReal = ((CrashFile*)pFile)->pRealFile;
+  return pReal->pMethods->xShmMap(pReal, iRegion, sz, w, pp);
 }

 static const sqlite3_io_methods CrashFileVtab = {
@@ -191,7 +191,7 @@ static int devsymDeviceCharacteristics(sqlite3_file *pFile){
 */
 static int devsymShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
  devsym_file *p = (devsym_file *)pFile;
-  return sqlite3OsShmLock(p->pReal, ofst, n, flags);
+  return p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
 }
 static int devsymShmMap(
  sqlite3_file *pFile, 
@@ -201,15 +201,15 @@ static int devsymShmMap(
  void volatile **pp
 ){
  devsym_file *p = (devsym_file *)pFile;
-  return sqlite3OsShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
+  return p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
 }
 static void devsymShmBarrier(sqlite3_file *pFile){
  devsym_file *p = (devsym_file *)pFile;
-  sqlite3OsShmBarrier(p->pReal);
+  p->pReal->pMethods->xShmBarrier(p->pReal);
 }
 static int devsymShmUnmap(sqlite3_file *pFile, int delFlag){
  devsym_file *p = (devsym_file *)pFile;
-  return sqlite3OsShmUnmap(p->pReal, delFlag);
+  return p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
 }


@@ -168,7 +168,7 @@ static int SQLITE_TCLAPI hexio_write(
  if( Tcl_GetIntFromObj(interp, objv[2], &offset) ) return TCL_ERROR;
  zFile = Tcl_GetString(objv[1]);
  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[3], &nIn);
-  aOut = sqlite3_malloc( nIn/2 );
+  aOut = sqlite3_malloc( 1 + nIn/2 );
  if( aOut==0 ){
    return TCL_ERROR;
  }
@@ -213,7 +213,7 @@ static int SQLITE_TCLAPI hexio_get_int(
    return TCL_ERROR;
  }
  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
-  aOut = sqlite3_malloc( nIn/2 );
+  aOut = sqlite3_malloc( 1 + nIn/2 );
  if( aOut==0 ){
    return TCL_ERROR;
  }
@@ -309,7 +309,7 @@ static int SQLITE_TCLAPI utf8_to_utf8(
    return TCL_ERROR;
  }
  zOrig = (unsigned char *)Tcl_GetStringFromObj(objv[1], &n);
-  z = sqlite3_malloc( n+3 );
+  z = sqlite3_malloc( n+4 );
  n = sqlite3TestHexToBin(zOrig, n, z);
  z[n] = 0;
  nOut = sqlite3Utf8To8(z);
@@ -895,7 +895,8 @@ static int tvfsShmMap(
  Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);

  if( p->isFullshm ){
-    return sqlite3OsShmMap(pFd->pReal, iPage, pgsz, isWrite, pp);
+    sqlite3_file *pReal = pFd->pReal;
+    return pReal->pMethods->xShmMap(pReal, iPage, pgsz, isWrite, pp);
  }

  if( 0==pFd->pShm ){
@@ -945,7 +946,8 @@ static int tvfsShmLock(
  char zLock[80];

  if( p->isFullshm ){
-    return sqlite3OsShmLock(pFd->pReal, ofst, n, flags);
+    sqlite3_file *pReal = pFd->pReal;
+    return pReal->pMethods->xShmLock(pReal, ofst, n, flags);
  }

  if( p->pScript && p->mask&TESTVFS_SHMLOCK_MASK ){
@@ -1009,7 +1011,8 @@ static void tvfsShmBarrier(sqlite3_file *pFile){
  }

  if( p->isFullshm ){
-    sqlite3OsShmBarrier(pFd->pReal);
+    sqlite3_file *pReal = pFd->pReal;
+    pReal->pMethods->xShmBarrier(pReal);
    return;
  }
 }
@@ -1025,7 +1028,8 @@ static int tvfsShmUnmap(
  TestvfsFd **ppFd;

  if( p->isFullshm ){
-    return sqlite3OsShmUnmap(pFd->pReal, deleteFlag);
+    sqlite3_file *pReal = pFd->pReal;
+    return pReal->pMethods->xShmUnmap(pReal, deleteFlag);
  }

  if( !pBuffer ) return SQLITE_OK;
@@ -157,22 +157,11 @@ void sqlite3BeginTrigger(
  pTab = sqlite3SrcListLookup(pParse, pTableName);
  if( !pTab ){
    /* The table does not exist. */
-    if( db->init.iDb==1 ){
-      /* Ticket #3810.
-      ** Normally, whenever a table is dropped, all associated triggers are
-      ** dropped too.  But if a TEMP trigger is created on a non-TEMP table
-      ** and the table is dropped by a different database connection, the
-      ** trigger is not visible to the database connection that does the
-      ** drop so the trigger cannot be dropped.  This results in an
-      ** "orphaned trigger" - a trigger whose associated table is missing.
-      */
-      db->init.orphanTrigger = 1;
-    }
-    goto trigger_cleanup;
+    goto trigger_orphan_error;
  }
  if( IsVirtual(pTab) ){
    sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables");
-    goto trigger_cleanup;
+    goto trigger_orphan_error;
  }

  /* Check that the trigger name is not reserved and that no trigger of the
@@ -210,12 +199,12 @@ void sqlite3BeginTrigger(
  if( pTab->pSelect && tr_tm!=TK_INSTEAD ){
    sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", 
        (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName, 0);
-    goto trigger_cleanup;
+    goto trigger_orphan_error;
  }
  if( !pTab->pSelect && tr_tm==TK_INSTEAD ){
    sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF"
        " trigger on table: %S", pTableName, 0);
-    goto trigger_cleanup;
+    goto trigger_orphan_error;
  }

 #ifndef SQLITE_OMIT_AUTHORIZATION
@@ -275,6 +264,23 @@ trigger_cleanup:
  }else{
    assert( pParse->pNewTrigger==pTrigger );
  }
+  return;
+
+trigger_orphan_error:
+  if( db->init.iDb==1 ){
+    /* Ticket #3810.
+    ** Normally, whenever a table is dropped, all associated triggers are
+    ** dropped too.  But if a TEMP trigger is created on a non-TEMP table
+    ** and the table is dropped by a different database connection, the
+    ** trigger is not visible to the database connection that does the
+    ** drop so the trigger cannot be dropped.  This results in an
+    ** "orphaned trigger" - a trigger whose associated table is missing.
+    **
+    ** 2020-11-05 see also https://sqlite.org/forum/forumpost/157dc791df
+    */
+    db->init.orphanTrigger = 1;
+  }
+  goto trigger_cleanup;
 }

 /*
@@ -234,7 +234,7 @@ static void updateFromSelect(
 #endif
      pList = sqlite3ExprListAppend(pParse, pList, pNew);
    }
-    eDest = SRT_Upfrom;
+    eDest = IsVirtual(pTab) ? SRT_Table : SRT_Upfrom;
  }else if( pTab->pSelect ){
    for(i=0; i<pTab->nCol; i++){
      pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i));
@@ -651,6 +651,8 @@ void sqlite3Update(

  if( nChangeFrom==0 && HasRowid(pTab) ){
    sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
+    iEph = pParse->nTab++;
+    addrOpen = sqlite3VdbeAddOp3(v, OP_OpenEphemeral, iEph, 0, regRowSet);
  }else{
    assert( pPk!=0 || HasRowid(pTab) );
    nPk = pPk ? pPk->nKeyCol : 0;
@@ -705,7 +707,7 @@ void sqlite3Update(
      ** be deleted as a result of REPLACE conflict handling. Any of these
      ** things might disturb a cursor being used to scan through the table
      ** or index, causing a single-pass approach to malfunction.  */
-      flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
+      flags = WHERE_ONEPASS_DESIRED;
      if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
        flags |= WHERE_ONEPASS_MULTIROW;
      }
@@ -742,9 +744,10 @@ void sqlite3Update(
      ** leave it in register regOldRowid.  */
      sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
      if( eOnePass==ONEPASS_OFF ){
-        /* We need to use regRowSet, so reallocate aRegIdx[nAllIdx] */
        aRegIdx[nAllIdx] = ++pParse->nMem;
-        sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
+        sqlite3VdbeAddOp3(v, OP_Insert, iEph, regRowSet, regOldRowid);
+      }else{
+        if( ALWAYS(addrOpen) ) sqlite3VdbeChangeToNoop(v, addrOpen);
      }
    }else{
      /* Read the PK of the current row into an array of registers. In
@@ -832,8 +835,9 @@ void sqlite3Update(
        VdbeCoverage(v);
      }
    }else{
-      labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet,labelBreak,
-                               regOldRowid);
+      sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
+      labelContinue = sqlite3VdbeMakeLabel(pParse);
+      addrTop = sqlite3VdbeAddOp2(v, OP_Rowid, iEph, regOldRowid);
      VdbeCoverage(v);
      sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid);
      VdbeCoverage(v);
@@ -1083,11 +1087,9 @@ void sqlite3Update(
  }else if( eOnePass==ONEPASS_MULTI ){
    sqlite3VdbeResolveLabel(v, labelContinue);
    sqlite3WhereEnd(pWInfo);
-  }else if( pPk || nChangeFrom ){
+  }else{
    sqlite3VdbeResolveLabel(v, labelContinue);
    sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v);
-  }else{
-    sqlite3VdbeGoto(v, labelContinue);
  }
  sqlite3VdbeResolveLabel(v, labelBreak);

@@ -1187,12 +1189,26 @@ static void updateVirtualTable(
  regArg = pParse->nMem + 1;
  pParse->nMem += nArg;
  if( pSrc->nSrc>1 ){
+    Index *pPk = 0;
    Expr *pRow;
    ExprList *pList;
-    if( pRowid ){
-      pRow = sqlite3ExprDup(db, pRowid, 0);
+    if( HasRowid(pTab) ){
+      if( pRowid ){
+        pRow = sqlite3ExprDup(db, pRowid, 0);
+      }else{
+        pRow = sqlite3PExpr(pParse, TK_ROW, 0, 0);
+      }
    }else{
-      pRow = sqlite3PExpr(pParse, TK_ROW, 0, 0);
+      i16 iPk;      /* PRIMARY KEY column */
+      pPk = sqlite3PrimaryKeyIndex(pTab);
+      assert( pPk!=0 );
+      assert( pPk->nKeyCol==1 );
+      iPk = pPk->aiColumn[0];
+      if( aXRef[iPk]>=0 ){
+        pRow = sqlite3ExprDup(db, pChanges->a[aXRef[iPk]].pExpr, 0);
+      }else{
+        pRow = exprRowColumn(pParse, iPk);
+      }
    }
    pList = sqlite3ExprListAppend(pParse, 0, pRow);

@@ -1206,7 +1222,7 @@ static void updateVirtualTable(
      }
    }

-    updateFromSelect(pParse, ephemTab, 0, pList, pSrc, pWhere, 0, 0);
+    updateFromSelect(pParse, ephemTab, pPk, pList, pSrc, pWhere, 0, 0);
    sqlite3ExprListDelete(db, pList);
    eOnePass = ONEPASS_OFF;
  }else{
@@ -681,6 +681,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
    incr = 1;
  }else{
    incr = 2;
+    length &= ~1;
    assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
    for(i=3-enc; i<length && zNum[i]==0; i+=2){}
    nonNum = i<length;
@@ -352,8 +352,8 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
       BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
    };

-    assert( 1==sqlite3BtreeIsInTrans(pTemp) );
-    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );
+    assert( SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pTemp) );
+    assert( pOut!=0 || SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pMain) );

    /* Copy Btree meta values */
    for(i=0; i<ArraySize(aCopy); i+=2){
@@ -3487,7 +3487,8 @@ case OP_AutoCommit: {
 ** active.
 ** If P2 is non-zero, then a write-transaction is started, or if a 
 ** read-transaction is already active, it is upgraded to a write-transaction.
-** If P2 is zero, then a read-transaction is started.
+** If P2 is zero, then a read-transaction is started.  If P2 is 2 or more
+** then an exclusive transaction is started.
 **
 ** P1 is the index of the database file on which the transaction is
 ** started.  Index 0 is the main database file and index 1 is the
@@ -3521,6 +3522,7 @@ case OP_Transaction: {

  assert( p->bIsReader );
  assert( p->readOnly==0 || pOp->p2==0 );
+  assert( pOp->p2>=0 && pOp->p2<=2 );
  assert( pOp->p1>=0 && pOp->p1<db->nDb );
  assert( DbMaskTest(p->btreeMask, pOp->p1) );
  if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){
@@ -3546,7 +3548,7 @@ case OP_Transaction: {
     && pOp->p2
     && (db->autoCommit==0 || db->nVdbeRead>1) 
    ){
-      assert( sqlite3BtreeIsInTrans(pBt) );
+      assert( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE );
      if( p->iStatement==0 ){
        assert( db->nStatement>=0 && db->nSavepoint>=0 );
        db->nStatement++; 
@@ -3899,7 +3901,7 @@ case OP_OpenDup: {
 }


-/* Opcode: OpenEphemeral P1 P2 * P4 P5
+/* Opcode: OpenEphemeral P1 P2 P3 P4 P5
 ** Synopsis: nColumn=P2
 **
 ** Open a new cursor P1 to a transient table.
@@ -3919,6 +3921,10 @@ case OP_OpenDup: {
 ** in btree.h.  These flags control aspects of the operation of
 ** the btree.  The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are
 ** added automatically.
+**
+** If P3 is positive, then reg[P3] is modified slightly so that it
+** can be used as zero-length data for OP_Insert.  This is an optimization
+** that avoids an extra OP_Blob opcode to initialize that register.
 */
 /* Opcode: OpenAutoindex P1 P2 * P4 *
 ** Synopsis: nColumn=P2
@@ -3941,6 +3947,15 @@ case OP_OpenEphemeral: {
      SQLITE_OPEN_TRANSIENT_DB;
  assert( pOp->p1>=0 );
  assert( pOp->p2>=0 );
+  if( pOp->p3>0 ){
+    /* Make register reg[P3] into a value that can be used as the data
+    ** form sqlite3BtreeInsert() where the length of the data is zero. */
+    assert( pOp->p2==0 ); /* Only used when number of columns is zero */
+    assert( pOp->opcode==OP_OpenEphemeral );
+    assert( aMem[pOp->p3].flags & MEM_Null );
+    aMem[pOp->p3].n = 0;
+    aMem[pOp->p3].z = "";
+  }
  pCx = p->apCsr[pOp->p1];
  if( pCx && pCx->pBtx ){
    /* If the ephermeral table is already open, erase all existing content
@@ -4383,22 +4398,172 @@ seek_not_found:
  break;
 }

-/* Opcode: SeekHit P1 P2 * * *
-** Synopsis: seekHit=P2
+
+/* Opcode: SeekScan  P1 P2 * * *
+** Synopsis: Scan-ahead up to P1 rows
 **
-** Set the seekHit flag on cursor P1 to the value in P2.
-** The seekHit flag is used by the IfNoHope opcode.
+** This opcode is a prefix opcode to OP_SeekGE.  In other words, this
+** opcode must be immediately followed by OP_SeekGE. This constraint is
+** checked by assert() statements.
 **
-** P1 must be a valid b-tree cursor.  P2 must be a boolean value,
-** either 0 or 1.
+** This opcode uses the P1 through P4 operands of the subsequent
+** OP_SeekGE.  In the text that follows, the operands of the subsequent
+** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4.   Only
+** the P1 and P2 operands of this opcode are also used, and  are called
+** This.P1 and This.P2.
+**
+** This opcode helps to optimize IN operators on a multi-column index
+** where the IN operator is on the later terms of the index by avoiding
+** unnecessary seeks on the btree, substituting steps to the next row
+** of the b-tree instead.  A correct answer is obtained if this opcode
+** is omitted or is a no-op.
+**
+** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which
+** is the desired entry that we want the cursor SeekGE.P1 to be pointing
+** to.  Call this SeekGE.P4/P5 row the "target".
+**
+** If the SeekGE.P1 cursor is not currently pointing to a valid row,
+** then this opcode is a no-op and control passes through into the OP_SeekGE.
+**
+** If the SeekGE.P1 cursor is pointing to a valid row, then that row
+** might be the target row, or it might be near and slightly before the
+** target row.  This opcode attempts to position the cursor on the target
+** row by, perhaps by invoking sqlite3BtreeStep() on the cursor
+** between 0 and This.P1 times.
+**
+** There are three possible outcomes from this opcode:<ol>
+**
+** <li> If after This.P1 steps, the cursor is still point to a place that
+**      is earlier in the btree than the target row,
+**      then fall through into the subsquence OP_SeekGE opcode.
+**
+** <li> If the cursor is successfully moved to the target row by 0 or more
+**      sqlite3BtreeNext() calls, then jump to This.P2, which will land just
+**      past the OP_IdxGT opcode that follows the OP_SeekGE.
+**
+** <li> If the cursor ends up past the target row (indicating the the target
+**      row does not exist in the btree) then jump to SeekOP.P2. 
+** </ol>
+*/
+case OP_SeekScan: {
+  VdbeCursor *pC;
+  int res;
+  int nStep;
+  UnpackedRecord r;
+
+  assert( pOp[1].opcode==OP_SeekGE );
+
+  /* pOp->p2 points to the first instruction past the OP_IdxGT that
+  ** follows the OP_SeekGE.  */
+  assert( pOp->p2>=(int)(pOp-aOp)+2 );
+  assert( aOp[pOp->p2-1].opcode==OP_IdxGT );
+  assert( pOp[1].p1==aOp[pOp->p2-1].p1 );
+  assert( pOp[1].p2==aOp[pOp->p2-1].p2 );
+  assert( pOp[1].p3==aOp[pOp->p2-1].p3 );
+
+  assert( pOp->p1>0 );
+  pC = p->apCsr[pOp[1].p1];
+  assert( pC!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( !pC->isTable );
+  if( !sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) ){
+#ifdef SQLITE_DEBUG
+     if( db->flags&SQLITE_VdbeTrace ){
+       printf("... cursor not valid - fall through\n");
+     }        
+#endif
+    break;
+  }
+  nStep = pOp->p1;
+  assert( nStep>=1 );
+  r.pKeyInfo = pC->pKeyInfo;
+  r.nField = (u16)pOp[1].p4.i;
+  r.default_rc = 0;
+  r.aMem = &aMem[pOp[1].p3];
+#ifdef SQLITE_DEBUG
+  {
+    int i;
+    for(i=0; i<r.nField; i++){
+      assert( memIsValid(&r.aMem[i]) );
+      REGISTER_TRACE(pOp[1].p3+i, &aMem[pOp[1].p3+i]);
+    }
+  }
+#endif
+  res = 0;  /* Not needed.  Only used to silence a warning. */
+  while(1){
+    rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res);
+    if( rc ) goto abort_due_to_error;
+    if( res>0 ){
+      seekscan_search_fail:
+#ifdef SQLITE_DEBUG
+      if( db->flags&SQLITE_VdbeTrace ){
+        printf("... %d steps and then skip\n", pOp->p1 - nStep);
+      }        
+#endif
+      VdbeBranchTaken(1,3);
+      pOp++;
+      goto jump_to_p2;
+    }
+    if( res==0 ){
+#ifdef SQLITE_DEBUG
+      if( db->flags&SQLITE_VdbeTrace ){
+        printf("... %d steps and then success\n", pOp->p1 - nStep);
+      }        
+#endif
+      VdbeBranchTaken(2,3);
+      goto jump_to_p2;
+      break;
+    }
+    if( nStep<=0 ){
+#ifdef SQLITE_DEBUG
+      if( db->flags&SQLITE_VdbeTrace ){
+        printf("... fall through after %d steps\n", pOp->p1);
+      }        
+#endif
+      VdbeBranchTaken(0,3);
+      break;
+    }
+    nStep--;
+    rc = sqlite3BtreeNext(pC->uc.pCursor, 0);
+    if( rc ){
+      if( rc==SQLITE_DONE ){
+        rc = SQLITE_OK;
+        goto seekscan_search_fail;
+      }else{
+        goto abort_due_to_error;
+      }
+    }
+  }
+  
+  break;
+}
+
+
+/* Opcode: SeekHit P1 P2 P3 * *
+** Synopsis: set P2<=seekHit<=P3
+**
+** Increase or decrease the seekHit value for cursor P1, if necessary,
+** so that it is no less than P2 and no greater than P3.
+**
+** The seekHit integer represents the maximum of terms in an index for which
+** there is known to be at least one match.  If the seekHit value is smaller
+** than the total number of equality terms in an index lookup, then the
+** OP_IfNoHope opcode might run to see if the IN loop can be abandoned
+** early, thus saving work.  This is part of the IN-early-out optimization.
+**
+** P1 must be a valid b-tree cursor.
 */
 case OP_SeekHit: {
  VdbeCursor *pC;
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pOp->p2==0 || pOp->p2==1 );
-  pC->seekHit = pOp->p2 & 1;
+  assert( pOp->p3>=pOp->p2 );
+  if( pC->seekHit<pOp->p2 ){
+    pC->seekHit = pOp->p2;
+  }else if( pC->seekHit>pOp->p3 ){
+    pC->seekHit = pOp->p3;
+  }
  break;
 }

@@ -4456,16 +4621,20 @@ case OP_IfNotOpen: {        /* jump */
 ** Synopsis: key=r[P3@P4]
 **
 ** Register P3 is the first of P4 registers that form an unpacked
-** record.
+** record.  Cursor P1 is an index btree.  P2 is a jump destination.
+** In other words, the operands to this opcode are the same as the
+** operands to OP_NotFound and OP_IdxGT.
 **
-** Cursor P1 is on an index btree.  If the seekHit flag is set on P1, then
-** this opcode is a no-op.  But if the seekHit flag of P1 is clear, then
-** check to see if there is any entry in P1 that matches the
-** prefix identified by P3 and P4.  If no entry matches the prefix,
-** jump to P2.  Otherwise fall through.
+** This opcode is an optimization attempt only.  If this opcode always
+** falls through, the correct answer is still obtained, but extra works
+** is performed.
 **
-** This opcode behaves like OP_NotFound if the seekHit
-** flag is clear and it behaves like OP_Noop if the seekHit flag is set.
+** A value of N in the seekHit flag of cursor P1 means that there exists
+** a key P3:N that will match some record in the index.  We want to know
+** if it is possible for a record P3:P4 to match some record in the
+** index.  If it is not possible, we can skips some work.  So if seekHit
+** is less than P4, attempt to find out if a match is possible by running
+** OP_NotFound.
 **
 ** This opcode is used in IN clause processing for a multi-column key.
 ** If an IN clause is attached to an element of the key other than the
@@ -4507,7 +4676,7 @@ case OP_IfNoHope: {     /* jump, in3 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  if( pC->seekHit ) break;
+  if( pC->seekHit>=pOp->p4.i ) break;
  /* Fall through into OP_NotFound */
  /* no break */ deliberate_fall_through
 }
@@ -4589,6 +4758,7 @@ case OP_Found: {        /* jump, in3 */
  }else{
    VdbeBranchTaken(takeJump||alreadyExists==0,2);
    if( takeJump || !alreadyExists ) goto jump_to_p2;
+    if( pOp->opcode==OP_IfNoHope ) pC->seekHit = pOp->p4.i;
  }
  break;
 }
@@ -4945,7 +5115,7 @@ case OP_Insert: {

  if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey;
-  assert( pData->flags & (MEM_Blob|MEM_Str) );
+  assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 );
  x.pData = pData->z;
  x.nData = pData->n;
  seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
@@ -5821,7 +5991,7 @@ case OP_FinishSeek: {
  break;
 }

-/* Opcode: IdxGE P1 P2 P3 P4 P5
+/* Opcode: IdxGE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
 ** The P4 register values beginning with P3 form an unpacked index 
@@ -5832,7 +6002,7 @@ case OP_FinishSeek: {
 ** If the P1 index entry is greater than or equal to the key value
 ** then jump to P2.  Otherwise fall through to the next instruction.
 */
-/* Opcode: IdxGT P1 P2 P3 P4 P5
+/* Opcode: IdxGT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
 ** The P4 register values beginning with P3 form an unpacked index 
@@ -5843,7 +6013,7 @@ case OP_FinishSeek: {
 ** If the P1 index entry is greater than the key value
 ** then jump to P2.  Otherwise fall through to the next instruction.
 */
-/* Opcode: IdxLT P1 P2 P3 P4 P5
+/* Opcode: IdxLT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
 ** The P4 register values beginning with P3 form an unpacked index 
@@ -5854,7 +6024,7 @@ case OP_FinishSeek: {
 ** If the P1 index entry is less than the key value then jump to P2.
 ** Otherwise fall through to the next instruction.
 */
-/* Opcode: IdxLE P1 P2 P3 P4 P5
+/* Opcode: IdxLE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
 ** The P4 register values beginning with P3 form an unpacked index 
@@ -5880,7 +6050,6 @@ case OP_IdxGE:  {       /* jump */
  assert( pC->eCurType==CURTYPE_BTREE );
  assert( pC->uc.pCursor!=0);
  assert( pC->deferredMoveto==0 );
-  assert( pOp->p5==0 || pOp->p5==1 );
  assert( pOp->p4type==P4_INT32 );
  r.pKeyInfo = pC->pKeyInfo;
  r.nField = (u16)pOp->p4.i;
@@ -5901,8 +6070,31 @@ case OP_IdxGE:  {       /* jump */
    }
  }
 #endif
-  res = 0;  /* Not needed.  Only used to silence a warning. */
-  rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res);
+
+  /* Inlined version of sqlite3VdbeIdxKeyCompare() */
+  {
+    i64 nCellKey = 0;
+    BtCursor *pCur;
+    Mem m;
+
+    assert( pC->eCurType==CURTYPE_BTREE );
+    pCur = pC->uc.pCursor;
+    assert( sqlite3BtreeCursorIsValid(pCur) );
+    nCellKey = sqlite3BtreePayloadSize(pCur);
+    /* nCellKey will always be between 0 and 0xffffffff because of the way
+    ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
+    if( nCellKey<=0 || nCellKey>0x7fffffff ){
+      rc = SQLITE_CORRUPT_BKPT;
+      goto abort_due_to_error;
+    }
+    sqlite3VdbeMemInit(&m, db, 0);
+    rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m);
+    if( rc ) goto abort_due_to_error;
+    res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, &r, 0);
+    sqlite3VdbeMemRelease(&m);
+  }
+  /* End of inlined sqlite3VdbeIdxKeyCompare() */
+
  assert( (OP_IdxLE&1)==(OP_IdxLT&1) && (OP_IdxGE&1)==(OP_IdxGT&1) );
  if( (pOp->opcode&1)==(OP_IdxLT&1) ){
    assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT );
@@ -5912,7 +6104,7 @@ case OP_IdxGE:  {       /* jump */
    res++;
  }
  VdbeBranchTaken(res>0,2);
-  if( rc ) goto abort_due_to_error;
+  assert( rc==SQLITE_OK );
  if( res>0 ) goto jump_to_p2;
  break;
 }
@@ -7038,7 +7230,7 @@ case OP_JournalMode: {    /* out2 */
      /* Open a transaction on the database file. Regardless of the journal
      ** mode, this transaction always uses a rollback journal.
      */
-      assert( sqlite3BtreeIsInTrans(pBt)==0 );
+      assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE );
      if( rc==SQLITE_OK ){
        rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1));
      }
@@ -7978,7 +8170,11 @@ default: {          /* This is really OP_Noop, OP_Explain */
  ** an error of some kind.
  */
 abort_due_to_error:
-  if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT;
+  if( db->mallocFailed ){
+    rc = SQLITE_NOMEM_BKPT;
+  }else if( rc==SQLITE_IOERR_CORRUPTFS ){
+    rc = SQLITE_CORRUPT_BKPT;
+  }
  assert( rc );
  if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){
    sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc));
@@ -86,7 +86,7 @@ struct VdbeCursor {
  Bool isEphemeral:1;     /* True for an ephemeral table */
  Bool useRandomRowid:1;  /* Generate new record numbers semi-randomly */
  Bool isOrdered:1;       /* True if the table is not BTREE_UNORDERED */
-  Bool seekHit:1;         /* See the OP_SeekHit and OP_IfNoHope opcodes */
+  u16 seekHit;            /* See the OP_SeekHit and OP_IfNoHope opcodes */
  Btree *pBtx;            /* Separate file holding temporary table */
  i64 seqCount;           /* Sequence counter */
  u32 *aAltMap;           /* Mapping from table to index column numbers */
@@ -475,6 +475,7 @@ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
  int j;
  sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
  for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
+  sqlite3MayAbort(p->pParse);
 }

 /*
@@ -703,7 +704,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
    if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename 
     || opcode==OP_VDestroy
     || opcode==OP_VCreate
-     || (opcode==OP_ParseSchema && pOp->p4.z==0)
+     || opcode==OP_ParseSchema
     || ((opcode==OP_Halt || opcode==OP_HaltIfNull) 
      && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort))
    ){
@@ -1521,7 +1522,7 @@ char *sqlite3VdbeDisplayComment(
              sqlite3_str_appendf(&x, "%d", v1);
            }else if( pCtx->argc>1 ){
              sqlite3_str_appendf(&x, "%d..%d", v1, v1+pCtx->argc-1);
-            }else{
+            }else if( x.accError==0 ){
              assert( x.nChar>2 );
              x.nChar -= 2;
              ii++;
@@ -2663,7 +2664,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
  */ 
  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ 
    Btree *pBt = db->aDb[i].pBt;
-    if( sqlite3BtreeIsInTrans(pBt) ){
+    if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
      /* Whether or not a database might need a super-journal depends upon
      ** its journal mode (among other things).  This matrix determines which
      ** journal modes use a super-journal and which do not */
@@ -2798,7 +2799,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
    */
    for(i=0; i<db->nDb; i++){
      Btree *pBt = db->aDb[i].pBt;
-      if( sqlite3BtreeIsInTrans(pBt) ){
+      if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
        char const *zFile = sqlite3BtreeGetJournalname(pBt);
        if( zFile==0 ){
          continue;  /* Ignore TEMP and :memory: databases */
@@ -4294,9 +4295,12 @@ SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){
 static int sqlite3IntFloatCompare(i64 i, double r){
  if( sizeof(LONGDOUBLE_TYPE)>8 ){
    LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i;
+    testcase( x<r );
+    testcase( x>r );
+    testcase( x==r );
    if( x<r ) return -1;
-    if( x>r ) return +1;
-    return 0;
+    if( x>r ) return +1;  /*NO_TEST*/ /* work around bugs in gcov */
+    return 0;             /*NO_TEST*/ /* work around bugs in gcov */
  }else{
    i64 y;
    double s;
@@ -970,13 +970,16 @@ int sqlite3VdbeSorterInit(
  if( pSorter==0 ){
    rc = SQLITE_NOMEM_BKPT;
  }else{
+    Btree *pBt = db->aDb[0].pBt;
    pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
    memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
    pKeyInfo->db = 0;
    if( nField && nWorker==0 ){
      pKeyInfo->nKeyField = nField;
    }
-    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
+    sqlite3BtreeEnter(pBt);
+    pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt);
+    sqlite3BtreeLeave(pBt);
    pSorter->nTask = nWorker + 1;
    pSorter->iPrev = (u8)(nWorker - 1);
    pSorter->bUseThreads = (pSorter->nTask>1);
@@ -235,6 +235,16 @@ static void createMask(WhereMaskSet *pMaskSet, int iCursor){
  pMaskSet->ix[pMaskSet->n++] = iCursor;
 }

+/*
+** If the right-hand branch of the expression is a TK_COLUMN, then return
+** a pointer to the right-hand branch.  Otherwise, return NULL.
+*/
+static Expr *whereRightSubexprIsColumn(Expr *p){
+  p = sqlite3ExprSkipCollateAndLikely(p->pRight);
+  if( ALWAYS(p!=0) && p->op==TK_COLUMN ) return p;
+  return 0;
+}
+
 /*
 ** Advance to the next WhereTerm that matches according to the criteria
 ** established when the pScan object was initialized by whereScanInit().
@@ -257,7 +267,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
    do{
      for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
        if( pTerm->leftCursor==iCur
-         && pTerm->u.leftColumn==iColumn
+         && pTerm->u.x.leftColumn==iColumn
         && (iColumn!=XN_EXPR
             || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
                                       pScan->pIdxExpr,iCur)==0)
@@ -265,8 +275,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
        ){
          if( (pTerm->eOperator & WO_EQUIV)!=0
           && pScan->nEquiv<ArraySize(pScan->aiCur)
-           && (pX = sqlite3ExprSkipCollateAndLikely(pTerm->pExpr->pRight))->op
-               ==TK_COLUMN
+           && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
          ){
            int j;
            for(j=0; j<pScan->nEquiv; j++){
@@ -462,7 +471,8 @@ static int findIndexCol(

  for(i=0; i<pList->nExpr; i++){
    Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
-    if( p->op==TK_COLUMN
+    if( ALWAYS(p!=0)
+     && p->op==TK_COLUMN
     && p->iColumn==pIdx->aiColumn[iCol]
     && p->iTable==iBase
    ){
@@ -526,6 +536,7 @@ static int isDistinctRedundant(
  */
  for(i=0; i<pDistinct->nExpr; i++){
    Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
+    if( NEVER(p==0) ) continue;
    if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
  }

@@ -679,8 +690,8 @@ static int termCanDriveIndex(
    return 0;
  }
  if( (pTerm->prereqRight & notReady)!=0 ) return 0;
-  if( pTerm->u.leftColumn<0 ) return 0;
-  aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity;
+  if( pTerm->u.x.leftColumn<0 ) return 0;
+  aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
  if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
  testcase( pTerm->pExpr->op==TK_IS );
  return 1;
@@ -751,7 +762,7 @@ static void constructAutomaticIndex(
                                sqlite3ExprDup(pParse->db, pExpr, 0));
    }
    if( termCanDriveIndex(pTerm, pSrc, notReady) ){
-      int iCol = pTerm->u.leftColumn;
+      int iCol = pTerm->u.x.leftColumn;
      Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
      testcase( iCol==BMS );
      testcase( iCol==BMS-1 );
@@ -804,14 +815,14 @@ static void constructAutomaticIndex(
  idxCols = 0;
  for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
    if( termCanDriveIndex(pTerm, pSrc, notReady) ){
-      int iCol = pTerm->u.leftColumn;
+      int iCol = pTerm->u.x.leftColumn;
      Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
      testcase( iCol==BMS-1 );
      testcase( iCol==BMS );
      if( (idxCols & cMask)==0 ){
        Expr *pX = pTerm->pExpr;
        idxCols |= cMask;
-        pIdx->aiColumn[n] = pTerm->u.leftColumn;
+        pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
        pColl = sqlite3ExprCompareCollSeq(pParse, pX);
        assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
        pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
@@ -932,7 +943,7 @@ static sqlite3_index_info *allocateIndexInfo(
    testcase( pTerm->eOperator & WO_ALL );
    if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
    if( pTerm->wtFlags & TERM_VNULL ) continue;
-    assert( pTerm->u.leftColumn>=(-1) );
+    assert( pTerm->u.x.leftColumn>=(-1) );
    nTerm++;
  }

@@ -992,8 +1003,8 @@ static sqlite3_index_info *allocateIndexInfo(
    ){
      continue;
    }
-    assert( pTerm->u.leftColumn>=(-1) );
-    pIdxCons[j].iColumn = pTerm->u.leftColumn;
+    assert( pTerm->u.x.leftColumn>=(-1) );
+    pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
    pIdxCons[j].iTermOffset = i;
    op = pTerm->eOperator & WO_ALL;
    if( op==WO_IN ) op = WO_EQ;
@@ -1756,9 +1767,9 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
    if( pTerm->wtFlags & TERM_CODED  ) zType[3] = 'C';
    if( pTerm->eOperator & WO_SINGLE ){
      sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
-                       pTerm->leftCursor, pTerm->u.leftColumn);
+                       pTerm->leftCursor, pTerm->u.x.leftColumn);
    }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
-      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", 
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx", 
                       pTerm->u.pOrInfo->indexable);
    }else{
      sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
@@ -1772,8 +1783,8 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
      sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
        pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
    }
-    if( pTerm->iField ){
-      sqlite3DebugPrintf(" iField=%d", pTerm->iField);
+    if( pTerm->u.x.iField ){
+      sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
    }
    if( pTerm->iParent>=0 ){
      sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
@@ -2440,9 +2451,9 @@ static int whereLoopAddBtreeIndex(

  pNew = pBuilder->pNew;
  if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
-  WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d\n",
+  WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
                     pProbe->pTable->zName,pProbe->zName,
-                     pNew->u.btree.nEq, pNew->nSkip));
+                     pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));

  assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
  assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
@@ -2536,7 +2547,7 @@ static int whereLoopAddBtreeIndex(
        /* "x IN (value, value, ...)" */
        nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
      }
-      if( pProbe->hasStat1 ){
+      if( pProbe->hasStat1 && rLogSize>=10 ){
        LogEst M, logK, safetyMargin;
        /* Let:
        **   N = the total number of rows in the table
@@ -2555,7 +2566,8 @@ static int whereLoopAddBtreeIndex(
        ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
        ** with the index, as using an index has better worst-case behavior.
        ** If we do not have real sqlite_stat1 data, always prefer to use
-        ** the index.
+        ** the index.  Do not bother with this optimization on very small
+        ** tables (less than 2 rows) as it is pointless in that case.
        */
        M = pProbe->aiRowLogEst[saved_nEq];
        logK = estLog(nIn);
@@ -2564,7 +2576,7 @@ static int whereLoopAddBtreeIndex(
          WHERETRACE(0x40,
            ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n",
             saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
-          continue;
+          pNew->wsFlags |= WHERE_IN_SEEKSCAN;
        }else{
          WHERETRACE(0x40,
            ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n",
@@ -2810,6 +2822,7 @@ static int indexMightHelpWithOrderBy(
  if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
  for(ii=0; ii<pOB->nExpr; ii++){
    Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
+    if( NEVER(pExpr==0) ) continue;
    if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
      if( pExpr->iColumn<0 ) return 1;
      for(jj=0; jj<pIndex->nKeyCol; jj++){
@@ -3041,8 +3054,23 @@ static int whereLoopAddBtree(

      /* Full table scan */
      pNew->iSortIdx = b ? iSortIdx : 0;
-      /* TUNING: Cost of full table scan is (N*3.0). */
+      /* TUNING: Cost of full table scan is 3.0*N.  The 3.0 factor is an
+      ** extra cost designed to discourage the use of full table scans,
+      ** since index lookups have better worst-case performance if our
+      ** stat guesses are wrong.  Reduce the 3.0 penalty slightly
+      ** (to 2.75) if we have valid STAT4 information for the table.
+      ** At 2.75, a full table scan is preferred over using an index on
+      ** a column with just two distinct values where each value has about
+      ** an equal number of appearances.  Without STAT4 data, we still want
+      ** to use an index in that case, since the constraint might be for
+      ** the scarcer of the two values, and in that case an index lookup is
+      ** better.
+      */
+#ifdef SQLITE_ENABLE_STAT4
+      pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
+#else
      pNew->rRun = rSize + 16;
+#endif
      ApplyCostMultiplier(pNew->rRun, pTab->costMult);
      whereLoopOutputAdjust(pWC, pNew, rSize);
      rc = whereLoopInsert(pBuilder, pNew);
@@ -3773,6 +3801,7 @@ static i8 wherePathSatisfiesOrderBy(
    for(i=0; i<nOrderBy; i++){
      if( MASKBIT(i) & obSat ) continue;
      pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
+      if( NEVER(pOBExpr==0) ) continue;
      if( pOBExpr->op!=TK_COLUMN ) continue;
      if( pOBExpr->iTable!=iCur ) continue;
      pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
@@ -3899,6 +3928,7 @@ static i8 wherePathSatisfiesOrderBy(
          pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
          testcase( wctrlFlags & WHERE_GROUPBY );
          testcase( wctrlFlags & WHERE_DISTINCTBY );
+          if( NEVER(pOBExpr==0) ) continue;
          if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
          if( iColumn>=XN_ROWID ){
            if( pOBExpr->op!=TK_COLUMN ) continue;
@@ -4053,16 +4083,24 @@ static LogEst whereSortingCost(
  **   cost = (3.0 * N * log(N)) * (Y/X)
  **
  ** The (Y/X) term is implemented using stack variable rScale
-  ** below.  */
+  ** below.
+  */
  LogEst rScale, rSortCost;
  assert( nOrderBy>0 && 66==sqlite3LogEst(100) );
  rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
  rSortCost = nRow + rScale + 16;

  /* Multiple by log(M) where M is the number of output rows.
-  ** Use the LIMIT for M if it is smaller */
+  ** Use the LIMIT for M if it is smaller.  Or if this sort is for
+  ** a DISTINCT operator, M will be the number of distinct output
+  ** rows, so fudge it downwards a bit.
+  */
  if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){
    nRow = pWInfo->iLimit;
+  }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
+    /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
+    ** reduces the number of output rows by a factor of 2 */
+    if( nRow>10 ) nRow -= 10;  assert( 10==sqlite3LogEst(2) );
  }
  rSortCost += estLog(nRow);
  return rSortCost;
@@ -5189,6 +5227,7 @@ WhereInfo *sqlite3WhereBegin(
        if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
         && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
         && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
+         && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
         && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
         && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
        ){
@@ -5246,6 +5285,7 @@ WhereInfo *sqlite3WhereBegin(

  /* Done. */
  VdbeModuleComment((v, "Begin WHERE-core"));
+  pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
  return pWInfo;

  /* Jump here if malloc fails */
@@ -5289,6 +5329,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
  WhereLoop *pLoop;
  SrcList *pTabList = pWInfo->pTabList;
  sqlite3 *db = pParse->db;
+  int iEnd = sqlite3VdbeCurrentAddr(v);

  /* Generate loop termination code.
  */
@@ -5349,7 +5390,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
        sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
        if( pIn->eEndLoopOp!=OP_Noop ){
          if( pIn->nPrefix ){
-            assert( pLoop->wsFlags & WHERE_IN_EARLYOUT );
+            int bEarlyOut = 
+                (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
+                 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
            if( pLevel->iLeftJoin ){
              /* For LEFT JOIN queries, cursor pIn->iCur may not have been
              ** opened yet. This occurs for WHERE clauses such as
@@ -5360,12 +5403,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
              ** jump over the OP_Next or OP_Prev instruction about to
              ** be coded.  */
              sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, 
-                  sqlite3VdbeCurrentAddr(v) + 2 + 
-                     ((pLoop->wsFlags & WHERE_VIRTUALTABLE)==0)
-              );
+                  sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
              VdbeCoverage(v);
            }
-            if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ){
+            if( bEarlyOut ){
              sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
                  sqlite3VdbeCurrentAddr(v)+2,
                  pIn->iBase, pIn->nPrefix);
@@ -5426,7 +5467,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
  assert( pWInfo->nLevel<=pTabList->nSrc );
  for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
    int k, last;
-    VdbeOp *pOp;
+    VdbeOp *pOp, *pLastOp;
    Index *pIdx = 0;
    struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom];
    Table *pTab = pTabItem->pTab;
@@ -5484,20 +5525,31 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
      pIdx = pLevel->u.pCovidx;
    }
    if( pIdx
-     && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable))
     && !db->mallocFailed
    ){
-      last = sqlite3VdbeCurrentAddr(v);
-      k = pLevel->addrBody;
+      if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
+        last = iEnd;
+      }else{
+        last = pWInfo->iEndWhere;
+      }
+      k = pLevel->addrBody + 1;
 #ifdef SQLITE_DEBUG
      if( db->flags & SQLITE_VdbeAddopTrace ){
        printf("TRANSLATE opcodes in range %d..%d\n", k, last-1);
      }
+      /* Proof that the "+1" on the k value above is safe */
+      pOp = sqlite3VdbeGetOp(v, k - 1);
+      assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
+      assert( pOp->opcode!=OP_Rowid  || pOp->p1!=pLevel->iTabCur );
+      assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
 #endif
      pOp = sqlite3VdbeGetOp(v, k);
-      for(; k<last; k++, pOp++){
-        if( pOp->p1!=pLevel->iTabCur ) continue;
-        if( pOp->opcode==OP_Column
+      pLastOp = pOp + (last - k);
+      assert( pOp<pLastOp );
+      do{
+        if( pOp->p1!=pLevel->iTabCur ){
+          /* no-op */
+        }else if( pOp->opcode==OP_Column
 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
         || pOp->opcode==OP_Offset
 #endif
@@ -5528,7 +5580,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
          pOp->p1 = pLevel->iIdxCur;
          OpcodeRewriteTrace(db, k, pOp);
        }
-      }
+#ifdef SQLITE_DEBUG
+        k++;
+#endif
+      }while( (++pOp)<pLastOp );
 #ifdef SQLITE_DEBUG
      if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
 #endif
@@ -261,9 +261,11 @@ struct WhereTerm {
  u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
  int iParent;            /* Disable pWC->a[iParent] when this term disabled */
  int leftCursor;         /* Cursor number of X in "X <op> <expr>" */
-  int iField;             /* Field in (?,?,?) IN (SELECT...) vector */
  union {
-    int leftColumn;         /* Column number of X in "X <op> <expr>" */
+    struct {
+      int leftColumn;         /* Column number of X in "X <op> <expr>" */
+      int iField;             /* Field in (?,?,?) IN (SELECT...) vector */
+    } x;                    /* Opcode other than OP_OR or OP_AND */
    WhereOrInfo *pOrInfo;   /* Extra information if (eOperator & WO_OR)!=0 */
    WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */
  } u;
@@ -488,6 +490,7 @@ struct WhereInfo {
  unsigned sorted :1;          /* True if really sorted (not just grouped) */
  LogEst nRowOut;           /* Estimated number of output rows */
  int iTop;                 /* The very beginning of the WHERE loop */
+  int iEndWhere;            /* End of the WHERE clause itself */
  WhereLoop *pLoops;        /* List of all WhereLoop objects */
  WhereExprMod *pExprMods;  /* Expression modifications */
  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
@@ -616,5 +619,6 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 #define WHERE_PARTIALIDX   0x00020000  /* The automatic index is partial */
 #define WHERE_IN_EARLYOUT  0x00040000  /* Perhaps quit IN loops early */
 #define WHERE_BIGNULL_SORT 0x00080000  /* Column nEq of index is BIGNULL */
+#define WHERE_IN_SEEKSCAN  0x00100000  /* Seek-scan optimization for IN */

 #endif /* !defined(SQLITE_WHEREINT_H) */
@@ -427,7 +427,7 @@ static Expr *removeUnindexableInClauseTerms(

    for(i=iEq; i<pLoop->nLTerm; i++){
      if( pLoop->aLTerm[i]->pExpr==pX ){
-        int iField = pLoop->aLTerm[i]->iField - 1;
+        int iField = pLoop->aLTerm[i]->u.x.iField - 1;
        if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */
        pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr);
        pOrigRhs->a[iField].pExpr = 0;
@@ -570,6 +570,9 @@ static int codeEqualityTerm(
    if( pLevel->u.in.nIn==0 ){
      pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse);
    }
+    if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){
+      pLoop->wsFlags |= WHERE_IN_EARLYOUT;
+    }

    i = pLevel->u.in.nIn;
    pLevel->u.in.nIn += nEq;
@@ -596,7 +599,6 @@ static int codeEqualityTerm(
            if( iEq>0 ){
              pIn->iBase = iReg - i;
              pIn->nPrefix = i;
-              pLoop->wsFlags |= WHERE_IN_EARLYOUT;
            }else{
              pIn->nPrefix = 0;
            }
@@ -606,6 +608,14 @@ static int codeEqualityTerm(
          pIn++;
        }
      }
+      testcase( iEq>0
+                && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
+                && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 );
+      if( iEq>0 
+       && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0
+      ){
+        sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq);
+      }
    }else{
      pLevel->u.in.nIn = 0;
    }
@@ -1392,6 +1402,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                      pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC);
    VdbeCoverage(v);
    pLoop->u.vtab.needFree = 0;
+    /* An OOM inside of AddOp4(OP_VFilter) instruction above might have freed
+    ** the u.vtab.idxStr.  NULL it out to prevent a use-after-free */
+    if( db->mallocFailed ) pLoop->u.vtab.idxStr = 0;
    pLevel->p1 = iCur;
    pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext;
    pLevel->p2 = sqlite3VdbeCurrentAddr(v);
@@ -1650,6 +1663,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    u8 bStopAtNull = 0;          /* Add condition to terminate at NULLs */
    int omitTable;               /* True if we use the index only */
    int regBignull = 0;          /* big-null flag register */
+    int addrSeekScan = 0;        /* Opcode of the OP_SeekScan, if any */

    pIdx = pLoop->u.btree.pIndex;
    iIdxCur = pLevel->iIdxCur;
@@ -1788,9 +1802,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      ** above has already left the cursor sitting on the correct row,
      ** so no further seeking is needed */
    }else{
-      if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){
-        sqlite3VdbeAddOp1(v, OP_SeekHit, iIdxCur);
-      }
      if( regBignull ){
        sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull);
        VdbeComment((v, "NULL-scan pass ctr"));
@@ -1798,6 +1809,20 @@ Bitmask sqlite3WhereCodeOneLoopStart(

      op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
      assert( op!=0 );
+      if( (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 && op==OP_SeekGE ){
+        assert( regBignull==0 );
+        /* TUNING:  The OP_SeekScan opcode seeks to reduce the number
+        ** of expensive seek operations by replacing a single seek with
+        ** 1 or more step operations.  The question is, how many steps
+        ** should we try before giving up and going with a seek.  The cost
+        ** of a seek is proportional to the logarithm of the of the number
+        ** of entries in the tree, so basing the number of steps to try
+        ** on the estimated number of rows in the btree seems like a good
+        ** guess. */
+        addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, 
+                                         (pIdx->aiRowLogEst[0]+9)/10);
+        VdbeCoverage(v);
+      }
      sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
      VdbeCoverage(v);
      VdbeCoverageIf(v, op==OP_Rewind);  testcase( op==OP_Rewind );
@@ -1880,6 +1905,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      testcase( op==OP_IdxGE );  VdbeCoverageIf(v, op==OP_IdxGE );
      testcase( op==OP_IdxLT );  VdbeCoverageIf(v, op==OP_IdxLT );
      testcase( op==OP_IdxLE );  VdbeCoverageIf(v, op==OP_IdxLE );
+      if( addrSeekScan ) sqlite3VdbeJumpHere(v, addrSeekScan);
    }
    if( regBignull ){
      /* During a NULL-scan, check to see if we have reached the end of
@@ -1899,8 +1925,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
      testcase( op==OP_IdxLE );  VdbeCoverageIf(v, op==OP_IdxLE );
    }

-    if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){
-      sqlite3VdbeAddOp2(v, OP_SeekHit, iIdxCur, 1);
+    if( (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0 ){
+      sqlite3VdbeAddOp3(v, OP_SeekHit, iIdxCur, nEq, nEq);
    }

    /* Seek the table cursor, if required */
@@ -1909,17 +1935,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    if( omitTable ){
      /* pIdx is a covering index.  No need to access the main table. */
    }else if( HasRowid(pIdx->pTable) ){
-      if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)
-       || ( (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE)!=0
-           && (pWInfo->eOnePass==ONEPASS_SINGLE || pLoop->nLTerm==0) )
-      ){
-        iRowidReg = ++pParse->nMem;
-        sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
-        sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg);
-        VdbeCoverage(v);
-      }else{
-        codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur);
-      }
+      codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur);
    }else if( iCur!=iIdxCur ){
      Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
      iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol);
@@ -2046,7 +2062,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    int iRetInit;                             /* Address of regReturn init */
    int untestedTerms = 0;             /* Some terms not completely tested */
    int ii;                            /* Loop counter */
-    u16 wctrlFlags;                    /* Flags for sub-WHERE clause */
    Expr *pAndExpr = 0;                /* An ".. AND (...)" expression */
    Table *pTab = pTabItem->pTab;

@@ -2147,7 +2162,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
    ** eliminating duplicates from other WHERE clauses, the action for each
    ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
    */
-    wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
    ExplainQueryPlan((pParse, 1, "MULTI-INDEX OR"));
    for(ii=0; ii<pOrWc->nTerm; ii++){
      WhereTerm *pOrTerm = &pOrWc->a[ii];
@@ -2166,7 +2180,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
        ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1));
        WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
-                                      wctrlFlags, iCovCur);
+                                      WHERE_OR_SUBCLAUSE, iCovCur);
        assert( pSubWInfo || pParse->nErr || db->mallocFailed );
        if( pSubWInfo ){
          WhereLoop *pSubLoop;
@@ -2264,6 +2278,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
          }else{
            pCov = 0;
          }
+          if( sqlite3WhereUsesDeferredSeek(pSubWInfo) ){
+            pWInfo->bDeferredSeek = 1;
+          }

          /* Finish the loop through table entries that match term pOrTerm. */
          sqlite3WhereEnd(pSubWInfo);
@@ -2416,7 +2433,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
 #endif
    assert( !ExprHasProperty(pE, EP_FromJoin) );
    assert( (pTerm->prereqRight & pLevel->notReady)!=0 );
-    pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady,
+    pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.x.leftColumn, notReady,
                    WO_EQ|WO_IN|WO_IS, 0);
    if( pAlt==0 ) continue;
    if( pAlt->wtFlags & (TERM_CODED) ) continue;
@@ -798,7 +798,7 @@ static void exprAnalyzeOrTerm(
          assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) );
          continue;
        }
-        iColumn = pOrTerm->u.leftColumn;
+        iColumn = pOrTerm->u.x.leftColumn;
        iCursor = pOrTerm->leftCursor;
        pLeft = pOrTerm->pExpr->pLeft;
        break;
@@ -820,7 +820,7 @@ static void exprAnalyzeOrTerm(
        assert( pOrTerm->eOperator & WO_EQ );
        if( pOrTerm->leftCursor!=iCursor ){
          pOrTerm->wtFlags &= ~TERM_OR_OK;
-        }else if( pOrTerm->u.leftColumn!=iColumn || (iColumn==XN_EXPR 
+        }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR 
               && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1)
        )){
          okToChngToIN = 0;
@@ -855,7 +855,7 @@ static void exprAnalyzeOrTerm(
        if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue;
        assert( pOrTerm->eOperator & WO_EQ );
        assert( pOrTerm->leftCursor==iCursor );
-        assert( pOrTerm->u.leftColumn==iColumn );
+        assert( pOrTerm->u.x.leftColumn==iColumn );
        pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0);
        pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup);
        pLeft = pOrTerm->pExpr->pLeft;
@@ -1091,15 +1091,15 @@ static void exprAnalyze(
    Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
    u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;

-    if( pTerm->iField>0 ){
+    if( pTerm->u.x.iField>0 ){
      assert( op==TK_IN );
      assert( pLeft->op==TK_VECTOR );
-      pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr;
+      pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr;
    }

    if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){
      pTerm->leftCursor = aiCurCol[0];
-      pTerm->u.leftColumn = aiCurCol[1];
+      pTerm->u.x.leftColumn = aiCurCol[1];
      pTerm->eOperator = operatorMask(op) & opMask;
    }
    if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
@@ -1109,7 +1109,7 @@ static void exprAnalyze(
      WhereTerm *pNew;
      Expr *pDup;
      u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
-      assert( pTerm->iField==0 );
+      assert( pTerm->u.x.iField==0 );
      if( pTerm->leftCursor>=0 ){
        int idxNew;
        pDup = sqlite3ExprDup(db, pExpr, 0);
@@ -1135,7 +1135,7 @@ static void exprAnalyze(
      }
      pNew->wtFlags |= exprCommute(pParse, pDup);
      pNew->leftCursor = aiCurCol[0];
-      pNew->u.leftColumn = aiCurCol[1];
+      pNew->u.x.leftColumn = aiCurCol[1];
      testcase( (prereqLeft | extraRight) != prereqLeft );
      pNew->prereqRight = prereqLeft | extraRight;
      pNew->prereqAll = prereqAll;
@@ -1309,7 +1309,7 @@ static void exprAnalyze(
        pNewTerm = &pWC->a[idxNew];
        pNewTerm->prereqRight = prereqExpr;
        pNewTerm->leftCursor = pLeft->iTable;
-        pNewTerm->u.leftColumn = pLeft->iColumn;
+        pNewTerm->u.x.leftColumn = pLeft->iColumn;
        pNewTerm->eOperator = WO_AUX;
        pNewTerm->eMatchOp = eOp2;
        markTermAsChild(pWC, idxNew, idxTerm);
@@ -1356,13 +1356,13 @@ static void exprAnalyze(
  /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create
  ** a virtual term for each vector component. The expression object
  ** used by each such virtual term is pExpr (the full vector IN(...) 
-  ** expression). The WhereTerm.iField variable identifies the index within
+  ** expression). The WhereTerm.u.x.iField variable identifies the index within
  ** the vector on the LHS that the virtual term represents.
  **
  ** This only works if the RHS is a simple SELECT (not a compound) that does
  ** not use window functions.
  */
-  if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0
+  if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->u.x.iField==0
   && pExpr->pLeft->op==TK_VECTOR
   && pExpr->x.pSelect->pPrior==0
 #ifndef SQLITE_OMIT_WINDOWFUNC
@@ -1373,7 +1373,7 @@ static void exprAnalyze(
    for(i=0; i<sqlite3ExprVectorSize(pExpr->pLeft); i++){
      int idxNew;
      idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL);
-      pWC->a[idxNew].iField = i+1;
+      pWC->a[idxNew].u.x.iField = i+1;
      exprAnalyze(pSrc, pWC, idxNew);
      markTermAsChild(pWC, idxNew, idxTerm);
    }
@@ -1408,7 +1408,7 @@ static void exprAnalyze(
      pNewTerm = &pWC->a[idxNew];
      pNewTerm->prereqRight = 0;
      pNewTerm->leftCursor = pLeft->iTable;
-      pNewTerm->u.leftColumn = pLeft->iColumn;
+      pNewTerm->u.x.leftColumn = pLeft->iColumn;
      pNewTerm->eOperator = WO_GT;
      markTermAsChild(pWC, idxNew, idxTerm);
      pTerm = &pWC->a[idxTerm];
@@ -1451,6 +1451,7 @@ static void exprAnalyze(
 void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){
  Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pExpr);
  pWC->op = op;
+  assert( pE2!=0 || pExpr==0 );
  if( pE2==0 ) return;
  if( pE2->op!=op ){
    whereClauseInsert(pWC, pExpr, 0);