231 lines
6.4 KiB
C
231 lines
6.4 KiB
C
/*
|
|
** 2006 Oct 10
|
|
**
|
|
** The author disclaims copyright to this source code. In place of
|
|
** a legal notice, here is a blessing:
|
|
**
|
|
** May you do good and not evil.
|
|
** May you find forgiveness for yourself and forgive others.
|
|
** May you share freely, never taking more than you give.
|
|
**
|
|
******************************************************************************
|
|
**
|
|
** Implementation of the "simple" full-text-search tokenizer.
|
|
*/
|
|
|
|
/*
|
|
** The code in this file is only compiled if:
|
|
**
|
|
** * The FTS2 module is being built as an extension
|
|
** (in which case SQLITE_CORE is not defined), or
|
|
**
|
|
** * The FTS2 module is being built into the core of
|
|
** SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
|
|
*/
|
|
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
|
|
|
|
|
|
#include <assert.h>
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <ctype.h>
|
|
|
|
#include "fts2_tokenizer.h"
|
|
|
|
typedef struct simple_tokenizer {
|
|
sqlite3_tokenizer base;
|
|
char delim[128]; /* flag ASCII delimiters */
|
|
} simple_tokenizer;
|
|
|
|
typedef struct simple_tokenizer_cursor {
|
|
sqlite3_tokenizer_cursor base;
|
|
const char *pInput; /* input we are tokenizing */
|
|
int nBytes; /* size of the input */
|
|
int iOffset; /* current position in pInput */
|
|
int iToken; /* index of next token to be returned */
|
|
char *pToken; /* storage for current token */
|
|
int nTokenAllocated; /* space allocated to zToken buffer */
|
|
} simple_tokenizer_cursor;
|
|
|
|
|
|
/* Forward declaration */
|
|
static const sqlite3_tokenizer_module simpleTokenizerModule;
|
|
|
|
static int simpleDelim(simple_tokenizer *t, unsigned char c){
|
|
return c<0x80 && t->delim[c];
|
|
}
|
|
|
|
/*
|
|
** Create a new tokenizer instance.
|
|
*/
|
|
static int simpleCreate(
|
|
int argc, const char * const *argv,
|
|
sqlite3_tokenizer **ppTokenizer
|
|
){
|
|
simple_tokenizer *t;
|
|
|
|
t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
|
|
if( t==NULL ) return SQLITE_NOMEM;
|
|
memset(t, 0, sizeof(*t));
|
|
|
|
/* TODO(shess) Delimiters need to remain the same from run to run,
|
|
** else we need to reindex. One solution would be a meta-table to
|
|
** track such information in the database, then we'd only want this
|
|
** information on the initial create.
|
|
*/
|
|
if( argc>1 ){
|
|
int i, n = strlen(argv[1]);
|
|
for(i=0; i<n; i++){
|
|
unsigned char ch = argv[1][i];
|
|
/* We explicitly don't support UTF-8 delimiters for now. */
|
|
if( ch>=0x80 ){
|
|
sqlite3_free(t);
|
|
return SQLITE_ERROR;
|
|
}
|
|
t->delim[ch] = 1;
|
|
}
|
|
} else {
|
|
/* Mark non-alphanumeric ASCII characters as delimiters */
|
|
int i;
|
|
for(i=1; i<0x80; i++){
|
|
t->delim[i] = !isalnum(i);
|
|
}
|
|
}
|
|
|
|
*ppTokenizer = &t->base;
|
|
return SQLITE_OK;
|
|
}
|
|
|
|
/*
|
|
** Destroy a tokenizer
|
|
*/
|
|
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
|
|
sqlite3_free(pTokenizer);
|
|
return SQLITE_OK;
|
|
}
|
|
|
|
/*
|
|
** Prepare to begin tokenizing a particular string. The input
|
|
** string to be tokenized is pInput[0..nBytes-1]. A cursor
|
|
** used to incrementally tokenize this string is returned in
|
|
** *ppCursor.
|
|
*/
|
|
static int simpleOpen(
|
|
sqlite3_tokenizer *pTokenizer, /* The tokenizer */
|
|
const char *pInput, int nBytes, /* String to be tokenized */
|
|
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
|
|
){
|
|
simple_tokenizer_cursor *c;
|
|
|
|
c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
|
|
if( c==NULL ) return SQLITE_NOMEM;
|
|
|
|
c->pInput = pInput;
|
|
if( pInput==0 ){
|
|
c->nBytes = 0;
|
|
}else if( nBytes<0 ){
|
|
c->nBytes = (int)strlen(pInput);
|
|
}else{
|
|
c->nBytes = nBytes;
|
|
}
|
|
c->iOffset = 0; /* start tokenizing at the beginning */
|
|
c->iToken = 0;
|
|
c->pToken = NULL; /* no space allocated, yet. */
|
|
c->nTokenAllocated = 0;
|
|
|
|
*ppCursor = &c->base;
|
|
return SQLITE_OK;
|
|
}
|
|
|
|
/*
|
|
** Close a tokenization cursor previously opened by a call to
|
|
** simpleOpen() above.
|
|
*/
|
|
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
|
|
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
|
|
sqlite3_free(c->pToken);
|
|
sqlite3_free(c);
|
|
return SQLITE_OK;
|
|
}
|
|
|
|
/*
|
|
** Extract the next token from a tokenization cursor. The cursor must
|
|
** have been opened by a prior call to simpleOpen().
|
|
*/
|
|
static int simpleNext(
|
|
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */
|
|
const char **ppToken, /* OUT: *ppToken is the token text */
|
|
int *pnBytes, /* OUT: Number of bytes in token */
|
|
int *piStartOffset, /* OUT: Starting offset of token */
|
|
int *piEndOffset, /* OUT: Ending offset of token */
|
|
int *piPosition /* OUT: Position integer of token */
|
|
){
|
|
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
|
|
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
|
|
unsigned char *p = (unsigned char *)c->pInput;
|
|
|
|
while( c->iOffset<c->nBytes ){
|
|
int iStartOffset;
|
|
|
|
/* Scan past delimiter characters */
|
|
while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
|
|
c->iOffset++;
|
|
}
|
|
|
|
/* Count non-delimiter characters. */
|
|
iStartOffset = c->iOffset;
|
|
while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
|
|
c->iOffset++;
|
|
}
|
|
|
|
if( c->iOffset>iStartOffset ){
|
|
int i, n = c->iOffset-iStartOffset;
|
|
if( n>c->nTokenAllocated ){
|
|
c->nTokenAllocated = n+20;
|
|
c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
|
|
if( c->pToken==NULL ) return SQLITE_NOMEM;
|
|
}
|
|
for(i=0; i<n; i++){
|
|
/* TODO(shess) This needs expansion to handle UTF-8
|
|
** case-insensitivity.
|
|
*/
|
|
unsigned char ch = p[iStartOffset+i];
|
|
c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
|
|
}
|
|
*ppToken = c->pToken;
|
|
*pnBytes = n;
|
|
*piStartOffset = iStartOffset;
|
|
*piEndOffset = c->iOffset;
|
|
*piPosition = c->iToken++;
|
|
|
|
return SQLITE_OK;
|
|
}
|
|
}
|
|
return SQLITE_DONE;
|
|
}
|
|
|
|
/*
|
|
** The set of routines that implement the simple tokenizer
|
|
*/
|
|
static const sqlite3_tokenizer_module simpleTokenizerModule = {
|
|
0,
|
|
simpleCreate,
|
|
simpleDestroy,
|
|
simpleOpen,
|
|
simpleClose,
|
|
simpleNext,
|
|
};
|
|
|
|
/*
|
|
** Allocate a new simple tokenizer. Return a pointer to the new
|
|
** tokenizer in *ppModule
|
|
*/
|
|
void sqlite3Fts2SimpleTokenizerModule(
|
|
sqlite3_tokenizer_module const**ppModule
|
|
){
|
|
*ppModule = &simpleTokenizerModule;
|
|
}
|
|
|
|
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
|