1 /* 2 ** The author disclaims copyright to this source code. 3 ** 4 ************************************************************************* 5 ** Implementation of the "simple" full-text-search tokenizer. 6 */ 7 8 #include <assert.h> 9 #if !defined(__APPLE__) 10 #include <malloc.h> 11 #else 12 #include <stdlib.h> 13 #endif 14 #include <stdio.h> 15 #include <string.h> 16 #include <ctype.h> 17 18 #include "tokenizer.h" 19 20 /* Duplicate a string; the caller must free() the returned string. 21 * (We don't use strdup() since it's not part of the standard C library and 22 * may not be available everywhere.) */ 23 /* TODO(shess) Copied from fulltext.c, consider util.c for such 24 ** things. */ 25 static char *string_dup(const char *s){ 26 char *str = malloc(strlen(s) + 1); 27 strcpy(str, s); 28 return str; 29 } 30 31 typedef struct simple_tokenizer { 32 sqlite3_tokenizer base; 33 const char *zDelim; /* token delimiters */ 34 } simple_tokenizer; 35 36 typedef struct simple_tokenizer_cursor { 37 sqlite3_tokenizer_cursor base; 38 const char *pInput; /* input we are tokenizing */ 39 int nBytes; /* size of the input */ 40 const char *pCurrent; /* current position in pInput */ 41 int iToken; /* index of next token to be returned */ 42 char *zToken; /* storage for current token */ 43 int nTokenBytes; /* actual size of current token */ 44 int nTokenAllocated; /* space allocated to zToken buffer */ 45 } simple_tokenizer_cursor; 46 47 static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */ 48 49 static int simpleCreate( 50 int argc, const char **argv, 51 sqlite3_tokenizer **ppTokenizer 52 ){ 53 simple_tokenizer *t; 54 55 t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer)); 56 /* TODO(shess) Delimiters need to remain the same from run to run, 57 ** else we need to reindex. One solution would be a meta-table to 58 ** track such information in the database, then we'd only want this 59 ** information on the initial create. 60 */ 61 if( argc>1 ){ 62 t->zDelim = string_dup(argv[1]); 63 } else { 64 /* Build a string excluding alphanumeric ASCII characters */ 65 char zDelim[0x80]; /* nul-terminated, so nul not a member */ 66 int i, j; 67 for(i=1, j=0; i<0x80; i++){ 68 if( !isalnum(i) ){ 69 zDelim[j++] = i; 70 } 71 } 72 zDelim[j++] = '\0'; 73 assert( j<=sizeof(zDelim) ); 74 t->zDelim = string_dup(zDelim); 75 } 76 77 *ppTokenizer = &t->base; 78 return SQLITE_OK; 79 } 80 81 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ 82 simple_tokenizer *t = (simple_tokenizer *) pTokenizer; 83 84 free((void *) t->zDelim); 85 free(t); 86 87 return SQLITE_OK; 88 } 89 90 static int simpleOpen( 91 sqlite3_tokenizer *pTokenizer, 92 const char *pInput, int nBytes, 93 sqlite3_tokenizer_cursor **ppCursor 94 ){ 95 simple_tokenizer_cursor *c; 96 97 c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor)); 98 c->pInput = pInput; 99 c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes; 100 c->pCurrent = c->pInput; /* start tokenizing at the beginning */ 101 c->iToken = 0; 102 c->zToken = NULL; /* no space allocated, yet. */ 103 c->nTokenBytes = 0; 104 c->nTokenAllocated = 0; 105 106 *ppCursor = &c->base; 107 return SQLITE_OK; 108 } 109 110 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ 111 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 112 113 if( NULL!=c->zToken ){ 114 free(c->zToken); 115 } 116 free(c); 117 118 return SQLITE_OK; 119 } 120 121 static int simpleNext( 122 sqlite3_tokenizer_cursor *pCursor, 123 const char **ppToken, int *pnBytes, 124 int *piStartOffset, int *piEndOffset, int *piPosition 125 ){ 126 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 127 simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; 128 int ii; 129 130 while( c->pCurrent-c->pInput<c->nBytes ){ 131 int n = (int) strcspn(c->pCurrent, t->zDelim); 132 if( n>0 ){ 133 if( n+1>c->nTokenAllocated ){ 134 c->zToken = realloc(c->zToken, n+1); 135 } 136 for(ii=0; ii<n; ii++){ 137 /* TODO(shess) This needs expansion to handle UTF-8 138 ** case-insensitivity. 139 */ 140 char ch = c->pCurrent[ii]; 141 c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch; 142 } 143 c->zToken[n] = '\0'; 144 *ppToken = c->zToken; 145 *pnBytes = n; 146 *piStartOffset = (int) (c->pCurrent-c->pInput); 147 *piEndOffset = *piStartOffset+n; 148 *piPosition = c->iToken++; 149 c->pCurrent += n + 1; 150 151 return SQLITE_OK; 152 } 153 c->pCurrent += n + 1; 154 /* TODO(shess) could strspn() to skip delimiters en masse. Needs 155 ** to happen in two places, though, which is annoying. 156 */ 157 } 158 return SQLITE_DONE; 159 } 160 161 static sqlite3_tokenizer_module simpleTokenizerModule = { 162 0, 163 simpleCreate, 164 simpleDestroy, 165 simpleOpen, 166 simpleClose, 167 simpleNext, 168 }; 169 170 void get_simple_tokenizer_module( 171 sqlite3_tokenizer_module **ppModule 172 ){ 173 *ppModule = &simpleTokenizerModule; 174 } 175
