1 /* 2 ** 2007 June 22 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ****************************************************************************** 12 ** 13 ** This is part of an SQLite module implementing full-text search. 14 ** This particular file implements the generic tokenizer interface. 15 */ 16 17 /* 18 ** The code in this file is only compiled if: 19 ** 20 ** * The FTS2 module is being built as an extension 21 ** (in which case SQLITE_CORE is not defined), or 22 ** 23 ** * The FTS2 module is being built into the core of 24 ** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). 25 */ 26 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) 27 28 29 #include "sqlite3.h" 30 #include "sqlite3ext.h" 31 #ifndef SQLITE_CORE 32 SQLITE_EXTENSION_INIT1 33 #endif 34 35 #include "fts2_hash.h" 36 #include "fts2_tokenizer.h" 37 #include <assert.h> 38 #include <stddef.h> 39 40 /* 41 ** Implementation of the SQL scalar function for accessing the underlying 42 ** hash table. This function may be called as follows: 43 ** 44 ** SELECT <function-name>(<key-name>); 45 ** SELECT <function-name>(<key-name>, <pointer>); 46 ** 47 ** where <function-name> is the name passed as the second argument 48 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer'). 49 ** 50 ** If the <pointer> argument is specified, it must be a blob value 51 ** containing a pointer to be stored as the hash data corresponding 52 ** to the string <key-name>. If <pointer> is not specified, then 53 ** the string <key-name> must already exist in the has table. Otherwise, 54 ** an error is returned. 55 ** 56 ** Whether or not the <pointer> argument is specified, the value returned 57 ** is a blob containing the pointer stored as the hash data corresponding 58 ** to string <key-name> (after the hash-table is updated, if applicable). 59 */ 60 static void scalarFunc( 61 sqlite3_context *context, 62 int argc, 63 sqlite3_value **argv 64 ){ 65 fts2Hash *pHash; 66 void *pPtr = 0; 67 const unsigned char *zName; 68 int nName; 69 70 assert( argc==1 || argc==2 ); 71 72 pHash = (fts2Hash *)sqlite3_user_data(context); 73 74 zName = sqlite3_value_text(argv[0]); 75 nName = sqlite3_value_bytes(argv[0])+1; 76 77 if( argc==2 ){ 78 void *pOld; 79 int n = sqlite3_value_bytes(argv[1]); 80 if( n!=sizeof(pPtr) ){ 81 sqlite3_result_error(context, "argument type mismatch", -1); 82 return; 83 } 84 pPtr = *(void **)sqlite3_value_blob(argv[1]); 85 pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr); 86 if( pOld==pPtr ){ 87 sqlite3_result_error(context, "out of memory", -1); 88 return; 89 } 90 }else{ 91 pPtr = sqlite3Fts2HashFind(pHash, zName, nName); 92 if( !pPtr ){ 93 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); 94 sqlite3_result_error(context, zErr, -1); 95 sqlite3_free(zErr); 96 return; 97 } 98 } 99 100 sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); 101 } 102 103 #ifdef SQLITE_TEST 104 105 #include <tcl.h> 106 #include <string.h> 107 108 /* 109 ** Implementation of a special SQL scalar function for testing tokenizers 110 ** designed to be used in concert with the Tcl testing framework. This 111 ** function must be called with two arguments: 112 ** 113 ** SELECT <function-name>(<key-name>, <input-string>); 114 ** SELECT <function-name>(<key-name>, <pointer>); 115 ** 116 ** where <function-name> is the name passed as the second argument 117 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer') 118 ** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test'). 119 ** 120 ** The return value is a string that may be interpreted as a Tcl 121 ** list. For each token in the <input-string>, three elements are 122 ** added to the returned list. The first is the token position, the 123 ** second is the token text (folded, stemmed, etc.) and the third is the 124 ** substring of <input-string> associated with the token. For example, 125 ** using the built-in "simple" tokenizer: 126 ** 127 ** SELECT fts_tokenizer_test('simple', 'I don't see how'); 128 ** 129 ** will return the string: 130 ** 131 ** "{0 i I 1 dont don't 2 see see 3 how how}" 132 ** 133 */ 134 static void testFunc( 135 sqlite3_context *context, 136 int argc, 137 sqlite3_value **argv 138 ){ 139 fts2Hash *pHash; 140 sqlite3_tokenizer_module *p; 141 sqlite3_tokenizer *pTokenizer = 0; 142 sqlite3_tokenizer_cursor *pCsr = 0; 143 144 const char *zErr = 0; 145 146 const char *zName; 147 int nName; 148 const char *zInput; 149 int nInput; 150 151 const char *zArg = 0; 152 153 const char *zToken; 154 int nToken; 155 int iStart; 156 int iEnd; 157 int iPos; 158 159 Tcl_Obj *pRet; 160 161 assert( argc==2 || argc==3 ); 162 163 nName = sqlite3_value_bytes(argv[0]); 164 zName = (const char *)sqlite3_value_text(argv[0]); 165 nInput = sqlite3_value_bytes(argv[argc-1]); 166 zInput = (const char *)sqlite3_value_text(argv[argc-1]); 167 168 if( argc==3 ){ 169 zArg = (const char *)sqlite3_value_text(argv[1]); 170 } 171 172 pHash = (fts2Hash *)sqlite3_user_data(context); 173 p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1); 174 175 if( !p ){ 176 char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); 177 sqlite3_result_error(context, zErr, -1); 178 sqlite3_free(zErr); 179 return; 180 } 181 182 pRet = Tcl_NewObj(); 183 Tcl_IncrRefCount(pRet); 184 185 if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){ 186 zErr = "error in xCreate()"; 187 goto finish; 188 } 189 pTokenizer->pModule = p; 190 if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ 191 zErr = "error in xOpen()"; 192 goto finish; 193 } 194 pCsr->pTokenizer = pTokenizer; 195 196 while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ 197 Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); 198 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); 199 zToken = &zInput[iStart]; 200 nToken = iEnd-iStart; 201 Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); 202 } 203 204 if( SQLITE_OK!=p->xClose(pCsr) ){ 205 zErr = "error in xClose()"; 206 goto finish; 207 } 208 if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ 209 zErr = "error in xDestroy()"; 210 goto finish; 211 } 212 213 finish: 214 if( zErr ){ 215 sqlite3_result_error(context, zErr, -1); 216 }else{ 217 sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); 218 } 219 Tcl_DecrRefCount(pRet); 220 } 221 222 static 223 int registerTokenizer( 224 sqlite3 *db, 225 char *zName, 226 const sqlite3_tokenizer_module *p 227 ){ 228 int rc; 229 sqlite3_stmt *pStmt; 230 const char zSql[] = "SELECT fts2_tokenizer(?, ?)"; 231 232 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 233 if( rc!=SQLITE_OK ){ 234 return rc; 235 } 236 237 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); 238 sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); 239 sqlite3_step(pStmt); 240 241 return sqlite3_finalize(pStmt); 242 } 243 244 static 245 int queryFts2Tokenizer( 246 sqlite3 *db, 247 char *zName, 248 const sqlite3_tokenizer_module **pp 249 ){ 250 int rc; 251 sqlite3_stmt *pStmt; 252 const char zSql[] = "SELECT fts2_tokenizer(?)"; 253 254 *pp = 0; 255 rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 256 if( rc!=SQLITE_OK ){ 257 return rc; 258 } 259 260 sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); 261 if( SQLITE_ROW==sqlite3_step(pStmt) ){ 262 if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ 263 memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); 264 } 265 } 266 267 return sqlite3_finalize(pStmt); 268 } 269 270 void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); 271 272 /* 273 ** Implementation of the scalar function fts2_tokenizer_internal_test(). 274 ** This function is used for testing only, it is not included in the 275 ** build unless SQLITE_TEST is defined. 276 ** 277 ** The purpose of this is to test that the fts2_tokenizer() function 278 ** can be used as designed by the C-code in the queryFts2Tokenizer and 279 ** registerTokenizer() functions above. These two functions are repeated 280 ** in the README.tokenizer file as an example, so it is important to 281 ** test them. 282 ** 283 ** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar 284 ** function with no arguments. An assert() will fail if a problem is 285 ** detected. i.e.: 286 ** 287 ** SELECT fts2_tokenizer_internal_test(); 288 ** 289 */ 290 static void intTestFunc( 291 sqlite3_context *context, 292 int argc, 293 sqlite3_value **argv 294 ){ 295 int rc; 296 const sqlite3_tokenizer_module *p1; 297 const sqlite3_tokenizer_module *p2; 298 sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); 299 300 /* Test the query function */ 301 sqlite3Fts2SimpleTokenizerModule(&p1); 302 rc = queryFts2Tokenizer(db, "simple", &p2); 303 assert( rc==SQLITE_OK ); 304 assert( p1==p2 ); 305 rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2); 306 assert( rc==SQLITE_ERROR ); 307 assert( p2==0 ); 308 assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); 309 310 /* Test the storage function */ 311 rc = registerTokenizer(db, "nosuchtokenizer", p1); 312 assert( rc==SQLITE_OK ); 313 rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2); 314 assert( rc==SQLITE_OK ); 315 assert( p2==p1 ); 316 317 sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); 318 } 319 320 #endif 321 322 /* 323 ** Set up SQL objects in database db used to access the contents of 324 ** the hash table pointed to by argument pHash. The hash table must 325 ** been initialised to use string keys, and to take a private copy 326 ** of the key when a value is inserted. i.e. by a call similar to: 327 ** 328 ** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1); 329 ** 330 ** This function adds a scalar function (see header comment above 331 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is 332 ** defined at compilation time, a temporary virtual table (see header 333 ** comment above struct HashTableVtab) to the database schema. Both 334 ** provide read/write access to the contents of *pHash. 335 ** 336 ** The third argument to this function, zName, is used as the name 337 ** of both the scalar and, if created, the virtual table. 338 */ 339 int sqlite3Fts2InitHashTable( 340 sqlite3 *db, 341 fts2Hash *pHash, 342 const char *zName 343 ){ 344 int rc = SQLITE_OK; 345 void *p = (void *)pHash; 346 const int any = SQLITE_ANY; 347 char *zTest = 0; 348 char *zTest2 = 0; 349 350 #ifdef SQLITE_TEST 351 void *pdb = (void *)db; 352 zTest = sqlite3_mprintf("%s_test", zName); 353 zTest2 = sqlite3_mprintf("%s_internal_test", zName); 354 if( !zTest || !zTest2 ){ 355 rc = SQLITE_NOMEM; 356 } 357 #endif 358 359 if( rc!=SQLITE_OK 360 || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) 361 || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) 362 #ifdef SQLITE_TEST 363 || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) 364 || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) 365 || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) 366 #endif 367 ); 368 369 sqlite3_free(zTest); 370 sqlite3_free(zTest2); 371 return rc; 372 } 373 374 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ 375
