Home | History | Annotate | Download | only in fts2 
      1 /*
      2 ** 2007 June 22
      3 **
      4 ** The author disclaims copyright to this source code.  In place of
      5 ** a legal notice, here is a blessing:
      6 **
      7 **    May you do good and not evil.
      8 **    May you find forgiveness for yourself and forgive others.
      9 **    May you share freely, never taking more than you give.
     10 **
     11 *************************************************************************
     12 ** This file implements a tokenizer for fts2 based on the ICU library.
     13 **
     14 ** $Id: fts2_icu.c,v 1.3 2008/12/18 05:30:26 danielk1977 Exp $
     15 */
     16 
     17 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
     18 #ifdef SQLITE_ENABLE_ICU
     19 
     20 #include <assert.h>
     21 #include <string.h>
     22 #include "fts2_tokenizer.h"
     23 
     24 #include <unicode/ubrk.h>
     25 #include <unicode/ucol.h>
     26 #include <unicode/ustring.h>
     27 #include <unicode/utf16.h>
     28 
     29 typedef struct IcuTokenizer IcuTokenizer;
     30 typedef struct IcuCursor IcuCursor;
     31 
     32 struct IcuTokenizer {
     33   sqlite3_tokenizer base;
     34   char *zLocale;
     35 };
     36 
     37 struct IcuCursor {
     38   sqlite3_tokenizer_cursor base;
     39 
     40   UBreakIterator *pIter;      /* ICU break-iterator object */
     41   int nChar;                  /* Number of UChar elements in pInput */
     42   UChar *aChar;               /* Copy of input using utf-16 encoding */
     43   int *aOffset;               /* Offsets of each character in utf-8 input */
     44 
     45   int nBuffer;
     46   char *zBuffer;
     47 
     48   int iToken;
     49 };
     50 
     51 /*
     52 ** Create a new tokenizer instance.
     53 */
     54 static int icuCreate(
     55   int argc,                            /* Number of entries in argv[] */
     56   const char * const *argv,            /* Tokenizer creation arguments */
     57   sqlite3_tokenizer **ppTokenizer      /* OUT: Created tokenizer */
     58 ){
     59   IcuTokenizer *p;
     60   int n = 0;
     61 
     62   if( argc>0 ){
     63     n = strlen(argv[0])+1;
     64   }
     65   p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
     66   if( !p ){
     67     return SQLITE_NOMEM;
     68   }
     69   memset(p, 0, sizeof(IcuTokenizer));
     70 
     71   if( n ){
     72     p->zLocale = (char *)&p[1];
     73     memcpy(p->zLocale, argv[0], n);
     74   }
     75 
     76   *ppTokenizer = (sqlite3_tokenizer *)p;
     77 
     78   return SQLITE_OK;
     79 }
     80 
     81 /*
     82 ** Destroy a tokenizer
     83 */
     84 static int icuDestroy(sqlite3_tokenizer *pTokenizer){
     85   IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
     86   sqlite3_free(p);
     87   return SQLITE_OK;
     88 }
     89 
     90 /*
     91 ** Prepare to begin tokenizing a particular string.  The input
     92 ** string to be tokenized is pInput[0..nBytes-1].  A cursor
     93 ** used to incrementally tokenize this string is returned in
     94 ** *ppCursor.
     95 */
     96 static int icuOpen(
     97   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
     98   const char *zInput,                    /* Input string */
     99   int nInput,                            /* Length of zInput in bytes */
    100   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
    101 ){
    102   IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
    103   IcuCursor *pCsr;
    104 
    105   const int32_t opt = U_FOLD_CASE_DEFAULT;
    106   UErrorCode status = U_ZERO_ERROR;
    107   int nChar;
    108 
    109   UChar32 c;
    110   int iInput = 0;
    111   int iOut = 0;
    112 
    113   *ppCursor = 0;
    114 
    115   if( nInput<0 ){
    116     nInput = strlen(zInput);
    117   }
    118   nChar = nInput+1;
    119   pCsr = (IcuCursor *)sqlite3_malloc(
    120       sizeof(IcuCursor) +                /* IcuCursor */
    121       (nChar+1) * sizeof(int) +          /* IcuCursor.aOffset[] */
    122       nChar * sizeof(UChar)              /* IcuCursor.aChar[] */
    123   );
    124   if( !pCsr ){
    125     return SQLITE_NOMEM;
    126   }
    127   memset(pCsr, 0, sizeof(IcuCursor));
    128   pCsr->aOffset = (int *)&pCsr[1];
    129   pCsr->aChar = (UChar *)&pCsr->aOffset[nChar+1];
    130 
    131   pCsr->aOffset[iOut] = iInput;
    132   U8_NEXT(zInput, iInput, nInput, c);
    133   while( c>0 ){
    134     int isError = 0;
    135     c = u_foldCase(c, opt);
    136     U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
    137     if( isError ){
    138       sqlite3_free(pCsr);
    139       return SQLITE_ERROR;
    140     }
    141     pCsr->aOffset[iOut] = iInput;
    142 
    143     if( iInput<nInput ){
    144       U8_NEXT(zInput, iInput, nInput, c);
    145     }else{
    146       c = 0;
    147     }
    148   }
    149 
    150   pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
    151   if( !U_SUCCESS(status) ){
    152     sqlite3_free(pCsr);
    153     return SQLITE_ERROR;
    154   }
    155   pCsr->nChar = iOut;
    156 
    157   ubrk_first(pCsr->pIter);
    158   *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
    159   return SQLITE_OK;
    160 }
    161 
    162 /*
    163 ** Close a tokenization cursor previously opened by a call to icuOpen().
    164 */
    165 static int icuClose(sqlite3_tokenizer_cursor *pCursor){
    166   IcuCursor *pCsr = (IcuCursor *)pCursor;
    167   ubrk_close(pCsr->pIter);
    168   sqlite3_free(pCsr->zBuffer);
    169   sqlite3_free(pCsr);
    170   return SQLITE_OK;
    171 }
    172 
    173 /*
    174 ** Extract the next token from a tokenization cursor.
    175 */
    176 static int icuNext(
    177   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
    178   const char **ppToken,               /* OUT: *ppToken is the token text */
    179   int *pnBytes,                       /* OUT: Number of bytes in token */
    180   int *piStartOffset,                 /* OUT: Starting offset of token */
    181   int *piEndOffset,                   /* OUT: Ending offset of token */
    182   int *piPosition                     /* OUT: Position integer of token */
    183 ){
    184   IcuCursor *pCsr = (IcuCursor *)pCursor;
    185 
    186   int iStart = 0;
    187   int iEnd = 0;
    188   int nByte = 0;
    189 
    190   while( iStart==iEnd ){
    191     UChar32 c;
    192 
    193     iStart = ubrk_current(pCsr->pIter);
    194     iEnd = ubrk_next(pCsr->pIter);
    195     if( iEnd==UBRK_DONE ){
    196       return SQLITE_DONE;
    197     }
    198 
    199     while( iStart<iEnd ){
    200       int iWhite = iStart;
    201       U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
    202       if( u_isspace(c) ){
    203         iStart = iWhite;
    204       }else{
    205         break;
    206       }
    207     }
    208     assert(iStart<=iEnd);
    209   }
    210 
    211   do {
    212     UErrorCode status = U_ZERO_ERROR;
    213     if( nByte ){
    214       char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
    215       if( !zNew ){
    216         return SQLITE_NOMEM;
    217       }
    218       pCsr->zBuffer = zNew;
    219       pCsr->nBuffer = nByte;
    220     }
    221 
    222     u_strToUTF8(
    223         pCsr->zBuffer, pCsr->nBuffer, &nByte,    /* Output vars */
    224         &pCsr->aChar[iStart], iEnd-iStart,       /* Input vars */
    225         &status                                  /* Output success/failure */
    226     );
    227   } while( nByte>pCsr->nBuffer );
    228 
    229   *ppToken = pCsr->zBuffer;
    230   *pnBytes = nByte;
    231   *piStartOffset = pCsr->aOffset[iStart];
    232   *piEndOffset = pCsr->aOffset[iEnd];
    233   *piPosition = pCsr->iToken++;
    234 
    235   return SQLITE_OK;
    236 }
    237 
    238 /*
    239 ** The set of routines that implement the simple tokenizer
    240 */
    241 static const sqlite3_tokenizer_module icuTokenizerModule = {
    242   0,                           /* iVersion */
    243   icuCreate,                   /* xCreate  */
    244   icuDestroy,                  /* xCreate  */
    245   icuOpen,                     /* xOpen    */
    246   icuClose,                    /* xClose   */
    247   icuNext,                     /* xNext    */
    248 };
    249 
    250 /*
    251 ** Set *ppModule to point at the implementation of the ICU tokenizer.
    252 */
    253 void sqlite3Fts2IcuTokenizerModule(
    254   sqlite3_tokenizer_module const**ppModule
    255 ){
    256   *ppModule = &icuTokenizerModule;
    257 }
    258 
    259 #endif /* defined(SQLITE_ENABLE_ICU) */
    260 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
    261