Home | History | Annotate | Download | only in fts3 
      1 /*
      2 ** 2011 Jan 27
      3 **
      4 ** The author disclaims copyright to this source code.  In place of
      5 ** a legal notice, here is a blessing:
      6 **
      7 **    May you do good and not evil.
      8 **    May you find forgiveness for yourself and forgive others.
      9 **    May you share freely, never taking more than you give.
     10 **
     11 ******************************************************************************
     12 **
     13 */
     14 
     15 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
     16 
     17 #include "fts3Int.h"
     18 #include <string.h>
     19 #include <assert.h>
     20 
     21 typedef struct Fts3auxTable Fts3auxTable;
     22 typedef struct Fts3auxCursor Fts3auxCursor;
     23 
     24 struct Fts3auxTable {
     25   sqlite3_vtab base;              /* Base class used by SQLite core */
     26   Fts3Table *pFts3Tab;
     27 };
     28 
     29 struct Fts3auxCursor {
     30   sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
     31   Fts3SegReaderCursor csr;        /* Must be right after "base" */
     32   Fts3SegFilter filter;
     33   char *zStop;
     34   int nStop;                      /* Byte-length of string zStop */
     35   int isEof;                      /* True if cursor is at EOF */
     36   sqlite3_int64 iRowid;           /* Current rowid */
     37 
     38   int iCol;                       /* Current value of 'col' column */
     39   int nStat;                      /* Size of aStat[] array */
     40   struct Fts3auxColstats {
     41     sqlite3_int64 nDoc;           /* 'documents' values for current csr row */
     42     sqlite3_int64 nOcc;           /* 'occurrences' values for current csr row */
     43   } *aStat;
     44 };
     45 
     46 /*
     47 ** Schema of the terms table.
     48 */
     49 #define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, col, documents, occurrences)"
     50 
     51 /*
     52 ** This function does all the work for both the xConnect and xCreate methods.
     53 ** These tables have no persistent representation of their own, so xConnect
     54 ** and xCreate are identical operations.
     55 */
     56 static int fts3auxConnectMethod(
     57   sqlite3 *db,                    /* Database connection */
     58   void *pUnused,                  /* Unused */
     59   int argc,                       /* Number of elements in argv array */
     60   const char * const *argv,       /* xCreate/xConnect argument array */
     61   sqlite3_vtab **ppVtab,          /* OUT: New sqlite3_vtab object */
     62   char **pzErr                    /* OUT: sqlite3_malloc'd error message */
     63 ){
     64   char const *zDb;                /* Name of database (e.g. "main") */
     65   char const *zFts3;              /* Name of fts3 table */
     66   int nDb;                        /* Result of strlen(zDb) */
     67   int nFts3;                      /* Result of strlen(zFts3) */
     68   int nByte;                      /* Bytes of space to allocate here */
     69   int rc;                         /* value returned by declare_vtab() */
     70   Fts3auxTable *p;                /* Virtual table object to return */
     71 
     72   UNUSED_PARAMETER(pUnused);
     73 
     74   /* The user should specify a single argument - the name of an fts3 table. */
     75   if( argc!=4 ){
     76     *pzErr = sqlite3_mprintf(
     77         "wrong number of arguments to fts4aux constructor"
     78     );
     79     return SQLITE_ERROR;
     80   }
     81 
     82   zDb = argv[1];
     83   nDb = strlen(zDb);
     84   zFts3 = argv[3];
     85   nFts3 = strlen(zFts3);
     86 
     87   rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
     88   if( rc!=SQLITE_OK ) return rc;
     89 
     90   nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
     91   p = (Fts3auxTable *)sqlite3_malloc(nByte);
     92   if( !p ) return SQLITE_NOMEM;
     93   memset(p, 0, nByte);
     94 
     95   p->pFts3Tab = (Fts3Table *)&p[1];
     96   p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
     97   p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
     98   p->pFts3Tab->db = db;
     99 
    100   memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
    101   memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
    102   sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
    103 
    104   *ppVtab = (sqlite3_vtab *)p;
    105   return SQLITE_OK;
    106 }
    107 
    108 /*
    109 ** This function does the work for both the xDisconnect and xDestroy methods.
    110 ** These tables have no persistent representation of their own, so xDisconnect
    111 ** and xDestroy are identical operations.
    112 */
    113 static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){
    114   Fts3auxTable *p = (Fts3auxTable *)pVtab;
    115   Fts3Table *pFts3 = p->pFts3Tab;
    116   int i;
    117 
    118   /* Free any prepared statements held */
    119   for(i=0; i<SizeofArray(pFts3->aStmt); i++){
    120     sqlite3_finalize(pFts3->aStmt[i]);
    121   }
    122   sqlite3_free(pFts3->zSegmentsTbl);
    123   sqlite3_free(p);
    124   return SQLITE_OK;
    125 }
    126 
    127 #define FTS4AUX_EQ_CONSTRAINT 1
    128 #define FTS4AUX_GE_CONSTRAINT 2
    129 #define FTS4AUX_LE_CONSTRAINT 4
    130 
    131 /*
    132 ** xBestIndex - Analyze a WHERE and ORDER BY clause.
    133 */
    134 static int fts3auxBestIndexMethod(
    135   sqlite3_vtab *pVTab,
    136   sqlite3_index_info *pInfo
    137 ){
    138   int i;
    139   int iEq = -1;
    140   int iGe = -1;
    141   int iLe = -1;
    142 
    143   UNUSED_PARAMETER(pVTab);
    144 
    145   /* This vtab delivers always results in "ORDER BY term ASC" order. */
    146   if( pInfo->nOrderBy==1
    147    && pInfo->aOrderBy[0].iColumn==0
    148    && pInfo->aOrderBy[0].desc==0
    149   ){
    150     pInfo->orderByConsumed = 1;
    151   }
    152 
    153   /* Search for equality and range constraints on the "term" column. */
    154   for(i=0; i<pInfo->nConstraint; i++){
    155     if( pInfo->aConstraint[i].usable && pInfo->aConstraint[i].iColumn==0 ){
    156       int op = pInfo->aConstraint[i].op;
    157       if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i;
    158       if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i;
    159       if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i;
    160       if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i;
    161       if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i;
    162     }
    163   }
    164 
    165   if( iEq>=0 ){
    166     pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT;
    167     pInfo->aConstraintUsage[iEq].argvIndex = 1;
    168     pInfo->estimatedCost = 5;
    169   }else{
    170     pInfo->idxNum = 0;
    171     pInfo->estimatedCost = 20000;
    172     if( iGe>=0 ){
    173       pInfo->idxNum += FTS4AUX_GE_CONSTRAINT;
    174       pInfo->aConstraintUsage[iGe].argvIndex = 1;
    175       pInfo->estimatedCost /= 2;
    176     }
    177     if( iLe>=0 ){
    178       pInfo->idxNum += FTS4AUX_LE_CONSTRAINT;
    179       pInfo->aConstraintUsage[iLe].argvIndex = 1 + (iGe>=0);
    180       pInfo->estimatedCost /= 2;
    181     }
    182   }
    183 
    184   return SQLITE_OK;
    185 }
    186 
    187 /*
    188 ** xOpen - Open a cursor.
    189 */
    190 static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
    191   Fts3auxCursor *pCsr;            /* Pointer to cursor object to return */
    192 
    193   UNUSED_PARAMETER(pVTab);
    194 
    195   pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor));
    196   if( !pCsr ) return SQLITE_NOMEM;
    197   memset(pCsr, 0, sizeof(Fts3auxCursor));
    198 
    199   *ppCsr = (sqlite3_vtab_cursor *)pCsr;
    200   return SQLITE_OK;
    201 }
    202 
    203 /*
    204 ** xClose - Close a cursor.
    205 */
    206 static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){
    207   Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
    208   Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
    209 
    210   sqlite3Fts3SegmentsClose(pFts3);
    211   sqlite3Fts3SegReaderFinish(&pCsr->csr);
    212   sqlite3_free((void *)pCsr->filter.zTerm);
    213   sqlite3_free(pCsr->zStop);
    214   sqlite3_free(pCsr->aStat);
    215   sqlite3_free(pCsr);
    216   return SQLITE_OK;
    217 }
    218 
    219 static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
    220   if( nSize>pCsr->nStat ){
    221     struct Fts3auxColstats *aNew;
    222     aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat,
    223         sizeof(struct Fts3auxColstats) * nSize
    224     );
    225     if( aNew==0 ) return SQLITE_NOMEM;
    226     memset(&aNew[pCsr->nStat], 0,
    227         sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat)
    228     );
    229     pCsr->aStat = aNew;
    230     pCsr->nStat = nSize;
    231   }
    232   return SQLITE_OK;
    233 }
    234 
    235 /*
    236 ** xNext - Advance the cursor to the next row, if any.
    237 */
    238 static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){
    239   Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
    240   Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
    241   int rc;
    242 
    243   /* Increment our pretend rowid value. */
    244   pCsr->iRowid++;
    245 
    246   for(pCsr->iCol++; pCsr->iCol<pCsr->nStat; pCsr->iCol++){
    247     if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK;
    248   }
    249 
    250   rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr);
    251   if( rc==SQLITE_ROW ){
    252     int i = 0;
    253     int nDoclist = pCsr->csr.nDoclist;
    254     char *aDoclist = pCsr->csr.aDoclist;
    255     int iCol;
    256 
    257     int eState = 0;
    258 
    259     if( pCsr->zStop ){
    260       int n = (pCsr->nStop<pCsr->csr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm;
    261       int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n);
    262       if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){
    263         pCsr->isEof = 1;
    264         return SQLITE_OK;
    265       }
    266     }
    267 
    268     if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM;
    269     memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat);
    270     iCol = 0;
    271 
    272     while( i<nDoclist ){
    273       sqlite3_int64 v = 0;
    274 
    275       i += sqlite3Fts3GetVarint(&aDoclist[i], &v);
    276       switch( eState ){
    277         /* State 0. In this state the integer just read was a docid. */
    278         case 0:
    279           pCsr->aStat[0].nDoc++;
    280           eState = 1;
    281           iCol = 0;
    282           break;
    283 
    284         /* State 1. In this state we are expecting either a 1, indicating
    285         ** that the following integer will be a column number, or the
    286         ** start of a position list for column 0.
    287         **
    288         ** The only difference between state 1 and state 2 is that if the
    289         ** integer encountered in state 1 is not 0 or 1, then we need to
    290         ** increment the column 0 "nDoc" count for this term.
    291         */
    292         case 1:
    293           assert( iCol==0 );
    294           if( v>1 ){
    295             pCsr->aStat[1].nDoc++;
    296           }
    297           eState = 2;
    298           /* fall through */
    299 
    300         case 2:
    301           if( v==0 ){       /* 0x00. Next integer will be a docid. */
    302             eState = 0;
    303           }else if( v==1 ){ /* 0x01. Next integer will be a column number. */
    304             eState = 3;
    305           }else{            /* 2 or greater. A position. */
    306             pCsr->aStat[iCol+1].nOcc++;
    307             pCsr->aStat[0].nOcc++;
    308           }
    309           break;
    310 
    311         /* State 3. The integer just read is a column number. */
    312         default: assert( eState==3 );
    313           iCol = (int)v;
    314           if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM;
    315           pCsr->aStat[iCol+1].nDoc++;
    316           eState = 2;
    317           break;
    318       }
    319     }
    320 
    321     pCsr->iCol = 0;
    322     rc = SQLITE_OK;
    323   }else{
    324     pCsr->isEof = 1;
    325   }
    326   return rc;
    327 }
    328 
    329 /*
    330 ** xFilter - Initialize a cursor to point at the start of its data.
    331 */
    332 static int fts3auxFilterMethod(
    333   sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
    334   int idxNum,                     /* Strategy index */
    335   const char *idxStr,             /* Unused */
    336   int nVal,                       /* Number of elements in apVal */
    337   sqlite3_value **apVal           /* Arguments for the indexing scheme */
    338 ){
    339   Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
    340   Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab;
    341   int rc;
    342   int isScan;
    343 
    344   UNUSED_PARAMETER(nVal);
    345 
    346   assert( idxStr==0 );
    347   assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0
    348        || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT
    349        || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT)
    350   );
    351   isScan = (idxNum!=FTS4AUX_EQ_CONSTRAINT);
    352 
    353   /* In case this cursor is being reused, close and zero it. */
    354   testcase(pCsr->filter.zTerm);
    355   sqlite3Fts3SegReaderFinish(&pCsr->csr);
    356   sqlite3_free((void *)pCsr->filter.zTerm);
    357   sqlite3_free(pCsr->aStat);
    358   memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr);
    359 
    360   pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
    361   if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
    362 
    363   if( idxNum&(FTS4AUX_EQ_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) ){
    364     const unsigned char *zStr = sqlite3_value_text(apVal[0]);
    365     if( zStr ){
    366       pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
    367       pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
    368       if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
    369     }
    370   }
    371   if( idxNum&FTS4AUX_LE_CONSTRAINT ){
    372     int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0;
    373     pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
    374     pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
    375     if( pCsr->zStop==0 ) return SQLITE_NOMEM;
    376   }
    377 
    378   rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
    379       pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
    380   );
    381   if( rc==SQLITE_OK ){
    382     rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
    383   }
    384 
    385   if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor);
    386   return rc;
    387 }
    388 
    389 /*
    390 ** xEof - Return true if the cursor is at EOF, or false otherwise.
    391 */
    392 static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){
    393   Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
    394   return pCsr->isEof;
    395 }
    396 
    397 /*
    398 ** xColumn - Return a column value.
    399 */
    400 static int fts3auxColumnMethod(
    401   sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
    402   sqlite3_context *pContext,      /* Context for sqlite3_result_xxx() calls */
    403   int iCol                        /* Index of column to read value from */
    404 ){
    405   Fts3auxCursor *p = (Fts3auxCursor *)pCursor;
    406 
    407   assert( p->isEof==0 );
    408   if( iCol==0 ){        /* Column "term" */
    409     sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT);
    410   }else if( iCol==1 ){  /* Column "col" */
    411     if( p->iCol ){
    412       sqlite3_result_int(pContext, p->iCol-1);
    413     }else{
    414       sqlite3_result_text(pContext, "*", -1, SQLITE_STATIC);
    415     }
    416   }else if( iCol==2 ){  /* Column "documents" */
    417     sqlite3_result_int64(pContext, p->aStat[p->iCol].nDoc);
    418   }else{                /* Column "occurrences" */
    419     sqlite3_result_int64(pContext, p->aStat[p->iCol].nOcc);
    420   }
    421 
    422   return SQLITE_OK;
    423 }
    424 
    425 /*
    426 ** xRowid - Return the current rowid for the cursor.
    427 */
    428 static int fts3auxRowidMethod(
    429   sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
    430   sqlite_int64 *pRowid            /* OUT: Rowid value */
    431 ){
    432   Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor;
    433   *pRowid = pCsr->iRowid;
    434   return SQLITE_OK;
    435 }
    436 
    437 /*
    438 ** Register the fts3aux module with database connection db. Return SQLITE_OK
    439 ** if successful or an error code if sqlite3_create_module() fails.
    440 */
    441 int sqlite3Fts3InitAux(sqlite3 *db){
    442   static const sqlite3_module fts3aux_module = {
    443      0,                           /* iVersion      */
    444      fts3auxConnectMethod,        /* xCreate       */
    445      fts3auxConnectMethod,        /* xConnect      */
    446      fts3auxBestIndexMethod,      /* xBestIndex    */
    447      fts3auxDisconnectMethod,     /* xDisconnect   */
    448      fts3auxDisconnectMethod,     /* xDestroy      */
    449      fts3auxOpenMethod,           /* xOpen         */
    450      fts3auxCloseMethod,          /* xClose        */
    451      fts3auxFilterMethod,         /* xFilter       */
    452      fts3auxNextMethod,           /* xNext         */
    453      fts3auxEofMethod,            /* xEof          */
    454      fts3auxColumnMethod,         /* xColumn       */
    455      fts3auxRowidMethod,          /* xRowid        */
    456      0,                           /* xUpdate       */
    457      0,                           /* xBegin        */
    458      0,                           /* xSync         */
    459      0,                           /* xCommit       */
    460      0,                           /* xRollback     */
    461      0,                           /* xFindFunction */
    462      0                            /* xRename       */
    463   };
    464   int rc;                         /* Return code */
    465 
    466   rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0);
    467   return rc;
    468 }
    469 
    470 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
    471