Lines Matching defs:TOKEN
2373 ** complete if it ends with a semicolon token and is not a prefix of a
2376 ** independent tokens (they are part of the token in which they are
8156 Token Token;
8505 void *token; /* id that may be used to recursive triggers */
10412 ** Each token coming out of the lexer is an instance of
10415 ** Note if Token.z==0 then Token.dyn and Token.n are undefined and
10416 ** may contain random values. Do not make any assumptions about Token.dyn
10417 ** and Token.n when Token.z==0.
10419 struct Token {
10420 const char *z; /* Text of the token. Not NULL-terminated! */
10421 unsigned int n; /* Number of characters in this token */
10487 ** Expr.op is the opcode. The integer parser token codes are reused
10494 ** or TK_STRING), then Expr.token contains the text of the SQL literal. If
10495 ** the expression is a variable (TK_VARIABLE), then Expr.token contains the
10497 ** then Expr.token contains the name of the function.
10551 char *zToken; /* Token value. Zero terminated and dequoted */
10598 #define EP_DblQuoted 0x0040 /* token.z was originally in "..." */
10664 char *zName; /* Token associated with this expression */
11162 Token sNameToken; /* Token with unqualified schema object name */
11163 Token sLastToken; /* The last token parsed */
11165 Token sArg; /* Complete text of a module argument */
11256 * target -> A token holding the quoted name of the table to insert into.
11264 * target -> A token holding the quoted name of the table to delete from.
11269 * target -> A token holding the quoted name of the table to update rows of.
11282 Token target; /* Target table for DELETE, UPDATE, INSERT */
11300 const Token *pName; /* Name of the container - used for error messages */
11585 SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
11588 SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*);
11590 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
11594 SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
11599 SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
11605 SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
11606 SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*);
11610 SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*);
11612 SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*);
11613 SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,Select*);
11633 SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
11653 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
11656 SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
11657 SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
11658 Token*, Select*, Expr*, IdList*);
11659 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);
11665 SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
11666 Token*, int, int);
11707 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*);
11722 Token*);
11762 SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*,
11764 SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*);
11775 SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
11777 SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8);
11778 SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*);
11795 SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*);
11796 SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int);
11812 SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*);
11870 SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
11877 SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*);
11908 SQLITE_PRIVATE const Token sqlite3IntTokens[];
11916 SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*);
11918 SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
11928 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *);
11932 SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*);
11934 SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*);
11970 SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*);
12016 SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int);
12017 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*);
12019 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*);
12372 SQLITE_PRIVATE const Token sqlite3IntTokens[] = {
12950 void *token; /* Copy of SubProgram.token */
19399 #define etTOKEN 12 /* a pointer to a Token structure */
19998 Token *pToken = va_arg(ap, Token*);
21152 ** %T Insert a token
21185 ** %T Insert a token
59388 ** token (i.e. "5", "5.1", "'a string'"). If the expression can
64164 int n; /* Length of a token prefix */
64165 int nToken; /* Length of the parameter token */
65357 void *t; /* Token identifying trigger */
70070 void *t; /* Token identifying trigger */
70086 ** single trigger all have the same value for the SubProgram.token
70089 u.cc.t = u.cc.pProgram->token;
70090 for(u.cc.pFrame=p->pFrame; u.cc.pFrame && u.cc.pFrame->token!=u.cc.t; u.cc.pFrame=u.cc.pFrame->pParent);
70133 u.cc.pFrame->token = u.cc.pProgram->token;
74504 SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){
74793 ** If dequote is true, then the token (if it exists) is dequoted.
74795 ** parameter is ignored if pToken is NULL or if the token does not
74800 ** can be translated into a 32-bit integer, then the token is not
74808 const Token *pToken, /* Token argument. Might be NULL */
74851 ** Allocate a new expression node from a zero-terminated token that has
74857 const char *zToken /* Token argument. Might be NULL */
74859 Token x;
74912 const Token *pToken /* Argument token */
74942 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){
75082 ** dupedExprNodeSize() Size of Expr + space for token
75083 ** dupedExprSize() Expr + token + subtree components
75463 Token *pName, /* Name to be added */
76810 /* Expressions of the form: CAST(pLeft AS token) */
77453 /* Expressions of the form: CAST(pLeft AS token) */
77863 ** This code depends on the fact that certain token values (ex: TK_EQ)
78512 int token;
78513 Token tname;
78523 ** statement is that the table name is the first non-space token that
78524 ** is immediately followed by a TK_LP or TK_USING token.
78533 /* Store the token that zCsr points to in tname. */
78537 /* Advance zCsr to the next token. Store that token type in 'token',
78542 len = sqlite3GetToken(zCsr, &token);
78543 } while( token==TK_SPACE );
78545 } while( token!=TK_LP && token!=TK_USING );
78581 unsigned const char *z; /* Pointer to token */
78582 int n; /* Length of token z */
78583 int token; /* Type of token */
78587 n = sqlite3GetToken(z, &token);
78588 if( token==TK_REFERENCES ){
78592 n = sqlite3GetToken(z, &token);
78593 }while( token==TK_SPACE );
78632 int token;
78633 Token tname;
78643 ** statement is that the table name is the first token that is immediatedly
78655 /* Store the token that zCsr points to in tname. */
78659 /* Advance zCsr to the next token. Store that token type in 'token',
78664 len = sqlite3GetToken(zCsr, &token);
78665 }while( token==TK_SPACE );
78670 ** token is read and 'dist' equals 2, the condition stated above
78678 if( token==TK_DOT || token==TK_ON ){
78681 } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) );
78683 /* Variable tname now contains the token that is the old table-name
78863 Token *pName /* The new table name. */
79090 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
80069 SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){
80076 Token *pTableName;
80850 const Token *pName /* Name of the view, trigger, or index */
81801 ** Given a token, return a string that consists of the text of that
81802 ** token. Space to hold the returned string
81807 ** surround the body of the token are removed.
81813 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){
81860 ** The token *pName contains the name of a database (either "main" or
81865 SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){
81886 ** This routine sets the *ppUnqual pointer to point at the token (pName1 or
81892 Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */
81893 Token *pName2, /* The "yyy" in the name "xxx.yyy" */
81894 Token **pUnqual /* Write the unqualified object name here */
81955 Token *pName1, /* First part of the name of the table or view */
81956 Token *pName2, /* Second part of the name of the table or view */
81967 Token *pName; /* Unqualified name of the table to create */
81981 ** The call below sets the pName pointer to point at the token (pName1 or
82176 SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){
82295 ** parsing a CREATE TABLE statement. The pFirst token is the first
82296 ** token in the sequence of tokens that describe the type of the
82297 ** column currently under construction. pLast is the last token
82302 SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){
82451 SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){
82689 Token *pCons, /* The ',' token after the last column defn. */
82690 Token *pEnd, /* The final ')' token in the CREATE TABLE */
82896 Token *pBegin, /* The CREATE token that begins the statement */
82897 Token *pName1, /* The token that holds the name of the view */
82898 Token *pName2, /* The token that holds the name of the view */
82906 Token sEnd;
82908 Token *pName = 0;
82933 ** This will force all the Expr.token.z values to be dynamically
83146 ** token for additional information.
83430 Token *pTo, /* Name of the other table */
83696 Token *pName1, /* First part of index name. May be NULL */
83697 Token *pName2, /* Second part of index name. May be NULL */
83701 Token *pStart, /* The CREATE token that begins this statement */
83702 Token *pEnd, /* The ")" that closes the CREATE INDEX statement */
83712 Token nullId; /* Fake token for an empty ID list */
83718 Token *pName = 0; /* Unqualified name of the index to create */
84320 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){
84471 Token *pTable, /* Table to append */
84472 Token *pDatabase /* Database of the table */
84491 Token *pTemp = pDatabase;
84546 ** alias token. If the term is a subquery, then pSubquery is the
84557 Token *pTable, /* Name of the table to add to the FROM clause */
84558 Token *pDatabase, /* Name of the database containing pTable */
84559 Token *pAlias, /* The right-hand side of the AS subexpression */
84598 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
84703 SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){
84957 SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
84965 Token *pObjName; /* Name of the table or index to be reindexed */
88777 Token tOld = { "old", 3 }; /* Literal "old" token */
88778 Token tNew = { "new", 3 }; /* Literal "new" token */
88779 Token tFromCol; /* Name of column in child table */
88780 Token tToCol; /* Name of column in parent table */
88851 Token tFrom;
92404 Token *pId1, /* First part of [database.]id field */
92405 Token *pId2, /* Second part of [database.]id field, or NULL */
92406 Token *pValue, /* Token for <value>, or NULL */
92412 Token *pId; /* Pointer to <id> token */
94665 SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
94667 Token *apAll[3];
94668 Token *p;
97889 Token sColname; /* Computed column name as a token */
99432 Token *pName1, /* The name of the trigger */
99433 Token *pName2, /* The name of the trigger */
99447 Token *pName; /* The unqualified db name */
99624 Token *pAll /* Token that describes the complete CREATE TRIGGER */
99631 Token nameToken; /* Trigger name for error reporting */
99722 Token *pName /* The target name */
99746 Token *pTableName, /* Name of the table into which we insert */
99779 Token *pTableName, /* Name of the table to be updated */
99804 Token *pTableName, /* The table from which rows are deleted */
100020 ** Convert the pStep->target token into a SrcList and return a pointer
100031 TriggerStep *pStep /* The trigger containing the target token */
100258 pProgram->token = (void *)pTrigger;
101778 Token *pName1, /* Name of new table, or database name */
101779 Token *pName2, /* Name of new table or NULL */
101780 Token *pModuleName, /* Name of the module for the virtual table */
101834 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
101913 ** The parser calls this routine when it sees the first token
101923 ** The parser calls this routine for each token after the first token
101926 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){
101927 Token *pArg = &pParse->sArg;
102001 ** columns of the table to see if any of them contain the token "hidden".
102002 ** If so, set the Column.isHidden flag and remove the token from
107866 Token eOperator; /* "like" or "glob" or "regexp" */
107884 struct AttachKey { int type; Token key; };
107899 Token *pStart, Token *pEnd){
107908 static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){
107936 Token *pPostOp /* The operand token for setting the span */
107961 Token *pPreOp /* The operand token for setting the span */
107967 /* Next is all token values, in a form suitable for use by makeheaders.
107994 ** original value of the token will not parse.
108020 #define sqlite3ParserTOKENTYPE Token
108071 ** current state and lookahead token. These tables are used to implement
108079 ** token onto the stack and goto state N.
108594 ** appears in the grammar, then ID becomes a fallback token for X, Y,
108596 ** but it does not parse, the type of the token is changed to ID and
108676 ** + The value of the token stored at this level of the stack.
108677 ** (In other words, the "major" token.)
108681 ** It is sometimes called the "minor" token.
108685 YYCODETYPE major; /* The major token value. This is the code
108686 ** number for the token at this stack level */
108687 YYMINORTYPE minor; /* The user-supplied minor token value. This
108688 ** is the value of the token */
109300 ** If there is a destructor routine associated with the token which
109303 ** Return the major token number for the symbol popped.
109364 ** look-ahead token iLookAhead.
109366 ** If the look-ahead token is YYNOCODE, then check to see if the action is
109372 YYCODETYPE iLookAhead /* The look-ahead token */
109386 YYCODETYPE iFallback; /* Fallback token */
109429 ** look-ahead token iLookAhead.
109431 ** If the look-ahead token is YYNOCODE, then check to see if the action is
109437 YYCODETYPE iLookAhead /* The look-ahead token */
109488 int yyMajor, /* The major token to shift in */
109489 YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */
110824 Token all;
111094 int yymajor, /* The major type of the error token */
111095 YYMINORTYPE yyminor /* The minor type of the error token */
111098 #define TOKEN (yyminor.yy0)
111101 assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */
111102 sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
111127 ** The second argument is the major token number. The third is
111128 ** the minor token. The fourth optional argument is whatever the
111135 ** <li> The major token number.
111136 ** <li> The minor token number.
111145 int yymajor, /* The major token code number */
111146 sqlite3ParserTOKENTYPE yyminor /* The value for the token */
111208 ** grammar defines an error token "ERROR".
111232 fprintf(yyTraceFILE,"%sDiscard input token %s\n",
111275 ** * Report an error message, and throw away the input token.
111277 ** * If the input token is $, then fail the parse.
111353 ** it is a keyword. If it is a keyword, the token code of that keyword is
111679 ** Return the length of the token that begins at z[0].
111680 ** Store the token type in *tokenType before returning.
111962 int tokenType; /* type of the next token */
111963 int lastTokenParsed = -1; /* type of the previous token */
112009 *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"",
112136 ** Token types used by the sqlite3_complete() routine. See the header
112189 ** (2) tkOTHER Any other SQL token.
112205 u8 token; /* Value of the next token */
112212 /* Token: */
112228 /* Token: */
112239 token = tkSEMI;
112247 token = tkWS;
112252 token = tkOTHER;
112259 token = tkWS;
112264 token = tkOTHER;
112269 token = tkWS;
112276 token = tkOTHER;
112286 token = tkOTHER;
112298 token = tkOTHER;
112303 token = tkCREATE;
112305 token
112311 token = tkTRIGGER;
112313 token = tkTEMP;
112315 token = tkTEMP;
112317 token = tkOTHER;
112323 token = tkEND;
112327 token = tkEXPLAIN;
112331 token = tkOTHER;
112336 token = tkOTHER;
112344 token = tkOTHER;
112349 state = trans[state][token];
115923 ** given term. Doclists hold docids and associated token positions.
115949 ** memory. A "position" is an index of a token in the token stream
116296 ** Retrieve the next token from the tokenizer cursor pCursor. This
116302 ** normalized version of the token (i.e. after any case-folding and/or
116304 ** of this buffer in bytes. The input text that generated the token is
116307 ** byte of the token in the input buffer. *piEndOffset should be set
116308 ** to the index of the first byte just past the end of the token in
116321 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
116322 int *piStartOffset, /* OUT: Byte offset of token in input buffer */
116323 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
116746 ** sequence. A single token is the base case and the most common case.
116751 char *z; /* Text of the token */
116753 token ends with a "*" character */
116754 int bFirst; /* True if token must appear at position 0 */
116759 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
116760 Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
116774 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
118653 token 'b' would consist of:
118663 ** when the *pp1 token appears before the *pp2 token, but not more than nToken
118670 int nToken, /* Maximum difference in token positions */
118786 ** lists are the offsets of the last token in each phrase (tokens "1" and "2"
118795 int nRight, /* Maximum difference in token positions */
118796 int nLeft, /* Maximum difference in token positions */
119075 ** a token in position 0 (of any column). If so, it writes argument iDelta
119473 Fts3PhraseToken *pTok, /* Token to query for */
120275 ** Allocate an Fts3MultiSegReader for each token in the expression headed
120322 ** Arguments pList/nList contain the doclist for token iToken of phrase p.
120332 int iToken, /* Token pList/nList corresponds to */
120429 int iPrev = -1; /* Token number of previous deferred token */
120704 ** each phrase in the expression (subject to deferred token processing).
120738 ** token in the FTS expression.
120748 Fts3Phrase *pPhrase; /* The phrase the token belongs to */
120749 int iToken; /* Position of token in phrase */
120750 Fts3PhraseToken *pToken; /* The token itself */
120753 int iCol; /* The column the token must match */
120875 ** called on each token to defer. Otherwise, an SQLite error code is
120897 ** causing any problems, the deferred token optimization is completely
120905 ** only 1 token, exit early. No tokens to defer in this case. */
120921 ** to retrieve the entire doclist for the token from the full-text index.
120924 ** a. The cheapest token in the entire query (i.e. the one visited by the
120927 ** b. Part of a multi-token phrase.
120929 ** After each token doclist is loaded, merge it with the others from the
120932 ** any phrase doclist for which 1 or more token doclists have been loaded.
120936 ** Then, for each token, defer it if loading the doclist would result in
120943 Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
120945 /* Set pTC to point to the cheapest remaining token. */
120971 /* Either this is the cheapest token in the entire query, or it is
120972 ** part of a multi-token phrase. Either way, the entire doclist will
121013 /* Allocate a MultiSegReader for each token in the expression. */
121151 ** where "*" may match any single token. The position list in this case
121482 ** token. Then, see if this row is really a match, considering deferred
121494 /* Free the position-lists accumulated for each deferred token above. */
122365 ** c) The old syntax supports the "-" token qualifier. This is not
122387 ** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
122395 ** phrase ::= TOKEN.
122396 ** phrase ::= COLUMN:TOKEN.
122397 ** phrase ::= "TOKEN TOKEN TOKEN...".
122494 ** Extract the next token from buffer z (length n) using the tokenizer
122497 ** single token and set *ppExpr to point to it. If the end of the buffer is
122498 ** reached before a token is found, set *ppExpr to zero. It is the
122613 ** Fts3PhraseToken structures token buffers are all stored as a single
122803 ** user has supplied a token such as "ORacle". Continue.
122844 /* If control flows to this point, this must be a regular token, or
122845 ** the end of the input. Read a regular token using the sqlite3_tokenizer
122847 ** column specifier for the token.
122850 ** with a quoted phrase, only with a single token. Not sure if this was
123782 int iToken; /* index of next token to be returned */
123783 char *zToken; /* storage for current token */
124291 ** Characters that can be part of a token. We assume any character
124293 ** part of a token. In other words, delimiters all must have
124307 ** Extract the next token from a tokenization cursor. The cursor must
124312 const char **pzToken, /* OUT: *pzToken is the token text */
124313 int *pnBytes, /* OUT: Number of bytes in token */
124314 int *piStartOffset, /* OUT: Starting offset of token */
124315 int *piEndOffset, /* OUT: Ending offset of token */
124316 int *piPosition /* OUT: Position integer of token */
124493 /* Find the start of the next token. */
124602 ** list. For each token in the <input-string>, three elements are
124603 ** added to the returned list. The first is the token position, the
124604 ** second is the token text (folded, stemmed, etc.) and the third is the
124605 ** substring of <input-string> associated with the token. For example,
124914 int iToken; /* index of next token to be returned */
124915 char *pToken; /* storage for current token */
125023 ** Extract the next token from a tokenization cursor. The cursor must
125028 const char **ppToken, /* OUT: *ppToken is the token text */
125029 int *pnBytes, /* OUT: Number of bytes in token */
125030 int *piStartOffset, /* OUT: Starting offset of token */
125031 int *piEndOffset, /* OUT: Ending offset of token */
125032 int *piPosition /* OUT: Position integer of token */
125193 Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
125194 int iCol; /* Column token must occur in */
125769 ** happen if there was no previous entry for this token.
125814 ** zText==0. In this case, add zero token entries to the hash table and
128367 char const *zToken; /* Buffer containing token */
128368 int nToken; /* Number of bytes in token */
128370 int iPos; /* Position of token in zText */
128426 ** Add an entry for token pToken to the pCsr->pDeferred list.
128430 Fts3PhraseToken *pToken, /* Token to defer */
128431 int iCol /* Column that token must appear in (or -1) */
128720 int iCurrent; /* First token of current snippet */
128734 int iPos; /* Index of first token in snippet */
128967 int *piToken, /* OUT: First token of proposed snippet */
128972 int iStart = pIter->iCurrent; /* First token of snippet */
129197 int *piPos, /* IN/OUT: First token of snippet */
129220 int iCurrent = 0; /* Token counter */
129270 int iCurrent = 0; /* Current token number of document */
129271 int iEnd = 0; /* Byte offset of end of current token */
129273 int iPos = pFragment->iPos; /* First token of snippet */
129290 /* Open a token cursor on the document. */
129298 int iBegin; /* Offset in zDoc of start of token */
129299 int iFin; /* Offset in zDoc of end of token */
129305 /* Special case - the last token of the snippet is also the last token
129307 ** of the previous token and the end of the document to the output.
130060 int iMinPos = 0x7FFFFFFF; /* Position of next token */
130061 TermOffset *pTerm = 0; /* TermOffset associated with next token */
134121 ** Extract the next token from a tokenization cursor.
134125 const char **ppToken, /* OUT: *ppToken is the token text */
134126 int *pnBytes, /* OUT: Number of bytes in token */
134127 int *piStartOffset, /* OUT: Starting offset of token */
134128 int *piEndOffset, /* OUT: Ending offset of token */
134129 int *piPosition /* OUT: Position integer of token */
