Lines Matching defs:TOKEN
2373 ** complete if it ends with a semicolon token and is not a prefix of a
2376 ** independent tokens (they are part of the token in which they are
8156 Token Token;
8505 void *token; /* id that may be used to recursive triggers */
10412 ** Each token coming out of the lexer is an instance of
10415 ** Note if Token.z==0 then Token.dyn and Token.n are undefined and
10416 ** may contain random values. Do not make any assumptions about Token.dyn
10417 ** and Token.n when Token.z==0.
10419 struct Token {
10420 const char *z; /* Text of the token. Not NULL-terminated! */
10421 unsigned int n; /* Number of characters in this token */
10487 ** Expr.op is the opcode. The integer parser token codes are reused
10494 ** or TK_STRING), then Expr.token contains the text of the SQL literal. If
10495 ** the expression is a variable (TK_VARIABLE), then Expr.token contains the
10497 ** then Expr.token contains the name of the function.
10551 char *zToken; /* Token value. Zero terminated and dequoted */
10598 #define EP_DblQuoted 0x0040 /* token.z was originally in "..." */
10664 char *zName; /* Token associated with this expression */
11162 Token sNameToken; /* Token with unqualified schema object name */
11163 Token sLastToken; /* The last token parsed */
11165 Token sArg; /* Complete text of a module argument */
11256 * target -> A token holding the quoted name of the table to insert into.
11264 * target -> A token holding the quoted name of the table to delete from.
11269 * target -> A token holding the quoted name of the table to update rows of.
11282 Token target; /* Target table for DELETE, UPDATE, INSERT */
11300 const Token *pName; /* Name of the container - used for error messages */
11585 SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
11588 SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*);
11590 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
11594 SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
11599 SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
11605 SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
11606 SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*);
11610 SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*);
11612 SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*);
11613 SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,Select*);
11633 SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
11653 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
11656 SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
11657 SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
11658 Token*, Select*, Expr*, IdList*);
11659 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);
11665 SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
11666 Token*, int, int);
11707 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*);
11722 Token*);
11762 SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*,
11764 SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*);
11775 SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
11777 SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8);
11778 SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*);
11795 SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*);
11796 SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int);
11812 SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*);
11870 SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
11877 SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*);
11908 SQLITE_PRIVATE const Token sqlite3IntTokens[];
11916 SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*);
11918 SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
11928 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *);
11932 SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*);
11934 SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*);
11970 SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*);
12016 SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int);
12017 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*);
12019 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*);
12372 SQLITE_PRIVATE const Token sqlite3IntTokens[] = {
12950 void *token; /* Copy of SubProgram.token */
19399 #define etTOKEN 12 /* a pointer to a Token structure */
19998 Token *pToken = va_arg(ap, Token*);
21152 ** %T Insert a token
21185 ** %T Insert a token
59360 ** token (i.e. "5", "5.1", "'a string'"). If the expression can
64136 int n; /* Length of a token prefix */
64137 int nToken; /* Length of the parameter token */
65329 void *t; /* Token identifying trigger */
70042 void *t; /* Token identifying trigger */
70058 ** single trigger all have the same value for the SubProgram.token
70061 u.cc.t = u.cc.pProgram->token;
70062 for(u.cc.pFrame=p->pFrame; u.cc.pFrame && u.cc.pFrame->token!=u.cc.t; u.cc.pFrame=u.cc.pFrame->pParent);
70105 u.cc.pFrame->token = u.cc.pProgram->token;
74468 SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){
74757 ** If dequote is true, then the token (if it exists) is dequoted.
74759 ** parameter is ignored if pToken is NULL or if the token does not
74764 ** can be translated into a 32-bit integer, then the token is not
74772 const Token *pToken, /* Token argument. Might be NULL */
74815 ** Allocate a new expression node from a zero-terminated token that has
74821 const char *zToken /* Token argument. Might be NULL */
74823 Token x;
74876 const Token *pToken /* Argument token */
74906 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){
75046 ** dupedExprNodeSize() Size of Expr + space for token
75047 ** dupedExprSize() Expr + token + subtree components
75427 Token *pName, /* Name to be added */
76774 /* Expressions of the form: CAST(pLeft AS token) */
77417 /* Expressions of the form: CAST(pLeft AS token) */
77827 ** This code depends on the fact that certain token values (ex: TK_EQ)
78476 int token;
78477 Token tname;
78487 ** statement is that the table name is the first non-space token that
78488 ** is immediately followed by a TK_LP or TK_USING token.
78497 /* Store the token that zCsr points to in tname. */
78501 /* Advance zCsr to the next token. Store that token type in 'token',
78506 len = sqlite3GetToken(zCsr, &token);
78507 } while( token==TK_SPACE );
78509 token!=TK_LP && token!=TK_USING );
78545 unsigned const char *z; /* Pointer to token */
78546 int n; /* Length of token z */
78547 int token; /* Type of token */
78551 n = sqlite3GetToken(z, &token);
78552 if( token==TK_REFERENCES ){
78556 n = sqlite3GetToken(z, &token);
78557 }while( token==TK_SPACE );
78596 int token;
78597 Token tname;
78607 ** statement is that the table name is the first token that is immediatedly
78619 /* Store the token that zCsr points to in tname. */
78623 /* Advance zCsr to the next token. Store that token type in 'token',
78628 len = sqlite3GetToken(zCsr, &token);
78629 }while( token==TK_SPACE );
78634 ** token is read and 'dist' equals 2, the condition stated above
78642 if( token==TK_DOT || token==TK_ON ){
78645 } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) );
78647 /* Variable tname now contains the token that is the old table-name
78827 Token *pName /* The new table name. */
79054 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
80033 SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){
80040 Token *pTableName;
80814 const Token *pName /* Name of the view, trigger, or index */
81765 ** Given a token, return a string that consists of the text of that
81766 ** token. Space to hold the returned string
81771 ** surround the body of the token are removed.
81777 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){
81824 ** The token *pName contains the name of a database (either "main" or
81829 SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){
81850 ** This routine sets the *ppUnqual pointer to point at the token (pName1 or
81856 Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */
81857 Token *pName2, /* The "yyy" in the name "xxx.yyy" */
81858 Token **pUnqual /* Write the unqualified object name here */
81919 Token *pName1, /* First part of the name of the table or view */
81920 Token *pName2, /* Second part of the name of the table or view */
81931 Token *pName; /* Unqualified name of the table to create */
81945 ** The call below sets the pName pointer to point at the token (pName1 or
82140 SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){
82259 ** parsing a CREATE TABLE statement. The pFirst token is the first
82260 ** token in the sequence of tokens that describe the type of the
82261 ** column currently under construction. pLast is the last token
82266 SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){
82415 SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){
82653 Token *pCons, /* The ',' token after the last column defn. */
82654 Token *pEnd, /* The final ')' token in the CREATE TABLE */
82860 Token *pBegin, /* The CREATE token that begins the statement */
82861 Token *pName1, /* The token that holds the name of the view */
82862 Token *pName2, /* The token that holds the name of the view */
82870 Token sEnd;
82872 Token *pName = 0;
82897 ** This will force all the Expr.token.z values to be dynamically
83110 ** token for additional information.
83394 Token *pTo, /* Name of the other table */
83660 Token *pName1, /* First part of index name. May be NULL */
83661 Token *pName2, /* Second part of index name. May be NULL */
83665 Token *pStart, /* The CREATE token that begins this statement */
83666 Token *pEnd, /* The ")" that closes the CREATE INDEX statement */
83676 Token nullId; /* Fake token for an empty ID list */
83682 Token *pName = 0; /* Unqualified name of the index to create */
84284 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){
84435 Token *pTable, /* Table to append */
84436 Token *pDatabase /* Database of the table */
84455 Token *pTemp = pDatabase;
84510 ** alias token. If the term is a subquery, then pSubquery is the
84521 Token *pTable, /* Name of the table to add to the FROM clause */
84522 Token *pDatabase, /* Name of the database containing pTable */
84523 Token *pAlias, /* The right-hand side of the AS subexpression */
84562 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
84667 SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){
84921 SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
84929 Token *pObjName; /* Name of the table or index to be reindexed */
88741 Token tOld = { "old", 3 }; /* Literal "old" token */
88742 Token tNew = { "new", 3 }; /* Literal "new" token */
88743 Token tFromCol; /* Name of column in child table */
88744 Token tToCol; /* Name of column in parent table */
88815 Token tFrom;
92368 Token *pId1, /* First part of [database.]id field */
92369 Token *pId2, /* Second part of [database.]id field, or NULL */
92370 Token *pValue, /* Token for <value>, or NULL */
92376 Token *pId; /* Pointer to <id> token */
94629 SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
94631 Token *apAll[3];
94632 Token *p;
97853 Token sColname; /* Computed column name as a token */
99396 Token *pName1, /* The name of the trigger */
99397 Token *pName2, /* The name of the trigger */
99411 Token *pName; /* The unqualified db name */
99588 Token *pAll /* Token that describes the complete CREATE TRIGGER */
99595 Token nameToken; /* Trigger name for error reporting */
99686 Token *pName /* The target name */
99710 Token *pTableName, /* Name of the table into which we insert */
99743 Token *pTableName, /* Name of the table to be updated */
99768 Token *pTableName, /* The table from which rows are deleted */
99984 ** Convert the pStep->target token into a SrcList and return a pointer
99995 TriggerStep *pStep /* The trigger containing the target token */
100222 pProgram->token = (void *)pTrigger;
101742 Token *pName1, /* Name of new table, or database name */
101743 Token *pName2, /* Name of new table or NULL */
101744 Token *pModuleName, /* Name of the module for the virtual table */
101798 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
101877 ** The parser calls this routine when it sees the first token
101887 ** The parser calls this routine for each token after the first token
101890 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){
101891 Token *pArg = &pParse->sArg;
101965 ** columns of the table to see if any of them contain the token "hidden".
101966 ** If so, set the Column.isHidden flag and remove the token from
107830 Token eOperator; /* "like" or "glob" or "regexp" */
107848 struct AttachKey { int type; Token key; };
107863 static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){
107872 static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){
107900 Token *pPostOp /* The operand token for setting the span */
107925 Token *pPreOp /* The operand token for setting the span */
107931 /* Next is all token values, in a form suitable for use by makeheaders.
107958 ** original value of the token will not parse.
107984 #define sqlite3ParserTOKENTYPE Token
108035 ** current state and lookahead token. These tables are used to implement
108043 ** token onto the stack and goto state N.
108558 ** appears in the grammar, then ID becomes a fallback token for X, Y,
108560 ** but it does not parse, the type of the token is changed to ID and
108640 ** + The value of the token stored at this level of the stack.
108641 ** (In other words, the "major" token.)
108645 ** It is sometimes called the "minor" token.
108649 YYCODETYPE major; /* The major token value. This is the code
108650 ** number for the token at this stack level */
108651 YYMINORTYPE minor; /* The user-supplied minor token value. This
108652 ** is the value of the token */
109264 ** If there is a destructor routine associated with the token which
109267 ** Return the major token number for the symbol popped.
109328 ** look-ahead token iLookAhead.
109330 ** If the look-ahead token is YYNOCODE, then check to see if the action is
109336 YYCODETYPE iLookAhead /* The look-ahead token */
109350 YYCODETYPE iFallback; /* Fallback token */
109393 ** look-ahead token iLookAhead.
109395 ** If the look-ahead token is YYNOCODE, then check to see if the action is
109401 YYCODETYPE iLookAhead /* The look-ahead token */
109452 int yyMajor, /* The major token to shift in */
109453 YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */
110788 Token all;
111058 int yymajor, /* The major type of the error token */
111059 YYMINORTYPE yyminor /* The minor type of the error token */
111062 #define TOKEN (yyminor.yy0)
111065 assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */
111066 sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
111091 ** The second argument is the major token number. The third is
111092 ** the minor token. The fourth optional argument is whatever the
111099 ** <li> The major token number.
111100 ** <li> The minor token number.
111109 int yymajor, /* The major token code number */
111110 sqlite3ParserTOKENTYPE yyminor /* The value for the token */
111172 ** grammar defines an error token "ERROR".
111196 fprintf(yyTraceFILE,"%sDiscard input token %s\n",
111239 ** * Report an error message, and throw away the input token.
111241 ** * If the input token is $, then fail the parse.
111317 ** it is a keyword. If it is a keyword, the token code of that keyword is
111643 ** Return the length of the token that begins at z[0].
111644 ** Store the token type in *tokenType before returning.
111926 int tokenType; /* type of the next token */
111927 int lastTokenParsed = -1; /* type of the previous token */
111973 *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"",
112100 ** Token types used by the sqlite3_complete() routine. See the header
112153 ** (2) tkOTHER Any other SQL token.
112169 u8 token; /* Value of the next token */
112176 /* Token: */
112192 /* Token: */
112203 token = tkSEMI;
112211 token = tkWS;
112216 token = tkOTHER;
112223 token = tkWS;
112228 token = tkOTHER;
112233 token = tkWS;
112240 token = tkOTHER;
112250 token = tkOTHER;
112262 token = tkOTHER;
112267 token = tkCREATE;
112269 token = tkOTHER;
112275 token = tkTRIGGER;
112277 token = tkTEMP;
112279 token = tkTEMP;
112281 token = tkOTHER;
112287 token = tkEND;
112291 token = tkEXPLAIN;
112295 token = tkOTHER;
112300 token = tkOTHER;
112308 token = tkOTHER;
112313 state = trans[state][token];
115887 ** given term. Doclists hold docids and associated token positions.
115913 ** memory. A "position" is an index of a token in the token stream
116260 ** Retrieve the next token from the tokenizer cursor pCursor. This
116266 ** normalized version of the token (i.e. after any case-folding and/or
116268 ** of this buffer in bytes. The input text that generated the token is
116271 ** byte of the token in the input buffer. *piEndOffset should be set
116272 ** to the index of the first byte just past the end of the token in
116285 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
116286 int *piStartOffset, /* OUT: Byte offset of token in input buffer */
116287 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
116710 ** sequence. A single token is the base case and the most common case.
116715 char *z; /* Text of the token */
116717 int isPrefix; /* True if token ends with a "*" character */
116718 int bFirst; /* True if token must appear at position 0 */
116723 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
116724 Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
116738 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
118617 ** Then the position list for this row for token 'b' would consist of:
118627 ** when the *pp1 token appears before the *pp2 token, but not more than nToken
118634 int nToken, /* Maximum difference in token positions */
118750 ** lists are the offsets of the last token in each phrase (tokens "1" and "2"
118759 int nRight, /* Maximum difference in token positions */
118760 int nLeft, /* Maximum difference in token positions */
119039 ** a token in position 0 (of any column). If so, it writes argument iDelta
119437 Fts3PhraseToken *pTok, /* Token to query for */
120227 ** Allocate an Fts3MultiSegReader for each token in the expression headed
120274 ** Arguments pList/nList contain the doclist for token iToken of phrase p.
120284 int iToken, /* Token pList/nList corresponds to */
120381 int iPrev = -1; /* Token number of previous deferred token */
120656 ** each phrase in the expression (subject to deferred token processing).
120690 ** token in the FTS expression.
120700 Fts3Phrase *pPhrase; /* The phrase the token belongs to */
120701 int iToken; /* Position of token in phrase */
120702 Fts3PhraseToken *pToken; /* The token itself */
120705 int iCol; /* The column the token must match */
120827 ** called on each token to defer. Otherwise, an SQLite error code is
120849 ** causing any problems, the deferred token optimization is completely
120857 ** only 1 token, exit early. No tokens to defer in this case. */
120873 ** to retrieve the entire doclist for the token from the full-text index.
120876 ** a. The cheapest token in the entire query (i.e. the one visited by the
120879 ** b. Part of a multi-token phrase.
120881 ** After each token doclist is loaded, merge it with the others from the
120884 ** any phrase doclist for which 1 or more token doclists have been loaded.
120888 ** Then, for each token, defer it if loading the doclist would result in
120895 Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */
120897 /* Set pTC to point to the cheapest remaining token. */
120923 /* Either this is the cheapest token in the entire query, or it is
120924 ** part of a multi-token phrase. Either way, the entire doclist will
120965 /* Allocate a MultiSegReader for each token in the expression. */
121103 ** where "*" may match any single token. The position list in this case
121434 ** token. Then, see if this row is really a match, considering deferred
121446 /* Free the position-lists accumulated for each deferred token above. */
122317 ** c) The old syntax supports the "-" token qualifier. This is not
122339 ** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
122347 ** phrase ::= TOKEN.
122348 ** phrase ::= COLUMN:TOKEN.
122349 ** phrase ::= "TOKEN TOKEN TOKEN...".
122446 ** Extract the next token from buffer z (length n) using the tokenizer
122449 ** single token and set *ppExpr to point to it. If the end of the buffer is
122450 ** reached before a token is found, set *ppExpr to zero. It is the
122565 ** Fts3PhraseToken structures token buffers are all stored as a single
122755 ** user has supplied a token such as "ORacle". Continue.
122796 /* If control flows to this point, this must be a regular token, or
122797 ** the end of the input. Read a regular token using the sqlite3_tokenizer
122799 ** column specifier for the token.
122802 ** with a quoted phrase, only with a single token. Not sure if this was
123734 int iToken; /* index of next token to be returned */
123735 char *zToken; /* storage for current token */
124243 ** Characters that can be part of a token. We assume any character
124245 ** part of a token. In other words, delimiters all must have
124259 ** Extract the next token from a tokenization cursor. The cursor must
124264 const char **pzToken, /* OUT: *pzToken is the token text */
124265 int *pnBytes, /* OUT: Number of bytes in token */
124266 int *piStartOffset, /* OUT: Starting offset of token */
124267 int *piEndOffset, /* OUT: Ending offset of token */
124268 int *piPosition /* OUT: Position integer of token */
124445 /* Find the start of the next token. */
124554 ** list. For each token in the <input-string>, three elements are
124555 ** added to the returned list. The first is the token position, the
124556 ** second is the token text (folded, stemmed, etc.) and the third is the
124557 ** substring of <input-string> associated with the token. For example,
124866 int iToken; /* index of next token to be returned */
124867 char *pToken; /* storage for current token */
124975 ** Extract the next token from a tokenization cursor. The cursor must
124980 const char **ppToken, /* OUT: *ppToken is the token text */
124981 int *pnBytes, /* OUT: Number of bytes in token */
124982 int *piStartOffset, /* OUT: Starting offset of token */
124983 int *piEndOffset, /* OUT: Ending offset of token */
124984 int *piPosition /* OUT: Position integer of token */
125145 Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
125146 int iCol; /* Column token must occur in */
125721 ** happen if there was no previous entry for this token.
125766 ** zText==0. In this case, add zero token entries to the hash table and
128319 char const *zToken; /* Buffer containing token */
128320 int nToken; /* Number of bytes in token */
128322 int iPos; /* Position of token in zText */
128378 ** Add an entry for token pToken to the pCsr->pDeferred list.
128382 Fts3PhraseToken *pToken, /* Token to defer */
128383 int iCol /* Column that token must appear in (or -1) */
128672 int iCurrent; /* First token of current snippet */
128686 int iPos; /* Index of first token in snippet */
128919 int *piToken, /* OUT: First token of proposed snippet */
128924 int iStart = pIter->iCurrent; /* First token of snippet */
129149 int *piPos, /* IN/OUT: First token of snippet */
129172 int iCurrent = 0; /* Token counter */
129222 int iCurrent = 0; /* Current token number of document */
129223 int iEnd = 0; /* Byte offset of end of current token */
129225 int iPos = pFragment->iPos; /* First token of snippet */
129242 /* Open a token cursor on the document. */
129250 int iBegin; /* Offset in zDoc of start of token */
129251 int iFin; /* Offset in zDoc of end of token */
129257 /* Special case - the last token of the snippet is also the last token
129259 ** of the previous token and the end of the document to the output.
130012 int iMinPos = 0x7FFFFFFF; /* Position of next token */
130013 TermOffset *pTerm = 0; /* TermOffset associated with next token */
134073 ** Extract the next token from a tokenization cursor.
134077 const char **ppToken, /* OUT: *ppToken is the token text */
134078 int *pnBytes, /* OUT: Number of bytes in token */
134079 int *piStartOffset, /* OUT: Starting offset of token */
134080 int *piEndOffset, /* OUT: Ending offset of token */
134081 int *piPosition /* OUT: Position integer of token */
