Lines Matching full:token
2004 ** complete if it ends with a semicolon token and is not a prefix of a
2007 ** independent tokens (they are part of the token in which they are
6810 typedef struct Token Token;
7157 void *token; /* id that may be used to recursive triggers */
8983 ** Each token coming out of the lexer is an instance of
8986 ** Note if Token.z==0 then Token.dyn and Token.n are undefined and
8987 ** may contain random values. Do not make any assumptions about Token.dyn
8988 ** and Token.n when Token.z==0.
8990 struct Token {
8991 const char *z; /* Text of the token. Not NULL-terminated! */
8992 unsigned int n; /* Number of characters in this token */
9059 ** Expr.op is the opcode. The integer parser token codes are reused
9066 ** or TK_STRING), then Expr.token contains the text of the SQL literal. If
9067 ** the expression is a variable (TK_VARIABLE), then Expr.token contains the
9069 ** then Expr.token contains the name of the function.
9123 char *zToken; /* Token value. Zero terminated and dequoted */
9170 #define EP_DblQuoted 0x0040 /* token.z was originally in "..." */
9237 char *zName; /* Token associated with this expression */
9698 Token sNameToken; /* Token with unqualified schema object name */
9699 Token sLastToken; /* The last token parsed */
9705 Token sArg; /* Complete text of a module argument */
9796 * target -> A token holding the quoted name of the table to insert into.
9804 * target -> A token holding the quoted name of the table to delete from.
9809 * target -> A token holding the quoted name of the table to update rows of.
9822 Token target; /* Target table for DELETE, UPDATE, INSERT */
9840 const Token *pName; /* Name of the container - used for error messages */
10086 SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
10089 SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*);
10091 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
10095 SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
10100 SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
10106 SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
10107 SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*);
10111 SQLITE_PRIVATE void sqlite3AddColumnType(Parse*,Token*);
10113 SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*);
10114 SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,Select*);
10130 SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
10149 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
10152 SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
10153 SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
10154 Token*, Select*, Expr*, IdList*);
10155 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);
10161 SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
10162 Token*, int, int);
10203 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*);
10216 SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*);
10256 SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*,
10258 SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*);
10269 SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*,
10271 SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8);
10272 SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*);
10289 SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*);
10290 SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int);
10308 SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*);
10365 SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
10371 SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *, Token *);
10397 SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*);
10399 SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
10409 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *);
10413 SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*);
10415 SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*);
10448 SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*);
10490 SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*);
10491 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*);
10493 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*);
16399 #define etTOKEN 12 /* a pointer to a Token structure */
17013 Token *pToken = va_arg(ap, Token*);
17634 void *token; /* Copy of SubProgram.token */
18564 ** %T Insert a token
18597 ** %T Insert a token
47305 ** token (i.e. "5", "5.1", "'a string'"). If the expression can
51861 int n; /* Length of a token prefix */
51862 int nToken; /* Length of the parameter token */
52904 void *t; /* Token identifying trigger */
57352 void *t; /* Token identifying trigger */
57368 ** single trigger all have the same value for the SubProgram.token
57371 u.by.t = u.by.pProgram->token;
57372 for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent);
57414 u.by.pFrame->token = u.by.pProgram->token;
60578 SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pCollName){
60876 ** If dequote is true, then the token (if it exists) is dequoted.
60878 ** parameter is ignored if pToken is NULL or if the token does not
60883 ** can be translated into a 32-bit integer, then the token is not
60891 const Token *pToken, /* Token argument. Might be NULL */
60932 ** Allocate a new expression node from a zero-terminated token that has
60938 const char *zToken /* Token argument. Might be NULL */
60940 Token x;
60993 const Token *pToken /* Argument token */
61020 SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){
61157 ** dupedExprNodeSize() Size of Expr + space for token
61158 token + subtree components
61531 Token *pName, /* Name to be added */
62225 static const Token one = { "1", 1 }; /* Token for literal value 1 */
62898 /* Expressions of the form: CAST(pLeft AS token) */
63672 ** This code depends on the fact that certain token values (ex: TK_EQ)
64296 int token;
64297 Token tname;
64307 ** statement is that the table name is the first non-space token that
64308 ** is immediately followed by a TK_LP or TK_USING token.
64317 /* Store the token that zCsr points to in tname. */
64321 /* Advance zCsr to the next token. Store that token type in 'token',
64326 len = sqlite3GetToken(zCsr, &token);
64327 } while( token==TK_SPACE );
64329 } while( token!=TK_LP && token!=TK_USING );
64365 unsigned const char *z; /* Pointer to token */
64366 int n; /* Length of token z */
64367 int token; /* Type of token */
64371 n = sqlite3GetToken(z, &token);
64372 if( token==TK_REFERENCES ){
64376 n = sqlite3GetToken(z, &token);
64377 }while( token==TK_SPACE );
64416 int token;
64417 Token tname;
64427 ** statement is that the table name is the first token that is immediatedly
64439 /* Store the token that zCsr points to in tname. */
64443 /* Advance zCsr to the next token. Store that token type in 'token',
64448 len = sqlite3GetToken(zCsr, &token);
64449 }while( token==TK_SPACE );
64454 ** token is read and 'dist' equals 2, the condition stated above
64462 if( token==TK_DOT || token==TK_ON ){
64465 } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) );
64467 /* Variable tname now contains the token that is the old table-name
64620 Token *pName /* The new table name. */
64846 SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
65434 SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){
65440 Token *pTableName;
66101 const Token *pName /* Name of the view, trigger, or index */
67045 ** Given a token, return a string that consists of the text of that
67046 ** token. Space to hold the returned string
67051 ** surround the body of the token are removed.
67057 SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){
67104 ** The token *pName contains the name of a database (either "main" or
67109 SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){
67130 ** This routine sets the *ppUnqual pointer to point at the token (pName1 or
67136 Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */
67137 Token *pName2, /* The "yyy" in the name "xxx.yyy" */
67138 Token **pUnqual /* Write the unqualified object name here */
67199 Token *pName1, /* First part of the name of the table or view */
67200 Token *pName2, /* Second part of the name of the table or view */
67211 Token *pName; /* Unqualified name of the table to create */
67225 ** The call below sets the pName pointer to point at the token (pName1 or
67414 SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){
67533 ** parsing a CREATE TABLE statement. The pFirst token is the first
67534 ** token in the sequence of tokens that describe the type of the
67535 ** column currently under construction. pLast is the last token
67540 SQLITE_PRIVATE void sqlite3AddColumnType(Parse *pParse, Token *pType){
67689 SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){
67926 Token *pCons, /* The ',' token after the last column defn. */
67927 Token *pEnd, /* The final ')' token in the CREATE TABLE */
68132 Token *pBegin, /* The CREATE token that begins the statement */
68133 Token *pName1, /* The token that holds the name of the view */
68134 Token *pName2, /* The token that holds the name of the view */
68142 Token sEnd;
68144 Token *pName;
68171 ** This will force all the Expr.token.z values to be dynamically
68377 ** token for additional information.
68629 Token *pTo, /* Name of the other table */
68853 Token *pName1, /* First part of index name. May be NULL */
68854 Token *pName2, /* Second part of index name. May be NULL */
68858 Token *pStart, /* The CREATE token that begins this statement */
68859 Token *pEnd, /* The ")" that closes the CREATE INDEX statement */
68869 Token nullId; /* Fake token for an empty ID list */
68875 Token *pName = 0; /* Unqualified name of the index to create */
69472 SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){
69626 Token *pTable, /* Table to append */
69627 Token *pDatabase /* Database of the table */
69646 Token *pTemp = pDatabase;
69701 ** alias token. If the term is a subquery, then pSubquery is the
69712 Token *pTable, /* Name of the table to add to the FROM clause */
69713 Token *pDatabase, /* Name of the database containing pTable */
69714 Token *pAlias, /* The right-hand side of the AS subexpression */
69753 SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
69863 SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){
70097 SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){
70105 Token *pObjName; /* Name of the table or index to be reindexed */
73775 Token tOld = { "old", 3 }; /* Literal "old" token */
73776 Token tNew = { "new", 3 }; /* Literal "new" token */
73777 Token tFromCol; /* Name of column in child table */
73778 Token tToCol; /* Name of column in parent table */
73849 Token tFrom;
77224 Token *pId1, /* First part of [database.]id field */
77225 Token *pId2, /* Second part of [database.]id field, or NULL */
77226 Token *pValue, /* Token for <value>, or NULL */
77232 Token *pId; /* Pointer to <id> token */
79363 SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
79365 Token *apAll[3];
79366 Token *p;
82447 Token sColname; /* Computed column name as a token */
83805 Token *pName1, /* The name of the trigger */
83806 Token *pName2, /* The name of the trigger */
83820 Token *pName; /* The unqualified db name */
83979 Token *pAll /* Token that describes the complete CREATE TRIGGER */
83986 Token nameToken; /* Trigger name for error reporting */
84078 Token *pName /* The target name */
84102 Token *pTableName, /* Name of the table into which we insert */
84135 Token *pTableName, /* Name of the table to be updated */
84160 Token *pTableName, /* The table from which rows are deleted */
84364 ** Convert the pStep->target token into a SrcList and return a pointer
84375 TriggerStep *pStep /* The trigger containing the target token */
84599 pProgram->token = (void *)pTrigger;
86080 Token *pName1, /* Name of new table, or database name */
86081 Token *pName2, /* Name of new table or NULL */
86082 Token *pModuleName /* Name of the module for the virtual table */
86135 SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
86214 ** The parser calls this routine when it sees the first token
86224 ** The parser calls this routine for each token after the first token
86227 SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){
86228 Token *pArg = &pParse->sArg;
86299 ** columns of the table to see if any of them contain the token "hidden".
86300 ** If so, set the Column.isHidden flag and remove the token from
90906 Token eOperator; /* "like" or "glob" or "regexp" */
90924 struct AttachKey { int type; Token key; };
90931 static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){
90940 static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){
90968 Token *pPostOp /* The operand token for setting the span */
90993 Token *pPreOp /* The operand token for setting the span */
90999 /* Next is all token values, in a form suitable for use by makeheaders.
91026 ** original value of the token will not parse.
91052 #define sqlite3ParserTOKENTYPE Token
91102 ** current state and lookahead token. These tables are used to implement
91110 ** token onto the stack and goto state N.
91621 ** appears in the grammar, then ID becomes a fallback token for X, Y,
91623 ** but it does not parse, the type of the token is changed to ID and
91703 ** + The value of the token stored at this level of the stack.
91704 ** (In other words, the "major" token.)
91708 ** It is sometimes called the "minor" token.
91712 YYCODETYPE major; /* The major token value. This is the code
91713 ** number for the token at this stack level */
91714 YYMINORTYPE minor; /* The user-supplied minor token value. This
91715 ** is the value of the token */
92325 ** If there is a destructor routine associated with the token which
92328 ** Return the major token number for the symbol popped.
92389 ** look-ahead token iLookAhead.
92391 ** If the look-ahead token is YYNOCODE, then check to see if the action is
92397 YYCODETYPE iLookAhead /* The look-ahead token */
92411 YYCODETYPE iFallback; /* Fallback token */
92454 ** look-ahead token iLookAhead.
92456 ** If the look-ahead token is YYNOCODE, then check to see if the action is
92462 YYCODETYPE iLookAhead /* The look-ahead token */
92514 int yyMajor, /* The major token to shift in */
92515 YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */
93806 Token all;
94078 int yymajor, /* The major type of the error token */
94079 YYMINORTYPE yyminor /* The minor type of the error token */
94082 #define TOKEN (yyminor.yy0)
94085 assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */
94086 sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN);
94112 ** The second argument is the major token number. The third is
94113 ** the minor token. The fourth optional argument is whatever the
94120 ** <li> The major token number.
94121 ** <li> The minor token number.
94130 int yymajor, /* The major token code number */
94131 sqlite3ParserTOKENTYPE yyminor /* The value for the token */
94171 assert( !yyendofinput ); /* Impossible to shift the $ token */
94190 ** grammar defines an error token "ERROR".
94214 fprintf(yyTraceFILE,"%sDiscard input token %s\n",
94257 ** * Report an error message, and throw away the input token.
94259 ** * If the input token is $, then fail the parse.
94334 ** it is a keyword. If it is a keyword, the token code of that keyword is
94660 ** Return the length of the token that begins at z[0].
94661 ** Store the token type in *tokenType before returning.
94944 int tokenType; /* type of the next token */
94945 int lastTokenParsed = -1; /* type of the previous token */
94992 *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"",
95118 ** Token types used by the sqlite3_complete() routine. See the header
95171 ** (2) tkOTHER Any other SQL token.
95187 u8 token; /* Value of the next token */
95194 /* Token: */
95210 /* Token: */
95221 token = tkSEMI;
95229 token = tkWS;
95234 token = tkOTHER;
95241 token = tkWS;
95246 token = tkOTHER;
95251 token = tkWS;
95258 token = tkOTHER;
95268 token = tkOTHER;
95280 token = tkOTHER;
95285 token = tkCREATE;
95287 token
95293 token = tkTRIGGER;
95295 token = tkTEMP;
95297 token = tkTEMP;
95299 token = tkOTHER;
95305 token = tkEND;
95309 token = tkEXPLAIN;
95313 token = tkOTHER;
95318 token = tkOTHER;
95326 token = tkOTHER;
95331 state = trans[state][token];
98217 ** token positions and offsets with docids.
98241 ** memory. A "position" is an index of a token in the token stream
98570 ** Retrieve the next token from the tokenizer cursor pCursor. This
98576 ** normalized version of the token (i.e. after any case-folding and/or
98578 ** of this buffer in bytes. The input text that generated the token is
98581 ** byte of the token in the input buffer. *piEndOffset should be set
98582 ** to the index of the first byte just past the end of the token in
98595 const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
98596 int *piStartOffset, /* OUT: Byte offset of token in input buffer */
98597 int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
98888 ** sequence. A single token is the base case and the most common case.
98897 char *z; /* Text of the token */
98899 int isPrefix; /* True if token ends in with a "*" character */
98900 } aToken[1]; /* One entry for each token in the phrase */
99867 int nToken, /* Maximum difference in token positions */
99979 int nRight, /* Maximum difference in token positions */
99980 int nLeft, /* Maximum difference in token positions */
100407 char *z = pTok->z; /* Next token of the phrase */
100409 int isPrefix = pTok->isPrefix;/* True if token is a prefix */
100410 char *pList; /* Pointer to token doclist */
101237 ** c) The old syntax supports the "-" token qualifier. This is not
101259 ** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
101267 ** phrase ::= TOKEN.
101268 ** phrase ::= COLUMN:TOKEN.
101269 ** phrase ::= "TOKEN TOKEN TOKEN...".
101314 ** Extract the next token from buffer z (length n) using the tokenizer
101317 ** single token and set *ppExpr to point to it. If the end of the buffer is
101318 ** reached before a token is found, set *ppExpr to zero. It is the
101592 ** user has supplied a token such as "ORacle". Continue.
101634 /* If control flows to this point, this must be a regular token, or
101635 ** the end of the input. Read a regular token using the sqlite3_tokenizer
101637 ** column specifier for the token.
101640 ** with a quoted phrase, only with a single token. Not sure if this was
102555 int iToken; /* index of next token to be returned */
102556 char *zToken; /* storage for current token */
103064 ** Characters that can be part of a token. We assume any character
103066 ** part of a token. In other words, delimiters all must have
103080 ** Extract the next token from a tokenization cursor. The cursor must
103085 const char **pzToken, /* OUT: *pzToken is the token text */
103086 int *pnBytes, /* OUT: Number of bytes in token */
103087 int *piStartOffset, /* OUT: Starting offset of token */
103088 int *piEndOffset, /* OUT: Ending offset of token */
103089 int *piPosition /* OUT: Position integer of token */
103265 /* Find the start of the next token. */
103384 ** list. For each token in the <input-string>, three elements are
103385 ** added to the returned list. The first is the token position, the
103386 ** second is the token text (folded, stemmed, etc.) and the third is the
103387 ** substring of <input-string> associated with the token. For example,
103686 int iToken; /* index of next token to be returned */
103687 char *pToken; /* storage for current token */
103792 ** Extract the next token from a tokenization cursor. The cursor must
103797 const char **ppToken, /* OUT: *ppToken is the token text */
103798 int *pnBytes, /* OUT: Number of bytes in token */
103799 int *piStartOffset, /* OUT: Starting offset of token */
103800 int *piEndOffset, /* OUT: Ending offset of token */
103801 int *piPosition /* OUT: Position integer of token */
104353 ** happen if there was no previous entry for this token.
106233 int iToken; /* The index of the matching document token */
106337 int iToken, /* Matching token in document */
106383 ** Move to the first token in the expression tree.
106450 const char *zToken; /* Next token from the tokenizer */
106456 unsigned int iRotor = 0; /* Index of current token */
106457 int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
106458 int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
106478 struct PhraseToken *pToken; /* Current token */
106511 ** operator. When this is called, pSnippet contains the list of token
106523 ** then when this function is called the Snippet contains token offsets
106528 ** the integer id of the left-most token in the expression tree headed by
106537 int *piLeft /* Index of left-most token in pExpr */
106552 ** set local variable iLeft to the token number of the left-most
106553 ** token in the right-hand phrase, and iRight to the right most
106554 ** token in the same phrase. For example, if we had:
106558 ** then iLeft will be set to 2 (token number of ghi) and nToken will
106580 ** search for an instance of token (iLeft-1) somewhere in the
107016 int iPos, /* Document token offset to load data for */
107127 int *piPos, /* OUT: Starting token for best snippet */
107201 /* Figure out how highly a snippet starting at token offset i scores
107259 int iPos, /* Index of first document token in snippet */
107279 /* Open a token cursor on the document. Read all tokens up to and
107280 ** including token iPos (the first token of the snippet). Set variable
107281 ** iStart to the byte offset in zDoc of the start of token iPos.
111074 ** Extract the next token from a tokenization cursor.
111078 const char **ppToken, /* OUT: *ppToken is the token text */
111079 int *pnBytes, /* OUT: Number of bytes in token */
111080 int *piStartOffset, /* OUT: Starting offset of token */
111081 int *piEndOffset, /* OUT: Ending offset of token */
111082 int *piPosition /* OUT: Position integer of token */
