Lines Matching refs:token
2 * \brief Defines the interface for a common token.
4 * All token streams should provide their tokens using an instance
5 * of this common token. A custom pointer is provided, wher you may attach
6 * a further structure to enhance the common token if you feel the need
7 * to do so. The C runtime will assume that a token provides implementations
45 /** How many tokens to allocate at once in the token factory
49 /* Base token types, which all lexer/parser tokens come after in sequence.
52 /** Indicator of an invalid token
58 /** Imaginary token type to cause a traversal of child nodes in a tree parser
62 /** Imaginary token type to signal the end of a stream of child nodes.
66 /** First token that can be used by users/generated code
71 /** End of file token
75 /** Default channel for a token
79 /** Reserved channel number for a HIDDEN token - a token that
88 // Indicates whether this token is carrying:
100 /** The definition of an ANTLR3 common token structure, which all implementations
101 * of a token stream should provide, installing any further structures in the
105 * Token streams are in essence provided by lexers or other programs that serve
110 /** The actual type of this token
114 /** Indicates that a token was produced from the token factory and therefore
116 * token factory is responsible for deleting it.
120 /// A string factory that we can use if we ever need the text of a token
125 /** The line number in the input stream where this token was derived from
130 * token resides starts.
134 /** The character position in the line that this token was derived from
138 /** The virtual channel that this token exists in.
142 /** Pointer to the input stream that this token originated in.
146 /** What the index of this token is, 0, 1, .., n-2, n-1 tokens
150 /** The character offset in the input stream where the text for this token
155 /** The character offset in the input stream where the text for this token
160 /// Indicates whether this token is carrying:
175 /// Pointer that is used when the token just has a pointer to
176 /// a char *, such as when a rewrite of an imaginary token supplies
183 /// Some token types actually do carry around their associated text, hence
191 * as the standard structure for a token, a number of user programmable
192 * elements are allowed in a token. This is one of them.
197 * as the standard structure for a token, a number of user programmable
198 * elements are allowed in a token. This is one of them.
203 * as the standard structure for a token, a number of user programmable
204 * elements are allowed in a token. This is one of them.
213 * token is destroyed.
221 /** Pointer to function that returns the text pointer of a token, use
222 * toString() if you want a pANTLR3_STRING version of the token.
224 pANTLR3_STRING (*getText)(struct ANTLR3_COMMON_TOKEN_struct * token);
227 * with a token. Imaginary tokens such as an ANTLR3_CLASSIC_TOKEN may actually
233 void (*setText)(struct ANTLR3_COMMON_TOKEN_struct * token, pANTLR3_STRING text);
236 * with a token. Imaginary tokens such as an ANTLR3_CLASSIC_TOKEN may actually
242 void (*setText8)(struct ANTLR3_COMMON_TOKEN_struct * token, pANTLR3_UINT8 text);
244 /** Pointer to a function that returns the token type of this token
246 ANTLR3_UINT32 (*getType)(struct ANTLR3_COMMON_TOKEN_struct * token);
248 /** Pointer to a function that sets the type of this token
250 void (*setType)(struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 ttype);
252 /** Pointer to a function that gets the 'line' number where this token resides
254 ANTLR3_UINT32 (*getLine)(struct ANTLR3_COMMON_TOKEN_struct * token);
256 /** Pointer to a function that sets the 'line' number where this token reside
258 void (*setLine)(struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 line);
260 /** Pointer to a function that gets the offset in the line where this token exists
262 ANTLR3_INT32 (*getCharPositionInLine) (struct ANTLR3_COMMON_TOKEN_struct * token);
264 /** Pointer to a function that sets the offset in the line where this token exists
266 void (*setCharPositionInLine) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_INT32 pos);
268 /** Pointer to a function that gets the channel that this token was placed in (parsers
271 ANTLR3_UINT32 (*getChannel) (struct ANTLR3_COMMON_TOKEN_struct * token);
273 /** Pointer to a function that sets the channel that this token should belong to
275 void (*setChannel) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_UINT32 channel);
277 /** Pointer to a function that returns an index 0...n-1 of the token in the token
280 ANTLR3_MARKER (*getTokenIndex) (struct ANTLR3_COMMON_TOKEN_struct * token);
282 /** Pointer to a function that can set the token index of this token in the token
285 void (*setTokenIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER);
287 /** Pointer to a function that gets the start index in the input stream for this token.
289 ANTLR3_MARKER (*getStartIndex) (struct ANTLR3_COMMON_TOKEN_struct * token);
291 /** Pointer to a function that sets the start index in the input stream for this token.
293 void (*setStartIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER index);
295 /** Pointer to a function that gets the stop index in the input stream for this token.
297 ANTLR3_MARKER (*getStopIndex) (struct ANTLR3_COMMON_TOKEN_struct * token);
299 /** Pointer to a function that sets the stop index in the input stream for this token.
301 void (*setStopIndex) (struct ANTLR3_COMMON_TOKEN_struct * token, ANTLR3_MARKER index);
303 /** Pointer to a function that returns this token as a text representation that can be
308 pANTLR3_STRING (*toString) (struct ANTLR3_COMMON_TOKEN_struct * token);
312 /** \brief ANTLR3 Token factory interface to create lots of tokens efficiently
329 /** The next token to throw out from the pool, will cause a new pool allocation
334 /** Trick to initialize tokens and their API quickly, we set up this token when the
335 * factory is created, then just copy the memory it uses into the new token.
339 /** Pointer to an input stream that is using this token factory (may be NULL)
344 /** Pointer to a function that returns a new token
