Lines Matching refs:tokens
41 * The most common stream of tokens is one where every token is buffered up
42 * and tokens are prefiltered for a certain channel (the parser will only
43 * see these tokens and cannot change the filter channel number during the
47 * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks>
60 protected List<IToken> tokens;
62 /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
65 /** <summary>Set<tokentype>; discard any tokens with this type</summary> */
68 /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
71 /** <summary>By default, track all incoming tokens</summary> */
78 * The index into the tokens list of the current token (next token
79 * to consume). p==-1 indicates that the tokens list is empty
86 tokens = new List<IToken>( 500 );
122 tokens.Clear();
128 * Load all tokens from the token source and put in tokens.
169 tokens.Add( t );
192 if ( p < tokens.Count )
202 int n = tokens.Count;
203 while ( i < n && ( (IToken)tokens[i] ).Channel != channel )
212 while ( i >= 0 && ( (IToken)tokens[i] ).Channel != channel )
221 * to force all tokens of type ttype to be on channel. For example,
256 return tokens;
265 * Given a start and stop index, return a List of all tokens in
266 * the token type BitSet. Return null if no tokens were found. This
267 * method looks at both on and off channel tokens.
276 if ( stop >= tokens.Count )
278 stop = tokens.Count - 1;
289 // list = tokens[start:stop]:{Token t, t.getType() in types}
293 IToken t = tokens[i];
336 if ( ( p + k - 1 ) >= tokens.Count )
338 return tokens[tokens.Count - 1];
340 //System.out.println(tokens.get(p+k-1));
343 // find k good tokens
346 // skip off-channel tokens
350 if ( i >= tokens.Count )
352 return tokens[tokens.Count - 1];
358 return (IToken)tokens[i];
361 /** <summary>Look backwards k tokens on-channel tokens</summary> */
380 // find k good tokens looking backwards
383 // skip off-channel tokens
391 return (IToken)tokens[i];
395 * Return absolute token i; ignore which channel the tokens are on;
396 * that is, count all tokens not just on-channel tokens.
401 return (IToken)tokens[i];
405 /** Get all tokens from start..stop inclusively */
416 return new List<IToken>(tokens.Skip(start).Take(count));
444 return tokens.Count;
491 return ToString( 0, tokens.Count - 1 );
504 if ( stop >= tokens.Count )
506 stop = tokens.Count - 1;
511 IToken t = tokens[i];
