Lines Matching refs:tokens
37 from antlr3.tokens import Token, CommonToken
246 @brief A stream of tokens accessing tokens from a TokenSource
258 i<0 indicates tokens in the past. So -1 is previous token and -2 is
259 two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken.
280 If you don't want to buffer up tokens, then this method makes no
291 Where is this stream pulling tokens from? This is not the name, but
300 Return the text of all tokens from start to stop, inclusive.
301 If the stream does not buffer all the tokens then it can just
609 @brief The most common stream of tokens
611 The most common stream of tokens is one where every token is buffered up
612 and tokens are prefiltered for a certain channel (the parser will only
613 see these tokens and cannot change the filter channel number during the
620 the tokens from.
622 @param channel Skip tokens on any channel but this one; this is how we
633 self.tokens = []
635 # Map<tokentype, channel> to override some Tokens' channel numbers
638 # Set<tokentype>; discard any tokens with this type
641 # Skip tokens on any channel but this one; this is how we skip
645 # By default, track all incoming tokens
648 # The index into the tokens list of the current token (next token
649 # to consume). p==-1 indicates that the tokens list is empty
667 self.tokens = []
679 Load all tokens from the token source and put in tokens.
712 self.tokens.append(t)
732 if self.p < len(self.tokens):
745 while self.tokens[i].channel != self.channel:
755 while i >= 0 and self.tokens[i].channel != self.channel:
764 to force all tokens of type ttype to be on channel. For example,
779 Given a start and stop index, return a list of all tokens in
780 the token type set. Return None if no tokens were found. This
781 method looks at both on and off channel tokens.
787 if stop is None or stop >= len(self.tokens):
788 stop = len(self.tokens) - 1
801 token for token in self.tokens[start:stop]
828 # find k good tokens
830 # skip off-channel tokens
838 return self.tokens[i]
844 """Look backwards k tokens on-channel tokens"""
857 # find k good tokens looking backwards
859 # skip off-channel tokens
866 return self.tokens[i]
871 Return absolute token i; ignore which channel the tokens are on;
872 that is, count all tokens not just on-channel tokens.
875 return self.tokens[i]
885 return self.tokens[start:stop+1]
903 return len(self.tokens)
943 stop = len(self.tokens) - 1
947 if stop >= len(self.tokens):
948 stop = len(self.tokens) - 1
950 return ''.join([t.text for t in self.tokens[start:stop+1]])
987 if self.stream.tokens[self.index].type != EOF:
988 buf.write(self.stream.tokens[self.index].text)
1032 all the time. As the buffer of tokens is converted to strings, the
1040 index i does not change the index values for tokens i+1..n-1.
1050 TokenRewriteStream tokens = new TokenRewriteStream(lex);
1051 T parser = new T(tokens);
1059 System.out.println(tokens.toString());
1069 tokens.insertAfter("pass1", t, "text to put after t");}
1070 tokens.insertAfter("pass2", u, "text after u");}
1071 System.out.println(tokens.toString("pass1"));
1072 System.out.println(tokens.toString("pass2"));
1198 if first > last or first < 0 or last < 0 or last >= len(self.tokens):
1201 % (first, last, len(self.tokens)))
1246 while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens):
1279 end = len(self.tokens) - 1
1284 if end >= len(self.tokens):
1285 end = len(self.tokens) - 1
1300 # Walk buffer, executing instructions and emitting tokens
1302 while i <= end and i < len(self.tokens):
1310 t = self.tokens[i]
1323 if end == len(self.tokens) - 1:
1328 if op.index >= len(self.tokens)-1:
1365 R.x-y.u = replace x-y indexed tokens with u
1386 add tokens in front of a method body '{' and then delete the method
1518 while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens):
