| /external/antlr/runtime/CSharp3/Sources/Antlr3.Runtime.Test/ |
| PreprocessorTests.cs | 51 List<IToken> tokens = tokenStream.GetTokens();
52 Assert.AreEqual(2, tokens.Count);
53 Assert.AreEqual(PreprocessorLexer.DELIMITED_COMMENT, tokens[0].Type);
54 Assert.AreEqual("/**/", tokens[0].Text);
55 Assert.AreEqual(PreprocessorLexer.EOF, tokens[1].Type);
|
| /external/deqp-deps/glslang/glslang/MachineIndependent/preprocessor/ |
| PpAtom.cpp | 97 } tokens[] = { member in namespace:__anon20614
163 // Add single character tokens to the atom table:
174 // Add multiple character scanner tokens :
175 for (size_t ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++)
176 addAtomFixed(tokens[ii].str, tokens[ii].val);
|
| /external/antlr/runtime/JavaScript/tests/functional/ |
| rhino-python.extensions | 33 tokens should be sent to the parser sometimes without a corresponding
48 This TokenStream normally just passes tokens through to the parser.
63 A queue of tokens is built up to hold multiple DEDENT tokens that
77 /** The queue of tokens */
78 this.tokens = [];
117 if (this.tokens.length>0 ) {
118 var t = this.tokens[0];
119 this.tokens.splice(0,1);
137 this.tokens = this.tokens.concat(hiddenTokens)
[all...] |
| /external/snakeyaml/src/test/java/org/pyyaml/ |
| CanonicalScanner.java | 26 import org.yaml.snakeyaml.tokens.AliasToken;
27 import org.yaml.snakeyaml.tokens.AnchorToken;
28 import org.yaml.snakeyaml.tokens.DirectiveToken;
29 import org.yaml.snakeyaml.tokens.DocumentStartToken;
30 import org.yaml.snakeyaml.tokens.FlowEntryToken;
31 import org.yaml.snakeyaml.tokens.FlowMappingEndToken;
32 import org.yaml.snakeyaml.tokens.FlowMappingStartToken;
33 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken;
34 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken;
35 import org.yaml.snakeyaml.tokens.KeyToken
52 public ArrayList<Token> tokens; field in class:CanonicalScanner
[all...] |
| /external/webp/src/utils/ |
| huffman_encode_utils.c | 262 HuffmanTreeToken* tokens,
266 tokens->code = value;
267 tokens->extra_bits = 0;
268 ++tokens;
275 tokens->code = value;
276 tokens->extra_bits = 0;
277 ++tokens;
281 tokens->code = 16;
282 tokens->extra_bits = repetitions - 3;
283 ++tokens;
[all...] |
| /external/antlr/runtime/CSharp2/Sources/Antlr3.Runtime/Antlr.Runtime/ |
| LegacyCommonTokenStream.cs | 43 * The most common stream of tokens is one where every token is buffered up
44 * and tokens are prefiltered for a certain channel (the parser will only
45 * see these tokens and cannot change the filter channel number during the
49 * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks>
61 protected List<IToken> tokens; field in class:Antlr.Runtime.LegacyCommonTokenStream
63 /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
66 /** <summary>Set<tokentype>; discard any tokens with this type</summary> */
69 /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
72 /** <summary>By default, track all incoming tokens</summary> */
79 * The index into the tokens list of the current token (next toke
[all...] |
| /external/antlr/runtime/CSharp3/Sources/Antlr3.Runtime/ |
| LegacyCommonTokenStream.cs | 41 * The most common stream of tokens is one where every token is buffered up
42 * and tokens are prefiltered for a certain channel (the parser will only
43 * see these tokens and cannot change the filter channel number during the
47 * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks>
60 protected List<IToken> tokens; field in class:Antlr.Runtime.LegacyCommonTokenStream
62 /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
65 /** <summary>Set<tokentype>; discard any tokens with this type</summary> */
68 /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
71 /** <summary>By default, track all incoming tokens</summary> */
78 * The index into the tokens list of the current token (next toke
[all...] |
| /external/antlr/runtime/Java/src/main/java/org/antlr/runtime/ |
| BufferedTokenStream.java | 35 /** Buffer all input tokens but do on-demand fetching of new tokens from
37 * proper lexing of future tokens. The ST template parser needs this,
43 * tokens to the parser. The stream can't ignore off-channel tokens.
47 * to confuse small moving window of tokens it uses for the full buffer.
57 protected List<Token> tokens = new ArrayList<Token>(100); field in class:BufferedTokenStream
62 /** The index into the tokens list of the current token (next token
63 * to consume). tokens[p] should be LT(1). p=-1 indicates need
117 public int size() { return tokens.size();
[all...] |
| LegacyCommonTokenStream.java | 32 /** The most common stream of tokens is one where every token is buffered up
33 * and tokens are prefiltered for a certain channel (the parser will only
34 * see these tokens and cannot change the filter channel number during the
37 * TODO: how to access the full token stream? How to track all tokens matched per rule?
45 protected List<Token> tokens; field in class:LegacyCommonTokenStream
47 /** Map<tokentype, channel> to override some Tokens' channel numbers */
50 /** Set<tokentype>; discard any tokens with this type */
53 /** Skip tokens on any channel but this one; this is how we skip whitespace... */
56 /** By default, track all incoming tokens */
64 /** The index into the tokens list of the current token (next toke
[all...] |