/external/antlr/runtime/CSharp3/Sources/Antlr3.Runtime.Test/ |
PreprocessorTests.cs | 51 List<IToken> tokens = tokenStream.GetTokens(); 52 Assert.AreEqual(2, tokens.Count); 53 Assert.AreEqual(PreprocessorLexer.DELIMITED_COMMENT, tokens[0].Type); 54 Assert.AreEqual("/**/", tokens[0].Text); 55 Assert.AreEqual(PreprocessorLexer.EOF, tokens[1].Type);
|
/external/deqp-deps/glslang/glslang/MachineIndependent/preprocessor/ |
PpAtom.cpp | 97 } tokens[] = { member in namespace:__anon20614 163 // Add single character tokens to the atom table: 174 // Add multiple character scanner tokens : 175 for (size_t ii = 0; ii < sizeof(tokens)/sizeof(tokens[0]); ii++) 176 addAtomFixed(tokens[ii].str, tokens[ii].val);
|
/external/antlr/runtime/JavaScript/tests/functional/ |
rhino-python.extensions | 33 tokens should be sent to the parser sometimes without a corresponding 48 This TokenStream normally just passes tokens through to the parser. 63 A queue of tokens is built up to hold multiple DEDENT tokens that 77 /** The queue of tokens */ 78 this.tokens = []; 117 if (this.tokens.length>0 ) { 118 var t = this.tokens[0]; 119 this.tokens.splice(0,1); 137 this.tokens = this.tokens.concat(hiddenTokens) [all...] |
/external/snakeyaml/src/test/java/org/pyyaml/ |
CanonicalScanner.java | 26 import org.yaml.snakeyaml.tokens.AliasToken; 27 import org.yaml.snakeyaml.tokens.AnchorToken; 28 import org.yaml.snakeyaml.tokens.DirectiveToken; 29 import org.yaml.snakeyaml.tokens.DocumentStartToken; 30 import org.yaml.snakeyaml.tokens.FlowEntryToken; 31 import org.yaml.snakeyaml.tokens.FlowMappingEndToken; 32 import org.yaml.snakeyaml.tokens.FlowMappingStartToken; 33 import org.yaml.snakeyaml.tokens.FlowSequenceEndToken; 34 import org.yaml.snakeyaml.tokens.FlowSequenceStartToken; 35 import org.yaml.snakeyaml.tokens.KeyToken 52 public ArrayList<Token> tokens; field in class:CanonicalScanner [all...] |
/external/webp/src/utils/ |
huffman_encode_utils.c | 262 HuffmanTreeToken* tokens, 266 tokens->code = value; 267 tokens->extra_bits = 0; 268 ++tokens; 275 tokens->code = value; 276 tokens->extra_bits = 0; 277 ++tokens; 281 tokens->code = 16; 282 tokens->extra_bits = repetitions - 3; 283 ++tokens; [all...] |
/external/antlr/runtime/CSharp2/Sources/Antlr3.Runtime/Antlr.Runtime/ |
LegacyCommonTokenStream.cs | 43 * The most common stream of tokens is one where every token is buffered up 44 * and tokens are prefiltered for a certain channel (the parser will only 45 * see these tokens and cannot change the filter channel number during the 49 * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> 61 protected List<IToken> tokens; field in class:Antlr.Runtime.LegacyCommonTokenStream 63 /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */ 66 /** <summary>Set<tokentype>; discard any tokens with this type</summary> */ 69 /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ 72 /** <summary>By default, track all incoming tokens</summary> */ 79 * The index into the tokens list of the current token (next toke [all...] |
/external/antlr/runtime/CSharp3/Sources/Antlr3.Runtime/ |
LegacyCommonTokenStream.cs | 41 * The most common stream of tokens is one where every token is buffered up 42 * and tokens are prefiltered for a certain channel (the parser will only 43 * see these tokens and cannot change the filter channel number during the 47 * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> 60 protected List<IToken> tokens; field in class:Antlr.Runtime.LegacyCommonTokenStream 62 /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */ 65 /** <summary>Set<tokentype>; discard any tokens with this type</summary> */ 68 /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ 71 /** <summary>By default, track all incoming tokens</summary> */ 78 * The index into the tokens list of the current token (next toke [all...] |
/external/antlr/runtime/Java/src/main/java/org/antlr/runtime/ |
BufferedTokenStream.java | 35 /** Buffer all input tokens but do on-demand fetching of new tokens from 37 * proper lexing of future tokens. The ST template parser needs this, 43 * tokens to the parser. The stream can't ignore off-channel tokens. 47 * to confuse small moving window of tokens it uses for the full buffer. 57 protected List<Token> tokens = new ArrayList<Token>(100); field in class:BufferedTokenStream 62 /** The index into the tokens list of the current token (next token 63 * to consume). tokens[p] should be LT(1). p=-1 indicates need 117 public int size() { return tokens.size(); [all...] |
LegacyCommonTokenStream.java | 32 /** The most common stream of tokens is one where every token is buffered up 33 * and tokens are prefiltered for a certain channel (the parser will only 34 * see these tokens and cannot change the filter channel number during the 37 * TODO: how to access the full token stream? How to track all tokens matched per rule? 45 protected List<Token> tokens; field in class:LegacyCommonTokenStream 47 /** Map<tokentype, channel> to override some Tokens' channel numbers */ 50 /** Set<tokentype>; discard any tokens with this type */ 53 /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 56 /** By default, track all incoming tokens */ 64 /** The index into the tokens list of the current token (next toke [all...] |