| /external/antlr/antlr-3.4/runtime/Ruby/test/unit/ |
| test-trees.rb | 163 r0 = new_node new_token(101)
164 r1 = new_node new_token(102)
166 r1.add_child(new_node new_token(103))
167 r2 = new_node new_token(106)
168 r2.add_child new_node( new_token 107 )
170 r0.add_child new_node( new_token 104 )
171 r0.add_child new_node( new_token 105 )
189 r0 = new_node new_token(101)
190 r1 = new_node new_token(102)
192 r1.add_child new_node( new_token 103
302 def new_token(type, opts = {}) method in class:TestTreeNodeStream
446 def new_token(type, opts = {}) method in class:TestCommonTreeNodeStream
782 def new_token(type, opts = {}) method in class:TestCommonTree
[all...] |
| test-streams.rb | 264 @source2.tokens << new_token( 10, :channel => ANTLR3::HIDDEN ) << new_token( 11 )
284 @source.tokens << new_token(12)
292 @source.tokens << new_token(12, :channel => ANTLR3::HIDDEN_CHANNEL) <<
293 new_token(13)
300 @source.tokens << new_token(12) <<
301 new_token(13, :channel => ANTLR3::HIDDEN_CHANNEL)
309 @source.tokens << new_token(12) << new_token(13)
318 @source.tokens << new_token(12) << new_token(13
404 def new_token(type, opts = {}) method in class:TestCommonTokenStream
[all...] |
| /external/libtextclassifier/smartselect/ |
| tokenizer.cc | 71 Token new_token("", 0, 0);
77 if (!new_token.value.empty()) {
78 result.push_back(new_token);
80 new_token = Token("", codepoint_index, codepoint_index);
83 new_token.value += std::string(
86 ++new_token.end;
89 if (!new_token.value.empty()) {
90 result.push_back(new_token);
92 new_token = Token("", codepoint_index + 1, codepoint_index + 1);
95 if (!new_token.value.empty())
[all...] |
| feature-processor.cc | 98 Token new_token(token_word.UTF8Substring(last_start, split_point),
103 current_pos = new_token.end;
105 replacement_tokens.push_back(new_token);
|
| /external/mesa3d/src/glsl/glcpp/ |
| glcpp-parse.y | 838 token_t *new_token = ralloc (copy, token_t);
839 *new_token = *node->token;
840 _token_list_append (copy, new_token);
[all...] |
