HomeSort by relevance Sort by last modified time
    Searched refs:Tokenizer (Results 1 - 25 of 74) sorted by null

1 2 3

  /external/chromium_org/third_party/protobuf/src/google/protobuf/io/
tokenizer_unittest.cc 40 #include <google/protobuf/io/tokenizer.h>
183 EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result));
198 Tokenizer::TokenType type;
208 { "hello", Tokenizer::TYPE_IDENTIFIER },
211 { "123", Tokenizer::TYPE_INTEGER },
212 { "0xab6", Tokenizer::TYPE_INTEGER },
213 { "0XAB6", Tokenizer::TYPE_INTEGER },
214 { "0X1234567", Tokenizer::TYPE_INTEGER },
215 { "0x89abcdef", Tokenizer::TYPE_INTEGER },
216 { "0x89ABCDEF", Tokenizer::TYPE_INTEGER }
    [all...]
tokenizer.h 52 class Tokenizer;
82 class LIBPROTOBUF_EXPORT Tokenizer {
84 // Construct a Tokenizer that reads and tokenizes text from the given
87 Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
88 ~Tokenizer();
191 // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
196 // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
206 // parsed by a Tokenizer, the result is undefined (possibly an assert
233 GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
367 inline const Tokenizer::Token& Tokenizer::current()
    [all...]
tokenizer.cc 91 #include <google/protobuf/io/tokenizer.h>
108 // For instance, Tokenizer::ConsumeZeroOrMore<Whitespace>() will eat
185 Tokenizer::Tokenizer(ZeroCopyInputStream* input,
208 Tokenizer::~Tokenizer() {
219 void Tokenizer::NextChar() {
240 void Tokenizer::Refresh() {
270 inline void Tokenizer::RecordTo(string* target) {
275 inline void Tokenizer::StopRecording()
    [all...]
  /external/chromium_org/third_party/angle/src/compiler/preprocessor/
Tokenizer.h 19 class Tokenizer : public Lexer
36 Tokenizer(Diagnostics *diagnostics);
37 ~Tokenizer();
48 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
generate_parser.sh 25 run_flex Tokenizer.l Tokenizer.cpp
27 patch --silent --forward < 64bit-tokenizer-safety.patch
DirectiveParser.h 20 class Tokenizer;
25 DirectiveParser(Tokenizer *tokenizer,
74 Tokenizer *mTokenizer;
Tokenizer.l 26 #include "Tokenizer.h"
74 %option extra-type="pp::Tokenizer::Context*"
270 Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(0)
275 Tokenizer::~Tokenizer()
280 bool Tokenizer::init(size_t count, const char * const string[], const int length[])
289 void Tokenizer::setFileNumber(int file)
296 void Tokenizer::setLineNumber(int line)
301 void Tokenizer::setMaxTokenSize(size_t maxTokenSize
    [all...]
Preprocessor.cpp 17 #include "Tokenizer.h"
26 Tokenizer tokenizer; member in struct:pp::PreprocessorImpl
33 tokenizer(diag),
34 directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
63 return mImpl->tokenizer.init(count, string, length);
115 mImpl->tokenizer.setMaxTokenSize(maxTokenSize);
  /external/chromium_org/tools/gyp/tools/Xcode/Specifications/
gyp.xclangspec 73 Tokenizer = "xcode.lang.gyp.lexer.toplevel";
108 Tokenizer = "xcode.lang.gyp.lexer";
121 Tokenizer = "xcode.lang.gyp.lexer";
133 Tokenizer = "xcode.lang.gyp.lexer";
144 Tokenizer = "xcode.lang.gyp.lexer";
155 Tokenizer = "xcode.lang.gyp.lexer";
168 Tokenizer = "xcode.lang.gyp.lexer";
183 Tokenizer = "xcode.lang.gyp.lexer";
  /external/chromium_org/tools/gn/
tokenizer_unittest.cc 8 #include "tools/gn/tokenizer.h"
23 std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err);
38 TEST(Tokenizer, Empty) {
43 std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err);
49 results = Tokenizer::Tokenize(&whitespace_input, &err);
53 TEST(Tokenizer, Identifier) {
60 TEST(Tokenizer, Integer) {
68 TEST(Tokenizer, IntegerNoSpace) {
76 TEST(Tokenizer, String) {
86 TEST(Tokenizer, Operator)
    [all...]
tokenizer.h 18 class Tokenizer {
27 // This is a helper function for error output so that the tokenizer's
45 // InputFile must outlive the tokenizer and all generated tokens.
46 explicit Tokenizer(const InputFile* input_file, Err* err);
47 ~Tokenizer();
91 DISALLOW_COPY_AND_ASSIGN(Tokenizer);
tokenizer.cc 5 #include "tools/gn/tokenizer.h"
71 Tokenizer::Tokenizer(const InputFile* input_file, Err* err)
80 Tokenizer::~Tokenizer() {
84 std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) {
85 Tokenizer t(input_file, err);
89 std::vector<Token> Tokenizer::Run() {
156 size_t Tokenizer::ByteOffsetOfNthLine(const base::StringPiece& buf, int n) {
176 bool Tokenizer::IsNewline(const base::StringPiece& buffer, size_t offset)
    [all...]
string_utils.cc 10 #include "tools/gn/tokenizer.h"
67 if (!Tokenizer::IsIdentifierFirstChar(input[*i])) {
78 while (*i < size && Tokenizer::IsIdentifierContinuingChar(input[*i]))
command_args.cc 22 #include "tools/gn/tokenizer.h"
50 if (offset == 0 || Tokenizer::IsNewline(data, offset))
56 if (Tokenizer::IsNewline(data, cur))
86 Tokenizer::ByteOffsetOfNthLine(data, location.line_number());
  /external/deqp/executor/
xeXMLParser.cpp 57 Tokenizer::Tokenizer (void)
65 Tokenizer::~Tokenizer (void)
69 void Tokenizer::clear (void)
77 void Tokenizer::error (const std::string& what)
82 void Tokenizer::feed (const deUint8* bytes, int numBytes)
98 int Tokenizer::getChar (int offset) const
108 void Tokenizer::advance (void)
338 void Tokenizer::getString (std::string& dst) cons
    [all...]
xeXMLParser.hpp 83 class Tokenizer
86 Tokenizer (void);
87 ~Tokenizer (void);
89 void clear (void); //!< Resets tokenizer to initial state.
103 Tokenizer (const Tokenizer& other);
104 Tokenizer& operator= (const Tokenizer& other);
190 Tokenizer m_tokenizer;
203 inline void Tokenizer::getTokenStr (std::string& dst) cons
    [all...]
  /external/clang/lib/ASTMatchers/Dynamic/
Parser.cpp 56 /// \brief Simple tokenizer for the parser.
287 const TokenInfo NameToken = Tokenizer->consumeNextToken();
289 if (Tokenizer->nextTokenKind() != TokenInfo::TK_OpenParen) {
297 if ((Tokenizer->nextTokenKind() == TokenInfo::TK_Comma ||
298 Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen ||
299 Tokenizer->nextTokenKind() == TokenInfo::TK_Eof) &&
319 const TokenInfo OpenToken = Tokenizer->consumeNextToken();
340 while (Tokenizer->nextTokenKind() != TokenInfo::TK_Eof) {
341 if (Tokenizer->nextTokenKind() == TokenInfo::TK_CloseParen) {
343 EndToken = Tokenizer->consumeNextToken()
    [all...]
  /external/libedit/src/
histedit.h 234 typedef struct tokenizer Tokenizer;
239 Tokenizer *tok_init(const char *);
240 void tok_end(Tokenizer *);
241 void tok_reset(Tokenizer *);
242 int tok_line(Tokenizer *, const LineInfo *,
244 int tok_str(Tokenizer *, const char *,
307 /* Wide character tokenizer support */
tokenizer.c 1 /* $NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $ */
41 static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
43 __RCSID("$NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $");
49 * tokenize.c: Bourne shell like tokenizer
74 struct TYPE(tokenizer) {
86 private void FUN(tok,finish)(TYPE(Tokenizer) *);
90 * Finish a word in the tokenizer.
93 FUN(tok,finish)(TYPE(Tokenizer) *tok)
107 * Initialize the tokenizer
109 public TYPE(Tokenizer) *
    [all...]
  /external/chromium_org/third_party/WebKit/Source/core/html/parser/
InputStreamPreprocessor.h 39 template <typename Tokenizer>
43 InputStreamPreprocessor(Tokenizer* tokenizer)
44 : m_tokenizer(tokenizer)
128 Tokenizer* m_tokenizer;
  /external/chromium_org/third_party/protobuf/src/google/protobuf/compiler/
parser.cc 46 #include <google/protobuf/io/tokenizer.h>
114 inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) {
119 return LookingAtType(io::Tokenizer::TYPE_END);
150 if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
161 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
163 if (!io::Tokenizer::ParseInteger(input_->current().text,
193 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
194 if (!io::Tokenizer::ParseInteger(input_->current().text, max_value,
209 if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) {
210 *output = io::Tokenizer::ParseFloat(input_->current().text)
    [all...]
parser.h 47 #include <google/protobuf/io/tokenizer.h>
73 bool Parse(io::Tokenizer* input, FileDescriptorProto* file);
150 inline bool LookingAtType(io::Tokenizer::TokenType token_type);
234 void StartAt(const io::Tokenizer::Token& token);
239 void EndAt(const io::Tokenizer::Token& token);
420 io::Tokenizer* input_;
  /external/clang/include/clang/ASTMatchers/Dynamic/
Parser.h 182 Parser(CodeTokenizer *Tokenizer, Sema *S,
194 CodeTokenizer *const Tokenizer;
  /external/doclava/src/com/google/doclava/apicheck/
ApiFile.java 67 final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray()); local
71 String token = tokenizer.getToken();
76 parsePackage(api, tokenizer);
78 throw new ApiParseException("expected package got " + token, tokenizer.getLine());
88 private static void parsePackage(ApiInfo api, Tokenizer tokenizer)
94 token = tokenizer.requireToken();
95 assertIdent(tokenizer, token)
    [all...]
  /external/chromium_org/third_party/protobuf/src/google/protobuf/
text_format.cc 50 #include <google/protobuf/io/tokenizer.h>
167 // This class makes use of the Protocol Message compiler's tokenizer found
168 // in //google/protobuf/io/tokenizer.h. Note that class's Parse
209 tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE);
223 if (LookingAtType(io::Tokenizer::TYPE_END)) {
238 return suc && LookingAtType(io::Tokenizer::TYPE_END);
307 // Consumes the current field (as returned by the tokenizer) on the
591 if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
616 if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
622 LookingAtType(io::Tokenizer::TYPE_INTEGER))
    [all...]

Completed in 3735 milliseconds

1 2 3