Lines Matching full:token
38 LiteralParser::TokenType LiteralParser::Lexer::lex(LiteralParserToken& token)
45 token.type = TokEnd;
46 token.start = token.end = m_ptr;
49 token.type = TokError;
50 token.start = m_ptr;
53 token.type = TokLBracket;
54 token.end = ++m_ptr;
57 token.type = TokRBracket;
58 token.end = ++m_ptr;
61 token.type = TokLParen;
62 token.end = ++m_ptr;
65 token.type = TokRParen;
66 token.end = ++m_ptr;
69 token.type = TokLBrace;
70 token.end = ++m_ptr;
73 token.type = TokRBrace;
74 token.end = ++m_ptr;
77 token.type = TokComma;
78 token.end = ++m_ptr;
81 token.type = TokColon;
82 token.end = ++m_ptr;
86 return lexString<StrictJSON>(token);
87 return lexString<NonStrictJSON>(token);
91 token.type = TokTrue;
92 token.end = m_ptr;
99 token.type = TokFalse;
100 token.end = m_ptr;
107 token.type = TokNull;
108 token.end = m_ptr;
123 return lexNumber(token);
134 template <LiteralParser::ParserMode mode> inline LiteralParser::TokenType LiteralParser::Lexer::lexString(LiteralParserToken& token)
203 token.stringToken = builder.build();
204 token.type = TokString;
205 token.end = ++m_ptr;
209 LiteralParser::TokenType LiteralParser::Lexer::lexNumber(LiteralParserToken& token)
268 token.type = TokNumber;
269 token.end = m_ptr;
270 Vector<char, 64> buffer(token.end - token.start + 1);
272 for (i = 0; i < token.end - token.start; i++) {
273 ASSERT(static_cast<char>(token.start[i]) == token.start[i]);
274 buffer[i] = static_cast<char>(token.start[i]);
278 token.numberToken = WTF::strtod(buffer.data(), &end);
279 ASSERT(buffer.data() + (token.end - token.start) == end);
