Lines Matching refs:token
78 * Makes sure that this.token contains a token if one is available.
84 HtmlToken token = readToken();
85 if (token == null) { return null; }
87 switch (token.type) {
94 if (state == State.SAW_EQ && HtmlTokenType.TAGEND == token.type) {
97 pushbackToken(token);
100 token.start, token.start, HtmlTokenType.ATTRVALUE);
115 if (HtmlTokenType.TEXT == token.type
116 || HtmlTokenType.UNESCAPED == token.type) {
117 token = collapseSubsequent(token);
121 if (HtmlTokenType.TEXT == token.type
122 && !token.tokenInContextMatches(input, "=")) {
124 token = HtmlInputSplitter.reclassify(
125 token, HtmlTokenType.ATTRNAME);
130 if (HtmlTokenType.TEXT == token.type) {
131 if (token.tokenInContextMatches(input, "=")) {
133 // Skip the '=' token
137 token = HtmlInputSplitter.reclassify(
138 token, HtmlTokenType.ATTRNAME);
145 if (HtmlTokenType.TEXT == token.type
146 || HtmlTokenType.QSTRING == token.type) {
147 if (HtmlTokenType.TEXT == token.type) {
151 token = collapseAttributeName(token);
154 token = HtmlInputSplitter.reclassify(
155 token, HtmlTokenType.ATTRVALUE);
163 return token;
167 * Collapses all the following tokens of the same type into this.token.
169 private HtmlToken collapseSubsequent(HtmlToken token) {
170 HtmlToken collapsed = token;
172 (next= peekToken(0)) != null && next.type == token.type;
179 private HtmlToken collapseAttributeName(HtmlToken token) {
208 if (nToMerge == 0) { return token; }
210 int end = token.end;
215 return HtmlToken.instance(token.start, end, HtmlTokenType.TEXT);
240 private void pushbackToken(HtmlToken token) {
241 lookahead.addFirst(token);
259 * A token stream that breaks a character stream into <tt>
291 * Make sure that there is a token ready to yield in this.token.
295 HtmlToken token = parseToken();
296 if (null == token) { return null; }
303 if (token.type != HtmlTokenType.SERVERCODE) {
305 token = reclassify(
306 token, (this.textEscapingMode == HtmlTextEscapingMode.RCDATA
311 switch (token.type) {
315 token.start + 1, token.end);
331 return token;
387 * This method returns a stream of tokens such that each token starts where
388 * the last token ended.
649 "Unexpectedly DONE while lexing HTML token stream");
704 static HtmlToken reclassify(HtmlToken token, HtmlTokenType type) {
705 return HtmlToken.instance(token.start, token.end, type);
711 * A TokenStream that lazily fetches one token at a time.
