Lines Matching refs:tokens
33 tokens should be sent to the parser sometimes without a corresponding
48 This TokenStream normally just passes tokens through to the parser.
63 A queue of tokens is built up to hold multiple DEDENT tokens that
77 /** The queue of tokens */
78 this.tokens = [];
117 if (this.tokens.length>0 ) {
118 var t = this.tokens[0];
119 this.tokens.splice(0,1);
137 this.tokens = this.tokens.concat(hiddenTokens);
140 this.tokens.push(t);
147 this.tokens = this.tokens.concat(hiddenTokens);
150 this.tokens.push(t);
158 this.tokens = this.tokens.concat(hiddenTokens);
178 this.tokens.push(indent);
188 this.tokens.push(dedent);
193 this.tokens.push(t);
