Lines Matching defs:token
25 private static final String LOG_TAG = "TOKEN";
37 * <li> If it's a space, finish a token unless we're being quoted
39 * <li> Otherwise, add it to the token being built
41 * <li> At EOL, we typically haven't added the final token to the (tokens) {@link ArrayList}
46 * <li> Otherwise, add the final token to (tokens)
61 StringBuilder token = new StringBuilder();
74 // inside a quotation; treat spaces as part of the token
75 token.append(aChar);
77 if (token.length() > 0) {
78 // this is the end of a non-empty token; dump it in our list of tokens,
80 Log.d(LOG_TAG, String.format("Finished token '%s'", token.toString()));
81 tokens.add(token.toString());
82 token.delete(0, token.length());
91 // default case: add the character to the token being built
92 token.append(aChar);
102 // Add the final token to the tokens array.
103 if (token.length() > 0) {
104 Log.v(LOG_TAG, String.format("Finished final token '%s'", token.toString()));
105 tokens.add(token.toString());
106 token.delete(0, token.length());
134 final String token = tokens[i];
135 final Matcher wsMatcher = wsPattern.matcher(token);
138 sb.append(token);
141 sb.append(token);
144 // don't output space after last token
