Lines Matching full:token
41 struct UString *token,
44 static UChar32 getNextChar (UCHARBUF *buf, UBool skipwhite, struct UString *token, UErrorCode *status);
45 static void seekUntilNewline (UCHARBUF *buf, struct UString *token, UErrorCode *status);
46 static void seekUntilEndOfComment (UCHARBUF *buf, struct UString *token, UErrorCode *status);
54 /* Read and return the next token from the stream. If the token is of
55 type eString, fill in the token parameter with the token. If the
56 token is eError, then the status parameter will contain the
63 struct UString *token,
64 uint32_t *linenumber, /* out: linenumber of token */
98 result = getStringToken(buf, c, token, status);
105 /* Copy a string token into the given UnicodeString. Upon entry, we
106 have already read the first character of the string token, which is
109 string, and copy them into the token parameter. The other
118 struct UString *token,
132 (if nothing else) as a string token. */
141 ustr_setlen(token, 0, status);
149 if (!lastStringWasQuoted && token->fLength > 0) {
150 ustr_ucat(token, SPACE, status);
193 ustr_uscat(token, pTarget,len, status);
210 if (token->fLength > 0) {
211 ustr_ucat(token, SPACE, status);
248 ustr_uscat(token, pTarget,len, status);
293 ustr_uscat(token, pTarget,len, status);
319 struct UString *token,
357 /* parse multi-line comment and store it in token*/
358 seekUntilEndOfComment(buf, token, status);
375 struct UString *token,
385 /* add the char to token */
386 if(token!=NULL){
387 ustr_u32cat(token, c, status);
393 struct UString *token,
416 /* add the char to token */
417 if(token!=NULL){
418 ustr_u32cat(token, c, status);
