Lines Matching full:token
252 * In unames.dat, it is marked as token[';']==-1 only if the
255 * So, it will be token[';']==-1 if we store U1.0 names/ISO comments
263 uint16_t token, tokenCount=*tokens++, bufferPos=0;
270 * if the semicolon byte value is a character, not a token number
290 * the semicolon byte value is a token number, therefore
298 /* write each letter directly, and write a token word per token */
312 token=tokens[c];
313 if(token==(uint16_t)(-2)) {
314 /* this is a lead byte for a double-byte token */
315 token=tokens[c<<8|*name++];
318 if(token==(uint16_t)(-1)) {
335 /* write token word */
336 uint8_t *tokenString=tokenStrings+token;
362 uint16_t token, tokenCount=*tokens++;
370 * if the semicolon byte value is a character, not a token number
381 * the semicolon byte value is a token number, therefore
389 /* compare each letter directly, and compare a token word per token */
405 token=tokens[c];
406 if(token==(uint16_t)(-2)) {
407 /* this is a lead byte for a double-byte token */
408 token=tokens[c<<8|*name++];
411 if(token==(uint16_t)(-1)) {
430 /* write token word */
431 uint8_t *tokenString=tokenStrings+token;
1319 uint16_t c, token;
1327 token=tokens[c];
1328 if(token==(uint16_t)(-2)) {
1329 /* this is a lead byte for a double-byte token */
1331 token=tokens[c];
1333 if(token==(uint16_t)(-1)) {
1338 /* count token word */
1340 /* use cached token length */
1343 tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token);
1347 tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token);
1775 * The token table contains non-negative entries for token bytes,
1947 /* read the first 512 tokens and make the token maps */
1992 * swap the token strings but not a possible padding byte after
1998 udata_printError(ds, "uchar_swapNames(token strings) failed\n");
2043 /* token lead byte: swap the trail byte, too */
