Lines Matching full:token
401 char16 *token;
405 token = utf16_strtok(to_tokenize, &token_size, &to_tokenize);
406 if (NULL == token) {
412 size_t lemma_size = utf16_strlen(token);
425 utf16_strcpy(lemma_arr_[i].hanzi_str, token);
430 token = utf16_strtok(to_tokenize, &token_size, &to_tokenize);
431 if (NULL == token) {
436 lemma_arr_[i].freq = utf16_atof(token);
446 token = utf16_strtok(to_tokenize, &token_size, &to_tokenize);
447 assert(NULL != token);
448 int gbk_flag = utf16_atoi(token);
467 token = utf16_strtok(to_tokenize, &token_size, &to_tokenize);
468 if (NULL == token) {
474 assert(utf16_strlen(token) <= kMaxPinyinSize);
476 utf16_strcpy_tochar(lemma_arr_[i].pinyin_str[hz_pos], token);
489 token = utf16_strtok(to_tokenize, &token_size, &to_tokenize);
490 if (spelling_not_support || NULL != token) {
