Lines Matching full:token
248 * It will split <data> on each instance of <delimiter> and insert each token
250 * token TEXT, source INTEGER, token_index INTEGER, tag (any type)
254 * One row is inserted for each token in <data>.
256 * In the first inserted row, 'token' is the hex collation key of
259 * 'token' will be set to the hex collation key of the I:th token (0-based).
264 * and one row for each token except the first one.
305 char * sql = sqlite3_mprintf("INSERT INTO %s (token, source%s%s) VALUES (?, ?%s%s);",
360 UChar * token = NULL;
366 token = origData;
373 uint32_t result = ucol_getSortKey(collator, token, -1, (uint8_t*)keybuf, sizeof(keybuf)-1);
412 } while ((token = u_strtok_r(NULL, delim, &state)) != NULL);
