Lines Matching refs:sourceToken
1474 UColToken *sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
1476 if (sourceToken == NULL) {
1480 sourceToken->rulesToParseHdl = &(src->source);
1481 sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
1482 sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
1484 sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
1485 sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
1488 sourceToken->flags = src->parsedToken.flags;
1495 uprv_free(sourceToken);
1498 sourceToken->prefix = 0;
1501 sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
1502 sourceToken->strength = UCOL_TOK_RESET;
1503 sourceToken->next = NULL;
1504 sourceToken->previous = NULL;
1505 sourceToken->noOfCEs = 0;
1506 sourceToken->noOfExpCEs = 0;
1507 sourceToken->listHeader = &src->lh[src->resultLen];
1514 src->lh[src->resultLen].reset = sourceToken;
1531 sourceToken->source = ((resetCharsOffset - src->parsedToken.charsOffset ) << 24) | src->parsedToken.charsOffset;
1538 uhash_put(src->tailored, sourceToken, sourceToken, status);
1540 return sourceToken;
1544 inline UColToken *getVirginBefore(UColTokenParser *src, UColToken *sourceToken, uint8_t strength, UParseError *parseError, UErrorCode *status) {
1553 if(sourceToken != NULL) {
1554 uprv_init_collIterate(src->UCA, src->source+((sourceToken->source)&0xFFFFFF), 1, &s, status);
1590 //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
1591 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1593 if(sourceToken == NULL) {
1607 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1669 //sourceToken = (UColToken *)uhash_iget(src->tailored, (int32_t)key);
1670 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1684 //if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
1706 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
1710 return sourceToken;
1758 UColToken *sourceToken = NULL;
1776 /* 4 Lookup each source in the CharsToToken map, and find a sourceToken */
1777 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1787 if(sourceToken == NULL) {
1788 /* If sourceToken is null, create new one, */
1789 sourceToken = (UColToken *)uprv_malloc(sizeof(UColToken));
1791 if (sourceToken == NULL) {
1795 sourceToken->rulesToParseHdl = &(src->source);
1796 sourceToken->source = src->parsedToken.charsLen << 24 | src->parsedToken.charsOffset;
1798 sourceToken->debugSource = *(src->source + src->parsedToken.charsOffset);
1800 sourceToken->prefix = src->parsedToken.prefixLen << 24 | src->parsedToken.prefixOffset;
1801 sourceToken->debugPrefix = *(src->source + src->parsedToken.prefixOffset);
1803 sourceToken->polarity = UCOL_TOK_POLARITY_POSITIVE; /* TODO: this should also handle reverse */
1804 sourceToken->next = NULL;
1805 sourceToken->previous = NULL;
1806 sourceToken->noOfCEs = 0;
1807 sourceToken->noOfExpCEs = 0;
1809 sourceToken->flags = src->parsedToken.flags;
1810 uhash_put(src->tailored, sourceToken, sourceToken, status);
1816 if(sourceToken->strength != UCOL_TOK_RESET && lastToken != sourceToken) {
1817 /* otherwise remove sourceToken from where it was. */
1818 if(sourceToken->next != NULL) {
1819 if(sourceToken->next->strength > sourceToken->strength) {
1820 sourceToken->next->strength = sourceToken->strength;
1822 sourceToken->next->previous = sourceToken->previous;
1824 sourceToken->listHeader->last = sourceToken->previous;
1827 if(sourceToken->previous != NULL) {
1828 sourceToken->previous->next = sourceToken->next;
1830 sourceToken->listHeader->first = sourceToken->next;
1832 sourceToken->next = NULL;
1833 sourceToken->previous = NULL;
1837 sourceToken->strength = src->parsedToken.strength;
1838 sourceToken->listHeader = lastToken->listHeader;
1845 || sourceToken->listHeader->first == 0) {
1847 insert sourceToken in the list. */
1848 if(sourceToken->listHeader->first == 0) {
1849 sourceToken->listHeader->first = sourceToken;
1850 sourceToken->listHeader->last = sourceToken;
1853 if(sourceToken->listHeader->first->strength <= sourceToken->strength) {
1854 sourceToken->next = sourceToken->listHeader->first;
1855 sourceToken->next->previous = sourceToken;
1856 sourceToken->listHeader->first = sourceToken;
1857 sourceToken->previous = NULL;
1859 lastToken = sourceToken->listHeader->first;
1860 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
1864 lastToken->next->previous = sourceToken;
1866 sourceToken->listHeader->last = sourceToken;
1868 sourceToken->previous = lastToken;
1869 sourceToken->next = lastToken->next;
1870 lastToken->next = sourceToken;
1875 if polarity (LAST) == polarity(relation), insert sourceToken after LAST,
1879 if(sourceToken != lastToken) {
1880 if(lastToken->polarity == sourceToken->polarity) {
1881 while(lastToken->next != NULL && lastToken->next->strength > sourceToken->strength) {
1884 sourceToken->previous = lastToken;
1886 lastToken->next->previous = sourceToken;
1888 sourceToken->listHeader->last = sourceToken;
1891 sourceToken->next = lastToken->next;
1892 lastToken->next = sourceToken;
1894 while(lastToken->previous != NULL && lastToken->previous->strength > sourceToken->strength) {
1897 sourceToken->next = lastToken;
1899 lastToken->previous->next = sourceToken;
1901 sourceToken->listHeader->first = sourceToken;
1903 sourceToken->previous = lastToken->previous;
1904 lastToken->previous = sourceToken;
1907 if(lastStrength < sourceToken->strength) {
1908 sourceToken->strength = lastStrength;
1916 src->varTop = sourceToken;
1924 sourceToken->expansion = src->parsedToken.extensionLen << 24 | src->parsedToken.extensionOffset;
1927 if(sourceToken->strength == UCOL_PRIMARY) { /* primary strength kills off the implicit expansion */
1929 } else if(sourceToken->expansion == 0) { /* if there is no expansion, implicit is just added to the token */
1930 sourceToken->expansion = expandNext;
1934 sourceToken->expansion = (uint32_t)(((expandNext >> 24) + src->parsedToken.extensionLen)<<24 | (uint32_t)(src->extraCurrent - src->source));
1940 if(sourceToken->expansion != 0) {
1941 sourceToken->debugExpansion = *(src->source + src->parsedToken.extensionOffset);
1943 sourceToken->debugExpansion = 0;
1952 if(beforeStrength != sourceToken->strength) {
1969 if(sourceToken == NULL) { /* this is a reset, but it might still be somewhere in the tailoring, in shorter form */
1971 while(searchCharsLen > 1 && sourceToken == NULL) {
1977 sourceToken = (UColToken *)uhash_get(src->tailored, &key);
1979 if(sourceToken != NULL) {
1987 if(sourceToken != NULL && sourceToken->strength != UCOL_TOK_RESET) {
1989 while(sourceToken->strength > strength && sourceToken->previous != NULL) {
1990 sourceToken = sourceToken->previous;
1993 if(sourceToken->strength == strength) {
1994 if(sourceToken->previous != NULL) {
1995 sourceTokensourceToken->previous;
1997 sourceToken = sourceToken->listHeader->reset;
2001 sourceToken = sourceToken->listHeader->reset;
2002 sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
2005 sourceToken = getVirginBefore(src, sourceToken, strength, parseError, status);
2038 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
2044 If sourceToken is null
2045 Create new list, create new sourceToken, make the baseCE from source, put
2046 the sourceToken in ListHeader of the new list */
2047 if(sourceToken == NULL) {
2078 sourceToken = ucol_tok_initAReset(src, expand, &expandNext, parseError, status);
2090 sourceToken = ucol_tok_initAReset(src, 0, &expandNext, parseError, status);
2097 /* 7 After all this, set LAST to point to sourceToken, and goto step 3. */
2098 lastToken = sourceToken;
