Lines Matching refs:tok
324 UColToken *tok = lh->first;
325 uint32_t tokStrength = tok->strength;
376 lh->fStrToken[tokStrength] = tok;
383 while(tok != NULL && tok->strength >= tokStrength) {
385 lh->lStrToken[tokStrength] = tok;
387 tok = tok->next;
398 if(tok != NULL) {
399 tokStrength = tok->strength;
445 static uint32_t ucol_getSimpleCEGenerator(ucolCEGenerator *g, UColToken *tok, uint32_t strength, UErrorCode *status) {
460 if(tok->next != NULL && tok->next->strength == strength) {
461 count = tok->next->toInsert;
473 static uint32_t ucol_getCEGenerator(ucolCEGenerator *g, uint32_t* lows, uint32_t* highs, UColToken *tok, uint32_t fStrength, UErrorCode *status) {
474 uint32_t strength = tok->strength;
486 uint32_t count = tok->toInsert;
662 U_CFUNC void ucol_doCE(UColTokenParser *src, uint32_t *CEparts, UColToken *tok, UErrorCode *status) {
692 tok->CEs[CEi] = value;
696 tok->noOfCEs = 1;
697 tok->CEs[0] = 0;
699 tok->noOfCEs = CEi;
705 if(tok->CEs[0] != 0) { // case bits should be set only for non-ignorables
706 tok->CEs[0] &= 0xFFFFFF3F; // Clean the case bits field
707 int32_t cSize = (tok->source & 0xFF000000) >> 24;
708 UChar *cPoints = (tok->source & 0x00FFFFFF) + src->source;
712 tok->CEs[0] |= ucol_uprv_getCaseBits(src->UCA, cPoints, cSize, status);
716 tok->CEs[0] |= (caseCE & 0xC0);
721 fprintf(stderr, "%04X str: %i, [%08X, %08X, %08X]: tok: ", tok->debugSource, tok->strength, CEparts[0] >> (32-8*noOfBytes[0]), CEparts[1] >> (32-8*noOfBytes[1]), CEparts[2]>> (32-8*noOfBytes[2]));
722 for(i = 0; i<tok->noOfCEs; i++) {
723 fprintf(stderr, "%08X ", tok->CEs[i]);
733 UColToken *tok = lh->last;
738 tok->toInsert = 1;
739 t[tok->strength] = 1;
741 while(tok->previous != NULL) {
742 if(tok->previous->strength < tok->strength) { /* going up */
743 t[tok->strength] = 0;
744 t[tok->previous->strength]++;
745 } else if(tok->previous->strength > tok->strength) { /* going down */
746 t[tok->previous->strength] = 1;
748 t[tok->strength]++;
750 tok=tok->previous;
751 tok->toInsert = t[tok->strength];
754 tok->toInsert = t[tok->strength];
764 tok=lh->first[UCOL_TOK_POLARITY_POSITIVE];
767 fprintf(stderr,"%i", tok->strength);
768 tok = tok->next;
769 } while(tok != NULL);
772 tok=lh->first[UCOL_TOK_POLARITY_POSITIVE];
775 fprintf(stderr,"%i", tok->toInsert);
776 tok = tok->next;
777 } while(tok != NULL);
780 tok = lh->first;
789 while (tok != NULL && U_SUCCESS(*status)) {
790 fStrength = tok->strength;
805 /*CEparts[UCOL_TERTIARY] = ucol_getCEGenerator(&Gens[2], lh->gapsLo[fStrength*3+2], lh->gapsHi[fStrength*3+2], tok, UCOL_TERTIARY); */
806 CEparts[UCOL_TERTIARY] = ucol_getCEGenerator(&Gens[UCOL_TERTIARY], lh->gapsLo, lh->gapsHi, tok, fStrength, status);
809 /*CEparts[1] = ucol_getCEGenerator(&Gens[1], lh->gapsLo[fStrength*3+1], lh->gapsHi[fStrength*3+1], tok, 1);*/
810 CEparts[UCOL_SECONDARY] = ucol_getCEGenerator(&Gens[UCOL_SECONDARY], lh->gapsLo, lh->gapsHi, tok, fStrength, status);
811 CEparts[UCOL_TERTIARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_TERTIARY], tok, UCOL_TERTIARY, status);
813 /*CEparts[UCOL_PRIMARY] = ucol_getCEGenerator(&Gens[0], lh->gapsLo[0], lh->gapsHi[0], tok, UCOL_PRIMARY);*/
814 CEparts[UCOL_PRIMARY] = ucol_getCEGenerator(&Gens[UCOL_PRIMARY], lh->gapsLo, lh->gapsHi, tok, fStrength, status);
815 CEparts[UCOL_SECONDARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_SECONDARY], tok, UCOL_SECONDARY, status);
816 CEparts[UCOL_TERTIARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_TERTIARY], tok, UCOL_TERTIARY, status);
819 if(tok->strength == UCOL_TERTIARY) {
821 } else if(tok->strength == UCOL_SECONDARY) {
823 CEparts[UCOL_TERTIARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_TERTIARY], tok, UCOL_TERTIARY, status);
824 } else if(tok->strength == UCOL_PRIMARY) {
826 CEparts[UCOL_SECONDARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_SECONDARY], tok, UCOL_SECONDARY, status);
827 CEparts[UCOL_TERTIARY] = ucol_getSimpleCEGenerator(&Gens[UCOL_TERTIARY], tok, UCOL_TERTIARY, status);
830 ucol_doCE(src, CEparts, tok, status);
831 tok = tok->next;
837 UColToken *tok = lh->first;
843 while(tok != NULL && U_SUCCESS(*status)) {
847 if(tok->expansion != 0) {
848 uint32_t len = tok->expansion >> 24;
850 uint32_t expOffset = tok->expansion & 0x00FFFFFF;
863 tok->expCEs[tok->noOfExpCEs + j] = expt->CEs[j];
865 tok->noOfExpCEs += noOfCEsToCopy;
888 tok->expCEs[tok->noOfExpCEs++] = order;
895 tok->noOfExpCEs = 0;
899 el.noOfCEs = tok->noOfCEs + tok->noOfExpCEs;
901 for(i = 0; i<tok->noOfCEs; i++) {
902 el.CEs[i] = tok->CEs[i];
904 for(i = 0; i<tok->noOfExpCEs; i++) {
905 el.CEs[i+tok->noOfCEs] = tok->expCEs[i];
913 if(tok->prefix != 0) { // we will just copy the prefix here, and adjust accordingly in the
916 el.prefixSize = tok->prefix>>24;
917 uprv_memcpy(el.prefix, src->source + (tok->prefix & 0x00FFFFFF), el.prefixSize*sizeof(UChar));
919 el.cSize = (tok->source >> 24)-(tok->prefix>>24);
920 uprv_memcpy(el.uchars, (tok->source & 0x00FFFFFF)+(tok
925 el.cSize = (tok->source >> 24);
926 uprv_memcpy(el.uchars, (tok->source & 0x00FFFFFF) + src->source, el.cSize*sizeof(UChar));
952 fprintf(stderr, "replaced CE for %04X with CE for %04X\n", el.cPoints[0], tok->debugSource);
957 tok = tok->next;
