1 // 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ********************************************************************** 5 * Copyright (C) 1999-2016, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ********************************************************************** 8 * Date Name Description 9 * 11/17/99 aliu Creation. 10 ********************************************************************** 11 */ 12 13 #include "unicode/utypes.h" 14 15 #if !UCONFIG_NO_TRANSLITERATION 16 17 #include "unicode/uobject.h" 18 #include "unicode/parseerr.h" 19 #include "unicode/parsepos.h" 20 #include "unicode/putil.h" 21 #include "unicode/uchar.h" 22 #include "unicode/ustring.h" 23 #include "unicode/uniset.h" 24 #include "unicode/utf16.h" 25 #include "cstring.h" 26 #include "funcrepl.h" 27 #include "hash.h" 28 #include "quant.h" 29 #include "rbt.h" 30 #include "rbt_data.h" 31 #include "rbt_pars.h" 32 #include "rbt_rule.h" 33 #include "strmatch.h" 34 #include "strrepl.h" 35 #include "unicode/symtable.h" 36 #include "tridpars.h" 37 #include "uvector.h" 38 #include "hash.h" 39 #include "patternprops.h" 40 #include "util.h" 41 #include "cmemory.h" 42 #include "uprops.h" 43 #include "putilimp.h" 44 45 // Operators 46 #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/ 47 #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/ 48 #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/ 49 #define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op 50 51 // Other special characters 52 #define QUOTE ((UChar)0x0027) /*'*/ 53 #define ESCAPE ((UChar)0x005C) /*\*/ 54 #define END_OF_RULE ((UChar)0x003B) /*;*/ 55 #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/ 56 57 #define SEGMENT_OPEN ((UChar)0x0028) /*(*/ 58 #define SEGMENT_CLOSE ((UChar)0x0029) /*)*/ 59 #define CONTEXT_ANTE ((UChar)0x007B) /*{*/ 60 #define CONTEXT_POST ((UChar)0x007D) /*}*/ 61 #define CURSOR_POS ((UChar)0x007C) /*|*/ 62 #define CURSOR_OFFSET ((UChar)0x0040) /*@*/ 63 #define ANCHOR_START ((UChar)0x005E) /*^*/ 64 #define KLEENE_STAR ((UChar)0x002A) /***/ 65 #define ONE_OR_MORE ((UChar)0x002B) /*+*/ 66 #define ZERO_OR_ONE ((UChar)0x003F) /*?*/ 67 68 #define DOT ((UChar)46) /*.*/ 69 70 static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]"; 71 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90, 72 108, 58, 93, 92, 114, 92, 110, 36, 93, 0 73 }; 74 75 // A function is denoted &Source-Target/Variant(text) 76 #define FUNCTION ((UChar)38) /*&*/ 77 78 // Aliases for some of the syntax characters. These are provided so 79 // transliteration rules can be expressed in XML without clashing with 80 // XML syntax characters '<', '>', and '&'. 81 #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow 82 #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow 83 #define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow 84 #define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta) 85 86 // Special characters disallowed at the top level 87 static const UChar ILLEGAL_TOP[] = {41,0}; // ")" 88 89 // Special characters disallowed within a segment 90 static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@" 91 92 // Special characters disallowed within a function argument 93 static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@" 94 95 // By definition, the ANCHOR_END special character is a 96 // trailing SymbolTable.SYMBOL_REF character. 97 // private static final char ANCHOR_END = '$'; 98 99 static const UChar gOPERATORS[] = { // "=><" 100 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, 101 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, 102 0 103 }; 104 105 static const UChar HALF_ENDERS[] = { // "=><;" 106 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP, 107 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP, 108 END_OF_RULE, 109 0 110 }; 111 112 // These are also used in Transliterator::toRules() 113 static const int32_t ID_TOKEN_LEN = 2; 114 static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':' 115 116 /* 117 commented out until we do real ::BEGIN/::END functionality 118 static const int32_t BEGIN_TOKEN_LEN = 5; 119 static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN' 120 121 static const int32_t END_TOKEN_LEN = 3; 122 static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END' 123 */ 124 125 U_NAMESPACE_BEGIN 126 127 //---------------------------------------------------------------------- 128 // BEGIN ParseData 129 //---------------------------------------------------------------------- 130 131 /** 132 * This class implements the SymbolTable interface. It is used 133 * during parsing to give UnicodeSet access to variables that 134 * have been defined so far. Note that it uses variablesVector, 135 * _not_ data.setVariables. 136 */ 137 class ParseData : public UMemory, public SymbolTable { 138 public: 139 const TransliterationRuleData* data; // alias 140 141 const UVector* variablesVector; // alias 142 143 const Hashtable* variableNames; // alias 144 145 ParseData(const TransliterationRuleData* data = 0, 146 const UVector* variablesVector = 0, 147 const Hashtable* variableNames = 0); 148 149 virtual ~ParseData(); 150 151 virtual const UnicodeString* lookup(const UnicodeString& s) const; 152 153 virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const; 154 155 virtual UnicodeString parseReference(const UnicodeString& text, 156 ParsePosition& pos, int32_t limit) const; 157 /** 158 * Return true if the given character is a matcher standin or a plain 159 * character (non standin). 160 */ 161 UBool isMatcher(UChar32 ch); 162 163 /** 164 * Return true if the given character is a replacer standin or a plain 165 * character (non standin). 166 */ 167 UBool isReplacer(UChar32 ch); 168 169 private: 170 ParseData(const ParseData &other); // forbid copying of this class 171 ParseData &operator=(const ParseData &other); // forbid copying of this class 172 }; 173 174 ParseData::ParseData(const TransliterationRuleData* d, 175 const UVector* sets, 176 const Hashtable* vNames) : 177 data(d), variablesVector(sets), variableNames(vNames) {} 178 179 ParseData::~ParseData() {} 180 181 /** 182 * Implement SymbolTable API. 183 */ 184 const UnicodeString* ParseData::lookup(const UnicodeString& name) const { 185 return (const UnicodeString*) variableNames->get(name); 186 } 187 188 /** 189 * Implement SymbolTable API. 190 */ 191 const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const { 192 // Note that we cannot use data.lookupSet() because the 193 // set array has not been constructed yet. 194 const UnicodeFunctor* set = NULL; 195 int32_t i = ch - data->variablesBase; 196 if (i >= 0 && i < variablesVector->size()) { 197 int32_t i = ch - data->variablesBase; 198 set = (i < variablesVector->size()) ? 199 (UnicodeFunctor*) variablesVector->elementAt(i) : 0; 200 } 201 return set; 202 } 203 204 /** 205 * Implement SymbolTable API. Parse out a symbol reference 206 * name. 207 */ 208 UnicodeString ParseData::parseReference(const UnicodeString& text, 209 ParsePosition& pos, int32_t limit) const { 210 int32_t start = pos.getIndex(); 211 int32_t i = start; 212 UnicodeString result; 213 while (i < limit) { 214 UChar c = text.charAt(i); 215 if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { 216 break; 217 } 218 ++i; 219 } 220 if (i == start) { // No valid name chars 221 return result; // Indicate failure with empty string 222 } 223 pos.setIndex(i); 224 text.extractBetween(start, i, result); 225 return result; 226 } 227 228 UBool ParseData::isMatcher(UChar32 ch) { 229 // Note that we cannot use data.lookup() because the 230 // set array has not been constructed yet. 231 int32_t i = ch - data->variablesBase; 232 if (i >= 0 && i < variablesVector->size()) { 233 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); 234 return f != NULL && f->toMatcher() != NULL; 235 } 236 return TRUE; 237 } 238 239 /** 240 * Return true if the given character is a replacer standin or a plain 241 * character (non standin). 242 */ 243 UBool ParseData::isReplacer(UChar32 ch) { 244 // Note that we cannot use data.lookup() because the 245 // set array has not been constructed yet. 246 int i = ch - data->variablesBase; 247 if (i >= 0 && i < variablesVector->size()) { 248 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i); 249 return f != NULL && f->toReplacer() != NULL; 250 } 251 return TRUE; 252 } 253 254 //---------------------------------------------------------------------- 255 // BEGIN RuleHalf 256 //---------------------------------------------------------------------- 257 258 /** 259 * A class representing one side of a rule. This class knows how to 260 * parse half of a rule. It is tightly coupled to the method 261 * RuleBasedTransliterator.Parser.parseRule(). 262 */ 263 class RuleHalf : public UMemory { 264 265 public: 266 267 UnicodeString text; 268 269 int32_t cursor; // position of cursor in text 270 int32_t ante; // position of ante context marker '{' in text 271 int32_t post; // position of post context marker '}' in text 272 273 // Record the offset to the cursor either to the left or to the 274 // right of the key. This is indicated by characters on the output 275 // side that allow the cursor to be positioned arbitrarily within 276 // the matching text. For example, abc{def} > | @@@ xyz; changes 277 // def to xyz and moves the cursor to before abc. Offset characters 278 // must be at the start or end, and they cannot move the cursor past 279 // the ante- or postcontext text. Placeholders are only valid in 280 // output text. The length of the ante and post context is 281 // determined at runtime, because of supplementals and quantifiers. 282 int32_t cursorOffset; // only nonzero on output side 283 284 // Position of first CURSOR_OFFSET on _right_. This will be -1 285 // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc. 286 int32_t cursorOffsetPos; 287 288 UBool anchorStart; 289 UBool anchorEnd; 290 291 /** 292 * The segment number from 1..n of the next '(' we see 293 * during parsing; 1-based. 294 */ 295 int32_t nextSegmentNumber; 296 297 TransliteratorParser& parser; 298 299 //-------------------------------------------------- 300 // Methods 301 302 RuleHalf(TransliteratorParser& parser); 303 ~RuleHalf(); 304 305 int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status); 306 307 int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, 308 UnicodeString& buf, 309 const UnicodeString& illegal, 310 UBool isSegment, 311 UErrorCode& status); 312 313 /** 314 * Remove context. 315 */ 316 void removeContext(); 317 318 /** 319 * Return true if this half looks like valid output, that is, does not 320 * contain quantifiers or other special input-only elements. 321 */ 322 UBool isValidOutput(TransliteratorParser& parser); 323 324 /** 325 * Return true if this half looks like valid input, that is, does not 326 * contain functions or other special output-only elements. 327 */ 328 UBool isValidInput(TransliteratorParser& parser); 329 330 int syntaxError(UErrorCode code, 331 const UnicodeString& rule, 332 int32_t start, 333 UErrorCode& status) { 334 return parser.syntaxError(code, rule, start, status); 335 } 336 337 private: 338 // Disallowed methods; no impl. 339 RuleHalf(const RuleHalf&); 340 RuleHalf& operator=(const RuleHalf&); 341 }; 342 343 RuleHalf::RuleHalf(TransliteratorParser& p) : 344 parser(p) 345 { 346 cursor = -1; 347 ante = -1; 348 post = -1; 349 cursorOffset = 0; 350 cursorOffsetPos = 0; 351 anchorStart = anchorEnd = FALSE; 352 nextSegmentNumber = 1; 353 } 354 355 RuleHalf::~RuleHalf() { 356 } 357 358 /** 359 * Parse one side of a rule, stopping at either the limit, 360 * the END_OF_RULE character, or an operator. 361 * @return the index after the terminating character, or 362 * if limit was reached, limit 363 */ 364 int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { 365 int32_t start = pos; 366 text.truncate(0); 367 pos = parseSection(rule, pos, limit, text, UnicodeString(TRUE, ILLEGAL_TOP, -1), FALSE, status); 368 369 if (cursorOffset > 0 && cursor != cursorOffsetPos) { 370 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); 371 } 372 373 return pos; 374 } 375 376 /** 377 * Parse a section of one side of a rule, stopping at either 378 * the limit, the END_OF_RULE character, an operator, or a 379 * segment close character. This method parses both a 380 * top-level rule half and a segment within such a rule half. 381 * It calls itself recursively to parse segments and nested 382 * segments. 383 * @param buf buffer into which to accumulate the rule pattern 384 * characters, either literal characters from the rule or 385 * standins for UnicodeMatcher objects including segments. 386 * @param illegal the set of special characters that is illegal during 387 * this parse. 388 * @param isSegment if true, then we've already seen a '(' and 389 * pos on entry points right after it. Accumulate everything 390 * up to the closing ')', put it in a segment matcher object, 391 * generate a standin for it, and add the standin to buf. As 392 * a side effect, update the segments vector with a reference 393 * to the segment matcher. This works recursively for nested 394 * segments. If isSegment is false, just accumulate 395 * characters into buf. 396 * @return the index after the terminating character, or 397 * if limit was reached, limit 398 */ 399 int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit, 400 UnicodeString& buf, 401 const UnicodeString& illegal, 402 UBool isSegment, UErrorCode& status) { 403 int32_t start = pos; 404 ParsePosition pp; 405 UnicodeString scratch; 406 UBool done = FALSE; 407 int32_t quoteStart = -1; // Most recent 'single quoted string' 408 int32_t quoteLimit = -1; 409 int32_t varStart = -1; // Most recent $variableReference 410 int32_t varLimit = -1; 411 int32_t bufStart = buf.length(); 412 413 while (pos < limit && !done) { 414 // Since all syntax characters are in the BMP, fetching 415 // 16-bit code units suffices here. 416 UChar c = rule.charAt(pos++); 417 if (PatternProps::isWhiteSpace(c)) { 418 // Ignore whitespace. Note that this is not Unicode 419 // spaces, but Java spaces -- a subset, representing 420 // whitespace likely to be seen in code. 421 continue; 422 } 423 if (u_strchr(HALF_ENDERS, c) != NULL) { 424 if (isSegment) { 425 // Unclosed segment 426 return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status); 427 } 428 break; 429 } 430 if (anchorEnd) { 431 // Text after a presumed end anchor is a syntax err 432 return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status); 433 } 434 if (UnicodeSet::resemblesPattern(rule, pos-1)) { 435 pp.setIndex(pos-1); // Backup to opening '[' 436 buf.append(parser.parseSet(rule, pp, status)); 437 if (U_FAILURE(status)) { 438 return syntaxError(U_MALFORMED_SET, rule, start, status); 439 } 440 pos = pp.getIndex(); 441 continue; 442 } 443 // Handle escapes 444 if (c == ESCAPE) { 445 if (pos == limit) { 446 return syntaxError(U_TRAILING_BACKSLASH, rule, start, status); 447 } 448 UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\' 449 if (escaped == (UChar32) -1) { 450 return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status); 451 } 452 if (!parser.checkVariableRange(escaped)) { 453 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); 454 } 455 buf.append(escaped); 456 continue; 457 } 458 // Handle quoted matter 459 if (c == QUOTE) { 460 int32_t iq = rule.indexOf(QUOTE, pos); 461 if (iq == pos) { 462 buf.append(c); // Parse [''] outside quotes as ['] 463 ++pos; 464 } else { 465 /* This loop picks up a run of quoted text of the 466 * form 'aaaa' each time through. If this run 467 * hasn't really ended ('aaaa''bbbb') then it keeps 468 * looping, each time adding on a new run. When it 469 * reaches the final quote it breaks. 470 */ 471 quoteStart = buf.length(); 472 for (;;) { 473 if (iq < 0) { 474 return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status); 475 } 476 scratch.truncate(0); 477 rule.extractBetween(pos, iq, scratch); 478 buf.append(scratch); 479 pos = iq+1; 480 if (pos < limit && rule.charAt(pos) == QUOTE) { 481 // Parse [''] inside quotes as ['] 482 iq = rule.indexOf(QUOTE, pos+1); 483 // Continue looping 484 } else { 485 break; 486 } 487 } 488 quoteLimit = buf.length(); 489 490 for (iq=quoteStart; iq<quoteLimit; ++iq) { 491 if (!parser.checkVariableRange(buf.charAt(iq))) { 492 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); 493 } 494 } 495 } 496 continue; 497 } 498 499 if (!parser.checkVariableRange(c)) { 500 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status); 501 } 502 503 if (illegal.indexOf(c) >= 0) { 504 syntaxError(U_ILLEGAL_CHARACTER, rule, start, status); 505 } 506 507 switch (c) { 508 509 //------------------------------------------------------ 510 // Elements allowed within and out of segments 511 //------------------------------------------------------ 512 case ANCHOR_START: 513 if (buf.length() == 0 && !anchorStart) { 514 anchorStart = TRUE; 515 } else { 516 return syntaxError(U_MISPLACED_ANCHOR_START, 517 rule, start, status); 518 } 519 break; 520 case SEGMENT_OPEN: 521 { 522 // bufSegStart is the offset in buf to the first 523 // character of the segment we are parsing. 524 int32_t bufSegStart = buf.length(); 525 526 // Record segment number now, since nextSegmentNumber 527 // will be incremented during the call to parseSection 528 // if there are nested segments. 529 int32_t segmentNumber = nextSegmentNumber++; // 1-based 530 531 // Parse the segment 532 pos = parseSection(rule, pos, limit, buf, UnicodeString(TRUE, ILLEGAL_SEG, -1), TRUE, status); 533 534 // After parsing a segment, the relevant characters are 535 // in buf, starting at offset bufSegStart. Extract them 536 // into a string matcher, and replace them with a 537 // standin for that matcher. 538 StringMatcher* m = 539 new StringMatcher(buf, bufSegStart, buf.length(), 540 segmentNumber, *parser.curData); 541 if (m == NULL) { 542 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 543 } 544 545 // Record and associate object and segment number 546 parser.setSegmentObject(segmentNumber, m, status); 547 buf.truncate(bufSegStart); 548 buf.append(parser.getSegmentStandin(segmentNumber, status)); 549 } 550 break; 551 case FUNCTION: 552 case ALT_FUNCTION: 553 { 554 int32_t iref = pos; 555 TransliteratorIDParser::SingleID* single = 556 TransliteratorIDParser::parseFilterID(rule, iref); 557 // The next character MUST be a segment open 558 if (single == NULL || 559 !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) { 560 return syntaxError(U_INVALID_FUNCTION, rule, start, status); 561 } 562 563 Transliterator *t = single->createInstance(); 564 delete single; 565 if (t == NULL) { 566 return syntaxError(U_INVALID_FUNCTION, rule, start, status); 567 } 568 569 // bufSegStart is the offset in buf to the first 570 // character of the segment we are parsing. 571 int32_t bufSegStart = buf.length(); 572 573 // Parse the segment 574 pos = parseSection(rule, iref, limit, buf, UnicodeString(TRUE, ILLEGAL_FUNC, -1), TRUE, status); 575 576 // After parsing a segment, the relevant characters are 577 // in buf, starting at offset bufSegStart. 578 UnicodeString output; 579 buf.extractBetween(bufSegStart, buf.length(), output); 580 FunctionReplacer *r = 581 new FunctionReplacer(t, new StringReplacer(output, parser.curData)); 582 if (r == NULL) { 583 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 584 } 585 586 // Replace the buffer contents with a stand-in 587 buf.truncate(bufSegStart); 588 buf.append(parser.generateStandInFor(r, status)); 589 } 590 break; 591 case SymbolTable::SYMBOL_REF: 592 // Handle variable references and segment references "$1" .. "$9" 593 { 594 // A variable reference must be followed immediately 595 // by a Unicode identifier start and zero or more 596 // Unicode identifier part characters, or by a digit 597 // 1..9 if it is a segment reference. 598 if (pos == limit) { 599 // A variable ref character at the end acts as 600 // an anchor to the context limit, as in perl. 601 anchorEnd = TRUE; 602 break; 603 } 604 // Parse "$1" "$2" .. "$9" .. (no upper limit) 605 c = rule.charAt(pos); 606 int32_t r = u_digit(c, 10); 607 if (r >= 1 && r <= 9) { 608 r = ICU_Utility::parseNumber(rule, pos, 10); 609 if (r < 0) { 610 return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, 611 rule, start, status); 612 } 613 buf.append(parser.getSegmentStandin(r, status)); 614 } else { 615 pp.setIndex(pos); 616 UnicodeString name = parser.parseData-> 617 parseReference(rule, pp, limit); 618 if (name.length() == 0) { 619 // This means the '$' was not followed by a 620 // valid name. Try to interpret it as an 621 // end anchor then. If this also doesn't work 622 // (if we see a following character) then signal 623 // an error. 624 anchorEnd = TRUE; 625 break; 626 } 627 pos = pp.getIndex(); 628 // If this is a variable definition statement, 629 // then the LHS variable will be undefined. In 630 // that case appendVariableDef() will append the 631 // special placeholder char variableLimit-1. 632 varStart = buf.length(); 633 parser.appendVariableDef(name, buf, status); 634 varLimit = buf.length(); 635 } 636 } 637 break; 638 case DOT: 639 buf.append(parser.getDotStandIn(status)); 640 break; 641 case KLEENE_STAR: 642 case ONE_OR_MORE: 643 case ZERO_OR_ONE: 644 // Quantifiers. We handle single characters, quoted strings, 645 // variable references, and segments. 646 // a+ matches aaa 647 // 'foo'+ matches foofoofoo 648 // $v+ matches xyxyxy if $v == xy 649 // (seg)+ matches segsegseg 650 { 651 if (isSegment && buf.length() == bufStart) { 652 // The */+ immediately follows '(' 653 return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status); 654 } 655 656 int32_t qstart, qlimit; 657 // The */+ follows an isolated character or quote 658 // or variable reference 659 if (buf.length() == quoteLimit) { 660 // The */+ follows a 'quoted string' 661 qstart = quoteStart; 662 qlimit = quoteLimit; 663 } else if (buf.length() == varLimit) { 664 // The */+ follows a $variableReference 665 qstart = varStart; 666 qlimit = varLimit; 667 } else { 668 // The */+ follows a single character, possibly 669 // a segment standin 670 qstart = buf.length() - 1; 671 qlimit = qstart + 1; 672 } 673 674 UnicodeFunctor *m = 675 new StringMatcher(buf, qstart, qlimit, 0, *parser.curData); 676 if (m == NULL) { 677 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 678 } 679 int32_t min = 0; 680 int32_t max = Quantifier::MAX; 681 switch (c) { 682 case ONE_OR_MORE: 683 min = 1; 684 break; 685 case ZERO_OR_ONE: 686 min = 0; 687 max = 1; 688 break; 689 // case KLEENE_STAR: 690 // do nothing -- min, max already set 691 } 692 m = new Quantifier(m, min, max); 693 if (m == NULL) { 694 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 695 } 696 buf.truncate(qstart); 697 buf.append(parser.generateStandInFor(m, status)); 698 } 699 break; 700 701 //------------------------------------------------------ 702 // Elements allowed ONLY WITHIN segments 703 //------------------------------------------------------ 704 case SEGMENT_CLOSE: 705 // assert(isSegment); 706 // We're done parsing a segment. 707 done = TRUE; 708 break; 709 710 //------------------------------------------------------ 711 // Elements allowed ONLY OUTSIDE segments 712 //------------------------------------------------------ 713 case CONTEXT_ANTE: 714 if (ante >= 0) { 715 return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status); 716 } 717 ante = buf.length(); 718 break; 719 case CONTEXT_POST: 720 if (post >= 0) { 721 return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status); 722 } 723 post = buf.length(); 724 break; 725 case CURSOR_POS: 726 if (cursor >= 0) { 727 return syntaxError(U_MULTIPLE_CURSORS, rule, start, status); 728 } 729 cursor = buf.length(); 730 break; 731 case CURSOR_OFFSET: 732 if (cursorOffset < 0) { 733 if (buf.length() > 0) { 734 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); 735 } 736 --cursorOffset; 737 } else if (cursorOffset > 0) { 738 if (buf.length() != cursorOffsetPos || cursor >= 0) { 739 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); 740 } 741 ++cursorOffset; 742 } else { 743 if (cursor == 0 && buf.length() == 0) { 744 cursorOffset = -1; 745 } else if (cursor < 0) { 746 cursorOffsetPos = buf.length(); 747 cursorOffset = 1; 748 } else { 749 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status); 750 } 751 } 752 break; 753 754 755 //------------------------------------------------------ 756 // Non-special characters 757 //------------------------------------------------------ 758 default: 759 // Disallow unquoted characters other than [0-9A-Za-z] 760 // in the printable ASCII range. These characters are 761 // reserved for possible future use. 762 if (c >= 0x0021 && c <= 0x007E && 763 !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) || 764 (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) || 765 (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) { 766 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); 767 } 768 buf.append(c); 769 break; 770 } 771 } 772 773 return pos; 774 } 775 776 /** 777 * Remove context. 778 */ 779 void RuleHalf::removeContext() { 780 //text = text.substring(ante < 0 ? 0 : ante, 781 // post < 0 ? text.length() : post); 782 if (post >= 0) { 783 text.remove(post); 784 } 785 if (ante >= 0) { 786 text.removeBetween(0, ante); 787 } 788 ante = post = -1; 789 anchorStart = anchorEnd = FALSE; 790 } 791 792 /** 793 * Return true if this half looks like valid output, that is, does not 794 * contain quantifiers or other special input-only elements. 795 */ 796 UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) { 797 for (int32_t i=0; i<text.length(); ) { 798 UChar32 c = text.char32At(i); 799 i += U16_LENGTH(c); 800 if (!transParser.parseData->isReplacer(c)) { 801 return FALSE; 802 } 803 } 804 return TRUE; 805 } 806 807 /** 808 * Return true if this half looks like valid input, that is, does not 809 * contain functions or other special output-only elements. 810 */ 811 UBool RuleHalf::isValidInput(TransliteratorParser& transParser) { 812 for (int32_t i=0; i<text.length(); ) { 813 UChar32 c = text.char32At(i); 814 i += U16_LENGTH(c); 815 if (!transParser.parseData->isMatcher(c)) { 816 return FALSE; 817 } 818 } 819 return TRUE; 820 } 821 822 //---------------------------------------------------------------------- 823 // PUBLIC API 824 //---------------------------------------------------------------------- 825 826 /** 827 * Constructor. 828 */ 829 TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) : 830 dataVector(statusReturn), 831 idBlockVector(statusReturn), 832 variablesVector(statusReturn), 833 segmentObjects(statusReturn) 834 { 835 idBlockVector.setDeleter(uprv_deleteUObject); 836 curData = NULL; 837 compoundFilter = NULL; 838 parseData = NULL; 839 variableNames.setValueDeleter(uprv_deleteUObject); 840 } 841 842 /** 843 * Destructor. 844 */ 845 TransliteratorParser::~TransliteratorParser() { 846 while (!dataVector.isEmpty()) 847 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); 848 delete compoundFilter; 849 delete parseData; 850 while (!variablesVector.isEmpty()) 851 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); 852 } 853 854 void 855 TransliteratorParser::parse(const UnicodeString& rules, 856 UTransDirection transDirection, 857 UParseError& pe, 858 UErrorCode& ec) { 859 if (U_SUCCESS(ec)) { 860 parseRules(rules, transDirection, ec); 861 pe = parseError; 862 } 863 } 864 865 /** 866 * Return the compound filter parsed by parse(). Caller owns result. 867 */ 868 UnicodeSet* TransliteratorParser::orphanCompoundFilter() { 869 UnicodeSet* f = compoundFilter; 870 compoundFilter = NULL; 871 return f; 872 } 873 874 //---------------------------------------------------------------------- 875 // Private implementation 876 //---------------------------------------------------------------------- 877 878 /** 879 * Parse the given string as a sequence of rules, separated by newline 880 * characters ('\n'), and cause this object to implement those rules. Any 881 * previous rules are discarded. Typically this method is called exactly 882 * once, during construction. 883 * @exception IllegalArgumentException if there is a syntax error in the 884 * rules 885 */ 886 void TransliteratorParser::parseRules(const UnicodeString& rule, 887 UTransDirection theDirection, 888 UErrorCode& status) 889 { 890 // Clear error struct 891 uprv_memset(&parseError, 0, sizeof(parseError)); 892 parseError.line = parseError.offset = -1; 893 894 UBool parsingIDs = TRUE; 895 int32_t ruleCount = 0; 896 897 while (!dataVector.isEmpty()) { 898 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0)); 899 } 900 if (U_FAILURE(status)) { 901 return; 902 } 903 904 idBlockVector.removeAllElements(); 905 curData = NULL; 906 direction = theDirection; 907 ruleCount = 0; 908 909 delete compoundFilter; 910 compoundFilter = NULL; 911 912 while (!variablesVector.isEmpty()) { 913 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0); 914 } 915 variableNames.removeAll(); 916 parseData = new ParseData(0, &variablesVector, &variableNames); 917 if (parseData == NULL) { 918 status = U_MEMORY_ALLOCATION_ERROR; 919 return; 920 } 921 922 dotStandIn = (UChar) -1; 923 924 UnicodeString *tempstr = NULL; // used for memory allocation error checking 925 UnicodeString str; // scratch 926 UnicodeString idBlockResult; 927 int32_t pos = 0; 928 int32_t limit = rule.length(); 929 930 // The compound filter offset is an index into idBlockResult. 931 // If it is 0, then the compound filter occurred at the start, 932 // and it is the offset to the _start_ of the compound filter 933 // pattern. Otherwise it is the offset to the _limit_ of the 934 // compound filter pattern within idBlockResult. 935 compoundFilter = NULL; 936 int32_t compoundFilterOffset = -1; 937 938 while (pos < limit && U_SUCCESS(status)) { 939 UChar c = rule.charAt(pos++); 940 if (PatternProps::isWhiteSpace(c)) { 941 // Ignore leading whitespace. 942 continue; 943 } 944 // Skip lines starting with the comment character 945 if (c == RULE_COMMENT_CHAR) { 946 pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1; 947 if (pos == 0) { 948 break; // No "\n" found; rest of rule is a commnet 949 } 950 continue; // Either fall out or restart with next line 951 } 952 953 // skip empty rules 954 if (c == END_OF_RULE) 955 continue; 956 957 // keep track of how many rules we've seen 958 ++ruleCount; 959 960 // We've found the start of a rule or ID. c is its first 961 // character, and pos points past c. 962 --pos; 963 // Look for an ID token. Must have at least ID_TOKEN_LEN + 1 964 // chars left. 965 if ((pos + ID_TOKEN_LEN + 1) <= limit && 966 rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) { 967 pos += ID_TOKEN_LEN; 968 c = rule.charAt(pos); 969 while (PatternProps::isWhiteSpace(c) && pos < limit) { 970 ++pos; 971 c = rule.charAt(pos); 972 } 973 974 int32_t p = pos; 975 976 if (!parsingIDs) { 977 if (curData != NULL) { 978 if (direction == UTRANS_FORWARD) 979 dataVector.addElement(curData, status); 980 else 981 dataVector.insertElementAt(curData, 0, status); 982 curData = NULL; 983 } 984 parsingIDs = TRUE; 985 } 986 987 TransliteratorIDParser::SingleID* id = 988 TransliteratorIDParser::parseSingleID(rule, p, direction, status); 989 if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) { 990 // Successful ::ID parse. 991 992 if (direction == UTRANS_FORWARD) { 993 idBlockResult.append(id->canonID).append(END_OF_RULE); 994 } else { 995 idBlockResult.insert(0, END_OF_RULE); 996 idBlockResult.insert(0, id->canonID); 997 } 998 999 } else { 1000 // Couldn't parse an ID. Try to parse a global filter 1001 int32_t withParens = -1; 1002 UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL); 1003 if (f != NULL) { 1004 if (ICU_Utility::parseChar(rule, p, END_OF_RULE) 1005 && (direction == UTRANS_FORWARD) == (withParens == 0)) 1006 { 1007 if (compoundFilter != NULL) { 1008 // Multiple compound filters 1009 syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status); 1010 delete f; 1011 } else { 1012 compoundFilter = f; 1013 compoundFilterOffset = ruleCount; 1014 } 1015 } else { 1016 delete f; 1017 } 1018 } else { 1019 // Invalid ::id 1020 // Can be parsed as neither an ID nor a global filter 1021 syntaxError(U_INVALID_ID, rule, pos, status); 1022 } 1023 } 1024 delete id; 1025 pos = p; 1026 } else { 1027 if (parsingIDs) { 1028 tempstr = new UnicodeString(idBlockResult); 1029 // NULL pointer check 1030 if (tempstr == NULL) { 1031 status = U_MEMORY_ALLOCATION_ERROR; 1032 return; 1033 } 1034 if (direction == UTRANS_FORWARD) 1035 idBlockVector.addElement(tempstr, status); 1036 else 1037 idBlockVector.insertElementAt(tempstr, 0, status); 1038 idBlockResult.remove(); 1039 parsingIDs = FALSE; 1040 curData = new TransliterationRuleData(status); 1041 // NULL pointer check 1042 if (curData == NULL) { 1043 status = U_MEMORY_ALLOCATION_ERROR; 1044 return; 1045 } 1046 parseData->data = curData; 1047 1048 // By default, rules use part of the private use area 1049 // E000..F8FF for variables and other stand-ins. Currently 1050 // the range F000..F8FF is typically sufficient. The 'use 1051 // variable range' pragma allows rule sets to modify this. 1052 setVariableRange(0xF000, 0xF8FF, status); 1053 } 1054 1055 if (resemblesPragma(rule, pos, limit)) { 1056 int32_t ppp = parsePragma(rule, pos, limit, status); 1057 if (ppp < 0) { 1058 syntaxError(U_MALFORMED_PRAGMA, rule, pos, status); 1059 } 1060 pos = ppp; 1061 // Parse a rule 1062 } else { 1063 pos = parseRule(rule, pos, limit, status); 1064 } 1065 } 1066 } 1067 1068 if (parsingIDs && idBlockResult.length() > 0) { 1069 tempstr = new UnicodeString(idBlockResult); 1070 // NULL pointer check 1071 if (tempstr == NULL) { 1072 status = U_MEMORY_ALLOCATION_ERROR; 1073 return; 1074 } 1075 if (direction == UTRANS_FORWARD) 1076 idBlockVector.addElement(tempstr, status); 1077 else 1078 idBlockVector.insertElementAt(tempstr, 0, status); 1079 } 1080 else if (!parsingIDs && curData != NULL) { 1081 if (direction == UTRANS_FORWARD) 1082 dataVector.addElement(curData, status); 1083 else 1084 dataVector.insertElementAt(curData, 0, status); 1085 } 1086 1087 if (U_SUCCESS(status)) { 1088 // Convert the set vector to an array 1089 int32_t i, dataVectorSize = dataVector.size(); 1090 for (i = 0; i < dataVectorSize; i++) { 1091 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); 1092 data->variablesLength = variablesVector.size(); 1093 if (data->variablesLength == 0) { 1094 data->variables = 0; 1095 } else { 1096 data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*)); 1097 // NULL pointer check 1098 if (data->variables == NULL) { 1099 status = U_MEMORY_ALLOCATION_ERROR; 1100 return; 1101 } 1102 data->variablesAreOwned = (i == 0); 1103 } 1104 1105 for (int32_t j = 0; j < data->variablesLength; j++) { 1106 data->variables[j] = 1107 static_cast<UnicodeFunctor *>(variablesVector.elementAt(j)); 1108 } 1109 1110 data->variableNames.removeAll(); 1111 int32_t pos = UHASH_FIRST; 1112 const UHashElement* he = variableNames.nextElement(pos); 1113 while (he != NULL) { 1114 UnicodeString* tempus = (UnicodeString*)(((UnicodeString*)(he->value.pointer))->clone()); 1115 if (tempus == NULL) { 1116 status = U_MEMORY_ALLOCATION_ERROR; 1117 return; 1118 } 1119 data->variableNames.put(*((UnicodeString*)(he->key.pointer)), 1120 tempus, status); 1121 he = variableNames.nextElement(pos); 1122 } 1123 } 1124 variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed 1125 1126 // Index the rules 1127 if (compoundFilter != NULL) { 1128 if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) || 1129 (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) { 1130 status = U_MISPLACED_COMPOUND_FILTER; 1131 } 1132 } 1133 1134 for (i = 0; i < dataVectorSize; i++) { 1135 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i); 1136 data->ruleSet.freeze(parseError, status); 1137 } 1138 if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) { 1139 idBlockVector.removeElementAt(0); 1140 } 1141 } 1142 } 1143 1144 /** 1145 * Set the variable range to [start, end] (inclusive). 1146 */ 1147 void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) { 1148 if (start > end || start < 0 || end > 0xFFFF) { 1149 status = U_MALFORMED_PRAGMA; 1150 return; 1151 } 1152 1153 curData->variablesBase = (UChar) start; 1154 if (dataVector.size() == 0) { 1155 variableNext = (UChar) start; 1156 variableLimit = (UChar) (end + 1); 1157 } 1158 } 1159 1160 /** 1161 * Assert that the given character is NOT within the variable range. 1162 * If it is, return FALSE. This is neccesary to ensure that the 1163 * variable range does not overlap characters used in a rule. 1164 */ 1165 UBool TransliteratorParser::checkVariableRange(UChar32 ch) const { 1166 return !(ch >= curData->variablesBase && ch < variableLimit); 1167 } 1168 1169 /** 1170 * Set the maximum backup to 'backup', in response to a pragma 1171 * statement. 1172 */ 1173 void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) { 1174 //TODO Finish 1175 } 1176 1177 /** 1178 * Begin normalizing all rules using the given mode, in response 1179 * to a pragma statement. 1180 */ 1181 void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) { 1182 //TODO Finish 1183 } 1184 1185 static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use " 1186 1187 static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;" 1188 1189 static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;" 1190 1191 static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;" 1192 1193 static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;" 1194 1195 /** 1196 * Return true if the given rule looks like a pragma. 1197 * @param pos offset to the first non-whitespace character 1198 * of the rule. 1199 * @param limit pointer past the last character of the rule. 1200 */ 1201 UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) { 1202 // Must start with /use\s/i 1203 return ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_USE, 4), NULL) >= 0; 1204 } 1205 1206 /** 1207 * Parse a pragma. This method assumes resemblesPragma() has 1208 * already returned true. 1209 * @param pos offset to the first non-whitespace character 1210 * of the rule. 1211 * @param limit pointer past the last character of the rule. 1212 * @return the position index after the final ';' of the pragma, 1213 * or -1 on failure. 1214 */ 1215 int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { 1216 int32_t array[2]; 1217 1218 // resemblesPragma() has already returned true, so we 1219 // know that pos points to /use\s/i; we can skip 4 characters 1220 // immediately 1221 pos += 4; 1222 1223 // Here are the pragmas we recognize: 1224 // use variable range 0xE000 0xEFFF; 1225 // use maximum backup 16; 1226 // use nfd rules; 1227 // use nfc rules; 1228 int p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_VARIABLE_RANGE, -1), array); 1229 if (p >= 0) { 1230 setVariableRange(array[0], array[1], status); 1231 return p; 1232 } 1233 1234 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_MAXIMUM_BACKUP, -1), array); 1235 if (p >= 0) { 1236 pragmaMaximumBackup(array[0]); 1237 return p; 1238 } 1239 1240 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFD_RULES, -1), NULL); 1241 if (p >= 0) { 1242 pragmaNormalizeRules(UNORM_NFD); 1243 return p; 1244 } 1245 1246 p = ICU_Utility::parsePattern(rule, pos, limit, UnicodeString(TRUE, PRAGMA_NFC_RULES, -1), NULL); 1247 if (p >= 0) { 1248 pragmaNormalizeRules(UNORM_NFC); 1249 return p; 1250 } 1251 1252 // Syntax error: unable to parse pragma 1253 return -1; 1254 } 1255 1256 /** 1257 * MAIN PARSER. Parse the next rule in the given rule string, starting 1258 * at pos. Return the index after the last character parsed. Do not 1259 * parse characters at or after limit. 1260 * 1261 * Important: The character at pos must be a non-whitespace character 1262 * that is not the comment character. 1263 * 1264 * This method handles quoting, escaping, and whitespace removal. It 1265 * parses the end-of-rule character. It recognizes context and cursor 1266 * indicators. Once it does a lexical breakdown of the rule at pos, it 1267 * creates a rule object and adds it to our rule list. 1268 */ 1269 int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) { 1270 // Locate the left side, operator, and right side 1271 int32_t start = pos; 1272 UChar op = 0; 1273 int32_t i; 1274 1275 // Set up segments data 1276 segmentStandins.truncate(0); 1277 segmentObjects.removeAllElements(); 1278 1279 // Use pointers to automatics to make swapping possible. 1280 RuleHalf _left(*this), _right(*this); 1281 RuleHalf* left = &_left; 1282 RuleHalf* right = &_right; 1283 1284 undefinedVariableName.remove(); 1285 pos = left->parse(rule, pos, limit, status); 1286 if (U_FAILURE(status)) { 1287 return start; 1288 } 1289 1290 if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) { 1291 return syntaxError(U_MISSING_OPERATOR, rule, start, status); 1292 } 1293 ++pos; 1294 1295 // Found an operator char. Check for forward-reverse operator. 1296 if (op == REVERSE_RULE_OP && 1297 (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) { 1298 ++pos; 1299 op = FWDREV_RULE_OP; 1300 } 1301 1302 // Translate alternate op characters. 1303 switch (op) { 1304 case ALT_FORWARD_RULE_OP: 1305 op = FORWARD_RULE_OP; 1306 break; 1307 case ALT_REVERSE_RULE_OP: 1308 op = REVERSE_RULE_OP; 1309 break; 1310 case ALT_FWDREV_RULE_OP: 1311 op = FWDREV_RULE_OP; 1312 break; 1313 } 1314 1315 pos = right->parse(rule, pos, limit, status); 1316 if (U_FAILURE(status)) { 1317 return start; 1318 } 1319 1320 if (pos < limit) { 1321 if (rule.charAt(--pos) == END_OF_RULE) { 1322 ++pos; 1323 } else { 1324 // RuleHalf parser must have terminated at an operator 1325 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status); 1326 } 1327 } 1328 1329 if (op == VARIABLE_DEF_OP) { 1330 // LHS is the name. RHS is a single character, either a literal 1331 // or a set (already parsed). If RHS is longer than one 1332 // character, it is either a multi-character string, or multiple 1333 // sets, or a mixture of chars and sets -- syntax error. 1334 1335 // We expect to see a single undefined variable (the one being 1336 // defined). 1337 if (undefinedVariableName.length() == 0) { 1338 // "Missing '$' or duplicate definition" 1339 return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status); 1340 } 1341 if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) { 1342 // "Malformed LHS" 1343 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); 1344 } 1345 if (left->anchorStart || left->anchorEnd || 1346 right->anchorStart || right->anchorEnd) { 1347 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status); 1348 } 1349 // We allow anything on the right, including an empty string. 1350 UnicodeString* value = new UnicodeString(right->text); 1351 // NULL pointer check 1352 if (value == NULL) { 1353 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 1354 } 1355 variableNames.put(undefinedVariableName, value, status); 1356 ++variableLimit; 1357 return pos; 1358 } 1359 1360 // If this is not a variable definition rule, we shouldn't have 1361 // any undefined variable names. 1362 if (undefinedVariableName.length() != 0) { 1363 return syntaxError(// "Undefined variable $" + undefinedVariableName, 1364 U_UNDEFINED_VARIABLE, 1365 rule, start, status); 1366 } 1367 1368 // Verify segments 1369 if (segmentStandins.length() > segmentObjects.size()) { 1370 syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status); 1371 } 1372 for (i=0; i<segmentStandins.length(); ++i) { 1373 if (segmentStandins.charAt(i) == 0) { 1374 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen 1375 } 1376 } 1377 for (i=0; i<segmentObjects.size(); ++i) { 1378 if (segmentObjects.elementAt(i) == NULL) { 1379 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen 1380 } 1381 } 1382 1383 // If the direction we want doesn't match the rule 1384 // direction, do nothing. 1385 if (op != FWDREV_RULE_OP && 1386 ((direction == UTRANS_FORWARD) != (op == FORWARD_RULE_OP))) { 1387 return pos; 1388 } 1389 1390 // Transform the rule into a forward rule by swapping the 1391 // sides if necessary. 1392 if (direction == UTRANS_REVERSE) { 1393 left = &_right; 1394 right = &_left; 1395 } 1396 1397 // Remove non-applicable elements in forward-reverse 1398 // rules. Bidirectional rules ignore elements that do not 1399 // apply. 1400 if (op == FWDREV_RULE_OP) { 1401 right->removeContext(); 1402 left->cursor = -1; 1403 left->cursorOffset = 0; 1404 } 1405 1406 // Normalize context 1407 if (left->ante < 0) { 1408 left->ante = 0; 1409 } 1410 if (left->post < 0) { 1411 left->post = left->text.length(); 1412 } 1413 1414 // Context is only allowed on the input side. Cursors are only 1415 // allowed on the output side. Segment delimiters can only appear 1416 // on the left, and references on the right. Cursor offset 1417 // cannot appear without an explicit cursor. Cursor offset 1418 // cannot place the cursor outside the limits of the context. 1419 // Anchors are only allowed on the input side. 1420 if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 || 1421 (right->cursorOffset != 0 && right->cursor < 0) || 1422 // - The following two checks were used to ensure that the 1423 // - the cursor offset stayed within the ante- or postcontext. 1424 // - However, with the addition of quantifiers, we have to 1425 // - allow arbitrary cursor offsets and do runtime checking. 1426 //(right->cursorOffset > (left->text.length() - left->post)) || 1427 //(-right->cursorOffset > left->ante) || 1428 right->anchorStart || right->anchorEnd || 1429 !left->isValidInput(*this) || !right->isValidOutput(*this) || 1430 left->ante > left->post) { 1431 1432 return syntaxError(U_MALFORMED_RULE, rule, start, status); 1433 } 1434 1435 // Flatten segment objects vector to an array 1436 UnicodeFunctor** segmentsArray = NULL; 1437 if (segmentObjects.size() > 0) { 1438 segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *)); 1439 // Null pointer check 1440 if (segmentsArray == NULL) { 1441 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 1442 } 1443 segmentObjects.toArray((void**) segmentsArray); 1444 } 1445 TransliterationRule* temptr = new TransliterationRule( 1446 left->text, left->ante, left->post, 1447 right->text, right->cursor, right->cursorOffset, 1448 segmentsArray, 1449 segmentObjects.size(), 1450 left->anchorStart, left->anchorEnd, 1451 curData, 1452 status); 1453 //Null pointer check 1454 if (temptr == NULL) { 1455 uprv_free(segmentsArray); 1456 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status); 1457 } 1458 1459 curData->ruleSet.addRule(temptr, status); 1460 1461 return pos; 1462 } 1463 1464 /** 1465 * Called by main parser upon syntax error. Search the rule string 1466 * for the probable end of the rule. Of course, if the error is that 1467 * the end of rule marker is missing, then the rule end will not be found. 1468 * In any case the rule start will be correctly reported. 1469 * @param msg error description 1470 * @param rule pattern string 1471 * @param start position of first character of current rule 1472 */ 1473 int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode, 1474 const UnicodeString& rule, 1475 int32_t pos, 1476 UErrorCode& status) 1477 { 1478 parseError.offset = pos; 1479 parseError.line = 0 ; /* we are not using line numbers */ 1480 1481 // for pre-context 1482 const int32_t LEN = U_PARSE_CONTEXT_LEN - 1; 1483 int32_t start = uprv_max(pos - LEN, 0); 1484 int32_t stop = pos; 1485 1486 rule.extract(start,stop-start,parseError.preContext); 1487 //null terminate the buffer 1488 parseError.preContext[stop-start] = 0; 1489 1490 //for post-context 1491 start = pos; 1492 stop = uprv_min(pos + LEN, rule.length()); 1493 1494 rule.extract(start,stop-start,parseError.postContext); 1495 //null terminate the buffer 1496 parseError.postContext[stop-start]= 0; 1497 1498 status = (UErrorCode)parseErrorCode; 1499 return pos; 1500 1501 } 1502 1503 /** 1504 * Parse a UnicodeSet out, store it, and return the stand-in character 1505 * used to represent it. 1506 */ 1507 UChar TransliteratorParser::parseSet(const UnicodeString& rule, 1508 ParsePosition& pos, 1509 UErrorCode& status) { 1510 UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status); 1511 // Null pointer check 1512 if (set == NULL) { 1513 status = U_MEMORY_ALLOCATION_ERROR; 1514 return (UChar)0x0000; // Return empty character with error. 1515 } 1516 set->compact(); 1517 return generateStandInFor(set, status); 1518 } 1519 1520 /** 1521 * Generate and return a stand-in for a new UnicodeFunctor. Store 1522 * the matcher (adopt it). 1523 */ 1524 UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) { 1525 // assert(obj != null); 1526 1527 // Look up previous stand-in, if any. This is a short list 1528 // (typical n is 0, 1, or 2); linear search is optimal. 1529 for (int32_t i=0; i<variablesVector.size(); ++i) { 1530 if (variablesVector.elementAt(i) == adopted) { // [sic] pointer comparison 1531 return (UChar) (curData->variablesBase + i); 1532 } 1533 } 1534 1535 if (variableNext >= variableLimit) { 1536 delete adopted; 1537 status = U_VARIABLE_RANGE_EXHAUSTED; 1538 return 0; 1539 } 1540 variablesVector.addElement(adopted, status); 1541 return variableNext++; 1542 } 1543 1544 /** 1545 * Return the standin for segment seg (1-based). 1546 */ 1547 UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) { 1548 // Special character used to indicate an empty spot 1549 UChar empty = curData->variablesBase - 1; 1550 while (segmentStandins.length() < seg) { 1551 segmentStandins.append(empty); 1552 } 1553 UChar c = segmentStandins.charAt(seg-1); 1554 if (c == empty) { 1555 if (variableNext >= variableLimit) { 1556 status = U_VARIABLE_RANGE_EXHAUSTED; 1557 return 0; 1558 } 1559 c = variableNext++; 1560 // Set a placeholder in the master variables vector that will be 1561 // filled in later by setSegmentObject(). We know that we will get 1562 // called first because setSegmentObject() will call us. 1563 variablesVector.addElement((void*) NULL, status); 1564 segmentStandins.setCharAt(seg-1, c); 1565 } 1566 return c; 1567 } 1568 1569 /** 1570 * Set the object for segment seg (1-based). 1571 */ 1572 void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) { 1573 // Since we call parseSection() recursively, nested 1574 // segments will result in segment i+1 getting parsed 1575 // and stored before segment i; be careful with the 1576 // vector handling here. 1577 if (segmentObjects.size() < seg) { 1578 segmentObjects.setSize(seg, status); 1579 } 1580 int32_t index = getSegmentStandin(seg, status) - curData->variablesBase; 1581 if (segmentObjects.elementAt(seg-1) != NULL || 1582 variablesVector.elementAt(index) != NULL) { 1583 // should never happen 1584 status = U_INTERNAL_TRANSLITERATOR_ERROR; 1585 return; 1586 } 1587 segmentObjects.setElementAt(adopted, seg-1); 1588 variablesVector.setElementAt(adopted, index); 1589 } 1590 1591 /** 1592 * Return the stand-in for the dot set. It is allocated the first 1593 * time and reused thereafter. 1594 */ 1595 UChar TransliteratorParser::getDotStandIn(UErrorCode& status) { 1596 if (dotStandIn == (UChar) -1) { 1597 UnicodeSet* tempus = new UnicodeSet(UnicodeString(TRUE, DOT_SET, -1), status); 1598 // Null pointer check. 1599 if (tempus == NULL) { 1600 status = U_MEMORY_ALLOCATION_ERROR; 1601 return (UChar)0x0000; 1602 } 1603 dotStandIn = generateStandInFor(tempus, status); 1604 } 1605 return dotStandIn; 1606 } 1607 1608 /** 1609 * Append the value of the given variable name to the given 1610 * UnicodeString. 1611 */ 1612 void TransliteratorParser::appendVariableDef(const UnicodeString& name, 1613 UnicodeString& buf, 1614 UErrorCode& status) { 1615 const UnicodeString* s = (const UnicodeString*) variableNames.get(name); 1616 if (s == NULL) { 1617 // We allow one undefined variable so that variable definition 1618 // statements work. For the first undefined variable we return 1619 // the special placeholder variableLimit-1, and save the variable 1620 // name. 1621 if (undefinedVariableName.length() == 0) { 1622 undefinedVariableName = name; 1623 if (variableNext >= variableLimit) { 1624 // throw new RuntimeException("Private use variables exhausted"); 1625 status = U_ILLEGAL_ARGUMENT_ERROR; 1626 return; 1627 } 1628 buf.append((UChar) --variableLimit); 1629 } else { 1630 //throw new IllegalArgumentException("Undefined variable $" 1631 // + name); 1632 status = U_ILLEGAL_ARGUMENT_ERROR; 1633 return; 1634 } 1635 } else { 1636 buf.append(*s); 1637 } 1638 } 1639 1640 /** 1641 * Glue method to get around access restrictions in C++. 1642 */ 1643 /*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) { 1644 return Transliterator::createBasicInstance(id, canonID); 1645 }*/ 1646 1647 U_NAMESPACE_END 1648 1649 U_CAPI int32_t 1650 utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) { 1651 U_NAMESPACE_USE 1652 1653 //const UChar *sourceStart = source; 1654 const UChar *targetStart = target; 1655 const UChar *sourceLimit = source+sourceLen; 1656 UChar *targetLimit = target+sourceLen; 1657 UChar32 c = 0; 1658 UBool quoted = FALSE; 1659 int32_t index; 1660 1661 uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR); 1662 1663 /* read the rules into the buffer */ 1664 while (source < sourceLimit) 1665 { 1666 index=0; 1667 U16_NEXT_UNSAFE(source, index, c); 1668 source+=index; 1669 if(c == QUOTE) { 1670 quoted = (UBool)!quoted; 1671 } 1672 else if (!quoted) { 1673 if (c == RULE_COMMENT_CHAR) { 1674 /* skip comments and all preceding spaces */ 1675 while (targetStart < target && *(target - 1) == 0x0020) { 1676 target--; 1677 } 1678 do { 1679 if (source == sourceLimit) { 1680 c = U_SENTINEL; 1681 break; 1682 } 1683 c = *(source++); 1684 } 1685 while (c != CR && c != LF); 1686 if (c < 0) { 1687 break; 1688 } 1689 } 1690 else if (c == ESCAPE && source < sourceLimit) { 1691 UChar32 c2 = *source; 1692 if (c2 == CR || c2 == LF) { 1693 /* A backslash at the end of a line. */ 1694 /* Since we're stripping lines, ignore the backslash. */ 1695 source++; 1696 continue; 1697 } 1698 if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */ 1699 int32_t escapeOffset = 0; 1700 UnicodeString escapedStr(source, 5); 1701 c2 = escapedStr.unescapeAt(escapeOffset); 1702 1703 if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0) 1704 { 1705 *status = U_PARSE_ERROR; 1706 return 0; 1707 } 1708 if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) { 1709 /* It was escaped for a reason. Write what it was suppose to be. */ 1710 source+=5; 1711 c = c2; 1712 } 1713 } 1714 else if (c2 == QUOTE) { 1715 /* \' seen. Make sure we don't do anything when we see it again. */ 1716 quoted = (UBool)!quoted; 1717 } 1718 } 1719 } 1720 if (c == CR || c == LF) 1721 { 1722 /* ignore spaces carriage returns, and all leading spaces on the next line. 1723 * and line feed unless in the form \uXXXX 1724 */ 1725 quoted = FALSE; 1726 while (source < sourceLimit) { 1727 c = *(source); 1728 if (c != CR && c != LF && c != 0x0020) { 1729 break; 1730 } 1731 source++; 1732 } 1733 continue; 1734 } 1735 1736 /* Append UChar * after dissembling if c > 0xffff*/ 1737 index=0; 1738 U16_APPEND_UNSAFE(target, index, c); 1739 target+=index; 1740 } 1741 if (target < targetLimit) { 1742 *target = 0; 1743 } 1744 return (int32_t)(target-targetStart); 1745 } 1746 1747 #endif /* #if !UCONFIG_NO_TRANSLITERATION */ 1748
