1 --- source/data/brkitr/word.txt 2010-09-17 18:22:35.000000000 -0700 2 +++ source/data/brkitr/word.txt 2011-01-25 18:14:09.445378000 -0800 3 @@ -29,29 +29,49 @@ 4 $Newline = [\p{Word_Break = Newline}]; 5 $Extend = [\p{Word_Break = Extend}]; 6 $Format = [\p{Word_Break = Format}]; 7 +$Hiragana = [:Hiragana:]; 8 $Katakana = [\p{Word_Break = Katakana}]; 9 +$Han = [:Han:]; 10 $ALetter = [\p{Word_Break = ALetter}]; 11 -$MidNumLet = [\p{Word_Break = MidNumLet}]; 12 +# Remove two full stop characters from $MidNumLet and add them to $MidNum 13 +# to break a hostname into its components at the cost of breaking 14 +# 'e.g.' and 'i.e.' as well. 15 +# $MidNumLet is used in rules 6/7 (rules of our interest) and rules 11/12. 16 +# Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected 17 +# while rules 6/7 are reverted to the old behavior we want. 18 +$MidNumLet = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]]; 19 $MidLetter = [\p{Word_Break = MidLetter}]; 20 -$MidNum = [\p{Word_Break = MidNum}]; 21 -$Numeric = [\p{Word_Break = Numeric}]; 22 +$MidNum = [\p{Word_Break = MidNum}[\u002E \uFF0E]]; 23 +$Numeric = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits 24 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; 25 26 +# Extra sets not to break 'HebrewLetter U+0022 HebrewLetter'. 27 +$HebrewLet = [\p{Word_Break = ALetter} & \p{Script = Hebrew} - [\u05F3]]; 28 +# U+05F3 is ALetter and U+05F4 is MidLetter so that they're covered by 29 +# the current rule 6/7. 30 +$HebrewMidLet = [\u0022]; 31 32 # Dictionary character set, for triggering language-based break engines. Currently 33 -# limited to LineBreak=Complex_Context. Note that this set only works in Unicode 34 -# 5.0 or later as the definition of Complex_Context was corrected to include all 35 +# limited to LineBreak=Complex_Context and CJK. Note that this set only works 36 +# in Unicode 5.0 or later as the definition of Complex_Context was corrected to include all 37 # characters requiring dictionary break. 38 39 -$dictionary = [:LineBreak = Complex_Context:]; 40 $Control = [\p{Grapheme_Cluster_Break = Control}]; 41 -$ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]]; # Note: default ALetter does not 42 - # include the dictionary characters. 43 +$HangulSyllable = [\uac00-\ud7a3]; 44 +$ComplexContext = [:LineBreak = Complex_Context:]; 45 +$KanaKanji = [$Han $Hiragana $Katakana]; 46 +$dictionaryCJK = [$KanaKanji $HangulSyllable]; 47 +$dictionary = [$ComplexContext $dictionaryCJK]; 48 + 49 +# leave CJK scripts out of ALetterPlus 50 +$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]]; 51 + 52 53 # 54 # Rules 4 Ignore Format and Extend characters, 55 # except when they appear at the beginning of a region of text. 56 # 57 +# TODO: check if handling of katakana in dictionary makes rules incorrect/void. 58 $KatakanaEx = $Katakana ($Extend | $Format)*; 59 $ALetterEx = $ALetterPlus ($Extend | $Format)*; 60 $MidNumLetEx = $MidNumLet ($Extend | $Format)*; 61 @@ -59,8 +79,8 @@ 62 $MidNumEx = $MidNum ($Extend | $Format)*; 63 $NumericEx = $Numeric ($Extend | $Format)*; 64 $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; 65 +$HebrewLetEx = $HebrewLet ($Extend | $Format)*; 66 67 -$Hiragana = [\p{script=Hiragana}]; 68 $Ideographic = [\p{Ideographic}]; 69 $HiraganaEx = $Hiragana ($Extend | $Format)*; 70 $IdeographicEx = $Ideographic ($Extend | $Format)*; 71 @@ -79,12 +99,14 @@ 72 # begins with a group of Format chars, or with a "word" consisting of a single 73 # char that is not in any of the listed word break categories followed by 74 # format char(s). 75 -[^$CR $LF $Newline]? ($Extend | $Format)+; 76 + # format char(s), or is not a CJK dictionary character. 77 +[^$CR $LF $Newline $dictionaryCJK]? ($Extend | $Format)+; 78 79 $NumericEx {100}; 80 $ALetterEx {200}; 81 -$KatakanaEx {300}; # note: these status values override those from rule 5 82 -$HiraganaEx {300}; # by virtual of being numerically larger. 83 +$HangulSyllable {200}; 84 +$KatakanaEx {400}; #originally 300 85 +$HiraganaEx {400}; #originally 300 86 $IdeographicEx {400}; # 87 88 # 89 @@ -96,6 +118,9 @@ 90 # rule 6 and 7 91 $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200}; 92 93 +# Chrome addition 94 +$HebrewLetEx $HebrewMidLet $HebrewLetEx {200}; 95 + 96 # rule 8 97 98 $NumericEx $NumericEx {100}; 99 @@ -114,19 +139,25 @@ 100 101 # rule 13 102 103 -$KatakanaEx $KatakanaEx {300}; 104 +# To be consistent with '$KanaKanji $KanaKanji', changed 105 +# from 300 to 400. 106 +# See also TestRuleStatus in intltest/rbbiapts.cpp 107 +$KatakanaEx $KatakanaEx {400}; 108 109 # rule 13a/b 110 111 $ALetterEx $ExtendNumLetEx {200}; # (13a) 112 $NumericEx $ExtendNumLetEx {100}; # (13a) 113 -$KatakanaEx $ExtendNumLetEx {300}; # (13a) 114 +$KatakanaEx $ExtendNumLetEx {400}; # (13a) 115 $ExtendNumLetEx $ExtendNumLetEx {200}; # (13a) 116 117 $ExtendNumLetEx $ALetterEx {200}; # (13b) 118 $ExtendNumLetEx $NumericEx {100}; # (13b) 119 -$ExtendNumLetEx $KatakanaEx {300}; # (13b) 120 - 121 +$ExtendNumLetEx $KatakanaEx {400}; # (13b) 122 + 123 +# special handling for CJK characters: chain for later dictionary segmentation 124 +$HangulSyllable $HangulSyllable {200}; 125 +$KanaKanji $KanaKanji {400}; #different rule status if both kanji and kana found 126 127 128 ## ------------------------------------------------- 129 @@ -139,13 +170,16 @@ 130 $BackMidNumEx = ($Format | $Extend)* $MidNum; 131 $BackMidLetterEx = ($Format | $Extend)* $MidLetter; 132 $BackKatakanaEx = ($Format | $Extend)* $Katakana; 133 +$BackHiraganaEx = ($Extend | $Format)* $Hiragana; 134 $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet; 135 +$BackHebrewLetEx = ($Format | $Extend)* $HebrewLet; 136 + 137 138 # rule 3 139 $LF $CR; 140 141 # rule 4 142 -($Format | $Extend)* [^$CR $LF $Newline]?; 143 +($Format | $Extend)* [^$CR $LF $Newline $dictionaryCJK]?; 144 145 # rule 5 146 147 @@ -155,6 +189,8 @@ 148 149 $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx; 150 151 +# Chrome addition 152 +$BackHebrewLetEx $HebrewMidLet $BackHebrewLetEx; 153 154 # rule 8 155 156 @@ -181,6 +217,10 @@ 157 $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx); 158 ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; 159 160 +# special handling for CJK characters: chain for later dictionary segmentation 161 +$HangulSyllable $HangulSyllable; 162 +$KanaKanji $KanaKanji; #different rule status if both kanji and kana found 163 + 164 ## ------------------------------------------------- 165 166 !!safe_reverse; 167 --- source/data/brkitr/line.txt 2010-09-17 18:22:35.000000000 -0700 168 +++ source/data/brkitr/line.txt 2011-07-22 13:46:19.923562000 -0700 169 @@ -11,6 +11,9 @@ 170 # TODO: Rule LB 8 remains as it was in Unicode 5.2 171 # This is only because of a limitation of ICU break engine implementation, 172 # not because the older behavior is desirable. 173 +# 174 +# CHROME: Hebrew tailoring was incorporatd as well as some 175 +# other minor changes (CL, OP, ID). 176 177 # 178 # Character Classes defined by TR 14. 179 @@ -55,15 +58,22 @@ 180 # 181 # See rule LB 19 for an example. 182 # 183 +$SmallHira = [\u3041 \u3043 \u3045 \u3047 \u3049 \u3063 \u3083 \u3085 \u3087 \u308E \u3095 \u3096]; 184 +$SmallKata = [\u30A1 \u30A3 \u30A5 \u30A7 \u30A9 \u30C3 \u30E3 \u30E5 \u30E7 \u30EE \u30F5 \u30F6]; 185 +$SmallKataExt = [\u31F0 \u31F1 \u31F2 \u31F3 \u31F4 \u31F5 \u31F6 \u31F7 \u31F8 \u31F9 \u31FA \u31FB \u31FC \u31FD \u31FE \u31FF]; 186 +$SmallKanaAndProlongedMark = [[$SmallHira] [$SmallKata] [$SmallKataExt] [\u30FC]]; 187 + 188 189 $AI = [:LineBreak = Ambiguous:]; 190 -$AL = [:LineBreak = Alphabetic:]; 191 -$BA = [:LineBreak = Break_After:]; 192 +$AL = [[:LineBreak = Alphabetic:] - [[:Hebrew:] & [:Letter:]] - [\u23B4\u23B5]]; 193 +$HL = [[:Hebrew:] & [:Letter:]]; 194 +$BA = [[:LineBreak = Break_After:] - [\u2010]]; 195 +$HH = [\u2010]; 196 $BB = [:LineBreak = Break_Before:]; 197 $BK = [:LineBreak = Mandatory_Break:]; 198 $B2 = [:LineBreak = Break_Both:]; 199 $CB = [:LineBreak = Contingent_Break:]; 200 -$CL = [:LineBreak = Close_Punctuation:]; 201 +$CL = [[:LineBreak = Close_Punctuation:] [\uFE51\uFE10\u23B5]]; 202 $CM = [:LineBreak = Combining_Mark:]; 203 $CP = [:LineBreak = Close_Parenthesis:]; 204 $CR = [:LineBreak = Carriage_Return:]; 205 @@ -72,17 +82,17 @@ 206 $HY = [:LineBreak = Hyphen:]; 207 $H2 = [:LineBreak = H2:]; 208 $H3 = [:LineBreak = H3:]; 209 -$ID = [:LineBreak = Ideographic:]; 210 +$ID = [[[:LineBreak = Ideographic:] - [\uFE51]] [$SmallKanaAndProlongedMark]]; 211 $IN = [:LineBreak = Inseperable:]; 212 -$IS = [:LineBreak = Infix_Numeric:]; 213 +$IS = [[:LineBreak = Infix_Numeric:] - [\uFE10]]; 214 $JL = [:LineBreak = JL:]; 215 $JV = [:LineBreak = JV:]; 216 $JT = [:LineBreak = JT:]; 217 $LF = [:LineBreak = Line_Feed:]; 218 $NL = [:LineBreak = Next_Line:]; 219 -$NS = [:LineBreak = Nonstarter:]; 220 +$NS = [[:LineBreak = Nonstarter:] - [$SmallKanaAndProlongedMark]]; 221 $NU = [:LineBreak = Numeric:]; 222 -$OP = [:LineBreak = Open_Punctuation:]; 223 +$OP = [[:LineBreak = Open_Punctuation:] \u23B4]; 224 $PO = [:LineBreak = Postfix_Numeric:]; 225 $PR = [:LineBreak = Prefix_Numeric:]; 226 $QU = [:LineBreak = Quotation:]; 227 @@ -108,13 +118,15 @@ 228 # XX (Unknown, unassigned) 229 # as $AL (Alphabetic) 230 # 231 -$ALPlus = [$AL $AI $SA $SG $XX]; 232 +$ALPlus = [$AL $HL $AI $SA $SG $XX]; 233 234 # 235 # Combining Marks. X $CM* behaves as if it were X. Rule LB6. 236 # 237 $ALcm = $ALPlus $CM*; 238 +$HLcm = $HL $CM*; 239 $BAcm = $BA $CM*; 240 +$HHcm = $HH $CM*; 241 $BBcm = $BB $CM*; 242 $B2cm = $B2 $CM*; 243 $CLcm = $CL $CM*; 244 @@ -148,6 +160,7 @@ 245 # 246 $ALPlus $CM+; 247 $BA $CM+; 248 +$HH $CM+; 249 $BB $CM+; 250 $B2 $CM+; 251 $CL $CM+; 252 @@ -190,7 +203,7 @@ 253 # so for this one case we need to manually list out longer sequences. 254 # 255 $AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP]; 256 -$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus]; 257 +$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus]; 258 $AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM]; 259 260 261 @@ -252,7 +265,7 @@ 262 # LB 12a Do not break before NBSP and related characters ... 263 # [^SP BA HY] x GL 264 # 265 -[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm; 266 +[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm; 267 $CM+ GLcm; 268 269 270 @@ -325,7 +338,7 @@ 271 # LB 21 x (BA | HY | NS) 272 # BB x 273 # 274 -$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm); 275 +$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm); 276 277 $BBcm [^$CB]; # $BB x 278 $BBcm $LB20NonBreaks $CM*; 279 @@ -381,6 +394,8 @@ 280 $CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL. 281 $CPcm ($ALcm | $NUcm); 282 283 +# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change 284 +$HLcm ($HY | $HH) $ALcm; 285 286 # 287 # Reverse Rules. 288 @@ -391,6 +406,7 @@ 289 290 $CM+ $ALPlus; 291 $CM+ $BA; 292 +$CM+ $HH; 293 $CM+ $BB; 294 $CM+ $B2; 295 $CM+ $CL; 296 @@ -479,7 +495,7 @@ 297 # LB 12a 298 # [^SP BA HY] x GL 299 # 300 -$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]]; 301 +$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]]; 302 303 # LB 12 304 # GL x 305 @@ -549,7 +565,7 @@ 306 # 307 308 # LB 21 309 -$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS) 310 +$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS) 311 312 $CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x . 313 [^$CB] $CM* $BB; # 314 @@ -597,6 +613,8 @@ 315 $CM* $OP $CM* ($ALPlus | $NU); 316 $CM* ($ALPlus | $NU) $CM* $CP; 317 318 +# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change 319 +$CM* $ALPlus ($HY | $HH) $CM* $HL; 320 321 ## ------------------------------------------------- 322 323 --- source/data/brkitr/brklocal.mk 1969-12-31 16:00:00.000000000 -0800 324 +++ source/data/brkitr/brklocal.mk 2011-01-25 17:54:01.149770000 -0800 325 @@ -0,0 +1,48 @@ 326 +# * Copyright (C) 1998-2010, International Business Machines 327 +# * Corporation and others. All Rights Reserved. 328 +BRK_RES_CLDR_VERSION = 1.9 329 +# A list of txt's to build 330 +# Note: 331 +# 332 +# If you are thinking of modifying this file, READ THIS. 333 +# 334 +# Instead of changing this file [unless you want to check it back in], 335 +# you should consider creating a 'brklocal.mk' file in this same directory. 336 +# Then, you can have your local changes remain even if you upgrade or 337 +# reconfigure ICU. 338 +# 339 +# Example 'brklocal.mk' files: 340 +# 341 +# * To add an additional locale to the list: 342 +# _____________________________________________________ 343 +# | BRK_RES_SOURCE_LOCAL = myLocale.txt ... 344 +# 345 +# * To REPLACE the default list and only build with a few 346 +# locales: 347 +# _____________________________________________________ 348 +# | BRK_RES_SOURCE = ar.txt ar_AE.txt en.txt de.txt zh.txt 349 +# 350 +# 351 +# Generated by LDML2ICUConverter, from LDML source files. 352 + 353 +# Aliases without a corresponding xx.xml file (see icu-config.xml & build.xml) 354 +BRK_RES_SYNTHETIC_ALIAS = 355 + 356 + 357 +# All aliases (to not be included under 'installed'), but not including root. 358 +BRK_RES_ALIAS_SOURCE = $(BRK_RES_SYNTHETIC_ALIAS) 359 + 360 + 361 +# List of compact trie dictionary files (ctd). 362 +BRK_CTD_SOURCE = thaidict.txt cjdict.txt 363 + 364 + 365 +# List of break iterator files (brk). 366 +# Chrome change: remove word_ja.txt and line_he.txt 367 +BRK_SOURCE = sent_el.txt word_POSIX.txt line_fi.txt char.txt word.txt line.txt sent.txt title.txt char_th.txt 368 + 369 + 370 +# Ordinary resources 371 +# Chrome change: remove ja.txt and he.txt 372 +BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt\ 373 + fi.txt th.txt 374
