1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/i18n/break_iterator.h" 6 7 #include "base/strings/string_piece.h" 8 #include "base/strings/stringprintf.h" 9 #include "base/strings/utf_string_conversions.h" 10 #include "testing/gtest/include/gtest/gtest.h" 11 12 namespace base { 13 namespace i18n { 14 15 TEST(BreakIteratorTest, BreakWordEmpty) { 16 string16 empty; 17 BreakIterator iter(empty, BreakIterator::BREAK_WORD); 18 ASSERT_TRUE(iter.Init()); 19 EXPECT_FALSE(iter.Advance()); 20 EXPECT_FALSE(iter.IsWord()); 21 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 22 EXPECT_FALSE(iter.IsWord()); 23 } 24 25 TEST(BreakIteratorTest, BreakWord) { 26 string16 space(UTF8ToUTF16(" ")); 27 string16 str(UTF8ToUTF16(" foo bar! \npouet boom")); 28 BreakIterator iter(str, BreakIterator::BREAK_WORD); 29 ASSERT_TRUE(iter.Init()); 30 EXPECT_TRUE(iter.Advance()); 31 EXPECT_FALSE(iter.IsWord()); 32 EXPECT_EQ(space, iter.GetString()); 33 EXPECT_TRUE(iter.Advance()); 34 EXPECT_TRUE(iter.IsWord()); 35 EXPECT_EQ(UTF8ToUTF16("foo"), iter.GetString()); 36 EXPECT_TRUE(iter.Advance()); 37 EXPECT_FALSE(iter.IsWord()); 38 EXPECT_EQ(space, iter.GetString()); 39 EXPECT_TRUE(iter.Advance()); 40 EXPECT_TRUE(iter.IsWord()); 41 EXPECT_EQ(UTF8ToUTF16("bar"), iter.GetString()); 42 EXPECT_TRUE(iter.Advance()); 43 EXPECT_FALSE(iter.IsWord()); 44 EXPECT_EQ(UTF8ToUTF16("!"), iter.GetString()); 45 EXPECT_TRUE(iter.Advance()); 46 EXPECT_FALSE(iter.IsWord()); 47 EXPECT_EQ(space, iter.GetString()); 48 EXPECT_TRUE(iter.Advance()); 49 EXPECT_FALSE(iter.IsWord()); 50 EXPECT_EQ(UTF8ToUTF16("\n"), iter.GetString()); 51 EXPECT_TRUE(iter.Advance()); 52 EXPECT_TRUE(iter.IsWord()); 53 EXPECT_EQ(UTF8ToUTF16("pouet"), iter.GetString()); 54 EXPECT_TRUE(iter.Advance()); 55 EXPECT_FALSE(iter.IsWord()); 56 EXPECT_EQ(space, iter.GetString()); 57 EXPECT_TRUE(iter.Advance()); 58 EXPECT_TRUE(iter.IsWord()); 59 EXPECT_EQ(UTF8ToUTF16("boom"), iter.GetString()); 60 EXPECT_FALSE(iter.Advance()); 61 EXPECT_FALSE(iter.IsWord()); 62 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 63 EXPECT_FALSE(iter.IsWord()); 64 } 65 66 TEST(BreakIteratorTest, BreakWide16) { 67 // Two greek words separated by space. 68 const string16 str(WideToUTF16( 69 L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" 70 L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2")); 71 const string16 word1(str.substr(0, 10)); 72 const string16 word2(str.substr(11, 5)); 73 BreakIterator iter(str, BreakIterator::BREAK_WORD); 74 ASSERT_TRUE(iter.Init()); 75 EXPECT_TRUE(iter.Advance()); 76 EXPECT_TRUE(iter.IsWord()); 77 EXPECT_EQ(word1, iter.GetString()); 78 EXPECT_TRUE(iter.Advance()); 79 EXPECT_FALSE(iter.IsWord()); 80 EXPECT_EQ(UTF8ToUTF16(" "), iter.GetString()); 81 EXPECT_TRUE(iter.Advance()); 82 EXPECT_TRUE(iter.IsWord()); 83 EXPECT_EQ(word2, iter.GetString()); 84 EXPECT_FALSE(iter.Advance()); 85 EXPECT_FALSE(iter.IsWord()); 86 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 87 EXPECT_FALSE(iter.IsWord()); 88 } 89 90 TEST(BreakIteratorTest, BreakWide32) { 91 // U+1D49C MATHEMATICAL SCRIPT CAPITAL A 92 const char* very_wide_char = "\xF0\x9D\x92\x9C"; 93 const string16 str( 94 UTF8ToUTF16(base::StringPrintf("%s a", very_wide_char))); 95 const string16 very_wide_word(str.substr(0, 2)); 96 97 BreakIterator iter(str, BreakIterator::BREAK_WORD); 98 ASSERT_TRUE(iter.Init()); 99 EXPECT_TRUE(iter.Advance()); 100 EXPECT_TRUE(iter.IsWord()); 101 EXPECT_EQ(very_wide_word, iter.GetString()); 102 EXPECT_TRUE(iter.Advance()); 103 EXPECT_FALSE(iter.IsWord()); 104 EXPECT_EQ(UTF8ToUTF16(" "), iter.GetString()); 105 EXPECT_TRUE(iter.Advance()); 106 EXPECT_TRUE(iter.IsWord()); 107 EXPECT_EQ(UTF8ToUTF16("a"), iter.GetString()); 108 EXPECT_FALSE(iter.Advance()); 109 EXPECT_FALSE(iter.IsWord()); 110 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 111 EXPECT_FALSE(iter.IsWord()); 112 } 113 114 TEST(BreakIteratorTest, BreakSpaceEmpty) { 115 string16 empty; 116 BreakIterator iter(empty, BreakIterator::BREAK_SPACE); 117 ASSERT_TRUE(iter.Init()); 118 EXPECT_FALSE(iter.Advance()); 119 EXPECT_FALSE(iter.IsWord()); 120 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 121 EXPECT_FALSE(iter.IsWord()); 122 } 123 124 TEST(BreakIteratorTest, BreakSpace) { 125 string16 str(UTF8ToUTF16(" foo bar! \npouet boom")); 126 BreakIterator iter(str, BreakIterator::BREAK_SPACE); 127 ASSERT_TRUE(iter.Init()); 128 EXPECT_TRUE(iter.Advance()); 129 EXPECT_FALSE(iter.IsWord()); 130 EXPECT_EQ(UTF8ToUTF16(" "), iter.GetString()); 131 EXPECT_TRUE(iter.Advance()); 132 EXPECT_FALSE(iter.IsWord()); 133 EXPECT_EQ(UTF8ToUTF16("foo "), iter.GetString()); 134 EXPECT_TRUE(iter.Advance()); 135 EXPECT_FALSE(iter.IsWord()); 136 EXPECT_EQ(UTF8ToUTF16("bar! \n"), iter.GetString()); 137 EXPECT_TRUE(iter.Advance()); 138 EXPECT_FALSE(iter.IsWord()); 139 EXPECT_EQ(UTF8ToUTF16("pouet "), iter.GetString()); 140 EXPECT_TRUE(iter.Advance()); 141 EXPECT_FALSE(iter.IsWord()); 142 EXPECT_EQ(UTF8ToUTF16("boom"), iter.GetString()); 143 EXPECT_FALSE(iter.Advance()); 144 EXPECT_FALSE(iter.IsWord()); 145 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 146 EXPECT_FALSE(iter.IsWord()); 147 } 148 149 TEST(BreakIteratorTest, BreakSpaceSP) { 150 string16 str(UTF8ToUTF16(" foo bar! \npouet boom ")); 151 BreakIterator iter(str, BreakIterator::BREAK_SPACE); 152 ASSERT_TRUE(iter.Init()); 153 EXPECT_TRUE(iter.Advance()); 154 EXPECT_FALSE(iter.IsWord()); 155 EXPECT_EQ(UTF8ToUTF16(" "), iter.GetString()); 156 EXPECT_TRUE(iter.Advance()); 157 EXPECT_FALSE(iter.IsWord()); 158 EXPECT_EQ(UTF8ToUTF16("foo "), iter.GetString()); 159 EXPECT_TRUE(iter.Advance()); 160 EXPECT_FALSE(iter.IsWord()); 161 EXPECT_EQ(UTF8ToUTF16("bar! \n"), iter.GetString()); 162 EXPECT_TRUE(iter.Advance()); 163 EXPECT_FALSE(iter.IsWord()); 164 EXPECT_EQ(UTF8ToUTF16("pouet "), iter.GetString()); 165 EXPECT_TRUE(iter.Advance()); 166 EXPECT_FALSE(iter.IsWord()); 167 EXPECT_EQ(UTF8ToUTF16("boom "), iter.GetString()); 168 EXPECT_FALSE(iter.Advance()); 169 EXPECT_FALSE(iter.IsWord()); 170 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 171 EXPECT_FALSE(iter.IsWord()); 172 } 173 174 TEST(BreakIteratorTest, BreakSpacekWide16) { 175 // Two Greek words. 176 const string16 str(WideToUTF16( 177 L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" 178 L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2")); 179 const string16 word1(str.substr(0, 11)); 180 const string16 word2(str.substr(11, 5)); 181 BreakIterator iter(str, BreakIterator::BREAK_SPACE); 182 ASSERT_TRUE(iter.Init()); 183 EXPECT_TRUE(iter.Advance()); 184 EXPECT_FALSE(iter.IsWord()); 185 EXPECT_EQ(word1, iter.GetString()); 186 EXPECT_TRUE(iter.Advance()); 187 EXPECT_FALSE(iter.IsWord()); 188 EXPECT_EQ(word2, iter.GetString()); 189 EXPECT_FALSE(iter.Advance()); 190 EXPECT_FALSE(iter.IsWord()); 191 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 192 EXPECT_FALSE(iter.IsWord()); 193 } 194 195 TEST(BreakIteratorTest, BreakSpaceWide32) { 196 // U+1D49C MATHEMATICAL SCRIPT CAPITAL A 197 const char* very_wide_char = "\xF0\x9D\x92\x9C"; 198 const string16 str( 199 UTF8ToUTF16(base::StringPrintf("%s a", very_wide_char))); 200 const string16 very_wide_word(str.substr(0, 3)); 201 202 BreakIterator iter(str, BreakIterator::BREAK_SPACE); 203 ASSERT_TRUE(iter.Init()); 204 EXPECT_TRUE(iter.Advance()); 205 EXPECT_FALSE(iter.IsWord()); 206 EXPECT_EQ(very_wide_word, iter.GetString()); 207 EXPECT_TRUE(iter.Advance()); 208 EXPECT_FALSE(iter.IsWord()); 209 EXPECT_EQ(UTF8ToUTF16("a"), iter.GetString()); 210 EXPECT_FALSE(iter.Advance()); 211 EXPECT_FALSE(iter.IsWord()); 212 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 213 EXPECT_FALSE(iter.IsWord()); 214 } 215 216 TEST(BreakIteratorTest, BreakLineEmpty) { 217 string16 empty; 218 BreakIterator iter(empty, BreakIterator::BREAK_NEWLINE); 219 ASSERT_TRUE(iter.Init()); 220 EXPECT_FALSE(iter.Advance()); 221 EXPECT_FALSE(iter.IsWord()); 222 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 223 EXPECT_FALSE(iter.IsWord()); 224 } 225 226 TEST(BreakIteratorTest, BreakLine) { 227 string16 nl(UTF8ToUTF16("\n")); 228 string16 str(UTF8ToUTF16("\nfoo bar!\n\npouet boom")); 229 BreakIterator iter(str, BreakIterator::BREAK_NEWLINE); 230 ASSERT_TRUE(iter.Init()); 231 EXPECT_TRUE(iter.Advance()); 232 EXPECT_FALSE(iter.IsWord()); 233 EXPECT_EQ(nl, iter.GetString()); 234 EXPECT_TRUE(iter.Advance()); 235 EXPECT_FALSE(iter.IsWord()); 236 EXPECT_EQ(UTF8ToUTF16("foo bar!\n"), iter.GetString()); 237 EXPECT_TRUE(iter.Advance()); 238 EXPECT_FALSE(iter.IsWord()); 239 EXPECT_EQ(nl, iter.GetString()); 240 EXPECT_TRUE(iter.Advance()); 241 EXPECT_FALSE(iter.IsWord()); 242 EXPECT_EQ(UTF8ToUTF16("pouet boom"), iter.GetString()); 243 EXPECT_FALSE(iter.Advance()); 244 EXPECT_FALSE(iter.IsWord()); 245 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 246 EXPECT_FALSE(iter.IsWord()); 247 } 248 249 TEST(BreakIteratorTest, BreakLineNL) { 250 string16 nl(UTF8ToUTF16("\n")); 251 string16 str(UTF8ToUTF16("\nfoo bar!\n\npouet boom\n")); 252 BreakIterator iter(str, BreakIterator::BREAK_NEWLINE); 253 ASSERT_TRUE(iter.Init()); 254 EXPECT_TRUE(iter.Advance()); 255 EXPECT_FALSE(iter.IsWord()); 256 EXPECT_EQ(nl, iter.GetString()); 257 EXPECT_TRUE(iter.Advance()); 258 EXPECT_FALSE(iter.IsWord()); 259 EXPECT_EQ(UTF8ToUTF16("foo bar!\n"), iter.GetString()); 260 EXPECT_TRUE(iter.Advance()); 261 EXPECT_FALSE(iter.IsWord()); 262 EXPECT_EQ(nl, iter.GetString()); 263 EXPECT_TRUE(iter.Advance()); 264 EXPECT_FALSE(iter.IsWord()); 265 EXPECT_EQ(UTF8ToUTF16("pouet boom\n"), iter.GetString()); 266 EXPECT_FALSE(iter.Advance()); 267 EXPECT_FALSE(iter.IsWord()); 268 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 269 EXPECT_FALSE(iter.IsWord()); 270 } 271 272 TEST(BreakIteratorTest, BreakLineWide16) { 273 // Two Greek words separated by newline. 274 const string16 str(WideToUTF16( 275 L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" 276 L"\x03bf\x03c2\x000a\x0399\x03c3\x03c4\x03cc\x03c2")); 277 const string16 line1(str.substr(0, 11)); 278 const string16 line2(str.substr(11, 5)); 279 BreakIterator iter(str, BreakIterator::BREAK_NEWLINE); 280 ASSERT_TRUE(iter.Init()); 281 EXPECT_TRUE(iter.Advance()); 282 EXPECT_FALSE(iter.IsWord()); 283 EXPECT_EQ(line1, iter.GetString()); 284 EXPECT_TRUE(iter.Advance()); 285 EXPECT_FALSE(iter.IsWord()); 286 EXPECT_EQ(line2, iter.GetString()); 287 EXPECT_FALSE(iter.Advance()); 288 EXPECT_FALSE(iter.IsWord()); 289 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 290 EXPECT_FALSE(iter.IsWord()); 291 } 292 293 TEST(BreakIteratorTest, BreakLineWide32) { 294 // U+1D49C MATHEMATICAL SCRIPT CAPITAL A 295 const char* very_wide_char = "\xF0\x9D\x92\x9C"; 296 const string16 str( 297 UTF8ToUTF16(base::StringPrintf("%s\na", very_wide_char))); 298 const string16 very_wide_line(str.substr(0, 3)); 299 BreakIterator iter(str, BreakIterator::BREAK_NEWLINE); 300 ASSERT_TRUE(iter.Init()); 301 EXPECT_TRUE(iter.Advance()); 302 EXPECT_FALSE(iter.IsWord()); 303 EXPECT_EQ(very_wide_line, iter.GetString()); 304 EXPECT_TRUE(iter.Advance()); 305 EXPECT_FALSE(iter.IsWord()); 306 EXPECT_EQ(UTF8ToUTF16("a"), iter.GetString()); 307 EXPECT_FALSE(iter.Advance()); 308 EXPECT_FALSE(iter.IsWord()); 309 EXPECT_FALSE(iter.Advance()); // Test unexpected advance after end. 310 EXPECT_FALSE(iter.IsWord()); 311 } 312 313 TEST(BreakIteratorTest, BreakCharacter) { 314 static const wchar_t* kCharacters[] = { 315 // An English word consisting of four ASCII characters. 316 L"w", L"o", L"r", L"d", L" ", 317 // A Hindi word (which means "Hindi") consisting of three Devanagari 318 // characters. 319 L"\x0939\x093F", L"\x0928\x094D", L"\x0926\x0940", L" ", 320 // A Thai word (which means "feel") consisting of three Thai characters. 321 L"\x0E23\x0E39\x0E49", L"\x0E2A\x0E36", L"\x0E01", L" ", 322 }; 323 std::vector<string16> characters; 324 string16 text; 325 for (size_t i = 0; i < arraysize(kCharacters); ++i) { 326 characters.push_back(WideToUTF16(kCharacters[i])); 327 text.append(characters.back()); 328 } 329 BreakIterator iter(text, BreakIterator::BREAK_CHARACTER); 330 ASSERT_TRUE(iter.Init()); 331 for (size_t i = 0; i < arraysize(kCharacters); ++i) { 332 EXPECT_TRUE(iter.Advance()); 333 EXPECT_EQ(characters[i], iter.GetString()); 334 } 335 } 336 337 } // namespace i18n 338 } // namespace base 339
