1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/string_tokenizer.h" 6 #include "testing/gtest/include/gtest/gtest.h" 7 8 using std::string; 9 10 namespace { 11 class StringTokenizerTest : public testing::Test {}; 12 } 13 14 TEST(StringTokenizerTest, Simple) { 15 string input = "this is a test"; 16 StringTokenizer t(input, " "); 17 18 EXPECT_TRUE(t.GetNext()); 19 EXPECT_EQ(string("this"), t.token()); 20 21 EXPECT_TRUE(t.GetNext()); 22 EXPECT_EQ(string("is"), t.token()); 23 24 EXPECT_TRUE(t.GetNext()); 25 EXPECT_EQ(string("a"), t.token()); 26 27 EXPECT_TRUE(t.GetNext()); 28 EXPECT_EQ(string("test"), t.token()); 29 30 EXPECT_FALSE(t.GetNext()); 31 } 32 33 TEST(StringTokenizerTest, Reset) { 34 string input = "this is a test"; 35 StringTokenizer t(input, " "); 36 37 for (int i = 0; i < 2; ++i) { 38 EXPECT_TRUE(t.GetNext()); 39 EXPECT_EQ(string("this"), t.token()); 40 41 EXPECT_TRUE(t.GetNext()); 42 EXPECT_EQ(string("is"), t.token()); 43 44 EXPECT_TRUE(t.GetNext()); 45 EXPECT_EQ(string("a"), t.token()); 46 47 EXPECT_TRUE(t.GetNext()); 48 EXPECT_EQ(string("test"), t.token()); 49 50 EXPECT_FALSE(t.GetNext()); 51 t.Reset(); 52 } 53 } 54 55 TEST(StringTokenizerTest, RetDelims) { 56 string input = "this is a test"; 57 StringTokenizer t(input, " "); 58 t.set_options(StringTokenizer::RETURN_DELIMS); 59 60 EXPECT_TRUE(t.GetNext()); 61 EXPECT_EQ(string("this"), t.token()); 62 63 EXPECT_TRUE(t.GetNext()); 64 EXPECT_EQ(string(" "), t.token()); 65 66 EXPECT_TRUE(t.GetNext()); 67 EXPECT_EQ(string("is"), t.token()); 68 69 EXPECT_TRUE(t.GetNext()); 70 EXPECT_EQ(string(" "), t.token()); 71 72 EXPECT_TRUE(t.GetNext()); 73 EXPECT_EQ(string("a"), t.token()); 74 75 EXPECT_TRUE(t.GetNext()); 76 EXPECT_EQ(string(" "), t.token()); 77 78 EXPECT_TRUE(t.GetNext()); 79 EXPECT_EQ(string("test"), t.token()); 80 81 EXPECT_FALSE(t.GetNext()); 82 } 83 84 TEST(StringTokenizerTest, ManyDelims) { 85 string input = "this: is, a-test"; 86 StringTokenizer t(input, ": ,-"); 87 88 EXPECT_TRUE(t.GetNext()); 89 EXPECT_EQ(string("this"), t.token()); 90 91 EXPECT_TRUE(t.GetNext()); 92 EXPECT_EQ(string("is"), t.token()); 93 94 EXPECT_TRUE(t.GetNext()); 95 EXPECT_EQ(string("a"), t.token()); 96 97 EXPECT_TRUE(t.GetNext()); 98 EXPECT_EQ(string("test"), t.token()); 99 100 EXPECT_FALSE(t.GetNext()); 101 } 102 103 TEST(StringTokenizerTest, ParseHeader) { 104 string input = "Content-Type: text/html ; charset=UTF-8"; 105 StringTokenizer t(input, ": ;="); 106 t.set_options(StringTokenizer::RETURN_DELIMS); 107 108 EXPECT_TRUE(t.GetNext()); 109 EXPECT_FALSE(t.token_is_delim()); 110 EXPECT_EQ(string("Content-Type"), t.token()); 111 112 EXPECT_TRUE(t.GetNext()); 113 EXPECT_TRUE(t.token_is_delim()); 114 EXPECT_EQ(string(":"), t.token()); 115 116 EXPECT_TRUE(t.GetNext()); 117 EXPECT_TRUE(t.token_is_delim()); 118 EXPECT_EQ(string(" "), t.token()); 119 120 EXPECT_TRUE(t.GetNext()); 121 EXPECT_FALSE(t.token_is_delim()); 122 EXPECT_EQ(string("text/html"), t.token()); 123 124 EXPECT_TRUE(t.GetNext()); 125 EXPECT_TRUE(t.token_is_delim()); 126 EXPECT_EQ(string(" "), t.token()); 127 128 EXPECT_TRUE(t.GetNext()); 129 EXPECT_TRUE(t.token_is_delim()); 130 EXPECT_EQ(string(";"), t.token()); 131 132 EXPECT_TRUE(t.GetNext()); 133 EXPECT_TRUE(t.token_is_delim()); 134 EXPECT_EQ(string(" "), t.token()); 135 136 EXPECT_TRUE(t.GetNext()); 137 EXPECT_FALSE(t.token_is_delim()); 138 EXPECT_EQ(string("charset"), t.token()); 139 140 EXPECT_TRUE(t.GetNext()); 141 EXPECT_TRUE(t.token_is_delim()); 142 EXPECT_EQ(string("="), t.token()); 143 144 EXPECT_TRUE(t.GetNext()); 145 EXPECT_FALSE(t.token_is_delim()); 146 EXPECT_EQ(string("UTF-8"), t.token()); 147 148 EXPECT_FALSE(t.GetNext()); 149 EXPECT_FALSE(t.token_is_delim()); 150 } 151 152 TEST(StringTokenizerTest, ParseQuotedString) { 153 string input = "foo bar 'hello world' baz"; 154 StringTokenizer t(input, " "); 155 t.set_quote_chars("'"); 156 157 EXPECT_TRUE(t.GetNext()); 158 EXPECT_EQ(string("foo"), t.token()); 159 160 EXPECT_TRUE(t.GetNext()); 161 EXPECT_EQ(string("bar"), t.token()); 162 163 EXPECT_TRUE(t.GetNext()); 164 EXPECT_EQ(string("'hello world'"), t.token()); 165 166 EXPECT_TRUE(t.GetNext()); 167 EXPECT_EQ(string("baz"), t.token()); 168 169 EXPECT_FALSE(t.GetNext()); 170 } 171 172 TEST(StringTokenizerTest, ParseQuotedString_Malformed) { 173 string input = "bar 'hello wo"; 174 StringTokenizer t(input, " "); 175 t.set_quote_chars("'"); 176 177 EXPECT_TRUE(t.GetNext()); 178 EXPECT_EQ(string("bar"), t.token()); 179 180 EXPECT_TRUE(t.GetNext()); 181 EXPECT_EQ(string("'hello wo"), t.token()); 182 183 EXPECT_FALSE(t.GetNext()); 184 } 185 186 TEST(StringTokenizerTest, ParseQuotedString_Multiple) { 187 string input = "bar 'hel\"lo\" wo' baz\""; 188 StringTokenizer t(input, " "); 189 t.set_quote_chars("'\""); 190 191 EXPECT_TRUE(t.GetNext()); 192 EXPECT_EQ(string("bar"), t.token()); 193 194 EXPECT_TRUE(t.GetNext()); 195 EXPECT_EQ(string("'hel\"lo\" wo'"), t.token()); 196 197 EXPECT_TRUE(t.GetNext()); 198 EXPECT_EQ(string("baz\""), t.token()); 199 200 EXPECT_FALSE(t.GetNext()); 201 } 202 203 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) { 204 string input = "foo 'don\\'t do that'"; 205 StringTokenizer t(input, " "); 206 t.set_quote_chars("'"); 207 208 EXPECT_TRUE(t.GetNext()); 209 EXPECT_EQ(string("foo"), t.token()); 210 211 EXPECT_TRUE(t.GetNext()); 212 EXPECT_EQ(string("'don\\'t do that'"), t.token()); 213 214 EXPECT_FALSE(t.GetNext()); 215 } 216 217 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) { 218 string input = "foo='a, b', bar"; 219 StringTokenizer t(input, ", "); 220 t.set_quote_chars("'"); 221 222 EXPECT_TRUE(t.GetNext()); 223 EXPECT_EQ(string("foo='a, b'"), t.token()); 224 225 EXPECT_TRUE(t.GetNext()); 226 EXPECT_EQ(string("bar"), t.token()); 227 228 EXPECT_FALSE(t.GetNext()); 229 } 230
