1 // (C) Copyright Gennadiy Rozental 2004-2008. 2 // Distributed under the Boost Software License, Version 1.0. 3 // (See accompanying file LICENSE_1_0.txt or copy at 4 // http://www.boost.org/LICENSE_1_0.txt) 5 6 // See http://www.boost.org/libs/test for the library home page. 7 // 8 // File : $RCSfile$ 9 // 10 // Version : $Revision: 54633 $ 11 // 12 // Description : token iterator for string and range tokenization 13 // *************************************************************************** 14 15 #ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER 16 #define BOOST_TOKEN_ITERATOR_HPP_071894GER 17 18 // Boost 19 #include <boost/config.hpp> 20 #include <boost/detail/workaround.hpp> 21 22 #include <boost/iterator/iterator_categories.hpp> 23 #include <boost/iterator/iterator_traits.hpp> 24 25 #include <boost/test/utils/iterator/input_iterator_facade.hpp> 26 #include <boost/test/utils/basic_cstring/basic_cstring.hpp> 27 #include <boost/test/utils/named_params.hpp> 28 #include <boost/test/utils/foreach.hpp> 29 30 // STL 31 #include <iosfwd> 32 #include <cctype> 33 34 #include <boost/test/detail/suppress_warnings.hpp> 35 36 //____________________________________________________________________________// 37 38 #ifdef BOOST_NO_STDC_NAMESPACE 39 namespace std{ using ::ispunct; using ::isspace; } 40 #endif 41 42 namespace boost { 43 44 namespace unit_test { 45 46 // ************************************************************************** // 47 // ************** ti_delimeter_type ************** // 48 // ************************************************************************** // 49 50 enum ti_delimeter_type { 51 dt_char, // character is delimeter if it among explicit list of some characters 52 dt_ispunct, // character is delimeter if it satisfies ispunct functor 53 dt_isspace, // character is delimeter if it satisfies isspace functor 54 dt_none // no character is delimeter 55 }; 56 57 namespace ut_detail { 58 59 // ************************************************************************** // 60 // ************** default_char_compare ************** // 61 // ************************************************************************** // 62 63 template<typename CharT> 64 class default_char_compare { 65 public: 66 bool operator()( CharT c1, CharT c2 ) 67 { 68 #ifdef BOOST_CLASSIC_IOSTREAMS 69 return std::string_char_traits<CharT>::eq( c1, c2 ); 70 #else 71 return std::char_traits<CharT>::eq( c1, c2 ); 72 #endif 73 } 74 }; 75 76 // ************************************************************************** // 77 // ************** delim_policy ************** // 78 // ************************************************************************** // 79 80 template<typename CharT,typename CharCompare> 81 class delim_policy { 82 typedef basic_cstring<CharT const> cstring; 83 public: 84 // Constructor 85 explicit delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() ) 86 : m_type( t ) 87 { 88 set_delimeters( d ); 89 } 90 91 void set_delimeters( ti_delimeter_type t ) { m_type = t; } 92 template<typename Src> 93 void set_delimeters( Src d ) 94 { 95 nfp::optionally_assign( m_delimeters, d ); 96 97 if( !m_delimeters.is_empty() ) 98 m_type = dt_char; 99 } 100 101 bool operator()( CharT c ) 102 { 103 switch( m_type ) { 104 case dt_char: { 105 BOOST_TEST_FOREACH( CharT, delim, m_delimeters ) 106 if( CharCompare()( delim, c ) ) 107 return true; 108 109 return false; 110 } 111 case dt_ispunct: 112 return (std::ispunct)( c ) != 0; 113 case dt_isspace: 114 return (std::isspace)( c ) != 0; 115 case dt_none: 116 return false; 117 } 118 119 return false; 120 } 121 122 private: 123 // Data members 124 cstring m_delimeters; 125 ti_delimeter_type m_type; 126 }; 127 128 // ************************************************************************** // 129 // ************** token_assigner ************** // 130 // ************************************************************************** // 131 132 template<typename TraversalTag> 133 struct token_assigner { 134 #if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 ) 135 template<typename Iterator, typename C, typename T> 136 static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t ) 137 { for( ; b != e; ++b ) t += *b; } 138 139 template<typename Iterator, typename C> 140 static void assign( Iterator b, Iterator e, basic_cstring<C>& t ) { t.assign( b, e ); } 141 #else 142 template<typename Iterator, typename Token> 143 static void assign( Iterator b, Iterator e, Token& t ) { t.assign( b, e ); } 144 #endif 145 template<typename Iterator, typename Token> 146 static void append_move( Iterator& b, Token& ) { ++b; } 147 }; 148 149 //____________________________________________________________________________// 150 151 template<> 152 struct token_assigner<single_pass_traversal_tag> { 153 template<typename Iterator, typename Token> 154 static void assign( Iterator b, Iterator e, Token& t ) {} 155 156 template<typename Iterator, typename Token> 157 static void append_move( Iterator& b, Token& t ) { t += *b; ++b; } 158 }; 159 160 } // namespace ut_detail 161 162 // ************************************************************************** // 163 // ************** modifiers ************** // 164 // ************************************************************************** // 165 166 namespace { 167 nfp::keyword<struct dropped_delimeters_t > dropped_delimeters; 168 nfp::keyword<struct kept_delimeters_t > kept_delimeters; 169 nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens; 170 nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens; 171 } 172 173 // ************************************************************************** // 174 // ************** token_iterator_base ************** // 175 // ************************************************************************** // 176 177 template<typename Derived, 178 typename CharT, 179 typename CharCompare = ut_detail::default_char_compare<CharT>, 180 typename ValueType = basic_cstring<CharT const>, 181 typename Reference = basic_cstring<CharT const>, 182 typename Traversal = forward_traversal_tag> 183 class token_iterator_base 184 : public input_iterator_facade<Derived,ValueType,Reference,Traversal> { 185 typedef basic_cstring<CharT const> cstring; 186 typedef ut_detail::delim_policy<CharT,CharCompare> delim_policy; 187 typedef input_iterator_facade<Derived,ValueType,Reference,Traversal> base; 188 189 protected: 190 // Constructor 191 explicit token_iterator_base() 192 : m_is_dropped( dt_isspace ) 193 , m_is_kept( dt_ispunct ) 194 , m_keep_empty_tokens( false ) 195 , m_tokens_left( static_cast<std::size_t>(-1) ) 196 , m_token_produced( false ) 197 { 198 } 199 200 template<typename Modifier> 201 void 202 apply_modifier( Modifier const& m ) 203 { 204 if( m.has( dropped_delimeters ) ) 205 m_is_dropped.set_delimeters( m[dropped_delimeters] ); 206 207 if( m.has( kept_delimeters ) ) 208 m_is_kept.set_delimeters( m[kept_delimeters] ); 209 210 if( m.has( keep_empty_tokens ) ) 211 m_keep_empty_tokens = true; 212 213 nfp::optionally_assign( m_tokens_left, m, max_tokens ); 214 } 215 216 template<typename Iter> 217 bool get( Iter& begin, Iter end ) 218 { 219 typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner; 220 Iter check_point; 221 222 this->m_value.clear(); 223 224 if( !m_keep_empty_tokens ) { 225 while( begin != end && m_is_dropped( *begin ) ) 226 ++begin; 227 228 if( begin == end ) 229 return false; 230 231 check_point = begin; 232 233 if( m_tokens_left == 1 ) 234 while( begin != end ) 235 Assigner::append_move( begin, this->m_value ); 236 else if( m_is_kept( *begin ) ) 237 Assigner::append_move( begin, this->m_value ); 238 else 239 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) ) 240 Assigner::append_move( begin, this->m_value ); 241 242 --m_tokens_left; 243 } 244 else { // m_keep_empty_tokens is true 245 check_point = begin; 246 247 if( begin == end ) { 248 if( m_token_produced ) 249 return false; 250 251 m_token_produced = true; 252 } 253 if( m_is_kept( *begin ) ) { 254 if( m_token_produced ) 255 Assigner::append_move( begin, this->m_value ); 256 257 m_token_produced = !m_token_produced; 258 } 259 else if( !m_token_produced && m_is_dropped( *begin ) ) 260 m_token_produced = true; 261 else { 262 if( m_is_dropped( *begin ) ) 263 check_point = ++begin; 264 265 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) ) 266 Assigner::append_move( begin, this->m_value ); 267 268 m_token_produced = true; 269 } 270 } 271 272 Assigner::assign( check_point, begin, this->m_value ); 273 274 return true; 275 } 276 277 private: 278 // Data members 279 delim_policy m_is_dropped; 280 delim_policy m_is_kept; 281 bool m_keep_empty_tokens; 282 std::size_t m_tokens_left; 283 bool m_token_produced; 284 }; 285 286 // ************************************************************************** // 287 // ************** basic_string_token_iterator ************** // 288 // ************************************************************************** // 289 290 template<typename CharT, 291 typename CharCompare = ut_detail::default_char_compare<CharT> > 292 class basic_string_token_iterator 293 : public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> { 294 typedef basic_cstring<CharT const> cstring; 295 typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base; 296 public: 297 explicit basic_string_token_iterator() {} 298 explicit basic_string_token_iterator( cstring src ) 299 : m_src( src ) 300 { 301 this->init(); 302 } 303 304 template<typename Src, typename Modifier> 305 basic_string_token_iterator( Src src, Modifier const& m ) 306 : m_src( src ) 307 { 308 this->apply_modifier( m ); 309 310 this->init(); 311 } 312 313 private: 314 friend class input_iterator_core_access; 315 316 // input iterator implementation 317 bool get() 318 { 319 typename cstring::iterator begin = m_src.begin(); 320 bool res = base::get( begin, m_src.end() ); 321 322 m_src.assign( begin, m_src.end() ); 323 324 return res; 325 } 326 327 // Data members 328 cstring m_src; 329 }; 330 331 typedef basic_string_token_iterator<char> string_token_iterator; 332 typedef basic_string_token_iterator<wchar_t> wstring_token_iterator; 333 334 // ************************************************************************** // 335 // ************** range_token_iterator ************** // 336 // ************************************************************************** // 337 338 template<typename Iter, 339 typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>, 340 typename ValueType = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>, 341 typename Reference = ValueType const&> 342 class range_token_iterator 343 : public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>, 344 typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> { 345 typedef basic_cstring<typename ValueType::value_type> cstring; 346 typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>, 347 typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base; 348 public: 349 explicit range_token_iterator() {} 350 explicit range_token_iterator( Iter begin, Iter end = Iter() ) 351 : m_begin( begin ), m_end( end ) 352 { 353 this->init(); 354 } 355 range_token_iterator( range_token_iterator const& rhs ) 356 : base( rhs ) 357 { 358 if( this->m_valid ) { 359 m_begin = rhs.m_begin; 360 m_end = rhs.m_end; 361 } 362 } 363 364 template<typename Modifier> 365 range_token_iterator( Iter begin, Iter end, Modifier const& m ) 366 : m_begin( begin ), m_end( end ) 367 { 368 this->apply_modifier( m ); 369 370 this->init(); 371 } 372 373 private: 374 friend class input_iterator_core_access; 375 376 // input iterator implementation 377 bool get() 378 { 379 return base::get( m_begin, m_end ); 380 } 381 382 // Data members 383 Iter m_begin; 384 Iter m_end; 385 }; 386 387 // ************************************************************************** // 388 // ************** make_range_token_iterator ************** // 389 // ************************************************************************** // 390 391 template<typename Iter> 392 inline range_token_iterator<Iter> 393 make_range_token_iterator( Iter begin, Iter end = Iter() ) 394 { 395 return range_token_iterator<Iter>( begin, end ); 396 } 397 398 //____________________________________________________________________________// 399 400 template<typename Iter,typename Modifier> 401 inline range_token_iterator<Iter> 402 make_range_token_iterator( Iter begin, Iter end, Modifier const& m ) 403 { 404 return range_token_iterator<Iter>( begin, end, m ); 405 } 406 407 //____________________________________________________________________________// 408 409 } // namespace unit_test 410 411 } // namespace boost 412 413 //____________________________________________________________________________// 414 415 #include <boost/test/detail/enable_warnings.hpp> 416 417 #endif // BOOST_TOKEN_ITERATOR_HPP_071894GER 418 419
