1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // http://code.google.com/p/protobuf/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 // from google3/strings/strutil.cc 32 33 #include <google/protobuf/stubs/strutil.h> 34 #include <errno.h> 35 #include <float.h> // FLT_DIG and DBL_DIG 36 #include <limits> 37 #include <limits.h> 38 #include <stdio.h> 39 #include <iterator> 40 41 #ifdef _WIN32 42 // MSVC has only _snprintf, not snprintf. 43 // 44 // MinGW has both snprintf and _snprintf, but they appear to be different 45 // functions. The former is buggy. When invoked like so: 46 // char buffer[32]; 47 // snprintf(buffer, 32, "%.*g\n", FLT_DIG, 1.23e10f); 48 // it prints "1.23000e+10". This is plainly wrong: %g should never print 49 // trailing zeros after the decimal point. For some reason this bug only 50 // occurs with some input values, not all. In any case, _snprintf does the 51 // right thing, so we use it. 52 #define snprintf _snprintf 53 #endif 54 55 namespace google { 56 namespace protobuf { 57 58 inline bool IsNaN(double value) { 59 // NaN is never equal to anything, even itself. 60 return value != value; 61 } 62 63 // These are defined as macros on some platforms. #undef them so that we can 64 // redefine them. 65 #undef isxdigit 66 #undef isprint 67 68 // The definitions of these in ctype.h change based on locale. Since our 69 // string manipulation is all in relation to the protocol buffer and C++ 70 // languages, we always want to use the C locale. So, we re-define these 71 // exactly as we want them. 72 inline bool isxdigit(char c) { 73 return ('0' <= c && c <= '9') || 74 ('a' <= c && c <= 'f') || 75 ('A' <= c && c <= 'F'); 76 } 77 78 inline bool isprint(char c) { 79 return c >= 0x20 && c <= 0x7E; 80 } 81 82 // ---------------------------------------------------------------------- 83 // StripString 84 // Replaces any occurrence of the character 'remove' (or the characters 85 // in 'remove') with the character 'replacewith'. 86 // ---------------------------------------------------------------------- 87 void StripString(string* s, const char* remove, char replacewith) { 88 const char * str_start = s->c_str(); 89 const char * str = str_start; 90 for (str = strpbrk(str, remove); 91 str != NULL; 92 str = strpbrk(str + 1, remove)) { 93 (*s)[str - str_start] = replacewith; 94 } 95 } 96 97 // ---------------------------------------------------------------------- 98 // StringReplace() 99 // Replace the "old" pattern with the "new" pattern in a string, 100 // and append the result to "res". If replace_all is false, 101 // it only replaces the first instance of "old." 102 // ---------------------------------------------------------------------- 103 104 void StringReplace(const string& s, const string& oldsub, 105 const string& newsub, bool replace_all, 106 string* res) { 107 if (oldsub.empty()) { 108 res->append(s); // if empty, append the given string. 109 return; 110 } 111 112 string::size_type start_pos = 0; 113 string::size_type pos; 114 do { 115 pos = s.find(oldsub, start_pos); 116 if (pos == string::npos) { 117 break; 118 } 119 res->append(s, start_pos, pos - start_pos); 120 res->append(newsub); 121 start_pos = pos + oldsub.size(); // start searching again after the "old" 122 } while (replace_all); 123 res->append(s, start_pos, s.length() - start_pos); 124 } 125 126 // ---------------------------------------------------------------------- 127 // StringReplace() 128 // Give me a string and two patterns "old" and "new", and I replace 129 // the first instance of "old" in the string with "new", if it 130 // exists. If "global" is true; call this repeatedly until it 131 // fails. RETURN a new string, regardless of whether the replacement 132 // happened or not. 133 // ---------------------------------------------------------------------- 134 135 string StringReplace(const string& s, const string& oldsub, 136 const string& newsub, bool replace_all) { 137 string ret; 138 StringReplace(s, oldsub, newsub, replace_all, &ret); 139 return ret; 140 } 141 142 // ---------------------------------------------------------------------- 143 // SplitStringUsing() 144 // Split a string using a character delimiter. Append the components 145 // to 'result'. 146 // 147 // Note: For multi-character delimiters, this routine will split on *ANY* of 148 // the characters in the string, not the entire string as a single delimiter. 149 // ---------------------------------------------------------------------- 150 template <typename ITR> 151 static inline 152 void SplitStringToIteratorUsing(const string& full, 153 const char* delim, 154 ITR& result) { 155 // Optimize the common case where delim is a single character. 156 if (delim[0] != '\0' && delim[1] == '\0') { 157 char c = delim[0]; 158 const char* p = full.data(); 159 const char* end = p + full.size(); 160 while (p != end) { 161 if (*p == c) { 162 ++p; 163 } else { 164 const char* start = p; 165 while (++p != end && *p != c); 166 *result++ = string(start, p - start); 167 } 168 } 169 return; 170 } 171 172 string::size_type begin_index, end_index; 173 begin_index = full.find_first_not_of(delim); 174 while (begin_index != string::npos) { 175 end_index = full.find_first_of(delim, begin_index); 176 if (end_index == string::npos) { 177 *result++ = full.substr(begin_index); 178 return; 179 } 180 *result++ = full.substr(begin_index, (end_index - begin_index)); 181 begin_index = full.find_first_not_of(delim, end_index); 182 } 183 } 184 185 void SplitStringUsing(const string& full, 186 const char* delim, 187 vector<string>* result) { 188 back_insert_iterator< vector<string> > it(*result); 189 SplitStringToIteratorUsing(full, delim, it); 190 } 191 192 // ---------------------------------------------------------------------- 193 // JoinStrings() 194 // This merges a vector of string components with delim inserted 195 // as separaters between components. 196 // 197 // ---------------------------------------------------------------------- 198 template <class ITERATOR> 199 static void JoinStringsIterator(const ITERATOR& start, 200 const ITERATOR& end, 201 const char* delim, 202 string* result) { 203 GOOGLE_CHECK(result != NULL); 204 result->clear(); 205 int delim_length = strlen(delim); 206 207 // Precompute resulting length so we can reserve() memory in one shot. 208 int length = 0; 209 for (ITERATOR iter = start; iter != end; ++iter) { 210 if (iter != start) { 211 length += delim_length; 212 } 213 length += iter->size(); 214 } 215 result->reserve(length); 216 217 // Now combine everything. 218 for (ITERATOR iter = start; iter != end; ++iter) { 219 if (iter != start) { 220 result->append(delim, delim_length); 221 } 222 result->append(iter->data(), iter->size()); 223 } 224 } 225 226 void JoinStrings(const vector<string>& components, 227 const char* delim, 228 string * result) { 229 JoinStringsIterator(components.begin(), components.end(), delim, result); 230 } 231 232 // ---------------------------------------------------------------------- 233 // UnescapeCEscapeSequences() 234 // This does all the unescaping that C does: \ooo, \r, \n, etc 235 // Returns length of resulting string. 236 // The implementation of \x parses any positive number of hex digits, 237 // but it is an error if the value requires more than 8 bits, and the 238 // result is truncated to 8 bits. 239 // 240 // The second call stores its errors in a supplied string vector. 241 // If the string vector pointer is NULL, it reports the errors with LOG(). 242 // ---------------------------------------------------------------------- 243 244 #define IS_OCTAL_DIGIT(c) (((c) >= '0') && ((c) <= '7')) 245 246 inline int hex_digit_to_int(char c) { 247 /* Assume ASCII. */ 248 assert('0' == 0x30 && 'A' == 0x41 && 'a' == 0x61); 249 assert(isxdigit(c)); 250 int x = static_cast<unsigned char>(c); 251 if (x > '9') { 252 x += 9; 253 } 254 return x & 0xf; 255 } 256 257 // Protocol buffers doesn't ever care about errors, but I don't want to remove 258 // the code. 259 #define LOG_STRING(LEVEL, VECTOR) GOOGLE_LOG_IF(LEVEL, false) 260 261 int UnescapeCEscapeSequences(const char* source, char* dest) { 262 return UnescapeCEscapeSequences(source, dest, NULL); 263 } 264 265 int UnescapeCEscapeSequences(const char* source, char* dest, 266 vector<string> *errors) { 267 GOOGLE_DCHECK(errors == NULL) << "Error reporting not implemented."; 268 269 char* d = dest; 270 const char* p = source; 271 272 // Small optimization for case where source = dest and there's no escaping 273 while ( p == d && *p != '\0' && *p != '\\' ) 274 p++, d++; 275 276 while (*p != '\0') { 277 if (*p != '\\') { 278 *d++ = *p++; 279 } else { 280 switch ( *++p ) { // skip past the '\\' 281 case '\0': 282 LOG_STRING(ERROR, errors) << "String cannot end with \\"; 283 *d = '\0'; 284 return d - dest; // we're done with p 285 case 'a': *d++ = '\a'; break; 286 case 'b': *d++ = '\b'; break; 287 case 'f': *d++ = '\f'; break; 288 case 'n': *d++ = '\n'; break; 289 case 'r': *d++ = '\r'; break; 290 case 't': *d++ = '\t'; break; 291 case 'v': *d++ = '\v'; break; 292 case '\\': *d++ = '\\'; break; 293 case '?': *d++ = '\?'; break; // \? Who knew? 294 case '\'': *d++ = '\''; break; 295 case '"': *d++ = '\"'; break; 296 case '0': case '1': case '2': case '3': // octal digit: 1 to 3 digits 297 case '4': case '5': case '6': case '7': { 298 char ch = *p - '0'; 299 if ( IS_OCTAL_DIGIT(p[1]) ) 300 ch = ch * 8 + *++p - '0'; 301 if ( IS_OCTAL_DIGIT(p[1]) ) // safe (and easy) to do this twice 302 ch = ch * 8 + *++p - '0'; // now points at last digit 303 *d++ = ch; 304 break; 305 } 306 case 'x': case 'X': { 307 if (!isxdigit(p[1])) { 308 if (p[1] == '\0') { 309 LOG_STRING(ERROR, errors) << "String cannot end with \\x"; 310 } else { 311 LOG_STRING(ERROR, errors) << 312 "\\x cannot be followed by non-hex digit: \\" << *p << p[1]; 313 } 314 break; 315 } 316 unsigned int ch = 0; 317 const char *hex_start = p; 318 while (isxdigit(p[1])) // arbitrarily many hex digits 319 ch = (ch << 4) + hex_digit_to_int(*++p); 320 if (ch > 0xFF) 321 LOG_STRING(ERROR, errors) << "Value of " << 322 "\\" << string(hex_start, p+1-hex_start) << " exceeds 8 bits"; 323 *d++ = ch; 324 break; 325 } 326 #if 0 // TODO(kenton): Support \u and \U? Requires runetochar(). 327 case 'u': { 328 // \uhhhh => convert 4 hex digits to UTF-8 329 char32 rune = 0; 330 const char *hex_start = p; 331 for (int i = 0; i < 4; ++i) { 332 if (isxdigit(p[1])) { // Look one char ahead. 333 rune = (rune << 4) + hex_digit_to_int(*++p); // Advance p. 334 } else { 335 LOG_STRING(ERROR, errors) 336 << "\\u must be followed by 4 hex digits: \\" 337 << string(hex_start, p+1-hex_start); 338 break; 339 } 340 } 341 d += runetochar(d, &rune); 342 break; 343 } 344 case 'U': { 345 // \Uhhhhhhhh => convert 8 hex digits to UTF-8 346 char32 rune = 0; 347 const char *hex_start = p; 348 for (int i = 0; i < 8; ++i) { 349 if (isxdigit(p[1])) { // Look one char ahead. 350 // Don't change rune until we're sure this 351 // is within the Unicode limit, but do advance p. 352 char32 newrune = (rune << 4) + hex_digit_to_int(*++p); 353 if (newrune > 0x10FFFF) { 354 LOG_STRING(ERROR, errors) 355 << "Value of \\" 356 << string(hex_start, p + 1 - hex_start) 357 << " exceeds Unicode limit (0x10FFFF)"; 358 break; 359 } else { 360 rune = newrune; 361 } 362 } else { 363 LOG_STRING(ERROR, errors) 364 << "\\U must be followed by 8 hex digits: \\" 365 << string(hex_start, p+1-hex_start); 366 break; 367 } 368 } 369 d += runetochar(d, &rune); 370 break; 371 } 372 #endif 373 default: 374 LOG_STRING(ERROR, errors) << "Unknown escape sequence: \\" << *p; 375 } 376 p++; // read past letter we escaped 377 } 378 } 379 *d = '\0'; 380 return d - dest; 381 } 382 383 // ---------------------------------------------------------------------- 384 // UnescapeCEscapeString() 385 // This does the same thing as UnescapeCEscapeSequences, but creates 386 // a new string. The caller does not need to worry about allocating 387 // a dest buffer. This should be used for non performance critical 388 // tasks such as printing debug messages. It is safe for src and dest 389 // to be the same. 390 // 391 // The second call stores its errors in a supplied string vector. 392 // If the string vector pointer is NULL, it reports the errors with LOG(). 393 // 394 // In the first and second calls, the length of dest is returned. In the 395 // the third call, the new string is returned. 396 // ---------------------------------------------------------------------- 397 int UnescapeCEscapeString(const string& src, string* dest) { 398 return UnescapeCEscapeString(src, dest, NULL); 399 } 400 401 int UnescapeCEscapeString(const string& src, string* dest, 402 vector<string> *errors) { 403 scoped_array<char> unescaped(new char[src.size() + 1]); 404 int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), errors); 405 GOOGLE_CHECK(dest); 406 dest->assign(unescaped.get(), len); 407 return len; 408 } 409 410 string UnescapeCEscapeString(const string& src) { 411 scoped_array<char> unescaped(new char[src.size() + 1]); 412 int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), NULL); 413 return string(unescaped.get(), len); 414 } 415 416 // ---------------------------------------------------------------------- 417 // CEscapeString() 418 // CHexEscapeString() 419 // Copies 'src' to 'dest', escaping dangerous characters using 420 // C-style escape sequences. This is very useful for preparing query 421 // flags. 'src' and 'dest' should not overlap. The 'Hex' version uses 422 // hexadecimal rather than octal sequences. 423 // Returns the number of bytes written to 'dest' (not including the \0) 424 // or -1 if there was insufficient space. 425 // 426 // Currently only \n, \r, \t, ", ', \ and !isprint() chars are escaped. 427 // ---------------------------------------------------------------------- 428 int CEscapeInternal(const char* src, int src_len, char* dest, 429 int dest_len, bool use_hex, bool utf8_safe) { 430 const char* src_end = src + src_len; 431 int used = 0; 432 bool last_hex_escape = false; // true if last output char was \xNN 433 434 for (; src < src_end; src++) { 435 if (dest_len - used < 2) // Need space for two letter escape 436 return -1; 437 438 bool is_hex_escape = false; 439 switch (*src) { 440 case '\n': dest[used++] = '\\'; dest[used++] = 'n'; break; 441 case '\r': dest[used++] = '\\'; dest[used++] = 'r'; break; 442 case '\t': dest[used++] = '\\'; dest[used++] = 't'; break; 443 case '\"': dest[used++] = '\\'; dest[used++] = '\"'; break; 444 case '\'': dest[used++] = '\\'; dest[used++] = '\''; break; 445 case '\\': dest[used++] = '\\'; dest[used++] = '\\'; break; 446 default: 447 // Note that if we emit \xNN and the src character after that is a hex 448 // digit then that digit must be escaped too to prevent it being 449 // interpreted as part of the character code by C. 450 if ((!utf8_safe || static_cast<uint8>(*src) < 0x80) && 451 (!isprint(*src) || 452 (last_hex_escape && isxdigit(*src)))) { 453 if (dest_len - used < 4) // need space for 4 letter escape 454 return -1; 455 sprintf(dest + used, (use_hex ? "\\x%02x" : "\\%03o"), 456 static_cast<uint8>(*src)); 457 is_hex_escape = use_hex; 458 used += 4; 459 } else { 460 dest[used++] = *src; break; 461 } 462 } 463 last_hex_escape = is_hex_escape; 464 } 465 466 if (dest_len - used < 1) // make sure that there is room for \0 467 return -1; 468 469 dest[used] = '\0'; // doesn't count towards return value though 470 return used; 471 } 472 473 int CEscapeString(const char* src, int src_len, char* dest, int dest_len) { 474 return CEscapeInternal(src, src_len, dest, dest_len, false, false); 475 } 476 477 // ---------------------------------------------------------------------- 478 // CEscape() 479 // CHexEscape() 480 // Copies 'src' to result, escaping dangerous characters using 481 // C-style escape sequences. This is very useful for preparing query 482 // flags. 'src' and 'dest' should not overlap. The 'Hex' version 483 // hexadecimal rather than octal sequences. 484 // 485 // Currently only \n, \r, \t, ", ', \ and !isprint() chars are escaped. 486 // ---------------------------------------------------------------------- 487 string CEscape(const string& src) { 488 const int dest_length = src.size() * 4 + 1; // Maximum possible expansion 489 scoped_array<char> dest(new char[dest_length]); 490 const int len = CEscapeInternal(src.data(), src.size(), 491 dest.get(), dest_length, false, false); 492 GOOGLE_DCHECK_GE(len, 0); 493 return string(dest.get(), len); 494 } 495 496 namespace strings { 497 498 string Utf8SafeCEscape(const string& src) { 499 const int dest_length = src.size() * 4 + 1; // Maximum possible expansion 500 scoped_array<char> dest(new char[dest_length]); 501 const int len = CEscapeInternal(src.data(), src.size(), 502 dest.get(), dest_length, false, true); 503 GOOGLE_DCHECK_GE(len, 0); 504 return string(dest.get(), len); 505 } 506 507 string CHexEscape(const string& src) { 508 const int dest_length = src.size() * 4 + 1; // Maximum possible expansion 509 scoped_array<char> dest(new char[dest_length]); 510 const int len = CEscapeInternal(src.data(), src.size(), 511 dest.get(), dest_length, true, false); 512 GOOGLE_DCHECK_GE(len, 0); 513 return string(dest.get(), len); 514 } 515 516 } // namespace strings 517 518 // ---------------------------------------------------------------------- 519 // strto32_adaptor() 520 // strtou32_adaptor() 521 // Implementation of strto[u]l replacements that have identical 522 // overflow and underflow characteristics for both ILP-32 and LP-64 523 // platforms, including errno preservation in error-free calls. 524 // ---------------------------------------------------------------------- 525 526 int32 strto32_adaptor(const char *nptr, char **endptr, int base) { 527 const int saved_errno = errno; 528 errno = 0; 529 const long result = strtol(nptr, endptr, base); 530 if (errno == ERANGE && result == LONG_MIN) { 531 return kint32min; 532 } else if (errno == ERANGE && result == LONG_MAX) { 533 return kint32max; 534 } else if (errno == 0 && result < kint32min) { 535 errno = ERANGE; 536 return kint32min; 537 } else if (errno == 0 && result > kint32max) { 538 errno = ERANGE; 539 return kint32max; 540 } 541 if (errno == 0) 542 errno = saved_errno; 543 return static_cast<int32>(result); 544 } 545 546 uint32 strtou32_adaptor(const char *nptr, char **endptr, int base) { 547 const int saved_errno = errno; 548 errno = 0; 549 const unsigned long result = strtoul(nptr, endptr, base); 550 if (errno == ERANGE && result == ULONG_MAX) { 551 return kuint32max; 552 } else if (errno == 0 && result > kuint32max) { 553 errno = ERANGE; 554 return kuint32max; 555 } 556 if (errno == 0) 557 errno = saved_errno; 558 return static_cast<uint32>(result); 559 } 560 561 // ---------------------------------------------------------------------- 562 // FastIntToBuffer() 563 // FastInt64ToBuffer() 564 // FastHexToBuffer() 565 // FastHex64ToBuffer() 566 // FastHex32ToBuffer() 567 // ---------------------------------------------------------------------- 568 569 // Offset into buffer where FastInt64ToBuffer places the end of string 570 // null character. Also used by FastInt64ToBufferLeft. 571 static const int kFastInt64ToBufferOffset = 21; 572 573 char *FastInt64ToBuffer(int64 i, char* buffer) { 574 // We could collapse the positive and negative sections, but that 575 // would be slightly slower for positive numbers... 576 // 22 bytes is enough to store -2**64, -18446744073709551616. 577 char* p = buffer + kFastInt64ToBufferOffset; 578 *p-- = '\0'; 579 if (i >= 0) { 580 do { 581 *p-- = '0' + i % 10; 582 i /= 10; 583 } while (i > 0); 584 return p + 1; 585 } else { 586 // On different platforms, % and / have different behaviors for 587 // negative numbers, so we need to jump through hoops to make sure 588 // we don't divide negative numbers. 589 if (i > -10) { 590 i = -i; 591 *p-- = '0' + i; 592 *p = '-'; 593 return p; 594 } else { 595 // Make sure we aren't at MIN_INT, in which case we can't say i = -i 596 i = i + 10; 597 i = -i; 598 *p-- = '0' + i % 10; 599 // Undo what we did a moment ago 600 i = i / 10 + 1; 601 do { 602 *p-- = '0' + i % 10; 603 i /= 10; 604 } while (i > 0); 605 *p = '-'; 606 return p; 607 } 608 } 609 } 610 611 // Offset into buffer where FastInt32ToBuffer places the end of string 612 // null character. Also used by FastInt32ToBufferLeft 613 static const int kFastInt32ToBufferOffset = 11; 614 615 // Yes, this is a duplicate of FastInt64ToBuffer. But, we need this for the 616 // compiler to generate 32 bit arithmetic instructions. It's much faster, at 617 // least with 32 bit binaries. 618 char *FastInt32ToBuffer(int32 i, char* buffer) { 619 // We could collapse the positive and negative sections, but that 620 // would be slightly slower for positive numbers... 621 // 12 bytes is enough to store -2**32, -4294967296. 622 char* p = buffer + kFastInt32ToBufferOffset; 623 *p-- = '\0'; 624 if (i >= 0) { 625 do { 626 *p-- = '0' + i % 10; 627 i /= 10; 628 } while (i > 0); 629 return p + 1; 630 } else { 631 // On different platforms, % and / have different behaviors for 632 // negative numbers, so we need to jump through hoops to make sure 633 // we don't divide negative numbers. 634 if (i > -10) { 635 i = -i; 636 *p-- = '0' + i; 637 *p = '-'; 638 return p; 639 } else { 640 // Make sure we aren't at MIN_INT, in which case we can't say i = -i 641 i = i + 10; 642 i = -i; 643 *p-- = '0' + i % 10; 644 // Undo what we did a moment ago 645 i = i / 10 + 1; 646 do { 647 *p-- = '0' + i % 10; 648 i /= 10; 649 } while (i > 0); 650 *p = '-'; 651 return p; 652 } 653 } 654 } 655 656 char *FastHexToBuffer(int i, char* buffer) { 657 GOOGLE_CHECK(i >= 0) << "FastHexToBuffer() wants non-negative integers, not " << i; 658 659 static const char *hexdigits = "0123456789abcdef"; 660 char *p = buffer + 21; 661 *p-- = '\0'; 662 do { 663 *p-- = hexdigits[i & 15]; // mod by 16 664 i >>= 4; // divide by 16 665 } while (i > 0); 666 return p + 1; 667 } 668 669 char *InternalFastHexToBuffer(uint64 value, char* buffer, int num_byte) { 670 static const char *hexdigits = "0123456789abcdef"; 671 buffer[num_byte] = '\0'; 672 for (int i = num_byte - 1; i >= 0; i--) { 673 buffer[i] = hexdigits[uint32(value) & 0xf]; 674 value >>= 4; 675 } 676 return buffer; 677 } 678 679 char *FastHex64ToBuffer(uint64 value, char* buffer) { 680 return InternalFastHexToBuffer(value, buffer, 16); 681 } 682 683 char *FastHex32ToBuffer(uint32 value, char* buffer) { 684 return InternalFastHexToBuffer(value, buffer, 8); 685 } 686 687 static inline char* PlaceNum(char* p, int num, char prev_sep) { 688 *p-- = '0' + num % 10; 689 *p-- = '0' + num / 10; 690 *p-- = prev_sep; 691 return p; 692 } 693 694 // ---------------------------------------------------------------------- 695 // FastInt32ToBufferLeft() 696 // FastUInt32ToBufferLeft() 697 // FastInt64ToBufferLeft() 698 // FastUInt64ToBufferLeft() 699 // 700 // Like the Fast*ToBuffer() functions above, these are intended for speed. 701 // Unlike the Fast*ToBuffer() functions, however, these functions write 702 // their output to the beginning of the buffer (hence the name, as the 703 // output is left-aligned). The caller is responsible for ensuring that 704 // the buffer has enough space to hold the output. 705 // 706 // Returns a pointer to the end of the string (i.e. the null character 707 // terminating the string). 708 // ---------------------------------------------------------------------- 709 710 static const char two_ASCII_digits[100][2] = { 711 {'0','0'}, {'0','1'}, {'0','2'}, {'0','3'}, {'0','4'}, 712 {'0','5'}, {'0','6'}, {'0','7'}, {'0','8'}, {'0','9'}, 713 {'1','0'}, {'1','1'}, {'1','2'}, {'1','3'}, {'1','4'}, 714 {'1','5'}, {'1','6'}, {'1','7'}, {'1','8'}, {'1','9'}, 715 {'2','0'}, {'2','1'}, {'2','2'}, {'2','3'}, {'2','4'}, 716 {'2','5'}, {'2','6'}, {'2','7'}, {'2','8'}, {'2','9'}, 717 {'3','0'}, {'3','1'}, {'3','2'}, {'3','3'}, {'3','4'}, 718 {'3','5'}, {'3','6'}, {'3','7'}, {'3','8'}, {'3','9'}, 719 {'4','0'}, {'4','1'}, {'4','2'}, {'4','3'}, {'4','4'}, 720 {'4','5'}, {'4','6'}, {'4','7'}, {'4','8'}, {'4','9'}, 721 {'5','0'}, {'5','1'}, {'5','2'}, {'5','3'}, {'5','4'}, 722 {'5','5'}, {'5','6'}, {'5','7'}, {'5','8'}, {'5','9'}, 723 {'6','0'}, {'6','1'}, {'6','2'}, {'6','3'}, {'6','4'}, 724 {'6','5'}, {'6','6'}, {'6','7'}, {'6','8'}, {'6','9'}, 725 {'7','0'}, {'7','1'}, {'7','2'}, {'7','3'}, {'7','4'}, 726 {'7','5'}, {'7','6'}, {'7','7'}, {'7','8'}, {'7','9'}, 727 {'8','0'}, {'8','1'}, {'8','2'}, {'8','3'}, {'8','4'}, 728 {'8','5'}, {'8','6'}, {'8','7'}, {'8','8'}, {'8','9'}, 729 {'9','0'}, {'9','1'}, {'9','2'}, {'9','3'}, {'9','4'}, 730 {'9','5'}, {'9','6'}, {'9','7'}, {'9','8'}, {'9','9'} 731 }; 732 733 char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { 734 int digits; 735 const char *ASCII_digits = NULL; 736 // The idea of this implementation is to trim the number of divides to as few 737 // as possible by using multiplication and subtraction rather than mod (%), 738 // and by outputting two digits at a time rather than one. 739 // The huge-number case is first, in the hopes that the compiler will output 740 // that case in one branch-free block of code, and only output conditional 741 // branches into it from below. 742 if (u >= 1000000000) { // >= 1,000,000,000 743 digits = u / 100000000; // 100,000,000 744 ASCII_digits = two_ASCII_digits[digits]; 745 buffer[0] = ASCII_digits[0]; 746 buffer[1] = ASCII_digits[1]; 747 buffer += 2; 748 sublt100_000_000: 749 u -= digits * 100000000; // 100,000,000 750 lt100_000_000: 751 digits = u / 1000000; // 1,000,000 752 ASCII_digits = two_ASCII_digits[digits]; 753 buffer[0] = ASCII_digits[0]; 754 buffer[1] = ASCII_digits[1]; 755 buffer += 2; 756 sublt1_000_000: 757 u -= digits * 1000000; // 1,000,000 758 lt1_000_000: 759 digits = u / 10000; // 10,000 760 ASCII_digits = two_ASCII_digits[digits]; 761 buffer[0] = ASCII_digits[0]; 762 buffer[1] = ASCII_digits[1]; 763 buffer += 2; 764 sublt10_000: 765 u -= digits * 10000; // 10,000 766 lt10_000: 767 digits = u / 100; 768 ASCII_digits = two_ASCII_digits[digits]; 769 buffer[0] = ASCII_digits[0]; 770 buffer[1] = ASCII_digits[1]; 771 buffer += 2; 772 sublt100: 773 u -= digits * 100; 774 lt100: 775 digits = u; 776 ASCII_digits = two_ASCII_digits[digits]; 777 buffer[0] = ASCII_digits[0]; 778 buffer[1] = ASCII_digits[1]; 779 buffer += 2; 780 done: 781 *buffer = 0; 782 return buffer; 783 } 784 785 if (u < 100) { 786 digits = u; 787 if (u >= 10) goto lt100; 788 *buffer++ = '0' + digits; 789 goto done; 790 } 791 if (u < 10000) { // 10,000 792 if (u >= 1000) goto lt10_000; 793 digits = u / 100; 794 *buffer++ = '0' + digits; 795 goto sublt100; 796 } 797 if (u < 1000000) { // 1,000,000 798 if (u >= 100000) goto lt1_000_000; 799 digits = u / 10000; // 10,000 800 *buffer++ = '0' + digits; 801 goto sublt10_000; 802 } 803 if (u < 100000000) { // 100,000,000 804 if (u >= 10000000) goto lt100_000_000; 805 digits = u / 1000000; // 1,000,000 806 *buffer++ = '0' + digits; 807 goto sublt1_000_000; 808 } 809 // we already know that u < 1,000,000,000 810 digits = u / 100000000; // 100,000,000 811 *buffer++ = '0' + digits; 812 goto sublt100_000_000; 813 } 814 815 char* FastInt32ToBufferLeft(int32 i, char* buffer) { 816 uint32 u = i; 817 if (i < 0) { 818 *buffer++ = '-'; 819 u = -i; 820 } 821 return FastUInt32ToBufferLeft(u, buffer); 822 } 823 824 char* FastUInt64ToBufferLeft(uint64 u64, char* buffer) { 825 int digits; 826 const char *ASCII_digits = NULL; 827 828 uint32 u = static_cast<uint32>(u64); 829 if (u == u64) return FastUInt32ToBufferLeft(u, buffer); 830 831 uint64 top_11_digits = u64 / 1000000000; 832 buffer = FastUInt64ToBufferLeft(top_11_digits, buffer); 833 u = u64 - (top_11_digits * 1000000000); 834 835 digits = u / 10000000; // 10,000,000 836 GOOGLE_DCHECK_LT(digits, 100); 837 ASCII_digits = two_ASCII_digits[digits]; 838 buffer[0] = ASCII_digits[0]; 839 buffer[1] = ASCII_digits[1]; 840 buffer += 2; 841 u -= digits * 10000000; // 10,000,000 842 digits = u / 100000; // 100,000 843 ASCII_digits = two_ASCII_digits[digits]; 844 buffer[0] = ASCII_digits[0]; 845 buffer[1] = ASCII_digits[1]; 846 buffer += 2; 847 u -= digits * 100000; // 100,000 848 digits = u / 1000; // 1,000 849 ASCII_digits = two_ASCII_digits[digits]; 850 buffer[0] = ASCII_digits[0]; 851 buffer[1] = ASCII_digits[1]; 852 buffer += 2; 853 u -= digits * 1000; // 1,000 854 digits = u / 10; 855 ASCII_digits = two_ASCII_digits[digits]; 856 buffer[0] = ASCII_digits[0]; 857 buffer[1] = ASCII_digits[1]; 858 buffer += 2; 859 u -= digits * 10; 860 digits = u; 861 *buffer++ = '0' + digits; 862 *buffer = 0; 863 return buffer; 864 } 865 866 char* FastInt64ToBufferLeft(int64 i, char* buffer) { 867 uint64 u = i; 868 if (i < 0) { 869 *buffer++ = '-'; 870 u = -i; 871 } 872 return FastUInt64ToBufferLeft(u, buffer); 873 } 874 875 // ---------------------------------------------------------------------- 876 // SimpleItoa() 877 // Description: converts an integer to a string. 878 // 879 // Return value: string 880 // ---------------------------------------------------------------------- 881 882 string SimpleItoa(int i) { 883 char buffer[kFastToBufferSize]; 884 return (sizeof(i) == 4) ? 885 FastInt32ToBuffer(i, buffer) : 886 FastInt64ToBuffer(i, buffer); 887 } 888 889 string SimpleItoa(unsigned int i) { 890 char buffer[kFastToBufferSize]; 891 return string(buffer, (sizeof(i) == 4) ? 892 FastUInt32ToBufferLeft(i, buffer) : 893 FastUInt64ToBufferLeft(i, buffer)); 894 } 895 896 string SimpleItoa(long i) { 897 char buffer[kFastToBufferSize]; 898 return (sizeof(i) == 4) ? 899 FastInt32ToBuffer(i, buffer) : 900 FastInt64ToBuffer(i, buffer); 901 } 902 903 string SimpleItoa(unsigned long i) { 904 char buffer[kFastToBufferSize]; 905 return string(buffer, (sizeof(i) == 4) ? 906 FastUInt32ToBufferLeft(i, buffer) : 907 FastUInt64ToBufferLeft(i, buffer)); 908 } 909 910 string SimpleItoa(long long i) { 911 char buffer[kFastToBufferSize]; 912 return (sizeof(i) == 4) ? 913 FastInt32ToBuffer(i, buffer) : 914 FastInt64ToBuffer(i, buffer); 915 } 916 917 string SimpleItoa(unsigned long long i) { 918 char buffer[kFastToBufferSize]; 919 return string(buffer, (sizeof(i) == 4) ? 920 FastUInt32ToBufferLeft(i, buffer) : 921 FastUInt64ToBufferLeft(i, buffer)); 922 } 923 924 // ---------------------------------------------------------------------- 925 // SimpleDtoa() 926 // SimpleFtoa() 927 // DoubleToBuffer() 928 // FloatToBuffer() 929 // We want to print the value without losing precision, but we also do 930 // not want to print more digits than necessary. This turns out to be 931 // trickier than it sounds. Numbers like 0.2 cannot be represented 932 // exactly in binary. If we print 0.2 with a very large precision, 933 // e.g. "%.50g", we get "0.2000000000000000111022302462515654042363167". 934 // On the other hand, if we set the precision too low, we lose 935 // significant digits when printing numbers that actually need them. 936 // It turns out there is no precision value that does the right thing 937 // for all numbers. 938 // 939 // Our strategy is to first try printing with a precision that is never 940 // over-precise, then parse the result with strtod() to see if it 941 // matches. If not, we print again with a precision that will always 942 // give a precise result, but may use more digits than necessary. 943 // 944 // An arguably better strategy would be to use the algorithm described 945 // in "How to Print Floating-Point Numbers Accurately" by Steele & 946 // White, e.g. as implemented by David M. Gay's dtoa(). It turns out, 947 // however, that the following implementation is about as fast as 948 // DMG's code. Furthermore, DMG's code locks mutexes, which means it 949 // will not scale well on multi-core machines. DMG's code is slightly 950 // more accurate (in that it will never use more digits than 951 // necessary), but this is probably irrelevant for most users. 952 // 953 // Rob Pike and Ken Thompson also have an implementation of dtoa() in 954 // third_party/fmt/fltfmt.cc. Their implementation is similar to this 955 // one in that it makes guesses and then uses strtod() to check them. 956 // Their implementation is faster because they use their own code to 957 // generate the digits in the first place rather than use snprintf(), 958 // thus avoiding format string parsing overhead. However, this makes 959 // it considerably more complicated than the following implementation, 960 // and it is embedded in a larger library. If speed turns out to be 961 // an issue, we could re-implement this in terms of their 962 // implementation. 963 // ---------------------------------------------------------------------- 964 965 string SimpleDtoa(double value) { 966 char buffer[kDoubleToBufferSize]; 967 return DoubleToBuffer(value, buffer); 968 } 969 970 string SimpleFtoa(float value) { 971 char buffer[kFloatToBufferSize]; 972 return FloatToBuffer(value, buffer); 973 } 974 975 static inline bool IsValidFloatChar(char c) { 976 return ('0' <= c && c <= '9') || 977 c == 'e' || c == 'E' || 978 c == '+' || c == '-'; 979 } 980 981 void DelocalizeRadix(char* buffer) { 982 // Fast check: if the buffer has a normal decimal point, assume no 983 // translation is needed. 984 if (strchr(buffer, '.') != NULL) return; 985 986 // Find the first unknown character. 987 while (IsValidFloatChar(*buffer)) ++buffer; 988 989 if (*buffer == '\0') { 990 // No radix character found. 991 return; 992 } 993 994 // We are now pointing at the locale-specific radix character. Replace it 995 // with '.'. 996 *buffer = '.'; 997 ++buffer; 998 999 if (!IsValidFloatChar(*buffer) && *buffer != '\0') { 1000 // It appears the radix was a multi-byte character. We need to remove the 1001 // extra bytes. 1002 char* target = buffer; 1003 do { ++buffer; } while (!IsValidFloatChar(*buffer) && *buffer != '\0'); 1004 memmove(target, buffer, strlen(buffer) + 1); 1005 } 1006 } 1007 1008 char* DoubleToBuffer(double value, char* buffer) { 1009 // DBL_DIG is 15 for IEEE-754 doubles, which are used on almost all 1010 // platforms these days. Just in case some system exists where DBL_DIG 1011 // is significantly larger -- and risks overflowing our buffer -- we have 1012 // this assert. 1013 GOOGLE_COMPILE_ASSERT(DBL_DIG < 20, DBL_DIG_is_too_big); 1014 1015 if (value == numeric_limits<double>::infinity()) { 1016 strcpy(buffer, "inf"); 1017 return buffer; 1018 } else if (value == -numeric_limits<double>::infinity()) { 1019 strcpy(buffer, "-inf"); 1020 return buffer; 1021 } else if (IsNaN(value)) { 1022 strcpy(buffer, "nan"); 1023 return buffer; 1024 } 1025 1026 int snprintf_result = 1027 snprintf(buffer, kDoubleToBufferSize, "%.*g", DBL_DIG, value); 1028 1029 // The snprintf should never overflow because the buffer is significantly 1030 // larger than the precision we asked for. 1031 GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kDoubleToBufferSize); 1032 1033 // We need to make parsed_value volatile in order to force the compiler to 1034 // write it out to the stack. Otherwise, it may keep the value in a 1035 // register, and if it does that, it may keep it as a long double instead 1036 // of a double. This long double may have extra bits that make it compare 1037 // unequal to "value" even though it would be exactly equal if it were 1038 // truncated to a double. 1039 volatile double parsed_value = strtod(buffer, NULL); 1040 if (parsed_value != value) { 1041 int snprintf_result = 1042 snprintf(buffer, kDoubleToBufferSize, "%.*g", DBL_DIG+2, value); 1043 1044 // Should never overflow; see above. 1045 GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kDoubleToBufferSize); 1046 } 1047 1048 DelocalizeRadix(buffer); 1049 return buffer; 1050 } 1051 1052 bool safe_strtof(const char* str, float* value) { 1053 char* endptr; 1054 errno = 0; // errno only gets set on errors 1055 #if defined(_WIN32) || defined (__hpux) // has no strtof() 1056 *value = strtod(str, &endptr); 1057 #else 1058 *value = strtof(str, &endptr); 1059 #endif 1060 return *str != 0 && *endptr == 0 && errno == 0; 1061 } 1062 1063 char* FloatToBuffer(float value, char* buffer) { 1064 // FLT_DIG is 6 for IEEE-754 floats, which are used on almost all 1065 // platforms these days. Just in case some system exists where FLT_DIG 1066 // is significantly larger -- and risks overflowing our buffer -- we have 1067 // this assert. 1068 GOOGLE_COMPILE_ASSERT(FLT_DIG < 10, FLT_DIG_is_too_big); 1069 1070 if (value == numeric_limits<double>::infinity()) { 1071 strcpy(buffer, "inf"); 1072 return buffer; 1073 } else if (value == -numeric_limits<double>::infinity()) { 1074 strcpy(buffer, "-inf"); 1075 return buffer; 1076 } else if (IsNaN(value)) { 1077 strcpy(buffer, "nan"); 1078 return buffer; 1079 } 1080 1081 int snprintf_result = 1082 snprintf(buffer, kFloatToBufferSize, "%.*g", FLT_DIG, value); 1083 1084 // The snprintf should never overflow because the buffer is significantly 1085 // larger than the precision we asked for. 1086 GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kFloatToBufferSize); 1087 1088 float parsed_value; 1089 if (!safe_strtof(buffer, &parsed_value) || parsed_value != value) { 1090 int snprintf_result = 1091 snprintf(buffer, kFloatToBufferSize, "%.*g", FLT_DIG+2, value); 1092 1093 // Should never overflow; see above. 1094 GOOGLE_DCHECK(snprintf_result > 0 && snprintf_result < kFloatToBufferSize); 1095 } 1096 1097 DelocalizeRadix(buffer); 1098 return buffer; 1099 } 1100 1101 // ---------------------------------------------------------------------- 1102 // NoLocaleStrtod() 1103 // This code will make you cry. 1104 // ---------------------------------------------------------------------- 1105 1106 // Returns a string identical to *input except that the character pointed to 1107 // by radix_pos (which should be '.') is replaced with the locale-specific 1108 // radix character. 1109 string LocalizeRadix(const char* input, const char* radix_pos) { 1110 // Determine the locale-specific radix character by calling sprintf() to 1111 // print the number 1.5, then stripping off the digits. As far as I can 1112 // tell, this is the only portable, thread-safe way to get the C library 1113 // to divuldge the locale's radix character. No, localeconv() is NOT 1114 // thread-safe. 1115 char temp[16]; 1116 int size = sprintf(temp, "%.1f", 1.5); 1117 GOOGLE_CHECK_EQ(temp[0], '1'); 1118 GOOGLE_CHECK_EQ(temp[size-1], '5'); 1119 GOOGLE_CHECK_LE(size, 6); 1120 1121 // Now replace the '.' in the input with it. 1122 string result; 1123 result.reserve(strlen(input) + size - 3); 1124 result.append(input, radix_pos); 1125 result.append(temp + 1, size - 2); 1126 result.append(radix_pos + 1); 1127 return result; 1128 } 1129 1130 double NoLocaleStrtod(const char* text, char** original_endptr) { 1131 // We cannot simply set the locale to "C" temporarily with setlocale() 1132 // as this is not thread-safe. Instead, we try to parse in the current 1133 // locale first. If parsing stops at a '.' character, then this is a 1134 // pretty good hint that we're actually in some other locale in which 1135 // '.' is not the radix character. 1136 1137 char* temp_endptr; 1138 double result = strtod(text, &temp_endptr); 1139 if (original_endptr != NULL) *original_endptr = temp_endptr; 1140 if (*temp_endptr != '.') return result; 1141 1142 // Parsing halted on a '.'. Perhaps we're in a different locale? Let's 1143 // try to replace the '.' with a locale-specific radix character and 1144 // try again. 1145 string localized = LocalizeRadix(text, temp_endptr); 1146 const char* localized_cstr = localized.c_str(); 1147 char* localized_endptr; 1148 result = strtod(localized_cstr, &localized_endptr); 1149 if ((localized_endptr - localized_cstr) > 1150 (temp_endptr - text)) { 1151 // This attempt got further, so replacing the decimal must have helped. 1152 // Update original_endptr to point at the right location. 1153 if (original_endptr != NULL) { 1154 // size_diff is non-zero if the localized radix has multiple bytes. 1155 int size_diff = localized.size() - strlen(text); 1156 // const_cast is necessary to match the strtod() interface. 1157 *original_endptr = const_cast<char*>( 1158 text + (localized_endptr - localized_cstr - size_diff)); 1159 } 1160 } 1161 1162 return result; 1163 } 1164 1165 } // namespace protobuf 1166 } // namespace google 1167