1 //===--- LiteralSupport.cpp - Code to parse and process literals ----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the NumericLiteralParser, CharLiteralParser, and 11 // StringLiteralParser interfaces. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "clang/Lex/LiteralSupport.h" 16 #include "clang/Basic/CharInfo.h" 17 #include "clang/Basic/TargetInfo.h" 18 #include "clang/Lex/LexDiagnostic.h" 19 #include "clang/Lex/Preprocessor.h" 20 #include "llvm/ADT/StringExtras.h" 21 #include "llvm/Support/ConvertUTF.h" 22 #include "llvm/Support/ErrorHandling.h" 23 24 using namespace clang; 25 26 static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) { 27 switch (kind) { 28 default: llvm_unreachable("Unknown token type!"); 29 case tok::char_constant: 30 case tok::string_literal: 31 case tok::utf8_string_literal: 32 return Target.getCharWidth(); 33 case tok::wide_char_constant: 34 case tok::wide_string_literal: 35 return Target.getWCharWidth(); 36 case tok::utf16_char_constant: 37 case tok::utf16_string_literal: 38 return Target.getChar16Width(); 39 case tok::utf32_char_constant: 40 case tok::utf32_string_literal: 41 return Target.getChar32Width(); 42 } 43 } 44 45 static CharSourceRange MakeCharSourceRange(const LangOptions &Features, 46 FullSourceLoc TokLoc, 47 const char *TokBegin, 48 const char *TokRangeBegin, 49 const char *TokRangeEnd) { 50 SourceLocation Begin = 51 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 52 TokLoc.getManager(), Features); 53 SourceLocation End = 54 Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin, 55 TokLoc.getManager(), Features); 56 return CharSourceRange::getCharRange(Begin, End); 57 } 58 59 /// \brief Produce a diagnostic highlighting some portion of a literal. 60 /// 61 /// Emits the diagnostic \p DiagID, highlighting the range of characters from 62 /// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be 63 /// a substring of a spelling buffer for the token beginning at \p TokBegin. 64 static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, 65 const LangOptions &Features, FullSourceLoc TokLoc, 66 const char *TokBegin, const char *TokRangeBegin, 67 const char *TokRangeEnd, unsigned DiagID) { 68 SourceLocation Begin = 69 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 70 TokLoc.getManager(), Features); 71 return Diags->Report(Begin, DiagID) << 72 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd); 73 } 74 75 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in 76 /// either a character or a string literal. 77 static unsigned ProcessCharEscape(const char *ThisTokBegin, 78 const char *&ThisTokBuf, 79 const char *ThisTokEnd, bool &HadError, 80 FullSourceLoc Loc, unsigned CharWidth, 81 DiagnosticsEngine *Diags, 82 const LangOptions &Features) { 83 const char *EscapeBegin = ThisTokBuf; 84 85 // Skip the '\' char. 86 ++ThisTokBuf; 87 88 // We know that this character can't be off the end of the buffer, because 89 // that would have been \", which would not have been the end of string. 90 unsigned ResultChar = *ThisTokBuf++; 91 switch (ResultChar) { 92 // These map to themselves. 93 case '\\': case '\'': case '"': case '?': break; 94 95 // These have fixed mappings. 96 case 'a': 97 // TODO: K&R: the meaning of '\\a' is different in traditional C 98 ResultChar = 7; 99 break; 100 case 'b': 101 ResultChar = 8; 102 break; 103 case 'e': 104 if (Diags) 105 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 106 diag::ext_nonstandard_escape) << "e"; 107 ResultChar = 27; 108 break; 109 case 'E': 110 if (Diags) 111 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 112 diag::ext_nonstandard_escape) << "E"; 113 ResultChar = 27; 114 break; 115 case 'f': 116 ResultChar = 12; 117 break; 118 case 'n': 119 ResultChar = 10; 120 break; 121 case 'r': 122 ResultChar = 13; 123 break; 124 case 't': 125 ResultChar = 9; 126 break; 127 case 'v': 128 ResultChar = 11; 129 break; 130 case 'x': { // Hex escape. 131 ResultChar = 0; 132 if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 133 if (Diags) 134 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 135 diag::err_hex_escape_no_digits) << "x"; 136 HadError = 1; 137 break; 138 } 139 140 // Hex escapes are a maximal series of hex digits. 141 bool Overflow = false; 142 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) { 143 int CharVal = llvm::hexDigitValue(ThisTokBuf[0]); 144 if (CharVal == -1) break; 145 // About to shift out a digit? 146 Overflow |= (ResultChar & 0xF0000000) ? true : false; 147 ResultChar <<= 4; 148 ResultChar |= CharVal; 149 } 150 151 // See if any bits will be truncated when evaluated as a character. 152 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 153 Overflow = true; 154 ResultChar &= ~0U >> (32-CharWidth); 155 } 156 157 // Check for overflow. 158 if (Overflow && Diags) // Too many digits to fit in 159 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 160 diag::err_hex_escape_too_large); 161 break; 162 } 163 case '0': case '1': case '2': case '3': 164 case '4': case '5': case '6': case '7': { 165 // Octal escapes. 166 --ThisTokBuf; 167 ResultChar = 0; 168 169 // Octal escapes are a series of octal digits with maximum length 3. 170 // "\0123" is a two digit sequence equal to "\012" "3". 171 unsigned NumDigits = 0; 172 do { 173 ResultChar <<= 3; 174 ResultChar |= *ThisTokBuf++ - '0'; 175 ++NumDigits; 176 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 && 177 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7'); 178 179 // Check for overflow. Reject '\777', but not L'\777'. 180 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 181 if (Diags) 182 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 183 diag::err_octal_escape_too_large); 184 ResultChar &= ~0U >> (32-CharWidth); 185 } 186 break; 187 } 188 189 // Otherwise, these are not valid escapes. 190 case '(': case '{': case '[': case '%': 191 // GCC accepts these as extensions. We warn about them as such though. 192 if (Diags) 193 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 194 diag::ext_nonstandard_escape) 195 << std::string(1, ResultChar); 196 break; 197 default: 198 if (!Diags) 199 break; 200 201 if (isPrintable(ResultChar)) 202 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 203 diag::ext_unknown_escape) 204 << std::string(1, ResultChar); 205 else 206 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 207 diag::ext_unknown_escape) 208 << "x" + llvm::utohexstr(ResultChar); 209 break; 210 } 211 212 return ResultChar; 213 } 214 215 static void appendCodePoint(unsigned Codepoint, 216 llvm::SmallVectorImpl<char> &Str) { 217 char ResultBuf[4]; 218 char *ResultPtr = ResultBuf; 219 bool Res = llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr); 220 (void)Res; 221 assert(Res && "Unexpected conversion failure"); 222 Str.append(ResultBuf, ResultPtr); 223 } 224 225 void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) { 226 for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) { 227 if (*I != '\\') { 228 Buf.push_back(*I); 229 continue; 230 } 231 232 ++I; 233 assert(*I == 'u' || *I == 'U'); 234 235 unsigned NumHexDigits; 236 if (*I == 'u') 237 NumHexDigits = 4; 238 else 239 NumHexDigits = 8; 240 241 assert(I + NumHexDigits <= E); 242 243 uint32_t CodePoint = 0; 244 for (++I; NumHexDigits != 0; ++I, --NumHexDigits) { 245 unsigned Value = llvm::hexDigitValue(*I); 246 assert(Value != -1U); 247 248 CodePoint <<= 4; 249 CodePoint += Value; 250 } 251 252 appendCodePoint(CodePoint, Buf); 253 --I; 254 } 255 } 256 257 /// ProcessUCNEscape - Read the Universal Character Name, check constraints and 258 /// return the UTF32. 259 static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 260 const char *ThisTokEnd, 261 uint32_t &UcnVal, unsigned short &UcnLen, 262 FullSourceLoc Loc, DiagnosticsEngine *Diags, 263 const LangOptions &Features, 264 bool in_char_string_literal = false) { 265 const char *UcnBegin = ThisTokBuf; 266 267 // Skip the '\u' char's. 268 ThisTokBuf += 2; 269 270 if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 271 if (Diags) 272 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 273 diag::err_hex_escape_no_digits) << StringRef(&ThisTokBuf[-1], 1); 274 return false; 275 } 276 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8); 277 unsigned short UcnLenSave = UcnLen; 278 for (; ThisTokBuf != ThisTokEnd && UcnLenSave; ++ThisTokBuf, UcnLenSave--) { 279 int CharVal = llvm::hexDigitValue(ThisTokBuf[0]); 280 if (CharVal == -1) break; 281 UcnVal <<= 4; 282 UcnVal |= CharVal; 283 } 284 // If we didn't consume the proper number of digits, there is a problem. 285 if (UcnLenSave) { 286 if (Diags) 287 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 288 diag::err_ucn_escape_incomplete); 289 return false; 290 } 291 292 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2] 293 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints 294 UcnVal > 0x10FFFF) { // maximum legal UTF32 value 295 if (Diags) 296 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 297 diag::err_ucn_escape_invalid); 298 return false; 299 } 300 301 // C++11 allows UCNs that refer to control characters and basic source 302 // characters inside character and string literals 303 if (UcnVal < 0xa0 && 304 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) { // $, @, ` 305 bool IsError = (!Features.CPlusPlus11 || !in_char_string_literal); 306 if (Diags) { 307 char BasicSCSChar = UcnVal; 308 if (UcnVal >= 0x20 && UcnVal < 0x7f) 309 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 310 IsError ? diag::err_ucn_escape_basic_scs : 311 diag::warn_cxx98_compat_literal_ucn_escape_basic_scs) 312 << StringRef(&BasicSCSChar, 1); 313 else 314 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 315 IsError ? diag::err_ucn_control_character : 316 diag::warn_cxx98_compat_literal_ucn_control_character); 317 } 318 if (IsError) 319 return false; 320 } 321 322 if (!Features.CPlusPlus && !Features.C99 && Diags) 323 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 324 diag::warn_ucn_not_valid_in_c89_literal); 325 326 return true; 327 } 328 329 /// MeasureUCNEscape - Determine the number of bytes within the resulting string 330 /// which this UCN will occupy. 331 static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 332 const char *ThisTokEnd, unsigned CharByteWidth, 333 const LangOptions &Features, bool &HadError) { 334 // UTF-32: 4 bytes per escape. 335 if (CharByteWidth == 4) 336 return 4; 337 338 uint32_t UcnVal = 0; 339 unsigned short UcnLen = 0; 340 FullSourceLoc Loc; 341 342 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, 343 UcnLen, Loc, nullptr, Features, true)) { 344 HadError = true; 345 return 0; 346 } 347 348 // UTF-16: 2 bytes for BMP, 4 bytes otherwise. 349 if (CharByteWidth == 2) 350 return UcnVal <= 0xFFFF ? 2 : 4; 351 352 // UTF-8. 353 if (UcnVal < 0x80) 354 return 1; 355 if (UcnVal < 0x800) 356 return 2; 357 if (UcnVal < 0x10000) 358 return 3; 359 return 4; 360 } 361 362 /// EncodeUCNEscape - Read the Universal Character Name, check constraints and 363 /// convert the UTF32 to UTF8 or UTF16. This is a subroutine of 364 /// StringLiteralParser. When we decide to implement UCN's for identifiers, 365 /// we will likely rework our support for UCN's. 366 static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 367 const char *ThisTokEnd, 368 char *&ResultBuf, bool &HadError, 369 FullSourceLoc Loc, unsigned CharByteWidth, 370 DiagnosticsEngine *Diags, 371 const LangOptions &Features) { 372 typedef uint32_t UTF32; 373 UTF32 UcnVal = 0; 374 unsigned short UcnLen = 0; 375 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen, 376 Loc, Diags, Features, true)) { 377 HadError = true; 378 return; 379 } 380 381 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) && 382 "only character widths of 1, 2, or 4 bytes supported"); 383 384 (void)UcnLen; 385 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported"); 386 387 if (CharByteWidth == 4) { 388 // FIXME: Make the type of the result buffer correct instead of 389 // using reinterpret_cast. 390 UTF32 *ResultPtr = reinterpret_cast<UTF32*>(ResultBuf); 391 *ResultPtr = UcnVal; 392 ResultBuf += 4; 393 return; 394 } 395 396 if (CharByteWidth == 2) { 397 // FIXME: Make the type of the result buffer correct instead of 398 // using reinterpret_cast. 399 UTF16 *ResultPtr = reinterpret_cast<UTF16*>(ResultBuf); 400 401 if (UcnVal <= (UTF32)0xFFFF) { 402 *ResultPtr = UcnVal; 403 ResultBuf += 2; 404 return; 405 } 406 407 // Convert to UTF16. 408 UcnVal -= 0x10000; 409 *ResultPtr = 0xD800 + (UcnVal >> 10); 410 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF); 411 ResultBuf += 4; 412 return; 413 } 414 415 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters"); 416 417 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8. 418 // The conversion below was inspired by: 419 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c 420 // First, we determine how many bytes the result will require. 421 typedef uint8_t UTF8; 422 423 unsigned short bytesToWrite = 0; 424 if (UcnVal < (UTF32)0x80) 425 bytesToWrite = 1; 426 else if (UcnVal < (UTF32)0x800) 427 bytesToWrite = 2; 428 else if (UcnVal < (UTF32)0x10000) 429 bytesToWrite = 3; 430 else 431 bytesToWrite = 4; 432 433 const unsigned byteMask = 0xBF; 434 const unsigned byteMark = 0x80; 435 436 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed 437 // into the first byte, depending on how many bytes follow. 438 static const UTF8 firstByteMark[5] = { 439 0x00, 0x00, 0xC0, 0xE0, 0xF0 440 }; 441 // Finally, we write the bytes into ResultBuf. 442 ResultBuf += bytesToWrite; 443 switch (bytesToWrite) { // note: everything falls through. 444 case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 445 case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 446 case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 447 case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); 448 } 449 // Update the buffer. 450 ResultBuf += bytesToWrite; 451 } 452 453 454 /// integer-constant: [C99 6.4.4.1] 455 /// decimal-constant integer-suffix 456 /// octal-constant integer-suffix 457 /// hexadecimal-constant integer-suffix 458 /// binary-literal integer-suffix [GNU, C++1y] 459 /// user-defined-integer-literal: [C++11 lex.ext] 460 /// decimal-literal ud-suffix 461 /// octal-literal ud-suffix 462 /// hexadecimal-literal ud-suffix 463 /// binary-literal ud-suffix [GNU, C++1y] 464 /// decimal-constant: 465 /// nonzero-digit 466 /// decimal-constant digit 467 /// octal-constant: 468 /// 0 469 /// octal-constant octal-digit 470 /// hexadecimal-constant: 471 /// hexadecimal-prefix hexadecimal-digit 472 /// hexadecimal-constant hexadecimal-digit 473 /// hexadecimal-prefix: one of 474 /// 0x 0X 475 /// binary-literal: 476 /// 0b binary-digit 477 /// 0B binary-digit 478 /// binary-literal binary-digit 479 /// integer-suffix: 480 /// unsigned-suffix [long-suffix] 481 /// unsigned-suffix [long-long-suffix] 482 /// long-suffix [unsigned-suffix] 483 /// long-long-suffix [unsigned-sufix] 484 /// nonzero-digit: 485 /// 1 2 3 4 5 6 7 8 9 486 /// octal-digit: 487 /// 0 1 2 3 4 5 6 7 488 /// hexadecimal-digit: 489 /// 0 1 2 3 4 5 6 7 8 9 490 /// a b c d e f 491 /// A B C D E F 492 /// binary-digit: 493 /// 0 494 /// 1 495 /// unsigned-suffix: one of 496 /// u U 497 /// long-suffix: one of 498 /// l L 499 /// long-long-suffix: one of 500 /// ll LL 501 /// 502 /// floating-constant: [C99 6.4.4.2] 503 /// TODO: add rules... 504 /// 505 NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, 506 SourceLocation TokLoc, 507 Preprocessor &PP) 508 : PP(PP), ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) { 509 510 // This routine assumes that the range begin/end matches the regex for integer 511 // and FP constants (specifically, the 'pp-number' regex), and assumes that 512 // the byte at "*end" is both valid and not part of the regex. Because of 513 // this, it doesn't have to check for 'overscan' in various places. 514 assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?"); 515 516 s = DigitsBegin = ThisTokBegin; 517 saw_exponent = false; 518 saw_period = false; 519 saw_ud_suffix = false; 520 isLong = false; 521 isUnsigned = false; 522 isLongLong = false; 523 isFloat = false; 524 isImaginary = false; 525 MicrosoftInteger = 0; 526 hadError = false; 527 528 if (*s == '0') { // parse radix 529 ParseNumberStartingWithZero(TokLoc); 530 if (hadError) 531 return; 532 } else { // the first digit is non-zero 533 radix = 10; 534 s = SkipDigits(s); 535 if (s == ThisTokEnd) { 536 // Done. 537 } else if (isHexDigit(*s) && !(*s == 'e' || *s == 'E')) { 538 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), 539 diag::err_invalid_decimal_digit) << StringRef(s, 1); 540 hadError = true; 541 return; 542 } else if (*s == '.') { 543 checkSeparator(TokLoc, s, CSK_AfterDigits); 544 s++; 545 saw_period = true; 546 checkSeparator(TokLoc, s, CSK_BeforeDigits); 547 s = SkipDigits(s); 548 } 549 if ((*s == 'e' || *s == 'E')) { // exponent 550 checkSeparator(TokLoc, s, CSK_AfterDigits); 551 const char *Exponent = s; 552 s++; 553 saw_exponent = true; 554 if (*s == '+' || *s == '-') s++; // sign 555 checkSeparator(TokLoc, s, CSK_BeforeDigits); 556 const char *first_non_digit = SkipDigits(s); 557 if (first_non_digit != s) { 558 s = first_non_digit; 559 } else { 560 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent - ThisTokBegin), 561 diag::err_exponent_has_no_digits); 562 hadError = true; 563 return; 564 } 565 } 566 } 567 568 SuffixBegin = s; 569 checkSeparator(TokLoc, s, CSK_AfterDigits); 570 571 // Parse the suffix. At this point we can classify whether we have an FP or 572 // integer constant. 573 bool isFPConstant = isFloatingLiteral(); 574 const char *ImaginarySuffixLoc = nullptr; 575 576 // Loop over all of the characters of the suffix. If we see something bad, 577 // we break out of the loop. 578 for (; s != ThisTokEnd; ++s) { 579 switch (*s) { 580 case 'f': // FP Suffix for "float" 581 case 'F': 582 if (!isFPConstant) break; // Error for integer constant. 583 if (isFloat || isLong) break; // FF, LF invalid. 584 isFloat = true; 585 continue; // Success. 586 case 'u': 587 case 'U': 588 if (isFPConstant) break; // Error for floating constant. 589 if (isUnsigned) break; // Cannot be repeated. 590 isUnsigned = true; 591 continue; // Success. 592 case 'l': 593 case 'L': 594 if (isLong || isLongLong) break; // Cannot be repeated. 595 if (isFloat) break; // LF invalid. 596 597 // Check for long long. The L's need to be adjacent and the same case. 598 if (s+1 != ThisTokEnd && s[1] == s[0]) { 599 if (isFPConstant) break; // long long invalid for floats. 600 isLongLong = true; 601 ++s; // Eat both of them. 602 } else { 603 isLong = true; 604 } 605 continue; // Success. 606 case 'i': 607 case 'I': 608 if (PP.getLangOpts().MicrosoftExt) { 609 if (isLong || isLongLong || MicrosoftInteger) 610 break; 611 612 // Allow i8, i16, i32, i64, and i128. 613 if (s + 1 != ThisTokEnd) { 614 switch (s[1]) { 615 case '8': 616 if (isFPConstant) break; 617 s += 2; // i8 suffix 618 MicrosoftInteger = 8; 619 break; 620 case '1': 621 if (isFPConstant) break; 622 if (s + 2 == ThisTokEnd) break; 623 if (s[2] == '6') { 624 s += 3; // i16 suffix 625 MicrosoftInteger = 16; 626 } 627 else if (s[2] == '2') { 628 if (s + 3 == ThisTokEnd) break; 629 if (s[3] == '8') { 630 s += 4; // i128 suffix 631 MicrosoftInteger = 128; 632 } 633 } 634 break; 635 case '3': 636 if (isFPConstant) break; 637 if (s + 2 == ThisTokEnd) break; 638 if (s[2] == '2') { 639 s += 3; // i32 suffix 640 MicrosoftInteger = 32; 641 } 642 break; 643 case '6': 644 if (isFPConstant) break; 645 if (s + 2 == ThisTokEnd) break; 646 if (s[2] == '4') { 647 s += 3; // i64 suffix 648 MicrosoftInteger = 64; 649 } 650 break; 651 default: 652 break; 653 } 654 if (MicrosoftInteger) 655 break; 656 } 657 } 658 // "i", "if", and "il" are user-defined suffixes in C++1y. 659 if (PP.getLangOpts().CPlusPlus1y && *s == 'i') 660 break; 661 // fall through. 662 case 'j': 663 case 'J': 664 if (isImaginary) break; // Cannot be repeated. 665 isImaginary = true; 666 ImaginarySuffixLoc = s; 667 continue; // Success. 668 } 669 // If we reached here, there was an error or a ud-suffix. 670 break; 671 } 672 673 if (s != ThisTokEnd) { 674 // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). 675 expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); 676 if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) { 677 // Any suffix pieces we might have parsed are actually part of the 678 // ud-suffix. 679 isLong = false; 680 isUnsigned = false; 681 isLongLong = false; 682 isFloat = false; 683 isImaginary = false; 684 MicrosoftInteger = 0; 685 686 saw_ud_suffix = true; 687 return; 688 } 689 690 // Report an error if there are any. 691 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), 692 isFPConstant ? diag::err_invalid_suffix_float_constant : 693 diag::err_invalid_suffix_integer_constant) 694 << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin); 695 hadError = true; 696 return; 697 } 698 699 if (isImaginary) { 700 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, 701 ImaginarySuffixLoc - ThisTokBegin), 702 diag::ext_imaginary_constant); 703 } 704 } 705 706 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved 707 /// suffixes as ud-suffixes, because the diagnostic experience is better if we 708 /// treat it as an invalid suffix. 709 bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts, 710 StringRef Suffix) { 711 if (!LangOpts.CPlusPlus11 || Suffix.empty()) 712 return false; 713 714 // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid. 715 if (Suffix[0] == '_') 716 return true; 717 718 // In C++11, there are no library suffixes. 719 if (!LangOpts.CPlusPlus1y) 720 return false; 721 722 // In C++1y, "s", "h", "min", "ms", "us", and "ns" are used in the library. 723 // Per tweaked N3660, "il", "i", and "if" are also used in the library. 724 return llvm::StringSwitch<bool>(Suffix) 725 .Cases("h", "min", "s", true) 726 .Cases("ms", "us", "ns", true) 727 .Cases("il", "i", "if", true) 728 .Default(false); 729 } 730 731 void NumericLiteralParser::checkSeparator(SourceLocation TokLoc, 732 const char *Pos, 733 CheckSeparatorKind IsAfterDigits) { 734 if (IsAfterDigits == CSK_AfterDigits) { 735 if (Pos == ThisTokBegin) 736 return; 737 --Pos; 738 } else if (Pos == ThisTokEnd) 739 return; 740 741 if (isDigitSeparator(*Pos)) 742 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin), 743 diag::err_digit_separator_not_between_digits) 744 << IsAfterDigits; 745 } 746 747 /// ParseNumberStartingWithZero - This method is called when the first character 748 /// of the number is found to be a zero. This means it is either an octal 749 /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or 750 /// a floating point number (01239.123e4). Eat the prefix, determining the 751 /// radix etc. 752 void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { 753 assert(s[0] == '0' && "Invalid method call"); 754 s++; 755 756 int c1 = s[0]; 757 int c2 = s[1]; 758 759 // Handle a hex number like 0x1234. 760 if ((c1 == 'x' || c1 == 'X') && (isHexDigit(c2) || c2 == '.')) { 761 s++; 762 radix = 16; 763 DigitsBegin = s; 764 s = SkipHexDigits(s); 765 bool noSignificand = (s == DigitsBegin); 766 if (s == ThisTokEnd) { 767 // Done. 768 } else if (*s == '.') { 769 s++; 770 saw_period = true; 771 const char *floatDigitsBegin = s; 772 checkSeparator(TokLoc, s, CSK_BeforeDigits); 773 s = SkipHexDigits(s); 774 noSignificand &= (floatDigitsBegin == s); 775 } 776 777 if (noSignificand) { 778 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), 779 diag::err_hexconstant_requires_digits); 780 hadError = true; 781 return; 782 } 783 784 // A binary exponent can appear with or with a '.'. If dotted, the 785 // binary exponent is required. 786 if (*s == 'p' || *s == 'P') { 787 checkSeparator(TokLoc, s, CSK_AfterDigits); 788 const char *Exponent = s; 789 s++; 790 saw_exponent = true; 791 if (*s == '+' || *s == '-') s++; // sign 792 const char *first_non_digit = SkipDigits(s); 793 if (first_non_digit == s) { 794 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 795 diag::err_exponent_has_no_digits); 796 hadError = true; 797 return; 798 } 799 checkSeparator(TokLoc, s, CSK_BeforeDigits); 800 s = first_non_digit; 801 802 if (!PP.getLangOpts().HexFloats) 803 PP.Diag(TokLoc, diag::ext_hexconstant_invalid); 804 } else if (saw_period) { 805 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 806 diag::err_hexconstant_requires_exponent); 807 hadError = true; 808 } 809 return; 810 } 811 812 // Handle simple binary numbers 0b01010 813 if ((c1 == 'b' || c1 == 'B') && (c2 == '0' || c2 == '1')) { 814 // 0b101010 is a C++1y / GCC extension. 815 PP.Diag(TokLoc, 816 PP.getLangOpts().CPlusPlus1y 817 ? diag::warn_cxx11_compat_binary_literal 818 : PP.getLangOpts().CPlusPlus 819 ? diag::ext_binary_literal_cxx1y 820 : diag::ext_binary_literal); 821 ++s; 822 radix = 2; 823 DigitsBegin = s; 824 s = SkipBinaryDigits(s); 825 if (s == ThisTokEnd) { 826 // Done. 827 } else if (isHexDigit(*s)) { 828 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 829 diag::err_invalid_binary_digit) << StringRef(s, 1); 830 hadError = true; 831 } 832 // Other suffixes will be diagnosed by the caller. 833 return; 834 } 835 836 // For now, the radix is set to 8. If we discover that we have a 837 // floating point constant, the radix will change to 10. Octal floating 838 // point constants are not permitted (only decimal and hexadecimal). 839 radix = 8; 840 DigitsBegin = s; 841 s = SkipOctalDigits(s); 842 if (s == ThisTokEnd) 843 return; // Done, simple octal number like 01234 844 845 // If we have some other non-octal digit that *is* a decimal digit, see if 846 // this is part of a floating point number like 094.123 or 09e1. 847 if (isDigit(*s)) { 848 const char *EndDecimal = SkipDigits(s); 849 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') { 850 s = EndDecimal; 851 radix = 10; 852 } 853 } 854 855 // If we have a hex digit other than 'e' (which denotes a FP exponent) then 856 // the code is using an incorrect base. 857 if (isHexDigit(*s) && *s != 'e' && *s != 'E') { 858 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 859 diag::err_invalid_octal_digit) << StringRef(s, 1); 860 hadError = true; 861 return; 862 } 863 864 if (*s == '.') { 865 s++; 866 radix = 10; 867 saw_period = true; 868 checkSeparator(TokLoc, s, CSK_BeforeDigits); 869 s = SkipDigits(s); // Skip suffix. 870 } 871 if (*s == 'e' || *s == 'E') { // exponent 872 checkSeparator(TokLoc, s, CSK_AfterDigits); 873 const char *Exponent = s; 874 s++; 875 radix = 10; 876 saw_exponent = true; 877 if (*s == '+' || *s == '-') s++; // sign 878 const char *first_non_digit = SkipDigits(s); 879 if (first_non_digit != s) { 880 checkSeparator(TokLoc, s, CSK_BeforeDigits); 881 s = first_non_digit; 882 } else { 883 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 884 diag::err_exponent_has_no_digits); 885 hadError = true; 886 return; 887 } 888 } 889 } 890 891 static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) { 892 switch (Radix) { 893 case 2: 894 return NumDigits <= 64; 895 case 8: 896 return NumDigits <= 64 / 3; // Digits are groups of 3 bits. 897 case 10: 898 return NumDigits <= 19; // floor(log10(2^64)) 899 case 16: 900 return NumDigits <= 64 / 4; // Digits are groups of 4 bits. 901 default: 902 llvm_unreachable("impossible Radix"); 903 } 904 } 905 906 /// GetIntegerValue - Convert this numeric literal value to an APInt that 907 /// matches Val's input width. If there is an overflow, set Val to the low bits 908 /// of the result and return true. Otherwise, return false. 909 bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) { 910 // Fast path: Compute a conservative bound on the maximum number of 911 // bits per digit in this radix. If we can't possibly overflow a 912 // uint64 based on that bound then do the simple conversion to 913 // integer. This avoids the expensive overflow checking below, and 914 // handles the common cases that matter (small decimal integers and 915 // hex/octal values which don't overflow). 916 const unsigned NumDigits = SuffixBegin - DigitsBegin; 917 if (alwaysFitsInto64Bits(radix, NumDigits)) { 918 uint64_t N = 0; 919 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr) 920 if (!isDigitSeparator(*Ptr)) 921 N = N * radix + llvm::hexDigitValue(*Ptr); 922 923 // This will truncate the value to Val's input width. Simply check 924 // for overflow by comparing. 925 Val = N; 926 return Val.getZExtValue() != N; 927 } 928 929 Val = 0; 930 const char *Ptr = DigitsBegin; 931 932 llvm::APInt RadixVal(Val.getBitWidth(), radix); 933 llvm::APInt CharVal(Val.getBitWidth(), 0); 934 llvm::APInt OldVal = Val; 935 936 bool OverflowOccurred = false; 937 while (Ptr < SuffixBegin) { 938 if (isDigitSeparator(*Ptr)) { 939 ++Ptr; 940 continue; 941 } 942 943 unsigned C = llvm::hexDigitValue(*Ptr++); 944 945 // If this letter is out of bound for this radix, reject it. 946 assert(C < radix && "NumericLiteralParser ctor should have rejected this"); 947 948 CharVal = C; 949 950 // Add the digit to the value in the appropriate radix. If adding in digits 951 // made the value smaller, then this overflowed. 952 OldVal = Val; 953 954 // Multiply by radix, did overflow occur on the multiply? 955 Val *= RadixVal; 956 OverflowOccurred |= Val.udiv(RadixVal) != OldVal; 957 958 // Add value, did overflow occur on the value? 959 // (a + b) ult b <=> overflow 960 Val += CharVal; 961 OverflowOccurred |= Val.ult(CharVal); 962 } 963 return OverflowOccurred; 964 } 965 966 llvm::APFloat::opStatus 967 NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) { 968 using llvm::APFloat; 969 970 unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin); 971 972 llvm::SmallString<16> Buffer; 973 StringRef Str(ThisTokBegin, n); 974 if (Str.find('\'') != StringRef::npos) { 975 Buffer.reserve(n); 976 std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer), 977 &isDigitSeparator); 978 Str = Buffer; 979 } 980 981 return Result.convertFromString(Str, APFloat::rmNearestTiesToEven); 982 } 983 984 985 /// \verbatim 986 /// user-defined-character-literal: [C++11 lex.ext] 987 /// character-literal ud-suffix 988 /// ud-suffix: 989 /// identifier 990 /// character-literal: [C++11 lex.ccon] 991 /// ' c-char-sequence ' 992 /// u' c-char-sequence ' 993 /// U' c-char-sequence ' 994 /// L' c-char-sequence ' 995 /// c-char-sequence: 996 /// c-char 997 /// c-char-sequence c-char 998 /// c-char: 999 /// any member of the source character set except the single-quote ', 1000 /// backslash \, or new-line character 1001 /// escape-sequence 1002 /// universal-character-name 1003 /// escape-sequence: 1004 /// simple-escape-sequence 1005 /// octal-escape-sequence 1006 /// hexadecimal-escape-sequence 1007 /// simple-escape-sequence: 1008 /// one of \' \" \? \\ \a \b \f \n \r \t \v 1009 /// octal-escape-sequence: 1010 /// \ octal-digit 1011 /// \ octal-digit octal-digit 1012 /// \ octal-digit octal-digit octal-digit 1013 /// hexadecimal-escape-sequence: 1014 /// \x hexadecimal-digit 1015 /// hexadecimal-escape-sequence hexadecimal-digit 1016 /// universal-character-name: [C++11 lex.charset] 1017 /// \u hex-quad 1018 /// \U hex-quad hex-quad 1019 /// hex-quad: 1020 /// hex-digit hex-digit hex-digit hex-digit 1021 /// \endverbatim 1022 /// 1023 CharLiteralParser::CharLiteralParser(const char *begin, const char *end, 1024 SourceLocation Loc, Preprocessor &PP, 1025 tok::TokenKind kind) { 1026 // At this point we know that the character matches the regex "(L|u|U)?'.*'". 1027 HadError = false; 1028 1029 Kind = kind; 1030 1031 const char *TokBegin = begin; 1032 1033 // Skip over wide character determinant. 1034 if (Kind != tok::char_constant) { 1035 ++begin; 1036 } 1037 1038 // Skip over the entry quote. 1039 assert(begin[0] == '\'' && "Invalid token lexed"); 1040 ++begin; 1041 1042 // Remove an optional ud-suffix. 1043 if (end[-1] != '\'') { 1044 const char *UDSuffixEnd = end; 1045 do { 1046 --end; 1047 } while (end[-1] != '\''); 1048 // FIXME: Don't bother with this if !tok.hasUCN(). 1049 expandUCNs(UDSuffixBuf, StringRef(end, UDSuffixEnd - end)); 1050 UDSuffixOffset = end - TokBegin; 1051 } 1052 1053 // Trim the ending quote. 1054 assert(end != begin && "Invalid token lexed"); 1055 --end; 1056 1057 // FIXME: The "Value" is an uint64_t so we can handle char literals of 1058 // up to 64-bits. 1059 // FIXME: This extensively assumes that 'char' is 8-bits. 1060 assert(PP.getTargetInfo().getCharWidth() == 8 && 1061 "Assumes char is 8 bits"); 1062 assert(PP.getTargetInfo().getIntWidth() <= 64 && 1063 (PP.getTargetInfo().getIntWidth() & 7) == 0 && 1064 "Assumes sizeof(int) on target is <= 64 and a multiple of char"); 1065 assert(PP.getTargetInfo().getWCharWidth() <= 64 && 1066 "Assumes sizeof(wchar) on target is <= 64"); 1067 1068 SmallVector<uint32_t, 4> codepoint_buffer; 1069 codepoint_buffer.resize(end - begin); 1070 uint32_t *buffer_begin = &codepoint_buffer.front(); 1071 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size(); 1072 1073 // Unicode escapes representing characters that cannot be correctly 1074 // represented in a single code unit are disallowed in character literals 1075 // by this implementation. 1076 uint32_t largest_character_for_kind; 1077 if (tok::wide_char_constant == Kind) { 1078 largest_character_for_kind = 1079 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth()); 1080 } else if (tok::utf16_char_constant == Kind) { 1081 largest_character_for_kind = 0xFFFF; 1082 } else if (tok::utf32_char_constant == Kind) { 1083 largest_character_for_kind = 0x10FFFF; 1084 } else { 1085 largest_character_for_kind = 0x7Fu; 1086 } 1087 1088 while (begin != end) { 1089 // Is this a span of non-escape characters? 1090 if (begin[0] != '\\') { 1091 char const *start = begin; 1092 do { 1093 ++begin; 1094 } while (begin != end && *begin != '\\'); 1095 1096 char const *tmp_in_start = start; 1097 uint32_t *tmp_out_start = buffer_begin; 1098 ConversionResult res = 1099 ConvertUTF8toUTF32(reinterpret_cast<UTF8 const **>(&start), 1100 reinterpret_cast<UTF8 const *>(begin), 1101 &buffer_begin, buffer_end, strictConversion); 1102 if (res != conversionOK) { 1103 // If we see bad encoding for unprefixed character literals, warn and 1104 // simply copy the byte values, for compatibility with gcc and 1105 // older versions of clang. 1106 bool NoErrorOnBadEncoding = isAscii(); 1107 unsigned Msg = diag::err_bad_character_encoding; 1108 if (NoErrorOnBadEncoding) 1109 Msg = diag::warn_bad_character_encoding; 1110 PP.Diag(Loc, Msg); 1111 if (NoErrorOnBadEncoding) { 1112 start = tmp_in_start; 1113 buffer_begin = tmp_out_start; 1114 for (; start != begin; ++start, ++buffer_begin) 1115 *buffer_begin = static_cast<uint8_t>(*start); 1116 } else { 1117 HadError = true; 1118 } 1119 } else { 1120 for (; tmp_out_start < buffer_begin; ++tmp_out_start) { 1121 if (*tmp_out_start > largest_character_for_kind) { 1122 HadError = true; 1123 PP.Diag(Loc, diag::err_character_too_large); 1124 } 1125 } 1126 } 1127 1128 continue; 1129 } 1130 // Is this a Universal Character Name escape? 1131 if (begin[1] == 'u' || begin[1] == 'U') { 1132 unsigned short UcnLen = 0; 1133 if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen, 1134 FullSourceLoc(Loc, PP.getSourceManager()), 1135 &PP.getDiagnostics(), PP.getLangOpts(), true)) { 1136 HadError = true; 1137 } else if (*buffer_begin > largest_character_for_kind) { 1138 HadError = true; 1139 PP.Diag(Loc, diag::err_character_too_large); 1140 } 1141 1142 ++buffer_begin; 1143 continue; 1144 } 1145 unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo()); 1146 uint64_t result = 1147 ProcessCharEscape(TokBegin, begin, end, HadError, 1148 FullSourceLoc(Loc,PP.getSourceManager()), 1149 CharWidth, &PP.getDiagnostics(), PP.getLangOpts()); 1150 *buffer_begin++ = result; 1151 } 1152 1153 unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front(); 1154 1155 if (NumCharsSoFar > 1) { 1156 if (isWide()) 1157 PP.Diag(Loc, diag::warn_extraneous_char_constant); 1158 else if (isAscii() && NumCharsSoFar == 4) 1159 PP.Diag(Loc, diag::ext_four_char_character_literal); 1160 else if (isAscii()) 1161 PP.Diag(Loc, diag::ext_multichar_character_literal); 1162 else 1163 PP.Diag(Loc, diag::err_multichar_utf_character_literal); 1164 IsMultiChar = true; 1165 } else { 1166 IsMultiChar = false; 1167 } 1168 1169 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0); 1170 1171 // Narrow character literals act as though their value is concatenated 1172 // in this implementation, but warn on overflow. 1173 bool multi_char_too_long = false; 1174 if (isAscii() && isMultiChar()) { 1175 LitVal = 0; 1176 for (size_t i = 0; i < NumCharsSoFar; ++i) { 1177 // check for enough leading zeros to shift into 1178 multi_char_too_long |= (LitVal.countLeadingZeros() < 8); 1179 LitVal <<= 8; 1180 LitVal = LitVal + (codepoint_buffer[i] & 0xFF); 1181 } 1182 } else if (NumCharsSoFar > 0) { 1183 // otherwise just take the last character 1184 LitVal = buffer_begin[-1]; 1185 } 1186 1187 if (!HadError && multi_char_too_long) { 1188 PP.Diag(Loc, diag::warn_char_constant_too_large); 1189 } 1190 1191 // Transfer the value from APInt to uint64_t 1192 Value = LitVal.getZExtValue(); 1193 1194 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1") 1195 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple 1196 // character constants are not sign extended in the this implementation: 1197 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC. 1198 if (isAscii() && NumCharsSoFar == 1 && (Value & 128) && 1199 PP.getLangOpts().CharIsSigned) 1200 Value = (signed char)Value; 1201 } 1202 1203 /// \verbatim 1204 /// string-literal: [C++0x lex.string] 1205 /// encoding-prefix " [s-char-sequence] " 1206 /// encoding-prefix R raw-string 1207 /// encoding-prefix: 1208 /// u8 1209 /// u 1210 /// U 1211 /// L 1212 /// s-char-sequence: 1213 /// s-char 1214 /// s-char-sequence s-char 1215 /// s-char: 1216 /// any member of the source character set except the double-quote ", 1217 /// backslash \, or new-line character 1218 /// escape-sequence 1219 /// universal-character-name 1220 /// raw-string: 1221 /// " d-char-sequence ( r-char-sequence ) d-char-sequence " 1222 /// r-char-sequence: 1223 /// r-char 1224 /// r-char-sequence r-char 1225 /// r-char: 1226 /// any member of the source character set, except a right parenthesis ) 1227 /// followed by the initial d-char-sequence (which may be empty) 1228 /// followed by a double quote ". 1229 /// d-char-sequence: 1230 /// d-char 1231 /// d-char-sequence d-char 1232 /// d-char: 1233 /// any member of the basic source character set except: 1234 /// space, the left parenthesis (, the right parenthesis ), 1235 /// the backslash \, and the control characters representing horizontal 1236 /// tab, vertical tab, form feed, and newline. 1237 /// escape-sequence: [C++0x lex.ccon] 1238 /// simple-escape-sequence 1239 /// octal-escape-sequence 1240 /// hexadecimal-escape-sequence 1241 /// simple-escape-sequence: 1242 /// one of \' \" \? \\ \a \b \f \n \r \t \v 1243 /// octal-escape-sequence: 1244 /// \ octal-digit 1245 /// \ octal-digit octal-digit 1246 /// \ octal-digit octal-digit octal-digit 1247 /// hexadecimal-escape-sequence: 1248 /// \x hexadecimal-digit 1249 /// hexadecimal-escape-sequence hexadecimal-digit 1250 /// universal-character-name: 1251 /// \u hex-quad 1252 /// \U hex-quad hex-quad 1253 /// hex-quad: 1254 /// hex-digit hex-digit hex-digit hex-digit 1255 /// \endverbatim 1256 /// 1257 StringLiteralParser:: 1258 StringLiteralParser(ArrayRef<Token> StringToks, 1259 Preprocessor &PP, bool Complain) 1260 : SM(PP.getSourceManager()), Features(PP.getLangOpts()), 1261 Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() :nullptr), 1262 MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown), 1263 ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) { 1264 init(StringToks); 1265 } 1266 1267 void StringLiteralParser::init(ArrayRef<Token> StringToks){ 1268 // The literal token may have come from an invalid source location (e.g. due 1269 // to a PCH error), in which case the token length will be 0. 1270 if (StringToks.empty() || StringToks[0].getLength() < 2) 1271 return DiagnoseLexingError(SourceLocation()); 1272 1273 // Scan all of the string portions, remember the max individual token length, 1274 // computing a bound on the concatenated string length, and see whether any 1275 // piece is a wide-string. If any of the string portions is a wide-string 1276 // literal, the result is a wide-string literal [C99 6.4.5p4]. 1277 assert(!StringToks.empty() && "expected at least one token"); 1278 MaxTokenLength = StringToks[0].getLength(); 1279 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!"); 1280 SizeBound = StringToks[0].getLength()-2; // -2 for "". 1281 Kind = StringToks[0].getKind(); 1282 1283 hadError = false; 1284 1285 // Implement Translation Phase #6: concatenation of string literals 1286 /// (C99 5.1.1.2p1). The common case is only one string fragment. 1287 for (unsigned i = 1; i != StringToks.size(); ++i) { 1288 if (StringToks[i].getLength() < 2) 1289 return DiagnoseLexingError(StringToks[i].getLocation()); 1290 1291 // The string could be shorter than this if it needs cleaning, but this is a 1292 // reasonable bound, which is all we need. 1293 assert(StringToks[i].getLength() >= 2 && "literal token is invalid!"); 1294 SizeBound += StringToks[i].getLength()-2; // -2 for "". 1295 1296 // Remember maximum string piece length. 1297 if (StringToks[i].getLength() > MaxTokenLength) 1298 MaxTokenLength = StringToks[i].getLength(); 1299 1300 // Remember if we see any wide or utf-8/16/32 strings. 1301 // Also check for illegal concatenations. 1302 if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) { 1303 if (isAscii()) { 1304 Kind = StringToks[i].getKind(); 1305 } else { 1306 if (Diags) 1307 Diags->Report(StringToks[i].getLocation(), 1308 diag::err_unsupported_string_concat); 1309 hadError = true; 1310 } 1311 } 1312 } 1313 1314 // Include space for the null terminator. 1315 ++SizeBound; 1316 1317 // TODO: K&R warning: "traditional C rejects string constant concatenation" 1318 1319 // Get the width in bytes of char/wchar_t/char16_t/char32_t 1320 CharByteWidth = getCharWidth(Kind, Target); 1321 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple"); 1322 CharByteWidth /= 8; 1323 1324 // The output buffer size needs to be large enough to hold wide characters. 1325 // This is a worst-case assumption which basically corresponds to L"" "long". 1326 SizeBound *= CharByteWidth; 1327 1328 // Size the temporary buffer to hold the result string data. 1329 ResultBuf.resize(SizeBound); 1330 1331 // Likewise, but for each string piece. 1332 SmallString<512> TokenBuf; 1333 TokenBuf.resize(MaxTokenLength); 1334 1335 // Loop over all the strings, getting their spelling, and expanding them to 1336 // wide strings as appropriate. 1337 ResultPtr = &ResultBuf[0]; // Next byte to fill in. 1338 1339 Pascal = false; 1340 1341 SourceLocation UDSuffixTokLoc; 1342 1343 for (unsigned i = 0, e = StringToks.size(); i != e; ++i) { 1344 const char *ThisTokBuf = &TokenBuf[0]; 1345 // Get the spelling of the token, which eliminates trigraphs, etc. We know 1346 // that ThisTokBuf points to a buffer that is big enough for the whole token 1347 // and 'spelled' tokens can only shrink. 1348 bool StringInvalid = false; 1349 unsigned ThisTokLen = 1350 Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features, 1351 &StringInvalid); 1352 if (StringInvalid) 1353 return DiagnoseLexingError(StringToks[i].getLocation()); 1354 1355 const char *ThisTokBegin = ThisTokBuf; 1356 const char *ThisTokEnd = ThisTokBuf+ThisTokLen; 1357 1358 // Remove an optional ud-suffix. 1359 if (ThisTokEnd[-1] != '"') { 1360 const char *UDSuffixEnd = ThisTokEnd; 1361 do { 1362 --ThisTokEnd; 1363 } while (ThisTokEnd[-1] != '"'); 1364 1365 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd); 1366 1367 if (UDSuffixBuf.empty()) { 1368 if (StringToks[i].hasUCN()) 1369 expandUCNs(UDSuffixBuf, UDSuffix); 1370 else 1371 UDSuffixBuf.assign(UDSuffix); 1372 UDSuffixToken = i; 1373 UDSuffixOffset = ThisTokEnd - ThisTokBuf; 1374 UDSuffixTokLoc = StringToks[i].getLocation(); 1375 } else { 1376 SmallString<32> ExpandedUDSuffix; 1377 if (StringToks[i].hasUCN()) { 1378 expandUCNs(ExpandedUDSuffix, UDSuffix); 1379 UDSuffix = ExpandedUDSuffix; 1380 } 1381 1382 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the 1383 // result of a concatenation involving at least one user-defined-string- 1384 // literal, all the participating user-defined-string-literals shall 1385 // have the same ud-suffix. 1386 if (UDSuffixBuf != UDSuffix) { 1387 if (Diags) { 1388 SourceLocation TokLoc = StringToks[i].getLocation(); 1389 Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix) 1390 << UDSuffixBuf << UDSuffix 1391 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc) 1392 << SourceRange(TokLoc, TokLoc); 1393 } 1394 hadError = true; 1395 } 1396 } 1397 } 1398 1399 // Strip the end quote. 1400 --ThisTokEnd; 1401 1402 // TODO: Input character set mapping support. 1403 1404 // Skip marker for wide or unicode strings. 1405 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') { 1406 ++ThisTokBuf; 1407 // Skip 8 of u8 marker for utf8 strings. 1408 if (ThisTokBuf[0] == '8') 1409 ++ThisTokBuf; 1410 } 1411 1412 // Check for raw string 1413 if (ThisTokBuf[0] == 'R') { 1414 ThisTokBuf += 2; // skip R" 1415 1416 const char *Prefix = ThisTokBuf; 1417 while (ThisTokBuf[0] != '(') 1418 ++ThisTokBuf; 1419 ++ThisTokBuf; // skip '(' 1420 1421 // Remove same number of characters from the end 1422 ThisTokEnd -= ThisTokBuf - Prefix; 1423 assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal"); 1424 1425 // Copy the string over 1426 if (CopyStringFragment(StringToks[i], ThisTokBegin, 1427 StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf))) 1428 hadError = true; 1429 } else { 1430 if (ThisTokBuf[0] != '"') { 1431 // The file may have come from PCH and then changed after loading the 1432 // PCH; Fail gracefully. 1433 return DiagnoseLexingError(StringToks[i].getLocation()); 1434 } 1435 ++ThisTokBuf; // skip " 1436 1437 // Check if this is a pascal string 1438 if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd && 1439 ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') { 1440 1441 // If the \p sequence is found in the first token, we have a pascal string 1442 // Otherwise, if we already have a pascal string, ignore the first \p 1443 if (i == 0) { 1444 ++ThisTokBuf; 1445 Pascal = true; 1446 } else if (Pascal) 1447 ThisTokBuf += 2; 1448 } 1449 1450 while (ThisTokBuf != ThisTokEnd) { 1451 // Is this a span of non-escape characters? 1452 if (ThisTokBuf[0] != '\\') { 1453 const char *InStart = ThisTokBuf; 1454 do { 1455 ++ThisTokBuf; 1456 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\'); 1457 1458 // Copy the character span over. 1459 if (CopyStringFragment(StringToks[i], ThisTokBegin, 1460 StringRef(InStart, ThisTokBuf - InStart))) 1461 hadError = true; 1462 continue; 1463 } 1464 // Is this a Universal Character Name escape? 1465 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') { 1466 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, 1467 ResultPtr, hadError, 1468 FullSourceLoc(StringToks[i].getLocation(), SM), 1469 CharByteWidth, Diags, Features); 1470 continue; 1471 } 1472 // Otherwise, this is a non-UCN escape character. Process it. 1473 unsigned ResultChar = 1474 ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError, 1475 FullSourceLoc(StringToks[i].getLocation(), SM), 1476 CharByteWidth*8, Diags, Features); 1477 1478 if (CharByteWidth == 4) { 1479 // FIXME: Make the type of the result buffer correct instead of 1480 // using reinterpret_cast. 1481 UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultPtr); 1482 *ResultWidePtr = ResultChar; 1483 ResultPtr += 4; 1484 } else if (CharByteWidth == 2) { 1485 // FIXME: Make the type of the result buffer correct instead of 1486 // using reinterpret_cast. 1487 UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultPtr); 1488 *ResultWidePtr = ResultChar & 0xFFFF; 1489 ResultPtr += 2; 1490 } else { 1491 assert(CharByteWidth == 1 && "Unexpected char width"); 1492 *ResultPtr++ = ResultChar & 0xFF; 1493 } 1494 } 1495 } 1496 } 1497 1498 if (Pascal) { 1499 if (CharByteWidth == 4) { 1500 // FIXME: Make the type of the result buffer correct instead of 1501 // using reinterpret_cast. 1502 UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultBuf.data()); 1503 ResultWidePtr[0] = GetNumStringChars() - 1; 1504 } else if (CharByteWidth == 2) { 1505 // FIXME: Make the type of the result buffer correct instead of 1506 // using reinterpret_cast. 1507 UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultBuf.data()); 1508 ResultWidePtr[0] = GetNumStringChars() - 1; 1509 } else { 1510 assert(CharByteWidth == 1 && "Unexpected char width"); 1511 ResultBuf[0] = GetNumStringChars() - 1; 1512 } 1513 1514 // Verify that pascal strings aren't too large. 1515 if (GetStringLength() > 256) { 1516 if (Diags) 1517 Diags->Report(StringToks.front().getLocation(), 1518 diag::err_pascal_string_too_long) 1519 << SourceRange(StringToks.front().getLocation(), 1520 StringToks.back().getLocation()); 1521 hadError = true; 1522 return; 1523 } 1524 } else if (Diags) { 1525 // Complain if this string literal has too many characters. 1526 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509; 1527 1528 if (GetNumStringChars() > MaxChars) 1529 Diags->Report(StringToks.front().getLocation(), 1530 diag::ext_string_too_long) 1531 << GetNumStringChars() << MaxChars 1532 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0) 1533 << SourceRange(StringToks.front().getLocation(), 1534 StringToks.back().getLocation()); 1535 } 1536 } 1537 1538 static const char *resyncUTF8(const char *Err, const char *End) { 1539 if (Err == End) 1540 return End; 1541 End = Err + std::min<unsigned>(getNumBytesForUTF8(*Err), End-Err); 1542 while (++Err != End && (*Err & 0xC0) == 0x80) 1543 ; 1544 return Err; 1545 } 1546 1547 /// \brief This function copies from Fragment, which is a sequence of bytes 1548 /// within Tok's contents (which begin at TokBegin) into ResultPtr. 1549 /// Performs widening for multi-byte characters. 1550 bool StringLiteralParser::CopyStringFragment(const Token &Tok, 1551 const char *TokBegin, 1552 StringRef Fragment) { 1553 const UTF8 *ErrorPtrTmp; 1554 if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp)) 1555 return false; 1556 1557 // If we see bad encoding for unprefixed string literals, warn and 1558 // simply copy the byte values, for compatibility with gcc and older 1559 // versions of clang. 1560 bool NoErrorOnBadEncoding = isAscii(); 1561 if (NoErrorOnBadEncoding) { 1562 memcpy(ResultPtr, Fragment.data(), Fragment.size()); 1563 ResultPtr += Fragment.size(); 1564 } 1565 1566 if (Diags) { 1567 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 1568 1569 FullSourceLoc SourceLoc(Tok.getLocation(), SM); 1570 const DiagnosticBuilder &Builder = 1571 Diag(Diags, Features, SourceLoc, TokBegin, 1572 ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()), 1573 NoErrorOnBadEncoding ? diag::warn_bad_string_encoding 1574 : diag::err_bad_string_encoding); 1575 1576 const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 1577 StringRef NextFragment(NextStart, Fragment.end()-NextStart); 1578 1579 // Decode into a dummy buffer. 1580 SmallString<512> Dummy; 1581 Dummy.reserve(Fragment.size() * CharByteWidth); 1582 char *Ptr = Dummy.data(); 1583 1584 while (!ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) { 1585 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 1586 NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 1587 Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin, 1588 ErrorPtr, NextStart); 1589 NextFragment = StringRef(NextStart, Fragment.end()-NextStart); 1590 } 1591 } 1592 return !NoErrorOnBadEncoding; 1593 } 1594 1595 void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) { 1596 hadError = true; 1597 if (Diags) 1598 Diags->Report(Loc, diag::err_lexing_string); 1599 } 1600 1601 /// getOffsetOfStringByte - This function returns the offset of the 1602 /// specified byte of the string data represented by Token. This handles 1603 /// advancing over escape sequences in the string. 1604 unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok, 1605 unsigned ByteNo) const { 1606 // Get the spelling of the token. 1607 SmallString<32> SpellingBuffer; 1608 SpellingBuffer.resize(Tok.getLength()); 1609 1610 bool StringInvalid = false; 1611 const char *SpellingPtr = &SpellingBuffer[0]; 1612 unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features, 1613 &StringInvalid); 1614 if (StringInvalid) 1615 return 0; 1616 1617 const char *SpellingStart = SpellingPtr; 1618 const char *SpellingEnd = SpellingPtr+TokLen; 1619 1620 // Handle UTF-8 strings just like narrow strings. 1621 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8') 1622 SpellingPtr += 2; 1623 1624 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' && 1625 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet"); 1626 1627 // For raw string literals, this is easy. 1628 if (SpellingPtr[0] == 'R') { 1629 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!"); 1630 // Skip 'R"'. 1631 SpellingPtr += 2; 1632 while (*SpellingPtr != '(') { 1633 ++SpellingPtr; 1634 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal"); 1635 } 1636 // Skip '('. 1637 ++SpellingPtr; 1638 return SpellingPtr - SpellingStart + ByteNo; 1639 } 1640 1641 // Skip over the leading quote 1642 assert(SpellingPtr[0] == '"' && "Should be a string literal!"); 1643 ++SpellingPtr; 1644 1645 // Skip over bytes until we find the offset we're looking for. 1646 while (ByteNo) { 1647 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!"); 1648 1649 // Step over non-escapes simply. 1650 if (*SpellingPtr != '\\') { 1651 ++SpellingPtr; 1652 --ByteNo; 1653 continue; 1654 } 1655 1656 // Otherwise, this is an escape character. Advance over it. 1657 bool HadError = false; 1658 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U') { 1659 const char *EscapePtr = SpellingPtr; 1660 unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd, 1661 1, Features, HadError); 1662 if (Len > ByteNo) { 1663 // ByteNo is somewhere within the escape sequence. 1664 SpellingPtr = EscapePtr; 1665 break; 1666 } 1667 ByteNo -= Len; 1668 } else { 1669 ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError, 1670 FullSourceLoc(Tok.getLocation(), SM), 1671 CharByteWidth*8, Diags, Features); 1672 --ByteNo; 1673 } 1674 assert(!HadError && "This method isn't valid on erroneous strings"); 1675 } 1676 1677 return SpellingPtr-SpellingStart; 1678 } 1679