1 /* Copyright (c) 2008-2010, Google Inc. 2 * All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are 6 * met: 7 * 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Neither the name of Google Inc. nor the names of its 11 * contributors may be used to endorse or promote products derived from 12 * this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 // This file is part of ThreadSanitizer, a dynamic data race detector. 28 // Author: Konstantin Serebryany. 29 // Author: Timur Iskhodzhanov. 30 // 31 // See ts_util.h for mode details. 32 33 #include "common_util.h" 34 #include "thread_sanitizer.h" 35 #include "ts_stats.h" 36 #include "ts_lock.h" 37 #include <stdarg.h> 38 39 FLAGS *G_flags = NULL; 40 41 #if defined(_MSC_VER) 42 43 #pragma comment(lib, "winmm.lib") 44 45 # ifdef TS_PIN 46 # include "pin.H" 47 # endif 48 namespace WINDOWS 49 { 50 // This is the way of including winows.h recommended by PIN docs. 51 #include<Windows.h> 52 } 53 int getpid() { return WINDOWS::GetCurrentProcessId(); } 54 #endif 55 56 #if defined(TS_VALGRIND) 57 size_t TimeInMilliSeconds() { 58 return VG_(read_millisecond_timer)(); 59 } 60 #else 61 // TODO(kcc): implement this. 62 size_t TimeInMilliSeconds() { 63 #ifdef __GNUC__ 64 return time(0) * 1000; 65 #else 66 return WINDOWS::timeGetTime(); 67 #endif 68 } 69 #endif 70 71 Stats *G_stats; 72 73 #ifndef TS_LLVM 74 bool GetNameAndOffsetOfGlobalObject(uintptr_t addr, 75 string *name, uintptr_t *offset) { 76 # ifdef TS_VALGRIND 77 const int kBufLen = 1023; 78 char buff[kBufLen+1]; 79 PtrdiffT off; 80 if (VG_(get_datasym_and_offset)(addr, reinterpret_cast<Char*>(buff), 81 kBufLen, &off)) { 82 *name = buff; 83 *offset = off; 84 return true; 85 } 86 return false; 87 # else 88 return false; 89 # endif // TS_VALGRIND 90 } 91 #endif // TS_LLVM 92 93 94 #ifndef TS_VALGRIND 95 void GetThreadStack(int tid, uintptr_t *min_addr, uintptr_t *max_addr) { 96 *min_addr = 0xfffa; 97 *max_addr = 0xfffb; 98 } 99 #endif 100 101 static int n_errs_found; 102 103 void SetNumberOfFoundErrors(int n_errs) { 104 n_errs_found = n_errs; 105 } 106 107 int GetNumberOfFoundErrors() { 108 return n_errs_found; 109 } 110 111 112 #if !defined(TS_VALGRIND) && !defined(TS_LLVM) 113 FILE *G_out = stderr; 114 #endif 115 116 #ifdef TS_LLVM 117 FILE *G_out; 118 #endif 119 120 static string RemoveUnsupportedFormat(const char *str) { 121 #ifdef _MSC_VER 122 // replace "%'" with "%" 123 string res; 124 size_t n = strlen(str); 125 if (n == 0) { 126 return ""; 127 } 128 res.reserve(n); 129 res.push_back(str[0]); 130 for (size_t i = 1; i < n; i++) { 131 if (str[i] == '\'' && *res.rbegin() == '%') continue; 132 res.push_back(str[i]); 133 } 134 return res; 135 #else 136 return str; 137 #endif 138 } 139 140 void Printf(const char *format, ...) { 141 #ifdef TS_VALGRIND 142 va_list args; 143 va_start(args, format); 144 VG_(vprintf)(format, args); 145 va_end(args); 146 #else 147 va_list args; 148 va_start(args, format); 149 vfprintf(G_out, RemoveUnsupportedFormat(format).c_str(), args); 150 fflush(G_out); 151 va_end(args); 152 #endif 153 } 154 155 // Like Print(), but prepend each line with ==XXXXX==, 156 // where XXXXX is the pid. 157 void Report(const char *format, ...) { 158 int buff_size = 1024*16; 159 char *buff = new char[buff_size]; 160 CHECK(buff); 161 DCHECK(G_flags); 162 163 va_list args; 164 165 while (1) { 166 va_start(args, format); 167 int ret = vsnprintf(buff, buff_size, 168 RemoveUnsupportedFormat(format).c_str(), args); 169 va_end(args); 170 if (ret < buff_size) break; 171 delete [] buff; 172 buff_size *= 2; 173 buff = new char[buff_size]; 174 CHECK(buff); 175 // Printf("Resized buff: %d\n", buff_size); 176 } 177 178 char pid_buff[100]; 179 snprintf(pid_buff, sizeof(pid_buff), "==%d== ", getpid()); 180 181 string res; 182 #ifndef TS_LLVM 183 int len = strlen(buff); 184 #else 185 int len = __real_strlen(buff); 186 #endif 187 bool last_was_new_line = true; 188 for (int i = 0; i < len; i++) { 189 if (G_flags->show_pid && last_was_new_line) 190 res += pid_buff; 191 last_was_new_line = (buff[i] == '\n'); 192 res += buff[i]; 193 } 194 195 delete [] buff; 196 197 Printf("%s", res.c_str()); 198 } 199 200 long my_strtol(const char *str, char **end, int base) { 201 #ifdef TS_VALGRIND 202 if (base == 16 || (base == 0 && str && str[0] == '0' && str[1] == 'x')) { 203 return VG_(strtoll16)((Char*)str, (Char**)end); 204 } 205 return VG_(strtoll10)((Char*)str, (Char**)end); 206 #else 207 return strtoll(str, end, base); 208 #endif 209 } 210 211 // Not thread-safe. Need to make it thread-local if we allow 212 // malloc to be called concurrently. 213 MallocCostCenterStack g_malloc_stack; 214 215 size_t GetVmSizeInMb() { 216 #ifdef VGO_linux 217 const char *path ="/proc/self/statm"; // see 'man proc' 218 uintptr_t counter = G_stats->read_proc_self_stats++; 219 if (counter >= 1024 && ((counter & (counter - 1)) == 0)) 220 Report("INFO: reading %s for %ld'th time\n", path, counter); 221 int fd = OpenFileReadOnly(path, false); 222 if (fd < 0) return 0; 223 char buff[128]; 224 int n_read = read(fd, buff, sizeof(buff) - 1); 225 buff[n_read] = 0; 226 close(fd); 227 char *end; 228 size_t vm_size_in_pages = my_strtol(buff, &end, 10); 229 return vm_size_in_pages >> 8; 230 #else 231 return 0; 232 #endif 233 } 234 235 static string StripTemplatesFromFunctionName(const string &fname) { 236 // Returns "" in case of error. 237 238 string ret; 239 size_t read_pointer = 0, braces_depth = 0; 240 241 while (read_pointer < fname.size()) { 242 size_t next_brace = fname.find_first_of("<>", read_pointer); 243 if (next_brace == fname.npos) { 244 if (braces_depth > 0) { 245 // This can happen on Visual Studio if we reach the ~2000 char limit. 246 CHECK(fname.size() > 256); 247 return ""; 248 } 249 ret += (fname.c_str() + read_pointer); 250 break; 251 } 252 253 if (braces_depth == 0) { 254 ret.append(fname, read_pointer, next_brace - read_pointer); 255 } 256 257 if (next_brace > 0) { 258 // We could have found one of the following operators. 259 const char *OP[] = {">>=", "<<=", 260 ">>", "<<", 261 ">=", "<=", 262 "->", "->*", 263 "<", ">"}; 264 265 bool operator_name = false; 266 for (size_t i = 0; i < TS_ARRAY_SIZE(OP); i++) { 267 size_t op_offset = ((string)OP[i]).find(fname[next_brace]); 268 if (op_offset == string::npos) 269 continue; 270 if (next_brace >= 8 + op_offset && // 8 == strlen("operator"); 271 "operator" == fname.substr(next_brace - (8 + op_offset), 8) && 272 OP[i] == fname.substr(next_brace - op_offset, strlen(OP[i]))) { 273 operator_name = true; 274 ret += OP[i] + op_offset; 275 next_brace += strlen(OP[i] + op_offset); 276 read_pointer = next_brace; 277 break; 278 } 279 } 280 281 if (operator_name) 282 continue; 283 } 284 285 if (fname[next_brace] == '<') { 286 braces_depth++; 287 read_pointer = next_brace + 1; 288 } else if (fname[next_brace] == '>') { 289 if (braces_depth == 0) { 290 // Going to `braces_depth == -1` IS possible at least for this function on Windows: 291 // "std::operator<<char,std::char_traits<char>,std::allocator<char> >". 292 // Oh, well... Return an empty string and let the caller decide. 293 return ""; 294 } 295 braces_depth--; 296 read_pointer = next_brace + 1; 297 } else 298 CHECK(0); 299 } 300 if (braces_depth != 0) { 301 CHECK(fname.size() > 256); 302 return ""; 303 } 304 return ret; 305 } 306 307 static string StripParametersFromFunctionName(const string &demangled) { 308 // Returns "" in case of error. 309 310 string fname = demangled; 311 312 // Strip stuff like "(***)" and "(anonymous namespace)" -> they are tricky. 313 size_t found = fname.npos; 314 while ((found = fname.find(", ")) != fname.npos) 315 fname.erase(found+1, 1); 316 while ((found = fname.find("(**")) != fname.npos) 317 fname.erase(found+2, 1); 318 while ((found = fname.find("(*)")) != fname.npos) 319 fname.erase(found, 3); 320 while ((found = fname.find("const()")) != fname.npos) 321 fname.erase(found+5, 2); 322 while ((found = fname.find("const volatile")) != fname.npos && 323 found > 1 && found + 14 == fname.size()) 324 fname.erase(found-1); 325 while ((found = fname.find("(anonymous namespace)")) != fname.npos) 326 fname.erase(found, 21); 327 328 if (fname.find_first_of("(") == fname.npos) 329 return fname; 330 DCHECK(count(fname.begin(), fname.end(), '(') == 331 count(fname.begin(), fname.end(), ')')); 332 333 string ret; 334 bool returns_fun_ptr = false; 335 size_t braces_depth = 0, read_pointer = 0; 336 337 size_t first_parenthesis = fname.find("("); 338 if (first_parenthesis != fname.npos) { 339 DCHECK(fname.find_first_of(")") != fname.npos); 340 DCHECK(fname.find_first_of(")") > first_parenthesis); 341 DCHECK(fname[first_parenthesis] == '('); 342 if (first_parenthesis + 2 < fname.size() && 343 fname[first_parenthesis - 1] == ' ' && 344 fname[first_parenthesis + 1] == '*' && 345 fname[first_parenthesis + 2] != ' ') { 346 // Return value type is a function pointer 347 read_pointer = first_parenthesis + 2; 348 while (fname[read_pointer] == '*' || fname[read_pointer] == '&') 349 read_pointer++; 350 braces_depth = 1; 351 returns_fun_ptr = true; 352 } 353 } 354 355 while (read_pointer < fname.size()) { 356 size_t next_brace = fname.find_first_of("()", read_pointer); 357 if (next_brace == fname.npos) { 358 if (braces_depth != 0) { 359 // Overflow? 360 return ""; 361 } 362 size_t _const = fname.find(" const", read_pointer); 363 if (_const == fname.npos) { 364 ret += (fname.c_str() + read_pointer); 365 } else { 366 CHECK(_const + 6 == fname.size()); 367 ret.append(fname, read_pointer, _const - read_pointer); 368 } 369 break; 370 } 371 372 if (braces_depth == (returns_fun_ptr ? 1 : 0)) { 373 ret.append(fname, read_pointer, next_brace - read_pointer); 374 returns_fun_ptr = false; 375 } 376 377 if (fname[next_brace] == '(') { 378 if (next_brace >= 8 && fname[next_brace+1] == ')' && 379 "operator" == fname.substr(next_brace - 8, 8)) { 380 ret += "()"; 381 read_pointer = next_brace + 2; 382 } else { 383 braces_depth++; 384 read_pointer = next_brace + 1; 385 } 386 } else if (fname[next_brace] == ')') { 387 CHECK(braces_depth > 0); 388 braces_depth--; 389 read_pointer = next_brace + 1; 390 } else 391 CHECK(0); 392 } 393 if (braces_depth != 0) 394 return ""; 395 396 // Special case: on Linux, Valgrind prepends the return type for template 397 // functions. And on Windows we may see `scalar deleting destructor'. 398 // And we may see "operaror new" etc. 399 // And some STL code inserts const& between the return type and the function 400 // name. 401 // Oh, well... 402 size_t space_or_tick; 403 while (ret != "") { 404 space_or_tick = ret.find_first_of("` "); 405 if (space_or_tick != ret.npos && ret[space_or_tick] == ' ' && 406 ret.substr(0, space_or_tick).find("operator") == string::npos) { 407 ret = ret.substr(space_or_tick + 1); 408 } else if (space_or_tick != ret.npos && space_or_tick + 1 == ret.size()) { 409 ret = ret.substr(0, space_or_tick); 410 } else { 411 break; 412 } 413 } 414 return ret; 415 } 416 417 string NormalizeFunctionName(const string &demangled) { 418 if (demangled[1] == '[' && strchr("+-=", demangled[0]) != NULL) { 419 // Objective-C function 420 return demangled; 421 } 422 423 if (demangled.find_first_of("<>()") == demangled.npos) { 424 // C function or a well-formatted function name. 425 return demangled; 426 } 427 428 if (demangled == "(below main)" || demangled == "(no symbols)") 429 return demangled; 430 431 const char* const MALFORMED = "(malformed frame)"; 432 433 string fname = StripTemplatesFromFunctionName(demangled); 434 if (fname.size() == 0) { 435 if (DEBUG_MODE) 436 Printf("PANIC: `%s`\n", demangled.c_str()); 437 return MALFORMED; 438 } 439 440 fname = StripParametersFromFunctionName(fname); 441 if (fname.size() == 0) { 442 CHECK(demangled.size() >= 256); 443 if (DEBUG_MODE) 444 Printf("PANIC: `%s`\n", demangled.c_str()); 445 return MALFORMED; 446 } 447 448 return fname; 449 } 450 451 void OpenFileWriteStringAndClose(const string &file_name, const string &str) { 452 #ifdef TS_VALGRIND 453 SysRes sres = VG_(open)((const Char*)file_name.c_str(), 454 VKI_O_WRONLY|VKI_O_CREAT|VKI_O_TRUNC, 455 VKI_S_IRUSR|VKI_S_IWUSR); 456 if (sr_isError(sres)) { 457 Report("WARNING: can not open file %s\n", file_name.c_str()); 458 exit(1); 459 } 460 int fd = sr_Res(sres); 461 write(fd, str.c_str(), str.size()); 462 close(fd); 463 #else 464 CHECK(0); 465 #endif 466 } 467 468 //--------- Sockets ------------------ {{{1 469 #if defined(TS_PIN) && defined(__GNUC__) 470 #include <sys/types.h> 471 #include <sys/socket.h> 472 #include <netinet/in.h> 473 #include <netdb.h> 474 FILE *OpenSocketForWriting(const string &host_and_port) { 475 size_t col = host_and_port.find(":"); 476 if (col == string::npos) return NULL; 477 string host = host_and_port.substr(0, col); 478 string port_str = host_and_port.substr(col + 1); 479 int sockfd; 480 struct sockaddr_in serv_addr; 481 struct hostent *server; 482 sockfd = socket(AF_INET, SOCK_STREAM, 0); 483 if (sockfd < 0) return NULL; 484 server = gethostbyname(host.c_str()); 485 if (server == 0) return NULL; 486 memset(&serv_addr, 0, sizeof(serv_addr)); 487 serv_addr.sin_family = AF_INET; 488 memcpy((char *)&serv_addr.sin_addr.s_addr, 489 (char *)server->h_addr, 490 server->h_length); 491 serv_addr.sin_port = htons(atoi(port_str.c_str())); 492 if (connect(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) 493 return NULL; 494 return fdopen(sockfd, "w"); 495 } 496 #else 497 FILE *OpenSocketForWriting(const string &host_and_port) { 498 return NULL; // unimplemented. 499 } 500 #endif 501 //--------- TSLock ------------------ {{{1 502 #ifdef _MSC_VER 503 //# define TS_LOCK_PIPE 504 # define TS_LOCK_PIN 505 #else 506 # define TS_LOCK_FUTEX 507 #endif 508 509 #if defined(TS_LOCK_PIPE) && defined(TS_PIN) 510 #ifdef __GNUC__ 511 #include <unistd.h> 512 // Lock based on pipe's send/receive. The idea (but not the code) 513 // is shamelessly stolen from valgrind's /coregrind/m_scheduler/sema.c 514 struct TSLock::Rep { 515 bool held; 516 char pipe_char; 517 int pipe_fd[2]; 518 519 void Write() { 520 char buf[2]; 521 buf[0] = pipe_char; 522 buf[1] = 0; 523 int res = write(pipe_fd[1], buf, 1); 524 CHECK(res == 1); 525 } 526 bool Read() { 527 char buf[2]; 528 buf[0] = 0; 529 buf[1] = 0; 530 int res = read(pipe_fd[0], buf, 1); 531 if (res != 1) 532 return false; 533 //Printf("rep::Read: %c\n", buf[0]); 534 535 pipe_char++; 536 if (pipe_char == 'Z' + 1) pipe_char = 'A'; 537 return true; 538 } 539 void Open() { 540 CHECK(0 == pipe(pipe_fd)); 541 CHECK(pipe_fd[0] != pipe_fd[1]); 542 pipe_char = 'A'; 543 } 544 void Close() { 545 close(pipe_fd[0]); 546 close(pipe_fd[1]); 547 } 548 }; 549 #elif defined(_MSC_VER) 550 struct TSLock::Rep { 551 bool held; 552 char pipe_char; 553 WINDOWS::HANDLE pipe_fd[2]; 554 void Write() { 555 char buf[2]; 556 buf[0] = pipe_char; 557 buf[1] = 0; 558 WINDOWS::DWORD n_written = 0; 559 int res = WINDOWS::WriteFile(pipe_fd[1], buf, 1, &n_written, NULL); 560 CHECK(res != 0 && n_written == 1); 561 } 562 bool Read() { 563 char buf[2]; 564 buf[0] = 0; 565 buf[1] = 0; 566 WINDOWS::DWORD n_read = 0; 567 int res = WINDOWS::ReadFile(pipe_fd[0], buf, 1, &n_read, NULL); 568 if (res == 0 && n_read == 0) 569 return false; 570 //Printf("rep::Read: %c\n", buf[0]); 571 572 pipe_char++; 573 if (pipe_char == 'Z' + 1) pipe_char = 'A'; 574 return true; 575 } 576 void Open() { 577 CHECK(WINDOWS::CreatePipe(&pipe_fd[0], &pipe_fd[1], NULL, 0)); 578 CHECK(pipe_fd[0] != pipe_fd[1]); 579 pipe_char = 'A'; 580 } 581 void Close() { 582 WINDOWS::CloseHandle(pipe_fd[0]); 583 WINDOWS::CloseHandle(pipe_fd[1]); 584 } 585 }; 586 #endif 587 588 TSLock::TSLock() { 589 rep_ = new Rep; 590 rep_->held = false; 591 rep_->Open(); 592 rep_->Write(); 593 } 594 TSLock::~TSLock() { 595 rep_->Close(); 596 } 597 void TSLock::Lock() { 598 while(rep_->Read() == false) 599 ; 600 rep_->held = true; 601 } 602 void TSLock::Unlock() { 603 rep_->held = false; 604 rep_->Write(); 605 } 606 void TSLock::AssertHeld() { 607 DCHECK(rep_->held); 608 } 609 #endif // __GNUC__ & TS_LOCK_PIPE 610 611 #if defined(TS_LOCK_PIN) && defined(TS_PIN) 612 #include "pin.H" 613 struct TSLock::Rep { 614 PIN_LOCK lock; 615 bool held; 616 }; 617 618 TSLock::TSLock() { 619 rep_ = new Rep(); 620 rep_->held = false; 621 InitLock(&rep_->lock); 622 } 623 TSLock::~TSLock() { 624 delete rep_; 625 } 626 void TSLock::Lock() { 627 GetLock(&rep_->lock, __LINE__); 628 rep_->held = true; 629 } 630 void TSLock::Unlock() { 631 rep_->held = false; 632 ReleaseLock(&rep_->lock); 633 } 634 void TSLock::AssertHeld() { 635 DCHECK(rep_->held); 636 } 637 #endif // TS_LOCK_PIN 638 639 #if defined(TS_WRAP_PTHREAD_LOCK) 640 #include "tsan_rtl_wrap.h" 641 642 struct TSLock::Rep { 643 pthread_mutex_t lock; 644 bool held; 645 }; 646 TSLock::TSLock() { 647 rep_ = new Rep(); 648 rep_->held = false; 649 __real_pthread_mutex_init(&rep_->lock, NULL); 650 } 651 TSLock::~TSLock() { 652 __real_pthread_mutex_destroy(&rep_->lock); 653 delete rep_; 654 } 655 void TSLock::Lock() { 656 __real_pthread_mutex_lock(&rep_->lock); 657 rep_->held = true; 658 } 659 void TSLock::Unlock() { 660 rep_->held = false; 661 __real_pthread_mutex_unlock(&rep_->lock); 662 } 663 void TSLock::AssertHeld() { 664 DCHECK(rep_->held); 665 } 666 #endif // TS_LLVM 667 668 #if defined(TS_LOCK_FUTEX) && defined(__GNUC__) && \ 669 (defined (TS_PIN) || defined (TS_LLVM)) 670 #include <linux/futex.h> 671 #include <sys/time.h> 672 #include <syscall.h> 673 674 // Simple futex-based lock. 675 // The idea is taken from "Futexes Are Tricky" by Ulrich Drepper 676 677 TSLock::TSLock() { 678 rep_ = 0; 679 ANNOTATE_BENIGN_RACE(&rep_, "Benign race on TSLock::rep_"); 680 ANNOTATE_RWLOCK_CREATE(this); 681 } 682 TSLock::~TSLock() { 683 ANNOTATE_RWLOCK_DESTROY(this); 684 DCHECK(rep_ == 0); 685 } 686 void TSLock::Lock() { 687 int *p = (int*)&rep_; 688 const int kSpinCount = 100; 689 DCHECK(kSpinCount > 0); 690 int c; 691 for (int i = 0; i < kSpinCount; i++) { 692 c = __sync_val_compare_and_swap(p, 0, 1); 693 if (c == 0) break; 694 } 695 if (c == 0) { 696 // The mutex was unlocked. Now it's ours. Done. 697 ANNOTATE_RWLOCK_ACQUIRED(this, /*is_w*/true); 698 return; 699 } 700 DCHECK(c == 1 || c == 2); 701 // We are going to block on this lock. Make sure others know that. 702 if (c != 2) { 703 c = __sync_lock_test_and_set(p, 2); 704 } 705 // Block. 706 int n_waits = 0; 707 while (c != 0) { 708 syscall(SYS_futex, p, FUTEX_WAIT, 2, 0, 0, 0); 709 n_waits++; 710 c = __sync_lock_test_and_set(p, 2); 711 } 712 ANNOTATE_RWLOCK_ACQUIRED(this, /*is_w*/true); 713 G_stats->futex_wait += n_waits; 714 } 715 void TSLock::Unlock() { 716 ANNOTATE_RWLOCK_RELEASED(this, /*is_w*/true); 717 int *p = (int*)&rep_; 718 DCHECK(*p == 1 || *p == 2); 719 int c = __sync_sub_and_fetch(p, 1); 720 DCHECK(c == 0 || c == 1); 721 if (c == 1) { 722 *p = 0; 723 syscall(SYS_futex, p, FUTEX_WAKE, 1, 0, 0, 0); 724 } 725 } 726 void TSLock::AssertHeld() { 727 DCHECK(rep_); 728 } 729 #endif 730 731 // Same as above to compile Go's rtl 732 // No annotations in this version: it should be simple as possible. 733 #if defined(TS_LOCK_FUTEX) && defined(__GNUC__) && \ 734 (defined (TS_GO)) 735 #include <linux/futex.h> // TODO(mpimenov): portability? 736 #include <sys/time.h> 737 #include <syscall.h> 738 739 // Simple futex-based lock. 740 // The idea is taken from "Futexes Are Tricky" by Ulrich Drepper 741 742 TSLock::TSLock() { 743 rep_ = 0; 744 } 745 TSLock::~TSLock() { 746 DCHECK(rep_ == 0); 747 } 748 void TSLock::Lock() { 749 int *p = (int*)&rep_; 750 const int kSpinCount = 100; 751 DCHECK(kSpinCount > 0); 752 int c; 753 for (int i = 0; i < kSpinCount; i++) { 754 c = __sync_val_compare_and_swap(p, 0, 1); 755 if (c == 0) break; 756 } 757 if (c == 0) { 758 // The mutex was unlocked. Now it's ours. Done. 759 return; 760 } 761 DCHECK(c == 1 || c == 2); 762 // We are going to block on this lock. Make sure others know that. 763 if (c != 2) { 764 c = __sync_lock_test_and_set(p, 2); 765 } 766 // Block. 767 int n_waits = 0; 768 while (c != 0) { 769 syscall(SYS_futex, p, FUTEX_WAIT, 2, 0, 0, 0); 770 n_waits++; 771 c = __sync_lock_test_and_set(p, 2); 772 } 773 G_stats->futex_wait += n_waits; 774 } 775 void TSLock::Unlock() { 776 int *p = (int*)&rep_; 777 DCHECK(*p == 1 || *p == 2); 778 int c = __sync_sub_and_fetch(p, 1); 779 DCHECK(c == 0 || c == 1); 780 if (c == 1) { 781 *p = 0; 782 syscall(SYS_futex, p, FUTEX_WAKE, 1, 0, 0, 0); 783 } 784 } 785 void TSLock::AssertHeld() { 786 DCHECK(rep_); 787 } 788 #endif // (TS_LOCK_FUTEX) (__GNUC__) && (TS_GO) 789 790 //--------------- Atomics ----------------- {{{1 791 #if defined (_MSC_VER) && TS_SERIALIZED == 0 792 uintptr_t AtomicExchange(uintptr_t *ptr, uintptr_t new_value) { 793 return _InterlockedExchange((volatile WINDOWS::LONG*)ptr, new_value); 794 } 795 796 void ReleaseStore(uintptr_t *ptr, uintptr_t value) { 797 *(volatile uintptr_t*)ptr = value; 798 // TODO(kcc): anything to add here? 799 } 800 801 int32_t NoBarrier_AtomicIncrement(int32_t* ptr) { 802 return _InterlockedIncrement((volatile WINDOWS::LONG *)ptr); 803 } 804 805 int32_t NoBarrier_AtomicDecrement(int32_t* ptr) { 806 return _InterlockedDecrement((volatile WINDOWS::LONG *)ptr); 807 } 808 #endif // _MSC_VER && TS_SERIALIZED 809 //--------------- YIELD ----------------- {{{1 810 #if defined (_MSC_VER) 811 #include <intrin.h> 812 void YIELD() { 813 WINDOWS::Sleep(0); 814 } 815 void PROCESSOR_YIELD() { 816 _mm_pause(); 817 } 818 #elif defined(TS_VALGRIND) 819 void YIELD() { 820 } 821 void PROCESSOR_YIELD() { 822 } 823 #elif defined(__GNUC__) 824 void YIELD() { 825 sched_yield(); 826 } 827 void PROCESSOR_YIELD() { 828 __asm__ __volatile__ ("pause"); 829 } 830 #else 831 #error "Unknown config" 832 #endif 833 834 // end. {{{1 835 // vim:shiftwidth=2:softtabstop=2:expandtab:tw=80 836
