1 //===-- tsan_interceptors_linux.cc ----------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer (TSan), a race detector. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "interception/interception.h" 15 #include "sanitizer_common/sanitizer_atomic.h" 16 #include "sanitizer_common/sanitizer_libc.h" 17 #include "sanitizer_common/sanitizer_placement_new.h" 18 #include "tsan_rtl.h" 19 #include "tsan_interface.h" 20 #include "tsan_platform.h" 21 #include "tsan_mman.h" 22 23 using namespace __tsan; // NOLINT 24 25 const int kSigCount = 128; 26 27 struct my_siginfo_t { 28 int opaque[128]; 29 }; 30 31 struct sigset_t { 32 u64 val[1024 / 8 / sizeof(u64)]; 33 }; 34 35 struct ucontext_t { 36 uptr opaque[117]; 37 }; 38 39 extern "C" int pthread_attr_init(void *attr); 40 extern "C" int pthread_attr_destroy(void *attr); 41 extern "C" int pthread_attr_getdetachstate(void *attr, int *v); 42 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); 43 extern "C" int pthread_attr_getstacksize(void *attr, uptr *stacksize); 44 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); 45 extern "C" int pthread_setspecific(unsigned key, const void *v); 46 extern "C" int pthread_mutexattr_gettype(void *a, int *type); 47 extern "C" int pthread_yield(); 48 extern "C" int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); 49 extern "C" int sigfillset(sigset_t *set); 50 extern "C" void *pthread_self(); 51 extern "C" void _exit(int status); 52 extern "C" int __cxa_atexit(void (*func)(void *arg), void *arg, void *dso); 53 extern "C" int *__errno_location(); 54 const int PTHREAD_MUTEX_RECURSIVE = 1; 55 const int PTHREAD_MUTEX_RECURSIVE_NP = 1; 56 const int kPthreadAttrSize = 56; 57 const int EINVAL = 22; 58 const int EBUSY = 16; 59 const int EPOLL_CTL_ADD = 1; 60 const int SIGILL = 4; 61 const int SIGABRT = 6; 62 const int SIGFPE = 8; 63 const int SIGSEGV = 11; 64 const int SIGPIPE = 13; 65 const int SIGBUS = 7; 66 void *const MAP_FAILED = (void*)-1; 67 const int PTHREAD_BARRIER_SERIAL_THREAD = -1; 68 const int MAP_FIXED = 0x10; 69 typedef long long_t; // NOLINT 70 71 typedef void (*sighandler_t)(int sig); 72 73 #define errno (*__errno_location()) 74 75 union pthread_attr_t { 76 char size[kPthreadAttrSize]; 77 void *align; 78 }; 79 80 struct sigaction_t { 81 union { 82 sighandler_t sa_handler; 83 void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx); 84 }; 85 sigset_t sa_mask; 86 int sa_flags; 87 void (*sa_restorer)(); 88 }; 89 90 const sighandler_t SIG_DFL = (sighandler_t)0; 91 const sighandler_t SIG_IGN = (sighandler_t)1; 92 const sighandler_t SIG_ERR = (sighandler_t)-1; 93 const int SA_SIGINFO = 4; 94 const int SIG_SETMASK = 2; 95 96 static sigaction_t sigactions[kSigCount]; 97 98 namespace __tsan { 99 struct SignalDesc { 100 bool armed; 101 bool sigaction; 102 my_siginfo_t siginfo; 103 ucontext_t ctx; 104 }; 105 106 struct SignalContext { 107 int int_signal_send; 108 int pending_signal_count; 109 SignalDesc pending_signals[kSigCount]; 110 }; 111 } 112 113 static SignalContext *SigCtx(ThreadState *thr) { 114 SignalContext *ctx = (SignalContext*)thr->signal_ctx; 115 if (ctx == 0 && thr->is_alive) { 116 ScopedInRtl in_rtl; 117 ctx = (SignalContext*)internal_alloc( 118 MBlockSignal, sizeof(*ctx)); 119 MemoryResetRange(thr, 0, (uptr)ctx, sizeof(*ctx)); 120 internal_memset(ctx, 0, sizeof(*ctx)); 121 thr->signal_ctx = ctx; 122 } 123 return ctx; 124 } 125 126 static unsigned g_thread_finalize_key; 127 128 static void process_pending_signals(ThreadState *thr); 129 130 class ScopedInterceptor { 131 public: 132 ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) 133 : thr_(thr) 134 , in_rtl_(thr->in_rtl) { 135 if (thr_->in_rtl == 0) { 136 Initialize(thr); 137 FuncEntry(thr, pc); 138 thr_->in_rtl++; 139 DPrintf("#%d: intercept %s()\n", thr_->tid, fname); 140 } else { 141 thr_->in_rtl++; 142 } 143 } 144 145 ~ScopedInterceptor() { 146 thr_->in_rtl--; 147 if (thr_->in_rtl == 0) { 148 FuncExit(thr_); 149 process_pending_signals(thr_); 150 } 151 CHECK_EQ(in_rtl_, thr_->in_rtl); 152 } 153 154 private: 155 ThreadState *const thr_; 156 const int in_rtl_; 157 }; 158 159 #define SCOPED_INTERCEPTOR_RAW(func, ...) \ 160 ThreadState *thr = cur_thread(); \ 161 StatInc(thr, StatInterceptor); \ 162 StatInc(thr, StatInt_##func); \ 163 ScopedInterceptor si(thr, #func, \ 164 (__sanitizer::uptr)__builtin_return_address(0)); \ 165 const uptr pc = (uptr)&func; \ 166 (void)pc; \ 167 /**/ 168 169 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ 170 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ 171 if (thr->in_rtl > 1) \ 172 return REAL(func)(__VA_ARGS__); \ 173 /**/ 174 175 #define SCOPED_INTERCEPTOR_LIBC(func, ...) \ 176 ThreadState *thr = cur_thread(); \ 177 StatInc(thr, StatInterceptor); \ 178 StatInc(thr, StatInt_##func); \ 179 ScopedInterceptor si(thr, #func, callpc); \ 180 const uptr pc = (uptr)&func; \ 181 (void)pc; \ 182 if (thr->in_rtl > 1) \ 183 return REAL(func)(__VA_ARGS__); \ 184 /**/ 185 186 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) 187 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) 188 189 // May be overriden by front-end. 190 extern "C" void WEAK __tsan_malloc_hook(void *ptr, uptr size) { 191 (void)ptr; 192 (void)size; 193 } 194 195 extern "C" void WEAK __tsan_free_hook(void *ptr) { 196 (void)ptr; 197 } 198 199 static void invoke_malloc_hook(void *ptr, uptr size) { 200 Context *ctx = CTX(); 201 ThreadState *thr = cur_thread(); 202 if (ctx == 0 || !ctx->initialized || thr->in_rtl) 203 return; 204 __tsan_malloc_hook(ptr, size); 205 } 206 207 static void invoke_free_hook(void *ptr) { 208 Context *ctx = CTX(); 209 ThreadState *thr = cur_thread(); 210 if (ctx == 0 || !ctx->initialized || thr->in_rtl) 211 return; 212 __tsan_free_hook(ptr); 213 } 214 215 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { 216 SCOPED_TSAN_INTERCEPTOR(sleep, sec); 217 unsigned res = sleep(sec); 218 AfterSleep(thr, pc); 219 return res; 220 } 221 222 TSAN_INTERCEPTOR(int, usleep, long_t usec) { 223 SCOPED_TSAN_INTERCEPTOR(usleep, usec); 224 int res = usleep(usec); 225 AfterSleep(thr, pc); 226 return res; 227 } 228 229 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { 230 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); 231 int res = nanosleep(req, rem); 232 AfterSleep(thr, pc); 233 return res; 234 } 235 236 class AtExitContext { 237 public: 238 AtExitContext() 239 : mtx_(MutexTypeAtExit, StatMtxAtExit) 240 , pos_() { 241 } 242 243 typedef void(*atexit_t)(); 244 245 int atexit(ThreadState *thr, uptr pc, atexit_t f) { 246 Lock l(&mtx_); 247 if (pos_ == kMaxAtExit) 248 return 1; 249 Release(thr, pc, (uptr)this); 250 stack_[pos_] = f; 251 pos_++; 252 return 0; 253 } 254 255 void exit(ThreadState *thr, uptr pc) { 256 CHECK_EQ(thr->in_rtl, 0); 257 for (;;) { 258 atexit_t f = 0; 259 { 260 Lock l(&mtx_); 261 if (pos_) { 262 pos_--; 263 f = stack_[pos_]; 264 ScopedInRtl in_rtl; 265 Acquire(thr, pc, (uptr)this); 266 } 267 } 268 if (f == 0) 269 break; 270 DPrintf("#%d: executing atexit func %p\n", thr->tid, f); 271 CHECK_EQ(thr->in_rtl, 0); 272 f(); 273 } 274 } 275 276 private: 277 static const int kMaxAtExit = 128; 278 Mutex mtx_; 279 atexit_t stack_[kMaxAtExit]; 280 int pos_; 281 }; 282 283 static AtExitContext *atexit_ctx; 284 285 static void finalize(void *arg) { 286 ThreadState * thr = cur_thread(); 287 uptr pc = 0; 288 atexit_ctx->exit(thr, pc); 289 { 290 ScopedInRtl in_rtl; 291 DestroyAndFree(atexit_ctx); 292 REAL(usleep)(flags()->atexit_sleep_ms * 1000); 293 } 294 int status = Finalize(cur_thread()); 295 if (status) 296 _exit(status); 297 } 298 299 TSAN_INTERCEPTOR(int, atexit, void (*f)()) { 300 SCOPED_TSAN_INTERCEPTOR(atexit, f); 301 return atexit_ctx->atexit(thr, pc, f); 302 return 0; 303 } 304 305 TSAN_INTERCEPTOR(void, longjmp, void *env, int val) { 306 SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); 307 TsanPrintf("ThreadSanitizer: longjmp() is not supported\n"); 308 Die(); 309 } 310 311 TSAN_INTERCEPTOR(void, siglongjmp, void *env, int val) { 312 SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); 313 TsanPrintf("ThreadSanitizer: siglongjmp() is not supported\n"); 314 Die(); 315 } 316 317 static uptr fd2addr(int fd) { 318 (void)fd; 319 static u64 addr; 320 return (uptr)&addr; 321 } 322 323 static uptr epollfd2addr(int fd) { 324 (void)fd; 325 static u64 addr; 326 return (uptr)&addr; 327 } 328 329 static uptr file2addr(char *path) { 330 (void)path; 331 static u64 addr; 332 return (uptr)&addr; 333 } 334 335 static uptr dir2addr(char *path) { 336 (void)path; 337 static u64 addr; 338 return (uptr)&addr; 339 } 340 341 TSAN_INTERCEPTOR(void*, malloc, uptr size) { 342 void *p = 0; 343 { 344 SCOPED_INTERCEPTOR_RAW(malloc, size); 345 p = user_alloc(thr, pc, size); 346 } 347 invoke_malloc_hook(p, size); 348 return p; 349 } 350 351 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { 352 void *p = 0; 353 { 354 SCOPED_INTERCEPTOR_RAW(calloc, size, n); 355 p = user_alloc(thr, pc, n * size); 356 internal_memset(p, 0, n * size); 357 } 358 invoke_malloc_hook(p, n * size); 359 return p; 360 } 361 362 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { 363 if (p) 364 invoke_free_hook(p); 365 { 366 SCOPED_INTERCEPTOR_RAW(realloc, p, size); 367 p = user_realloc(thr, pc, p, size); 368 } 369 invoke_malloc_hook(p, size); 370 return p; 371 } 372 373 TSAN_INTERCEPTOR(void, free, void *p) { 374 if (p == 0) 375 return; 376 invoke_free_hook(p); 377 SCOPED_INTERCEPTOR_RAW(free, p); 378 user_free(thr, pc, p); 379 } 380 381 TSAN_INTERCEPTOR(void, cfree, void *p) { 382 if (p == 0) 383 return; 384 invoke_free_hook(p); 385 SCOPED_INTERCEPTOR_RAW(cfree, p); 386 user_free(thr, pc, p); 387 } 388 389 TSAN_INTERCEPTOR(uptr, strlen, const char *s) { 390 SCOPED_TSAN_INTERCEPTOR(strlen, s); 391 uptr len = internal_strlen(s); 392 MemoryAccessRange(thr, pc, (uptr)s, len + 1, false); 393 return len; 394 } 395 396 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) { 397 SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size); 398 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 399 return internal_memset(dst, v, size); 400 } 401 402 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { 403 SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size); 404 MemoryAccessRange(thr, pc, (uptr)dst, size, true); 405 MemoryAccessRange(thr, pc, (uptr)src, size, false); 406 return internal_memcpy(dst, src, size); 407 } 408 409 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { 410 SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n); 411 int res = 0; 412 uptr len = 0; 413 for (; len < n; len++) { 414 if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len])) 415 break; 416 } 417 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); 418 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); 419 return res; 420 } 421 422 TSAN_INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { 423 SCOPED_TSAN_INTERCEPTOR(strcmp, s1, s2); 424 uptr len = 0; 425 for (; s1[len] && s2[len]; len++) { 426 if (s1[len] != s2[len]) 427 break; 428 } 429 MemoryAccessRange(thr, pc, (uptr)s1, len + 1, false); 430 MemoryAccessRange(thr, pc, (uptr)s2, len + 1, false); 431 return s1[len] - s2[len]; 432 } 433 434 TSAN_INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr n) { 435 SCOPED_TSAN_INTERCEPTOR(strncmp, s1, s2, n); 436 uptr len = 0; 437 for (; len < n && s1[len] && s2[len]; len++) { 438 if (s1[len] != s2[len]) 439 break; 440 } 441 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); 442 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); 443 return len == n ? 0 : s1[len] - s2[len]; 444 } 445 446 TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) { 447 SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n); 448 void *res = REAL(memchr)(s, c, n); 449 uptr len = res ? (char*)res - (char*)s + 1 : n; 450 MemoryAccessRange(thr, pc, (uptr)s, len, false); 451 return res; 452 } 453 454 TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) { 455 SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n); 456 MemoryAccessRange(thr, pc, (uptr)s, n, false); 457 return REAL(memrchr)(s, c, n); 458 } 459 460 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { 461 SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); 462 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 463 MemoryAccessRange(thr, pc, (uptr)src, n, false); 464 return REAL(memmove)(dst, src, n); 465 } 466 467 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { 468 SCOPED_TSAN_INTERCEPTOR(strchr, s, c); 469 char *res = REAL(strchr)(s, c); 470 uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1; 471 MemoryAccessRange(thr, pc, (uptr)s, len, false); 472 return res; 473 } 474 475 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { 476 SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); 477 char *res = REAL(strchrnul)(s, c); 478 uptr len = (char*)res - (char*)s + 1; 479 MemoryAccessRange(thr, pc, (uptr)s, len, false); 480 return res; 481 } 482 483 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { 484 SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); 485 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false); 486 return REAL(strrchr)(s, c); 487 } 488 489 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT 490 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT 491 uptr srclen = internal_strlen(src); 492 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); 493 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); 494 return REAL(strcpy)(dst, src); // NOLINT 495 } 496 497 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { 498 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); 499 uptr srclen = internal_strnlen(src, n); 500 MemoryAccessRange(thr, pc, (uptr)dst, n, true); 501 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); 502 return REAL(strncpy)(dst, src, n); 503 } 504 505 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) { 506 SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2); 507 const char *res = REAL(strstr)(s1, s2); 508 uptr len1 = internal_strlen(s1); 509 uptr len2 = internal_strlen(s2); 510 MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false); 511 MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false); 512 return res; 513 } 514 515 static bool fix_mmap_addr(void **addr, long_t sz, int flags) { 516 if (*addr) { 517 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { 518 if (flags & MAP_FIXED) { 519 errno = EINVAL; 520 return false; 521 } else { 522 *addr = 0; 523 } 524 } 525 } 526 return true; 527 } 528 529 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, 530 int flags, int fd, unsigned off) { 531 SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); 532 if (!fix_mmap_addr(&addr, sz, flags)) 533 return MAP_FAILED; 534 void *res = REAL(mmap)(addr, sz, prot, flags, fd, off); 535 if (res != MAP_FAILED) { 536 MemoryResetRange(thr, pc, (uptr)res, sz); 537 } 538 return res; 539 } 540 541 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, 542 int flags, int fd, u64 off) { 543 SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); 544 if (!fix_mmap_addr(&addr, sz, flags)) 545 return MAP_FAILED; 546 void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off); 547 if (res != MAP_FAILED) { 548 MemoryResetRange(thr, pc, (uptr)res, sz); 549 } 550 return res; 551 } 552 553 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { 554 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); 555 int res = REAL(munmap)(addr, sz); 556 return res; 557 } 558 559 #ifdef __LP64__ 560 561 // void *operator new(size_t) 562 TSAN_INTERCEPTOR(void*, _Znwm, uptr sz) { 563 void *p = 0; 564 { 565 SCOPED_TSAN_INTERCEPTOR(_Znwm, sz); 566 p = user_alloc(thr, pc, sz); 567 } 568 invoke_malloc_hook(p, sz); 569 return p; 570 } 571 572 // void *operator new(size_t, nothrow_t) 573 TSAN_INTERCEPTOR(void*, _ZnwmRKSt9nothrow_t, uptr sz) { 574 void *p = 0; 575 { 576 SCOPED_TSAN_INTERCEPTOR(_ZnwmRKSt9nothrow_t, sz); 577 p = user_alloc(thr, pc, sz); 578 } 579 invoke_malloc_hook(p, sz); 580 return p; 581 } 582 583 // void *operator new[](size_t) 584 TSAN_INTERCEPTOR(void*, _Znam, uptr sz) { 585 void *p = 0; 586 { 587 SCOPED_TSAN_INTERCEPTOR(_Znam, sz); 588 p = user_alloc(thr, pc, sz); 589 } 590 invoke_malloc_hook(p, sz); 591 return p; 592 } 593 594 // void *operator new[](size_t, nothrow_t) 595 TSAN_INTERCEPTOR(void*, _ZnamRKSt9nothrow_t, uptr sz) { 596 void *p = 0; 597 { 598 SCOPED_TSAN_INTERCEPTOR(_ZnamRKSt9nothrow_t, sz); 599 p = user_alloc(thr, pc, sz); 600 } 601 invoke_malloc_hook(p, sz); 602 return p; 603 } 604 605 #else 606 #error "Not implemented" 607 #endif 608 609 // void operator delete(void*) 610 TSAN_INTERCEPTOR(void, _ZdlPv, void *p) { 611 if (p == 0) 612 return; 613 invoke_free_hook(p); 614 SCOPED_TSAN_INTERCEPTOR(_ZdlPv, p); 615 user_free(thr, pc, p); 616 } 617 618 // void operator delete(void*, nothrow_t) 619 TSAN_INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *p) { 620 if (p == 0) 621 return; 622 invoke_free_hook(p); 623 SCOPED_TSAN_INTERCEPTOR(_ZdlPvRKSt9nothrow_t, p); 624 user_free(thr, pc, p); 625 } 626 627 // void operator delete[](void*) 628 TSAN_INTERCEPTOR(void, _ZdaPv, void *p) { 629 if (p == 0) 630 return; 631 invoke_free_hook(p); 632 SCOPED_TSAN_INTERCEPTOR(_ZdaPv, p); 633 user_free(thr, pc, p); 634 } 635 636 // void operator delete[](void*, nothrow_t) 637 TSAN_INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *p) { 638 if (p == 0) 639 return; 640 invoke_free_hook(p); 641 SCOPED_TSAN_INTERCEPTOR(_ZdaPvRKSt9nothrow_t, p); 642 user_free(thr, pc, p); 643 } 644 645 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { 646 SCOPED_TSAN_INTERCEPTOR(memalign, align, sz); 647 return user_alloc(thr, pc, sz, align); 648 } 649 650 TSAN_INTERCEPTOR(void*, valloc, uptr sz) { 651 SCOPED_TSAN_INTERCEPTOR(valloc, sz); 652 return user_alloc(thr, pc, sz, kPageSize); 653 } 654 655 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { 656 SCOPED_TSAN_INTERCEPTOR(pvalloc, sz); 657 sz = RoundUp(sz, kPageSize); 658 return user_alloc(thr, pc, sz, kPageSize); 659 } 660 661 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { 662 SCOPED_TSAN_INTERCEPTOR(posix_memalign, memptr, align, sz); 663 *memptr = user_alloc(thr, pc, sz, align); 664 return 0; 665 } 666 667 // Used in thread-safe function static initialization. 668 TSAN_INTERCEPTOR(int, __cxa_guard_acquire, char *m) { 669 SCOPED_TSAN_INTERCEPTOR(__cxa_guard_acquire, m); 670 int res = REAL(__cxa_guard_acquire)(m); 671 if (res) { 672 // This thread does the init. 673 } else { 674 Acquire(thr, pc, (uptr)m); 675 } 676 return res; 677 } 678 679 TSAN_INTERCEPTOR(void, __cxa_guard_release, char *m) { 680 SCOPED_TSAN_INTERCEPTOR(__cxa_guard_release, m); 681 Release(thr, pc, (uptr)m); 682 REAL(__cxa_guard_release)(m); 683 } 684 685 static void thread_finalize(void *v) { 686 uptr iter = (uptr)v; 687 if (iter > 1) { 688 if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { 689 TsanPrintf("ThreadSanitizer: failed to set thread key\n"); 690 Die(); 691 } 692 return; 693 } 694 { 695 ScopedInRtl in_rtl; 696 ThreadState *thr = cur_thread(); 697 ThreadFinish(thr); 698 SignalContext *sctx = thr->signal_ctx; 699 if (sctx) { 700 thr->signal_ctx = 0; 701 internal_free(sctx); 702 } 703 } 704 } 705 706 707 struct ThreadParam { 708 void* (*callback)(void *arg); 709 void *param; 710 atomic_uintptr_t tid; 711 }; 712 713 extern "C" void *__tsan_thread_start_func(void *arg) { 714 ThreadParam *p = (ThreadParam*)arg; 715 void* (*callback)(void *arg) = p->callback; 716 void *param = p->param; 717 int tid = 0; 718 { 719 ThreadState *thr = cur_thread(); 720 ScopedInRtl in_rtl; 721 if (pthread_setspecific(g_thread_finalize_key, (void*)4)) { 722 TsanPrintf("ThreadSanitizer: failed to set thread key\n"); 723 Die(); 724 } 725 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) 726 pthread_yield(); 727 atomic_store(&p->tid, 0, memory_order_release); 728 ThreadStart(thr, tid); 729 CHECK_EQ(thr->in_rtl, 1); 730 } 731 void *res = callback(param); 732 // Prevent the callback from being tail called, 733 // it mixes up stack traces. 734 volatile int foo = 42; 735 foo++; 736 return res; 737 } 738 739 TSAN_INTERCEPTOR(int, pthread_create, 740 void *th, void *attr, void *(*callback)(void*), void * param) { 741 SCOPED_TSAN_INTERCEPTOR(pthread_create, th, attr, callback, param); 742 pthread_attr_t myattr; 743 if (attr == 0) { 744 pthread_attr_init(&myattr); 745 attr = &myattr; 746 } 747 int detached = 0; 748 pthread_attr_getdetachstate(attr, &detached); 749 uptr stacksize = 0; 750 pthread_attr_getstacksize(attr, &stacksize); 751 // We place the huge ThreadState object into TLS, account for that. 752 const uptr minstacksize = GetTlsSize() + 128*1024; 753 if (stacksize < minstacksize) { 754 DPrintf("ThreadSanitizer: stacksize %zu->%zu\n", stacksize, minstacksize); 755 pthread_attr_setstacksize(attr, minstacksize); 756 } 757 ThreadParam p; 758 p.callback = callback; 759 p.param = param; 760 atomic_store(&p.tid, 0, memory_order_relaxed); 761 int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); 762 if (res == 0) { 763 int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); 764 CHECK_NE(tid, 0); 765 atomic_store(&p.tid, tid, memory_order_release); 766 while (atomic_load(&p.tid, memory_order_acquire) != 0) 767 pthread_yield(); 768 } 769 if (attr == &myattr) 770 pthread_attr_destroy(&myattr); 771 return res; 772 } 773 774 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { 775 SCOPED_TSAN_INTERCEPTOR(pthread_join, th, ret); 776 int tid = ThreadTid(thr, pc, (uptr)th); 777 int res = REAL(pthread_join)(th, ret); 778 if (res == 0) { 779 ThreadJoin(thr, pc, tid); 780 } 781 return res; 782 } 783 784 TSAN_INTERCEPTOR(int, pthread_detach, void *th) { 785 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); 786 int tid = ThreadTid(thr, pc, (uptr)th); 787 int res = REAL(pthread_detach)(th); 788 if (res == 0) { 789 ThreadDetach(thr, pc, tid); 790 } 791 return res; 792 } 793 794 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { 795 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); 796 int res = REAL(pthread_mutex_init)(m, a); 797 if (res == 0) { 798 bool recursive = false; 799 if (a) { 800 int type = 0; 801 if (pthread_mutexattr_gettype(a, &type) == 0) 802 recursive = (type == PTHREAD_MUTEX_RECURSIVE 803 || type == PTHREAD_MUTEX_RECURSIVE_NP); 804 } 805 MutexCreate(thr, pc, (uptr)m, false, recursive, false); 806 } 807 return res; 808 } 809 810 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { 811 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); 812 int res = REAL(pthread_mutex_destroy)(m); 813 if (res == 0 || res == EBUSY) { 814 MutexDestroy(thr, pc, (uptr)m); 815 } 816 return res; 817 } 818 819 TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { 820 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); 821 int res = REAL(pthread_mutex_lock)(m); 822 if (res == 0) { 823 MutexLock(thr, pc, (uptr)m); 824 } 825 return res; 826 } 827 828 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { 829 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); 830 int res = REAL(pthread_mutex_trylock)(m); 831 if (res == 0) { 832 MutexLock(thr, pc, (uptr)m); 833 } 834 return res; 835 } 836 837 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { 838 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); 839 int res = REAL(pthread_mutex_timedlock)(m, abstime); 840 if (res == 0) { 841 MutexLock(thr, pc, (uptr)m); 842 } 843 return res; 844 } 845 846 TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { 847 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); 848 MutexUnlock(thr, pc, (uptr)m); 849 int res = REAL(pthread_mutex_unlock)(m); 850 return res; 851 } 852 853 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { 854 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); 855 int res = REAL(pthread_spin_init)(m, pshared); 856 if (res == 0) { 857 MutexCreate(thr, pc, (uptr)m, false, false, false); 858 } 859 return res; 860 } 861 862 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { 863 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); 864 int res = REAL(pthread_spin_destroy)(m); 865 if (res == 0) { 866 MutexDestroy(thr, pc, (uptr)m); 867 } 868 return res; 869 } 870 871 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { 872 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); 873 int res = REAL(pthread_spin_lock)(m); 874 if (res == 0) { 875 MutexLock(thr, pc, (uptr)m); 876 } 877 return res; 878 } 879 880 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { 881 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); 882 int res = REAL(pthread_spin_trylock)(m); 883 if (res == 0) { 884 MutexLock(thr, pc, (uptr)m); 885 } 886 return res; 887 } 888 889 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { 890 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); 891 MutexUnlock(thr, pc, (uptr)m); 892 int res = REAL(pthread_spin_unlock)(m); 893 return res; 894 } 895 896 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { 897 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); 898 int res = REAL(pthread_rwlock_init)(m, a); 899 if (res == 0) { 900 MutexCreate(thr, pc, (uptr)m, true, false, false); 901 } 902 return res; 903 } 904 905 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { 906 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); 907 int res = REAL(pthread_rwlock_destroy)(m); 908 if (res == 0) { 909 MutexDestroy(thr, pc, (uptr)m); 910 } 911 return res; 912 } 913 914 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { 915 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); 916 int res = REAL(pthread_rwlock_rdlock)(m); 917 if (res == 0) { 918 MutexReadLock(thr, pc, (uptr)m); 919 } 920 return res; 921 } 922 923 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { 924 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); 925 int res = REAL(pthread_rwlock_tryrdlock)(m); 926 if (res == 0) { 927 MutexReadLock(thr, pc, (uptr)m); 928 } 929 return res; 930 } 931 932 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { 933 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); 934 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); 935 if (res == 0) { 936 MutexReadLock(thr, pc, (uptr)m); 937 } 938 return res; 939 } 940 941 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { 942 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); 943 int res = REAL(pthread_rwlock_wrlock)(m); 944 if (res == 0) { 945 MutexLock(thr, pc, (uptr)m); 946 } 947 return res; 948 } 949 950 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { 951 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); 952 int res = REAL(pthread_rwlock_trywrlock)(m); 953 if (res == 0) { 954 MutexLock(thr, pc, (uptr)m); 955 } 956 return res; 957 } 958 959 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { 960 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); 961 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); 962 if (res == 0) { 963 MutexLock(thr, pc, (uptr)m); 964 } 965 return res; 966 } 967 968 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { 969 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); 970 MutexReadOrWriteUnlock(thr, pc, (uptr)m); 971 int res = REAL(pthread_rwlock_unlock)(m); 972 return res; 973 } 974 975 TSAN_INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { 976 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, c, a); 977 int res = REAL(pthread_cond_init)(c, a); 978 return res; 979 } 980 981 TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) { 982 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c); 983 int res = REAL(pthread_cond_destroy)(c); 984 return res; 985 } 986 987 TSAN_INTERCEPTOR(int, pthread_cond_signal, void *c) { 988 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, c); 989 int res = REAL(pthread_cond_signal)(c); 990 return res; 991 } 992 993 TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) { 994 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, c); 995 int res = REAL(pthread_cond_broadcast)(c); 996 return res; 997 } 998 999 TSAN_INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { 1000 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, c, m); 1001 MutexUnlock(thr, pc, (uptr)m); 1002 int res = REAL(pthread_cond_wait)(c, m); 1003 MutexLock(thr, pc, (uptr)m); 1004 return res; 1005 } 1006 1007 TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { 1008 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime); 1009 MutexUnlock(thr, pc, (uptr)m); 1010 int res = REAL(pthread_cond_timedwait)(c, m, abstime); 1011 MutexLock(thr, pc, (uptr)m); 1012 return res; 1013 } 1014 1015 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { 1016 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); 1017 MemoryWrite1Byte(thr, pc, (uptr)b); 1018 int res = REAL(pthread_barrier_init)(b, a, count); 1019 return res; 1020 } 1021 1022 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { 1023 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); 1024 MemoryWrite1Byte(thr, pc, (uptr)b); 1025 int res = REAL(pthread_barrier_destroy)(b); 1026 return res; 1027 } 1028 1029 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { 1030 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); 1031 Release(thr, pc, (uptr)b); 1032 MemoryRead1Byte(thr, pc, (uptr)b); 1033 int res = REAL(pthread_barrier_wait)(b); 1034 MemoryRead1Byte(thr, pc, (uptr)b); 1035 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { 1036 Acquire(thr, pc, (uptr)b); 1037 } 1038 return res; 1039 } 1040 1041 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { 1042 SCOPED_TSAN_INTERCEPTOR(pthread_once, o, f); 1043 if (o == 0 || f == 0) 1044 return EINVAL; 1045 atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); 1046 u32 v = atomic_load(a, memory_order_acquire); 1047 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, 1048 memory_order_relaxed)) { 1049 const int old_in_rtl = thr->in_rtl; 1050 thr->in_rtl = 0; 1051 (*f)(); 1052 CHECK_EQ(thr->in_rtl, 0); 1053 thr->in_rtl = old_in_rtl; 1054 Release(thr, pc, (uptr)o); 1055 atomic_store(a, 2, memory_order_release); 1056 } else { 1057 while (v != 2) { 1058 pthread_yield(); 1059 v = atomic_load(a, memory_order_acquire); 1060 } 1061 Acquire(thr, pc, (uptr)o); 1062 } 1063 return 0; 1064 } 1065 1066 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) { 1067 SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value); 1068 int res = REAL(sem_init)(s, pshared, value); 1069 return res; 1070 } 1071 1072 TSAN_INTERCEPTOR(int, sem_destroy, void *s) { 1073 SCOPED_TSAN_INTERCEPTOR(sem_destroy, s); 1074 int res = REAL(sem_destroy)(s); 1075 return res; 1076 } 1077 1078 TSAN_INTERCEPTOR(int, sem_wait, void *s) { 1079 SCOPED_TSAN_INTERCEPTOR(sem_wait, s); 1080 int res = REAL(sem_wait)(s); 1081 if (res == 0) { 1082 Acquire(thr, pc, (uptr)s); 1083 } 1084 return res; 1085 } 1086 1087 TSAN_INTERCEPTOR(int, sem_trywait, void *s) { 1088 SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); 1089 int res = REAL(sem_trywait)(s); 1090 if (res == 0) { 1091 Acquire(thr, pc, (uptr)s); 1092 } 1093 return res; 1094 } 1095 1096 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { 1097 SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); 1098 int res = REAL(sem_timedwait)(s, abstime); 1099 if (res == 0) { 1100 Acquire(thr, pc, (uptr)s); 1101 } 1102 return res; 1103 } 1104 1105 TSAN_INTERCEPTOR(int, sem_post, void *s) { 1106 SCOPED_TSAN_INTERCEPTOR(sem_post, s); 1107 Release(thr, pc, (uptr)s); 1108 int res = REAL(sem_post)(s); 1109 return res; 1110 } 1111 1112 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { 1113 SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); 1114 int res = REAL(sem_getvalue)(s, sval); 1115 if (res == 0) { 1116 Acquire(thr, pc, (uptr)s); 1117 } 1118 return res; 1119 } 1120 1121 TSAN_INTERCEPTOR(long_t, read, int fd, void *buf, long_t sz) { 1122 SCOPED_TSAN_INTERCEPTOR(read, fd, buf, sz); 1123 int res = REAL(read)(fd, buf, sz); 1124 if (res >= 0) { 1125 Acquire(thr, pc, fd2addr(fd)); 1126 } 1127 return res; 1128 } 1129 1130 TSAN_INTERCEPTOR(long_t, pread, int fd, void *buf, long_t sz, unsigned off) { 1131 SCOPED_TSAN_INTERCEPTOR(pread, fd, buf, sz, off); 1132 int res = REAL(pread)(fd, buf, sz, off); 1133 if (res >= 0) { 1134 Acquire(thr, pc, fd2addr(fd)); 1135 } 1136 return res; 1137 } 1138 1139 TSAN_INTERCEPTOR(long_t, pread64, int fd, void *buf, long_t sz, u64 off) { 1140 SCOPED_TSAN_INTERCEPTOR(pread64, fd, buf, sz, off); 1141 int res = REAL(pread64)(fd, buf, sz, off); 1142 if (res >= 0) { 1143 Acquire(thr, pc, fd2addr(fd)); 1144 } 1145 return res; 1146 } 1147 1148 TSAN_INTERCEPTOR(long_t, readv, int fd, void *vec, int cnt) { 1149 SCOPED_TSAN_INTERCEPTOR(readv, fd, vec, cnt); 1150 int res = REAL(readv)(fd, vec, cnt); 1151 if (res >= 0) { 1152 Acquire(thr, pc, fd2addr(fd)); 1153 } 1154 return res; 1155 } 1156 1157 TSAN_INTERCEPTOR(long_t, preadv64, int fd, void *vec, int cnt, u64 off) { 1158 SCOPED_TSAN_INTERCEPTOR(preadv64, fd, vec, cnt, off); 1159 int res = REAL(preadv64)(fd, vec, cnt, off); 1160 if (res >= 0) { 1161 Acquire(thr, pc, fd2addr(fd)); 1162 } 1163 return res; 1164 } 1165 1166 TSAN_INTERCEPTOR(long_t, write, int fd, void *buf, long_t sz) { 1167 SCOPED_TSAN_INTERCEPTOR(write, fd, buf, sz); 1168 Release(thr, pc, fd2addr(fd)); 1169 int res = REAL(write)(fd, buf, sz); 1170 return res; 1171 } 1172 1173 TSAN_INTERCEPTOR(long_t, pwrite, int fd, void *buf, long_t sz, unsigned off) { 1174 SCOPED_TSAN_INTERCEPTOR(pwrite, fd, buf, sz, off); 1175 Release(thr, pc, fd2addr(fd)); 1176 int res = REAL(pwrite)(fd, buf, sz, off); 1177 return res; 1178 } 1179 1180 TSAN_INTERCEPTOR(long_t, pwrite64, int fd, void *buf, long_t sz, u64 off) { 1181 SCOPED_TSAN_INTERCEPTOR(pwrite64, fd, buf, sz, off); 1182 Release(thr, pc, fd2addr(fd)); 1183 int res = REAL(pwrite64)(fd, buf, sz, off); 1184 return res; 1185 } 1186 1187 TSAN_INTERCEPTOR(long_t, writev, int fd, void *vec, int cnt) { 1188 SCOPED_TSAN_INTERCEPTOR(writev, fd, vec, cnt); 1189 Release(thr, pc, fd2addr(fd)); 1190 int res = REAL(writev)(fd, vec, cnt); 1191 return res; 1192 } 1193 1194 TSAN_INTERCEPTOR(long_t, pwritev64, int fd, void *vec, int cnt, u64 off) { 1195 SCOPED_TSAN_INTERCEPTOR(pwritev64, fd, vec, cnt, off); 1196 Release(thr, pc, fd2addr(fd)); 1197 int res = REAL(pwritev64)(fd, vec, cnt, off); 1198 return res; 1199 } 1200 1201 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { 1202 SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); 1203 Release(thr, pc, fd2addr(fd)); 1204 int res = REAL(send)(fd, buf, len, flags); 1205 return res; 1206 } 1207 1208 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { 1209 SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); 1210 Release(thr, pc, fd2addr(fd)); 1211 int res = REAL(sendmsg)(fd, msg, flags); 1212 return res; 1213 } 1214 1215 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { 1216 SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); 1217 int res = REAL(recv)(fd, buf, len, flags); 1218 if (res >= 0) { 1219 Acquire(thr, pc, fd2addr(fd)); 1220 } 1221 return res; 1222 } 1223 1224 TSAN_INTERCEPTOR(long_t, recvmsg, int fd, void *msg, int flags) { 1225 SCOPED_TSAN_INTERCEPTOR(recvmsg, fd, msg, flags); 1226 int res = REAL(recvmsg)(fd, msg, flags); 1227 if (res >= 0) { 1228 Acquire(thr, pc, fd2addr(fd)); 1229 } 1230 return res; 1231 } 1232 1233 TSAN_INTERCEPTOR(int, unlink, char *path) { 1234 SCOPED_TSAN_INTERCEPTOR(unlink, path); 1235 Release(thr, pc, file2addr(path)); 1236 int res = REAL(unlink)(path); 1237 return res; 1238 } 1239 1240 TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) { 1241 SCOPED_TSAN_INTERCEPTOR(fopen, path, mode); 1242 void *res = REAL(fopen)(path, mode); 1243 Acquire(thr, pc, file2addr(path)); 1244 return res; 1245 } 1246 1247 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { 1248 SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); 1249 MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); 1250 return REAL(fread)(ptr, size, nmemb, f); 1251 } 1252 1253 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { 1254 SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); 1255 MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); 1256 return REAL(fwrite)(p, size, nmemb, f); 1257 } 1258 1259 TSAN_INTERCEPTOR(int, puts, const char *s) { 1260 SCOPED_TSAN_INTERCEPTOR(puts, s); 1261 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); 1262 return REAL(puts)(s); 1263 } 1264 1265 TSAN_INTERCEPTOR(int, rmdir, char *path) { 1266 SCOPED_TSAN_INTERCEPTOR(rmdir, path); 1267 Release(thr, pc, dir2addr(path)); 1268 int res = REAL(rmdir)(path); 1269 return res; 1270 } 1271 1272 TSAN_INTERCEPTOR(void*, opendir, char *path) { 1273 SCOPED_TSAN_INTERCEPTOR(opendir, path); 1274 void *res = REAL(opendir)(path); 1275 Acquire(thr, pc, dir2addr(path)); 1276 return res; 1277 } 1278 1279 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { 1280 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); 1281 if (op == EPOLL_CTL_ADD) { 1282 Release(thr, pc, epollfd2addr(epfd)); 1283 } 1284 int res = REAL(epoll_ctl)(epfd, op, fd, ev); 1285 return res; 1286 } 1287 1288 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { 1289 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); 1290 int res = REAL(epoll_wait)(epfd, ev, cnt, timeout); 1291 if (res > 0) { 1292 Acquire(thr, pc, epollfd2addr(epfd)); 1293 } 1294 return res; 1295 } 1296 1297 static void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, 1298 my_siginfo_t *info, void *ctx) { 1299 ThreadState *thr = cur_thread(); 1300 SignalContext *sctx = SigCtx(thr); 1301 // Don't mess with synchronous signals. 1302 if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || 1303 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || 1304 (sctx && sig == sctx->int_signal_send)) { 1305 CHECK(thr->in_rtl == 0 || thr->in_rtl == 1); 1306 int in_rtl = thr->in_rtl; 1307 thr->in_rtl = 0; 1308 CHECK_EQ(thr->in_signal_handler, false); 1309 thr->in_signal_handler = true; 1310 if (sigact) 1311 sigactions[sig].sa_sigaction(sig, info, ctx); 1312 else 1313 sigactions[sig].sa_handler(sig); 1314 CHECK_EQ(thr->in_signal_handler, true); 1315 thr->in_signal_handler = false; 1316 thr->in_rtl = in_rtl; 1317 return; 1318 } 1319 1320 if (sctx == 0) 1321 return; 1322 SignalDesc *signal = &sctx->pending_signals[sig]; 1323 if (signal->armed == false) { 1324 signal->armed = true; 1325 signal->sigaction = sigact; 1326 if (info) 1327 internal_memcpy(&signal->siginfo, info, sizeof(*info)); 1328 if (ctx) 1329 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); 1330 sctx->pending_signal_count++; 1331 } 1332 } 1333 1334 static void rtl_sighandler(int sig) { 1335 rtl_generic_sighandler(false, sig, 0, 0); 1336 } 1337 1338 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { 1339 rtl_generic_sighandler(true, sig, info, ctx); 1340 } 1341 1342 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { 1343 SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old); 1344 if (old) 1345 internal_memcpy(old, &sigactions[sig], sizeof(*old)); 1346 if (act == 0) 1347 return 0; 1348 internal_memcpy(&sigactions[sig], act, sizeof(*act)); 1349 sigaction_t newact; 1350 internal_memcpy(&newact, act, sizeof(newact)); 1351 sigfillset(&newact.sa_mask); 1352 if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { 1353 if (newact.sa_flags & SA_SIGINFO) 1354 newact.sa_sigaction = rtl_sigaction; 1355 else 1356 newact.sa_handler = rtl_sighandler; 1357 } 1358 int res = REAL(sigaction)(sig, &newact, 0); 1359 return res; 1360 } 1361 1362 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { 1363 sigaction_t act = {}; 1364 act.sa_handler = h; 1365 REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); 1366 act.sa_flags = 0; 1367 sigaction_t old = {}; 1368 int res = sigaction(sig, &act, &old); 1369 if (res) 1370 return SIG_ERR; 1371 return old.sa_handler; 1372 } 1373 1374 TSAN_INTERCEPTOR(int, raise, int sig) { 1375 SCOPED_TSAN_INTERCEPTOR(raise, sig); 1376 SignalContext *sctx = SigCtx(thr); 1377 CHECK_NE(sctx, 0); 1378 int prev = sctx->int_signal_send; 1379 sctx->int_signal_send = sig; 1380 int res = REAL(raise)(sig); 1381 CHECK_EQ(sctx->int_signal_send, sig); 1382 sctx->int_signal_send = prev; 1383 return res; 1384 } 1385 1386 TSAN_INTERCEPTOR(int, kill, int pid, int sig) { 1387 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); 1388 SignalContext *sctx = SigCtx(thr); 1389 CHECK_NE(sctx, 0); 1390 int prev = sctx->int_signal_send; 1391 if (pid == GetPid()) { 1392 sctx->int_signal_send = sig; 1393 } 1394 int res = REAL(kill)(pid, sig); 1395 if (pid == GetPid()) { 1396 CHECK_EQ(sctx->int_signal_send, sig); 1397 sctx->int_signal_send = prev; 1398 } 1399 return res; 1400 } 1401 1402 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { 1403 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); 1404 SignalContext *sctx = SigCtx(thr); 1405 CHECK_NE(sctx, 0); 1406 int prev = sctx->int_signal_send; 1407 if (tid == pthread_self()) { 1408 sctx->int_signal_send = sig; 1409 } 1410 int res = REAL(pthread_kill)(tid, sig); 1411 if (tid == pthread_self()) { 1412 CHECK_EQ(sctx->int_signal_send, sig); 1413 sctx->int_signal_send = prev; 1414 } 1415 return res; 1416 } 1417 1418 static void process_pending_signals(ThreadState *thr) { 1419 CHECK_EQ(thr->in_rtl, 0); 1420 SignalContext *sctx = SigCtx(thr); 1421 if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler) 1422 return; 1423 thr->in_signal_handler = true; 1424 sctx->pending_signal_count = 0; 1425 // These are too big for stack. 1426 static THREADLOCAL sigset_t emptyset, oldset; 1427 sigfillset(&emptyset); 1428 pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); 1429 for (int sig = 0; sig < kSigCount; sig++) { 1430 SignalDesc *signal = &sctx->pending_signals[sig]; 1431 if (signal->armed) { 1432 signal->armed = false; 1433 if (sigactions[sig].sa_handler != SIG_DFL 1434 && sigactions[sig].sa_handler != SIG_IGN) { 1435 // Insure that the handler does not spoil errno. 1436 const int saved_errno = errno; 1437 errno = 0; 1438 if (signal->sigaction) 1439 sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx); 1440 else 1441 sigactions[sig].sa_handler(sig); 1442 if (errno != 0) { 1443 ScopedInRtl in_rtl; 1444 StackTrace stack; 1445 uptr pc = signal->sigaction ? 1446 (uptr)sigactions[sig].sa_sigaction : 1447 (uptr)sigactions[sig].sa_handler; 1448 stack.Init(&pc, 1); 1449 ScopedReport rep(ReportTypeErrnoInSignal); 1450 rep.AddStack(&stack); 1451 OutputReport(rep, rep.GetReport()->stacks[0]); 1452 } 1453 errno = saved_errno; 1454 } 1455 } 1456 } 1457 pthread_sigmask(SIG_SETMASK, &oldset, 0); 1458 CHECK_EQ(thr->in_signal_handler, true); 1459 thr->in_signal_handler = false; 1460 } 1461 1462 namespace __tsan { 1463 1464 void InitializeInterceptors() { 1465 CHECK_GT(cur_thread()->in_rtl, 0); 1466 1467 // We need to setup it early, because functions like dlsym() can call it. 1468 REAL(memset) = internal_memset; 1469 REAL(memcpy) = internal_memcpy; 1470 REAL(memcmp) = internal_memcmp; 1471 1472 TSAN_INTERCEPT(longjmp); 1473 TSAN_INTERCEPT(siglongjmp); 1474 1475 TSAN_INTERCEPT(malloc); 1476 TSAN_INTERCEPT(calloc); 1477 TSAN_INTERCEPT(realloc); 1478 TSAN_INTERCEPT(free); 1479 TSAN_INTERCEPT(cfree); 1480 TSAN_INTERCEPT(mmap); 1481 TSAN_INTERCEPT(mmap64); 1482 TSAN_INTERCEPT(munmap); 1483 TSAN_INTERCEPT(memalign); 1484 TSAN_INTERCEPT(valloc); 1485 TSAN_INTERCEPT(pvalloc); 1486 TSAN_INTERCEPT(posix_memalign); 1487 1488 TSAN_INTERCEPT(_Znwm); 1489 TSAN_INTERCEPT(_ZnwmRKSt9nothrow_t); 1490 TSAN_INTERCEPT(_Znam); 1491 TSAN_INTERCEPT(_ZnamRKSt9nothrow_t); 1492 TSAN_INTERCEPT(_ZdlPv); 1493 TSAN_INTERCEPT(_ZdlPvRKSt9nothrow_t); 1494 TSAN_INTERCEPT(_ZdaPv); 1495 TSAN_INTERCEPT(_ZdaPvRKSt9nothrow_t); 1496 1497 TSAN_INTERCEPT(strlen); 1498 TSAN_INTERCEPT(memset); 1499 TSAN_INTERCEPT(memcpy); 1500 TSAN_INTERCEPT(strcmp); 1501 TSAN_INTERCEPT(memchr); 1502 TSAN_INTERCEPT(memrchr); 1503 TSAN_INTERCEPT(memmove); 1504 TSAN_INTERCEPT(memcmp); 1505 TSAN_INTERCEPT(strchr); 1506 TSAN_INTERCEPT(strchrnul); 1507 TSAN_INTERCEPT(strrchr); 1508 TSAN_INTERCEPT(strncmp); 1509 TSAN_INTERCEPT(strcpy); // NOLINT 1510 TSAN_INTERCEPT(strncpy); 1511 TSAN_INTERCEPT(strstr); 1512 1513 TSAN_INTERCEPT(__cxa_guard_acquire); 1514 TSAN_INTERCEPT(__cxa_guard_release); 1515 1516 TSAN_INTERCEPT(pthread_create); 1517 TSAN_INTERCEPT(pthread_join); 1518 TSAN_INTERCEPT(pthread_detach); 1519 1520 TSAN_INTERCEPT(pthread_mutex_init); 1521 TSAN_INTERCEPT(pthread_mutex_destroy); 1522 TSAN_INTERCEPT(pthread_mutex_lock); 1523 TSAN_INTERCEPT(pthread_mutex_trylock); 1524 TSAN_INTERCEPT(pthread_mutex_timedlock); 1525 TSAN_INTERCEPT(pthread_mutex_unlock); 1526 1527 TSAN_INTERCEPT(pthread_spin_init); 1528 TSAN_INTERCEPT(pthread_spin_destroy); 1529 TSAN_INTERCEPT(pthread_spin_lock); 1530 TSAN_INTERCEPT(pthread_spin_trylock); 1531 TSAN_INTERCEPT(pthread_spin_unlock); 1532 1533 TSAN_INTERCEPT(pthread_rwlock_init); 1534 TSAN_INTERCEPT(pthread_rwlock_destroy); 1535 TSAN_INTERCEPT(pthread_rwlock_rdlock); 1536 TSAN_INTERCEPT(pthread_rwlock_tryrdlock); 1537 TSAN_INTERCEPT(pthread_rwlock_timedrdlock); 1538 TSAN_INTERCEPT(pthread_rwlock_wrlock); 1539 TSAN_INTERCEPT(pthread_rwlock_trywrlock); 1540 TSAN_INTERCEPT(pthread_rwlock_timedwrlock); 1541 TSAN_INTERCEPT(pthread_rwlock_unlock); 1542 1543 TSAN_INTERCEPT(pthread_cond_init); 1544 TSAN_INTERCEPT(pthread_cond_destroy); 1545 TSAN_INTERCEPT(pthread_cond_signal); 1546 TSAN_INTERCEPT(pthread_cond_broadcast); 1547 TSAN_INTERCEPT(pthread_cond_wait); 1548 TSAN_INTERCEPT(pthread_cond_timedwait); 1549 1550 TSAN_INTERCEPT(pthread_barrier_init); 1551 TSAN_INTERCEPT(pthread_barrier_destroy); 1552 TSAN_INTERCEPT(pthread_barrier_wait); 1553 1554 TSAN_INTERCEPT(pthread_once); 1555 1556 TSAN_INTERCEPT(sem_init); 1557 TSAN_INTERCEPT(sem_destroy); 1558 TSAN_INTERCEPT(sem_wait); 1559 TSAN_INTERCEPT(sem_trywait); 1560 TSAN_INTERCEPT(sem_timedwait); 1561 TSAN_INTERCEPT(sem_post); 1562 TSAN_INTERCEPT(sem_getvalue); 1563 1564 TSAN_INTERCEPT(read); 1565 TSAN_INTERCEPT(pread); 1566 TSAN_INTERCEPT(pread64); 1567 TSAN_INTERCEPT(readv); 1568 TSAN_INTERCEPT(preadv64); 1569 TSAN_INTERCEPT(write); 1570 TSAN_INTERCEPT(pwrite); 1571 TSAN_INTERCEPT(pwrite64); 1572 TSAN_INTERCEPT(writev); 1573 TSAN_INTERCEPT(pwritev64); 1574 TSAN_INTERCEPT(send); 1575 TSAN_INTERCEPT(sendmsg); 1576 TSAN_INTERCEPT(recv); 1577 TSAN_INTERCEPT(recvmsg); 1578 1579 TSAN_INTERCEPT(unlink); 1580 TSAN_INTERCEPT(fopen); 1581 TSAN_INTERCEPT(fread); 1582 TSAN_INTERCEPT(fwrite); 1583 TSAN_INTERCEPT(puts); 1584 TSAN_INTERCEPT(rmdir); 1585 TSAN_INTERCEPT(opendir); 1586 1587 TSAN_INTERCEPT(epoll_ctl); 1588 TSAN_INTERCEPT(epoll_wait); 1589 1590 TSAN_INTERCEPT(sigaction); 1591 TSAN_INTERCEPT(signal); 1592 TSAN_INTERCEPT(raise); 1593 TSAN_INTERCEPT(kill); 1594 TSAN_INTERCEPT(pthread_kill); 1595 TSAN_INTERCEPT(sleep); 1596 TSAN_INTERCEPT(usleep); 1597 TSAN_INTERCEPT(nanosleep); 1598 1599 atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext))) 1600 AtExitContext(); 1601 1602 if (__cxa_atexit(&finalize, 0, 0)) { 1603 TsanPrintf("ThreadSanitizer: failed to setup atexit callback\n"); 1604 Die(); 1605 } 1606 1607 if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { 1608 TsanPrintf("ThreadSanitizer: failed to create thread key\n"); 1609 Die(); 1610 } 1611 } 1612 1613 void internal_start_thread(void(*func)(void *arg), void *arg) { 1614 void *th; 1615 REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); 1616 REAL(pthread_detach)(th); 1617 } 1618 1619 } // namespace __tsan 1620
