Home | History | Annotate | Download | only in rtl
      1 //===-- tsan_interceptors.cc ----------------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file is a part of ThreadSanitizer (TSan), a race detector.
     11 //
     12 // FIXME: move as many interceptors as possible into
     13 // sanitizer_common/sanitizer_common_interceptors.inc
     14 //===----------------------------------------------------------------------===//
     15 
     16 #include "sanitizer_common/sanitizer_atomic.h"
     17 #include "sanitizer_common/sanitizer_libc.h"
     18 #include "sanitizer_common/sanitizer_linux.h"
     19 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
     20 #include "sanitizer_common/sanitizer_placement_new.h"
     21 #include "sanitizer_common/sanitizer_stacktrace.h"
     22 #include "interception/interception.h"
     23 #include "tsan_interface.h"
     24 #include "tsan_platform.h"
     25 #include "tsan_suppressions.h"
     26 #include "tsan_rtl.h"
     27 #include "tsan_mman.h"
     28 #include "tsan_fd.h"
     29 
     30 using namespace __tsan;  // NOLINT
     31 
     32 const int kSigCount = 65;
     33 
     34 struct my_siginfo_t {
     35   // The size is determined by looking at sizeof of real siginfo_t on linux.
     36   u64 opaque[128 / sizeof(u64)];
     37 };
     38 
     39 struct ucontext_t {
     40   // The size is determined by looking at sizeof of real ucontext_t on linux.
     41   u64 opaque[936 / sizeof(u64) + 1];
     42 };
     43 
     44 extern "C" int pthread_attr_init(void *attr);
     45 extern "C" int pthread_attr_destroy(void *attr);
     46 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
     47 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
     48 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
     49 extern "C" int pthread_setspecific(unsigned key, const void *v);
     50 DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
     51 extern "C" int pthread_yield();
     52 extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set,
     53                                __sanitizer_sigset_t *oldset);
     54 // REAL(sigfillset) defined in common interceptors.
     55 DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set)
     56 DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
     57 extern "C" void *pthread_self();
     58 extern "C" void _exit(int status);
     59 extern "C" int *__errno_location();
     60 extern "C" int fileno_unlocked(void *stream);
     61 extern "C" void *__libc_malloc(uptr size);
     62 extern "C" void *__libc_calloc(uptr size, uptr n);
     63 extern "C" void *__libc_realloc(void *ptr, uptr size);
     64 extern "C" void __libc_free(void *ptr);
     65 extern "C" int mallopt(int param, int value);
     66 extern __sanitizer_FILE *stdout, *stderr;
     67 const int PTHREAD_MUTEX_RECURSIVE = 1;
     68 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
     69 const int EINVAL = 22;
     70 const int EBUSY = 16;
     71 const int EOWNERDEAD = 130;
     72 const int EPOLL_CTL_ADD = 1;
     73 const int SIGILL = 4;
     74 const int SIGABRT = 6;
     75 const int SIGFPE = 8;
     76 const int SIGSEGV = 11;
     77 const int SIGPIPE = 13;
     78 const int SIGTERM = 15;
     79 const int SIGBUS = 7;
     80 const int SIGSYS = 31;
     81 void *const MAP_FAILED = (void*)-1;
     82 const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
     83 const int MAP_FIXED = 0x10;
     84 typedef long long_t;  // NOLINT
     85 
     86 // From /usr/include/unistd.h
     87 # define F_ULOCK 0      /* Unlock a previously locked region.  */
     88 # define F_LOCK  1      /* Lock a region for exclusive use.  */
     89 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
     90 # define F_TEST  3      /* Test a region for other processes locks.  */
     91 
     92 typedef void (*sighandler_t)(int sig);
     93 
     94 #define errno (*__errno_location())
     95 
     96 struct sigaction_t {
     97   union {
     98     sighandler_t sa_handler;
     99     void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
    100   };
    101   __sanitizer_sigset_t sa_mask;
    102   int sa_flags;
    103   void (*sa_restorer)();
    104 };
    105 
    106 const sighandler_t SIG_DFL = (sighandler_t)0;
    107 const sighandler_t SIG_IGN = (sighandler_t)1;
    108 const sighandler_t SIG_ERR = (sighandler_t)-1;
    109 const int SA_SIGINFO = 4;
    110 const int SIG_SETMASK = 2;
    111 
    112 namespace std {
    113 struct nothrow_t {};
    114 }  // namespace std
    115 
    116 static sigaction_t sigactions[kSigCount];
    117 
    118 namespace __tsan {
    119 struct SignalDesc {
    120   bool armed;
    121   bool sigaction;
    122   my_siginfo_t siginfo;
    123   ucontext_t ctx;
    124 };
    125 
    126 struct SignalContext {
    127   int in_blocking_func;
    128   int int_signal_send;
    129   int pending_signal_count;
    130   SignalDesc pending_signals[kSigCount];
    131 };
    132 
    133 // The object is 64-byte aligned, because we want hot data to be located in
    134 // a single cache line if possible (it's accessed in every interceptor).
    135 static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
    136 static LibIgnore *libignore() {
    137   return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
    138 }
    139 
    140 void InitializeLibIgnore() {
    141   libignore()->Init(*GetSuppressionContext());
    142   libignore()->OnLibraryLoaded(0);
    143 }
    144 
    145 }  // namespace __tsan
    146 
    147 static SignalContext *SigCtx(ThreadState *thr) {
    148   SignalContext *ctx = (SignalContext*)thr->signal_ctx;
    149   if (ctx == 0 && thr->is_alive) {
    150     ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext");
    151     MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
    152     thr->signal_ctx = ctx;
    153   }
    154   return ctx;
    155 }
    156 
    157 static unsigned g_thread_finalize_key;
    158 
    159 class ScopedInterceptor {
    160  public:
    161   ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
    162   ~ScopedInterceptor();
    163  private:
    164   ThreadState *const thr_;
    165   const uptr pc_;
    166   bool in_ignored_lib_;
    167 };
    168 
    169 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
    170                                      uptr pc)
    171     : thr_(thr)
    172     , pc_(pc)
    173     , in_ignored_lib_(false) {
    174   if (!thr_->ignore_interceptors) {
    175     Initialize(thr);
    176     FuncEntry(thr, pc);
    177   }
    178   DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
    179   if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
    180     in_ignored_lib_ = true;
    181     thr_->in_ignored_lib = true;
    182     ThreadIgnoreBegin(thr_, pc_);
    183   }
    184 }
    185 
    186 ScopedInterceptor::~ScopedInterceptor() {
    187   if (in_ignored_lib_) {
    188     thr_->in_ignored_lib = false;
    189     ThreadIgnoreEnd(thr_, pc_);
    190   }
    191   if (!thr_->ignore_interceptors) {
    192     ProcessPendingSignals(thr_);
    193     FuncExit(thr_);
    194     CheckNoLocks(thr_);
    195   }
    196 }
    197 
    198 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
    199     ThreadState *thr = cur_thread(); \
    200     const uptr caller_pc = GET_CALLER_PC(); \
    201     ScopedInterceptor si(thr, #func, caller_pc); \
    202     const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
    203     (void)pc; \
    204 /**/
    205 
    206 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
    207     SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
    208     if (REAL(func) == 0) { \
    209       Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
    210       Die(); \
    211     }                                                    \
    212     if (thr->ignore_interceptors || thr->in_ignored_lib) \
    213       return REAL(func)(__VA_ARGS__); \
    214 /**/
    215 
    216 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
    217 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
    218 #define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
    219 
    220 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
    221 
    222 struct BlockingCall {
    223   explicit BlockingCall(ThreadState *thr)
    224       : ctx(SigCtx(thr)) {
    225     ctx->in_blocking_func++;
    226   }
    227 
    228   ~BlockingCall() {
    229     ctx->in_blocking_func--;
    230   }
    231 
    232   SignalContext *ctx;
    233 
    234   // When we are in a "blocking call", we process signals asynchronously
    235   // (right when they arrive). In this context we do not expect to be
    236   // executing any user/runtime code. The known interceptor sequence when
    237   // this is not true is: pthread_join -> munmap(stack). It's fine
    238   // to ignore munmap in this case -- we handle stack shadow separately.
    239   ScopedIgnoreInterceptors ignore_interceptors;
    240 };
    241 
    242 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
    243   SCOPED_TSAN_INTERCEPTOR(sleep, sec);
    244   unsigned res = BLOCK_REAL(sleep)(sec);
    245   AfterSleep(thr, pc);
    246   return res;
    247 }
    248 
    249 TSAN_INTERCEPTOR(int, usleep, long_t usec) {
    250   SCOPED_TSAN_INTERCEPTOR(usleep, usec);
    251   int res = BLOCK_REAL(usleep)(usec);
    252   AfterSleep(thr, pc);
    253   return res;
    254 }
    255 
    256 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
    257   SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
    258   int res = BLOCK_REAL(nanosleep)(req, rem);
    259   AfterSleep(thr, pc);
    260   return res;
    261 }
    262 
    263 class AtExitContext {
    264  public:
    265   AtExitContext()
    266     : mtx_(MutexTypeAtExit, StatMtxAtExit)
    267     , pos_() {
    268   }
    269 
    270   typedef void(*atexit_t)();
    271 
    272   int atexit(ThreadState *thr, uptr pc, bool is_on_exit,
    273              atexit_t f, void *arg) {
    274     Lock l(&mtx_);
    275     if (pos_ == kMaxAtExit)
    276       return 1;
    277     Release(thr, pc, (uptr)this);
    278     stack_[pos_] = f;
    279     args_[pos_] = arg;
    280     is_on_exits_[pos_] = is_on_exit;
    281     pos_++;
    282     return 0;
    283   }
    284 
    285   void exit(ThreadState *thr, uptr pc) {
    286     for (;;) {
    287       atexit_t f = 0;
    288       void *arg = 0;
    289       bool is_on_exit = false;
    290       {
    291         Lock l(&mtx_);
    292         if (pos_) {
    293           pos_--;
    294           f = stack_[pos_];
    295           arg = args_[pos_];
    296           is_on_exit = is_on_exits_[pos_];
    297           Acquire(thr, pc, (uptr)this);
    298         }
    299       }
    300       if (f == 0)
    301         break;
    302       DPrintf("#%d: executing atexit func %p\n", thr->tid, f);
    303       if (is_on_exit)
    304         ((void(*)(int status, void *arg))f)(0, arg);
    305       else
    306         ((void(*)(void *arg, void *dso))f)(arg, 0);
    307     }
    308   }
    309 
    310  private:
    311   static const int kMaxAtExit = 128;
    312   Mutex mtx_;
    313   atexit_t stack_[kMaxAtExit];
    314   void *args_[kMaxAtExit];
    315   bool is_on_exits_[kMaxAtExit];
    316   int pos_;
    317 };
    318 
    319 static AtExitContext *atexit_ctx;
    320 
    321 TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
    322   if (cur_thread()->in_symbolizer)
    323     return 0;
    324   // We want to setup the atexit callback even if we are in ignored lib
    325   // or after fork.
    326   SCOPED_INTERCEPTOR_RAW(atexit, f);
    327   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0);
    328 }
    329 
    330 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
    331   if (cur_thread()->in_symbolizer)
    332     return 0;
    333   SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
    334   return atexit_ctx->atexit(thr, pc, true, (void(*)())f, arg);
    335 }
    336 
    337 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
    338   if (cur_thread()->in_symbolizer)
    339     return 0;
    340   SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
    341   if (dso) {
    342     // Memory allocation in __cxa_atexit will race with free during exit,
    343     // because we do not see synchronization around atexit callback list.
    344     ThreadIgnoreBegin(thr, pc);
    345     int res = REAL(__cxa_atexit)(f, arg, dso);
    346     ThreadIgnoreEnd(thr, pc);
    347     return res;
    348   }
    349   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg);
    350 }
    351 
    352 // Cleanup old bufs.
    353 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
    354   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
    355     JmpBuf *buf = &thr->jmp_bufs[i];
    356     if (buf->sp <= sp) {
    357       uptr sz = thr->jmp_bufs.Size();
    358       thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1];
    359       thr->jmp_bufs.PopBack();
    360       i--;
    361     }
    362   }
    363 }
    364 
    365 static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
    366   if (thr->shadow_stack_pos == 0)  // called from libc guts during bootstrap
    367     return;
    368   // Cleanup old bufs.
    369   JmpBufGarbageCollect(thr, sp);
    370   // Remember the buf.
    371   JmpBuf *buf = thr->jmp_bufs.PushBack();
    372   buf->sp = sp;
    373   buf->mangled_sp = mangled_sp;
    374   buf->shadow_stack_pos = thr->shadow_stack_pos;
    375 }
    376 
    377 static void LongJmp(ThreadState *thr, uptr *env) {
    378   uptr mangled_sp = env[6];
    379   // Find the saved buf by mangled_sp.
    380   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
    381     JmpBuf *buf = &thr->jmp_bufs[i];
    382     if (buf->mangled_sp == mangled_sp) {
    383       CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
    384       // Unwind the stack.
    385       while (thr->shadow_stack_pos > buf->shadow_stack_pos)
    386         FuncExit(thr);
    387       JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
    388       return;
    389     }
    390   }
    391   Printf("ThreadSanitizer: can't find longjmp buf\n");
    392   CHECK(0);
    393 }
    394 
    395 // FIXME: put everything below into a common extern "C" block?
    396 extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
    397   SetJmp(cur_thread(), sp, mangled_sp);
    398 }
    399 
    400 // Not called.  Merely to satisfy TSAN_INTERCEPT().
    401 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
    402 int __interceptor_setjmp(void *env);
    403 extern "C" int __interceptor_setjmp(void *env) {
    404   CHECK(0);
    405   return 0;
    406 }
    407 
    408 // FIXME: any reason to have a separate declaration?
    409 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
    410 int __interceptor__setjmp(void *env);
    411 extern "C" int __interceptor__setjmp(void *env) {
    412   CHECK(0);
    413   return 0;
    414 }
    415 
    416 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
    417 int __interceptor_sigsetjmp(void *env);
    418 extern "C" int __interceptor_sigsetjmp(void *env) {
    419   CHECK(0);
    420   return 0;
    421 }
    422 
    423 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
    424 int __interceptor___sigsetjmp(void *env);
    425 extern "C" int __interceptor___sigsetjmp(void *env) {
    426   CHECK(0);
    427   return 0;
    428 }
    429 
    430 extern "C" int setjmp(void *env);
    431 extern "C" int _setjmp(void *env);
    432 extern "C" int sigsetjmp(void *env);
    433 extern "C" int __sigsetjmp(void *env);
    434 DEFINE_REAL(int, setjmp, void *env)
    435 DEFINE_REAL(int, _setjmp, void *env)
    436 DEFINE_REAL(int, sigsetjmp, void *env)
    437 DEFINE_REAL(int, __sigsetjmp, void *env)
    438 
    439 TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) {
    440   {
    441     SCOPED_TSAN_INTERCEPTOR(longjmp, env, val);
    442   }
    443   LongJmp(cur_thread(), env);
    444   REAL(longjmp)(env, val);
    445 }
    446 
    447 TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
    448   {
    449     SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val);
    450   }
    451   LongJmp(cur_thread(), env);
    452   REAL(siglongjmp)(env, val);
    453 }
    454 
    455 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
    456   if (cur_thread()->in_symbolizer)
    457     return __libc_malloc(size);
    458   void *p = 0;
    459   {
    460     SCOPED_INTERCEPTOR_RAW(malloc, size);
    461     p = user_alloc(thr, pc, size);
    462   }
    463   invoke_malloc_hook(p, size);
    464   return p;
    465 }
    466 
    467 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
    468   SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
    469   return user_alloc(thr, pc, sz, align);
    470 }
    471 
    472 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
    473   if (cur_thread()->in_symbolizer)
    474     return __libc_calloc(size, n);
    475   if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
    476     return AllocatorReturnNull();
    477   void *p = 0;
    478   {
    479     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
    480     p = user_alloc(thr, pc, n * size);
    481     if (p)
    482       internal_memset(p, 0, n * size);
    483   }
    484   invoke_malloc_hook(p, n * size);
    485   return p;
    486 }
    487 
    488 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
    489   if (cur_thread()->in_symbolizer)
    490     return __libc_realloc(p, size);
    491   if (p)
    492     invoke_free_hook(p);
    493   {
    494     SCOPED_INTERCEPTOR_RAW(realloc, p, size);
    495     p = user_realloc(thr, pc, p, size);
    496   }
    497   invoke_malloc_hook(p, size);
    498   return p;
    499 }
    500 
    501 TSAN_INTERCEPTOR(void, free, void *p) {
    502   if (p == 0)
    503     return;
    504   if (cur_thread()->in_symbolizer)
    505     return __libc_free(p);
    506   invoke_free_hook(p);
    507   SCOPED_INTERCEPTOR_RAW(free, p);
    508   user_free(thr, pc, p);
    509 }
    510 
    511 TSAN_INTERCEPTOR(void, cfree, void *p) {
    512   if (p == 0)
    513     return;
    514   if (cur_thread()->in_symbolizer)
    515     return __libc_free(p);
    516   invoke_free_hook(p);
    517   SCOPED_INTERCEPTOR_RAW(cfree, p);
    518   user_free(thr, pc, p);
    519 }
    520 
    521 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
    522   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
    523   return user_alloc_usable_size(p);
    524 }
    525 
    526 #define OPERATOR_NEW_BODY(mangled_name) \
    527   if (cur_thread()->in_symbolizer) \
    528     return __libc_malloc(size); \
    529   void *p = 0; \
    530   {  \
    531     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
    532     p = user_alloc(thr, pc, size); \
    533   }  \
    534   invoke_malloc_hook(p, size);  \
    535   return p;
    536 
    537 SANITIZER_INTERFACE_ATTRIBUTE
    538 void *operator new(__sanitizer::uptr size);
    539 void *operator new(__sanitizer::uptr size) {
    540   OPERATOR_NEW_BODY(_Znwm);
    541 }
    542 
    543 SANITIZER_INTERFACE_ATTRIBUTE
    544 void *operator new[](__sanitizer::uptr size);
    545 void *operator new[](__sanitizer::uptr size) {
    546   OPERATOR_NEW_BODY(_Znam);
    547 }
    548 
    549 SANITIZER_INTERFACE_ATTRIBUTE
    550 void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
    551 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
    552   OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
    553 }
    554 
    555 SANITIZER_INTERFACE_ATTRIBUTE
    556 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
    557 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
    558   OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
    559 }
    560 
    561 #define OPERATOR_DELETE_BODY(mangled_name) \
    562   if (ptr == 0) return;  \
    563   if (cur_thread()->in_symbolizer) \
    564     return __libc_free(ptr); \
    565   invoke_free_hook(ptr);  \
    566   SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
    567   user_free(thr, pc, ptr);
    568 
    569 SANITIZER_INTERFACE_ATTRIBUTE
    570 void operator delete(void *ptr) throw();
    571 void operator delete(void *ptr) throw() {
    572   OPERATOR_DELETE_BODY(_ZdlPv);
    573 }
    574 
    575 SANITIZER_INTERFACE_ATTRIBUTE
    576 void operator delete[](void *ptr) throw();
    577 void operator delete[](void *ptr) throw() {
    578   OPERATOR_DELETE_BODY(_ZdaPv);
    579 }
    580 
    581 SANITIZER_INTERFACE_ATTRIBUTE
    582 void operator delete(void *ptr, std::nothrow_t const&);
    583 void operator delete(void *ptr, std::nothrow_t const&) {
    584   OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
    585 }
    586 
    587 SANITIZER_INTERFACE_ATTRIBUTE
    588 void operator delete[](void *ptr, std::nothrow_t const&);
    589 void operator delete[](void *ptr, std::nothrow_t const&) {
    590   OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
    591 }
    592 
    593 TSAN_INTERCEPTOR(uptr, strlen, const char *s) {
    594   SCOPED_TSAN_INTERCEPTOR(strlen, s);
    595   uptr len = internal_strlen(s);
    596   MemoryAccessRange(thr, pc, (uptr)s, len + 1, false);
    597   return len;
    598 }
    599 
    600 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
    601   SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size);
    602   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
    603   return internal_memset(dst, v, size);
    604 }
    605 
    606 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
    607   SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size);
    608   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
    609   MemoryAccessRange(thr, pc, (uptr)src, size, false);
    610   return internal_memcpy(dst, src, size);
    611 }
    612 
    613 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) {
    614   SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n);
    615   int res = 0;
    616   uptr len = 0;
    617   for (; len < n; len++) {
    618     if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len]))
    619       break;
    620   }
    621   MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
    622   MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
    623   return res;
    624 }
    625 
    626 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) {
    627   SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n);
    628   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
    629   MemoryAccessRange(thr, pc, (uptr)src, n, false);
    630   return REAL(memmove)(dst, src, n);
    631 }
    632 
    633 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) {
    634   SCOPED_TSAN_INTERCEPTOR(strchr, s, c);
    635   char *res = REAL(strchr)(s, c);
    636   uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1;
    637   MemoryAccessRange(thr, pc, (uptr)s, len, false);
    638   return res;
    639 }
    640 
    641 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) {
    642   SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c);
    643   char *res = REAL(strchrnul)(s, c);
    644   uptr len = (char*)res - (char*)s + 1;
    645   MemoryAccessRange(thr, pc, (uptr)s, len, false);
    646   return res;
    647 }
    648 
    649 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) {
    650   SCOPED_TSAN_INTERCEPTOR(strrchr, s, c);
    651   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false);
    652   return REAL(strrchr)(s, c);
    653 }
    654 
    655 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) {  // NOLINT
    656   SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);  // NOLINT
    657   uptr srclen = internal_strlen(src);
    658   MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
    659   MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
    660   return REAL(strcpy)(dst, src);  // NOLINT
    661 }
    662 
    663 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
    664   SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
    665   uptr srclen = internal_strnlen(src, n);
    666   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
    667   MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
    668   return REAL(strncpy)(dst, src, n);
    669 }
    670 
    671 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) {
    672   SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2);
    673   const char *res = REAL(strstr)(s1, s2);
    674   uptr len1 = internal_strlen(s1);
    675   uptr len2 = internal_strlen(s2);
    676   MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false);
    677   MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false);
    678   return res;
    679 }
    680 
    681 TSAN_INTERCEPTOR(char*, strdup, const char *str) {
    682   SCOPED_TSAN_INTERCEPTOR(strdup, str);
    683   // strdup will call malloc, so no instrumentation is required here.
    684   return REAL(strdup)(str);
    685 }
    686 
    687 static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
    688   if (*addr) {
    689     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
    690       if (flags & MAP_FIXED) {
    691         errno = EINVAL;
    692         return false;
    693       } else {
    694         *addr = 0;
    695       }
    696     }
    697   }
    698   return true;
    699 }
    700 
    701 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot,
    702                          int flags, int fd, unsigned off) {
    703   SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off);
    704   if (!fix_mmap_addr(&addr, sz, flags))
    705     return MAP_FAILED;
    706   void *res = REAL(mmap)(addr, sz, prot, flags, fd, off);
    707   if (res != MAP_FAILED) {
    708     if (fd > 0)
    709       FdAccess(thr, pc, fd);
    710     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
    711   }
    712   return res;
    713 }
    714 
    715 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot,
    716                            int flags, int fd, u64 off) {
    717   SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off);
    718   if (!fix_mmap_addr(&addr, sz, flags))
    719     return MAP_FAILED;
    720   void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off);
    721   if (res != MAP_FAILED) {
    722     if (fd > 0)
    723       FdAccess(thr, pc, fd);
    724     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
    725   }
    726   return res;
    727 }
    728 
    729 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
    730   SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
    731   DontNeedShadowFor((uptr)addr, sz);
    732   int res = REAL(munmap)(addr, sz);
    733   return res;
    734 }
    735 
    736 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
    737   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
    738   return user_alloc(thr, pc, sz, align);
    739 }
    740 
    741 TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
    742   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
    743   return user_alloc(thr, pc, sz, align);
    744 }
    745 
    746 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
    747   SCOPED_INTERCEPTOR_RAW(valloc, sz);
    748   return user_alloc(thr, pc, sz, GetPageSizeCached());
    749 }
    750 
    751 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
    752   SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
    753   sz = RoundUp(sz, GetPageSizeCached());
    754   return user_alloc(thr, pc, sz, GetPageSizeCached());
    755 }
    756 
    757 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
    758   SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
    759   *memptr = user_alloc(thr, pc, sz, align);
    760   return 0;
    761 }
    762 
    763 // Used in thread-safe function static initialization.
    764 extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) {
    765   SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
    766   for (;;) {
    767     u32 cmp = atomic_load(g, memory_order_acquire);
    768     if (cmp == 0) {
    769       if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
    770         return 1;
    771     } else if (cmp == 1) {
    772       Acquire(thr, pc, (uptr)g);
    773       return 0;
    774     } else {
    775       internal_sched_yield();
    776     }
    777   }
    778 }
    779 
    780 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) {
    781   SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
    782   Release(thr, pc, (uptr)g);
    783   atomic_store(g, 1, memory_order_release);
    784 }
    785 
    786 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) {
    787   SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
    788   atomic_store(g, 0, memory_order_relaxed);
    789 }
    790 
    791 static void thread_finalize(void *v) {
    792   uptr iter = (uptr)v;
    793   if (iter > 1) {
    794     if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
    795       Printf("ThreadSanitizer: failed to set thread key\n");
    796       Die();
    797     }
    798     return;
    799   }
    800   {
    801     ThreadState *thr = cur_thread();
    802     ThreadFinish(thr);
    803     SignalContext *sctx = thr->signal_ctx;
    804     if (sctx) {
    805       thr->signal_ctx = 0;
    806       UnmapOrDie(sctx, sizeof(*sctx));
    807     }
    808   }
    809 }
    810 
    811 
    812 struct ThreadParam {
    813   void* (*callback)(void *arg);
    814   void *param;
    815   atomic_uintptr_t tid;
    816 };
    817 
    818 extern "C" void *__tsan_thread_start_func(void *arg) {
    819   ThreadParam *p = (ThreadParam*)arg;
    820   void* (*callback)(void *arg) = p->callback;
    821   void *param = p->param;
    822   int tid = 0;
    823   {
    824     ThreadState *thr = cur_thread();
    825     // Thread-local state is not initialized yet.
    826     ScopedIgnoreInterceptors ignore;
    827     if (pthread_setspecific(g_thread_finalize_key,
    828                             (void *)kPthreadDestructorIterations)) {
    829       Printf("ThreadSanitizer: failed to set thread key\n");
    830       Die();
    831     }
    832     while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
    833       pthread_yield();
    834     atomic_store(&p->tid, 0, memory_order_release);
    835     ThreadStart(thr, tid, GetTid());
    836   }
    837   void *res = callback(param);
    838   // Prevent the callback from being tail called,
    839   // it mixes up stack traces.
    840   volatile int foo = 42;
    841   foo++;
    842   return res;
    843 }
    844 
    845 TSAN_INTERCEPTOR(int, pthread_create,
    846     void *th, void *attr, void *(*callback)(void*), void * param) {
    847   SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
    848   if (ctx->after_multithreaded_fork) {
    849     if (flags()->die_after_fork) {
    850       Report("ThreadSanitizer: starting new threads after multi-threaded "
    851           "fork is not supported. Dying (set die_after_fork=0 to override)\n");
    852       Die();
    853     } else {
    854       VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
    855           "fork is not supported (pid %d). Continuing because of "
    856           "die_after_fork=0, but you are on your own\n", internal_getpid());
    857     }
    858   }
    859   __sanitizer_pthread_attr_t myattr;
    860   if (attr == 0) {
    861     pthread_attr_init(&myattr);
    862     attr = &myattr;
    863   }
    864   int detached = 0;
    865   REAL(pthread_attr_getdetachstate)(attr, &detached);
    866   AdjustStackSize(attr);
    867 
    868   ThreadParam p;
    869   p.callback = callback;
    870   p.param = param;
    871   atomic_store(&p.tid, 0, memory_order_relaxed);
    872   int res = -1;
    873   {
    874     // Otherwise we see false positives in pthread stack manipulation.
    875     ScopedIgnoreInterceptors ignore;
    876     ThreadIgnoreBegin(thr, pc);
    877     res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
    878     ThreadIgnoreEnd(thr, pc);
    879   }
    880   if (res == 0) {
    881     int tid = ThreadCreate(thr, pc, *(uptr*)th, detached);
    882     CHECK_NE(tid, 0);
    883     atomic_store(&p.tid, tid, memory_order_release);
    884     while (atomic_load(&p.tid, memory_order_acquire) != 0)
    885       pthread_yield();
    886   }
    887   if (attr == &myattr)
    888     pthread_attr_destroy(&myattr);
    889   return res;
    890 }
    891 
    892 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
    893   SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
    894   int tid = ThreadTid(thr, pc, (uptr)th);
    895   ThreadIgnoreBegin(thr, pc);
    896   int res = BLOCK_REAL(pthread_join)(th, ret);
    897   ThreadIgnoreEnd(thr, pc);
    898   if (res == 0) {
    899     ThreadJoin(thr, pc, tid);
    900   }
    901   return res;
    902 }
    903 
    904 TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
    905   SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
    906   int tid = ThreadTid(thr, pc, (uptr)th);
    907   int res = REAL(pthread_detach)(th);
    908   if (res == 0) {
    909     ThreadDetach(thr, pc, tid);
    910   }
    911   return res;
    912 }
    913 
    914 // Problem:
    915 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
    916 // pthread_cond_t has different size in the different versions.
    917 // If call new REAL functions for old pthread_cond_t, they will corrupt memory
    918 // after pthread_cond_t (old cond is smaller).
    919 // If we call old REAL functions for new pthread_cond_t, we will lose  some
    920 // functionality (e.g. old functions do not support waiting against
    921 // CLOCK_REALTIME).
    922 // Proper handling would require to have 2 versions of interceptors as well.
    923 // But this is messy, in particular requires linker scripts when sanitizer
    924 // runtime is linked into a shared library.
    925 // Instead we assume we don't have dynamic libraries built against old
    926 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
    927 // that allows to work with old libraries (but this mode does not support
    928 // some features, e.g. pthread_condattr_getpshared).
    929 static void *init_cond(void *c, bool force = false) {
    930   // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
    931   // So we allocate additional memory on the side large enough to hold
    932   // any pthread_cond_t object. Always call new REAL functions, but pass
    933   // the aux object to them.
    934   // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
    935   // first word of pthread_cond_t to zero.
    936   // It's all relevant only for linux.
    937   if (!common_flags()->legacy_pthread_cond)
    938     return c;
    939   atomic_uintptr_t *p = (atomic_uintptr_t*)c;
    940   uptr cond = atomic_load(p, memory_order_acquire);
    941   if (!force && cond != 0)
    942     return (void*)cond;
    943   void *newcond = WRAP(malloc)(pthread_cond_t_sz);
    944   internal_memset(newcond, 0, pthread_cond_t_sz);
    945   if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
    946       memory_order_acq_rel))
    947     return newcond;
    948   WRAP(free)(newcond);
    949   return (void*)cond;
    950 }
    951 
    952 struct CondMutexUnlockCtx {
    953   ThreadState *thr;
    954   uptr pc;
    955   void *m;
    956 };
    957 
    958 static void cond_mutex_unlock(CondMutexUnlockCtx *arg) {
    959   MutexLock(arg->thr, arg->pc, (uptr)arg->m);
    960 }
    961 
    962 INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
    963   void *cond = init_cond(c, true);
    964   SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a);
    965   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
    966   return REAL(pthread_cond_init)(cond, a);
    967 }
    968 
    969 INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
    970   void *cond = init_cond(c);
    971   SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
    972   MutexUnlock(thr, pc, (uptr)m);
    973   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
    974   CondMutexUnlockCtx arg = {thr, pc, m};
    975   // This ensures that we handle mutex lock even in case of pthread_cancel.
    976   // See test/tsan/cond_cancel.cc.
    977   int res = call_pthread_cancel_with_cleanup(
    978       (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait),
    979       cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg);
    980   if (res == errno_EOWNERDEAD)
    981     MutexRepair(thr, pc, (uptr)m);
    982   MutexLock(thr, pc, (uptr)m);
    983   return res;
    984 }
    985 
    986 INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
    987   void *cond = init_cond(c);
    988   SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
    989   MutexUnlock(thr, pc, (uptr)m);
    990   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
    991   CondMutexUnlockCtx arg = {thr, pc, m};
    992   // This ensures that we handle mutex lock even in case of pthread_cancel.
    993   // See test/tsan/cond_cancel.cc.
    994   int res = call_pthread_cancel_with_cleanup(
    995       REAL(pthread_cond_timedwait), cond, m, abstime,
    996       (void(*)(void *arg))cond_mutex_unlock, &arg);
    997   if (res == errno_EOWNERDEAD)
    998     MutexRepair(thr, pc, (uptr)m);
    999   MutexLock(thr, pc, (uptr)m);
   1000   return res;
   1001 }
   1002 
   1003 INTERCEPTOR(int, pthread_cond_signal, void *c) {
   1004   void *cond = init_cond(c);
   1005   SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond);
   1006   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
   1007   return REAL(pthread_cond_signal)(cond);
   1008 }
   1009 
   1010 INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
   1011   void *cond = init_cond(c);
   1012   SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond);
   1013   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
   1014   return REAL(pthread_cond_broadcast)(cond);
   1015 }
   1016 
   1017 INTERCEPTOR(int, pthread_cond_destroy, void *c) {
   1018   void *cond = init_cond(c);
   1019   SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond);
   1020   MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true);
   1021   int res = REAL(pthread_cond_destroy)(cond);
   1022   if (common_flags()->legacy_pthread_cond) {
   1023     // Free our aux cond and zero the pointer to not leave dangling pointers.
   1024     WRAP(free)(cond);
   1025     atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
   1026   }
   1027   return res;
   1028 }
   1029 
   1030 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
   1031   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
   1032   int res = REAL(pthread_mutex_init)(m, a);
   1033   if (res == 0) {
   1034     bool recursive = false;
   1035     if (a) {
   1036       int type = 0;
   1037       if (REAL(pthread_mutexattr_gettype)(a, &type) == 0)
   1038         recursive = (type == PTHREAD_MUTEX_RECURSIVE
   1039             || type == PTHREAD_MUTEX_RECURSIVE_NP);
   1040     }
   1041     MutexCreate(thr, pc, (uptr)m, false, recursive, false);
   1042   }
   1043   return res;
   1044 }
   1045 
   1046 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
   1047   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
   1048   int res = REAL(pthread_mutex_destroy)(m);
   1049   if (res == 0 || res == EBUSY) {
   1050     MutexDestroy(thr, pc, (uptr)m);
   1051   }
   1052   return res;
   1053 }
   1054 
   1055 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
   1056   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
   1057   int res = REAL(pthread_mutex_trylock)(m);
   1058   if (res == EOWNERDEAD)
   1059     MutexRepair(thr, pc, (uptr)m);
   1060   if (res == 0 || res == EOWNERDEAD)
   1061     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
   1062   return res;
   1063 }
   1064 
   1065 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
   1066   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
   1067   int res = REAL(pthread_mutex_timedlock)(m, abstime);
   1068   if (res == 0) {
   1069     MutexLock(thr, pc, (uptr)m);
   1070   }
   1071   return res;
   1072 }
   1073 
   1074 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
   1075   SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
   1076   int res = REAL(pthread_spin_init)(m, pshared);
   1077   if (res == 0) {
   1078     MutexCreate(thr, pc, (uptr)m, false, false, false);
   1079   }
   1080   return res;
   1081 }
   1082 
   1083 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
   1084   SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
   1085   int res = REAL(pthread_spin_destroy)(m);
   1086   if (res == 0) {
   1087     MutexDestroy(thr, pc, (uptr)m);
   1088   }
   1089   return res;
   1090 }
   1091 
   1092 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
   1093   SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
   1094   int res = REAL(pthread_spin_lock)(m);
   1095   if (res == 0) {
   1096     MutexLock(thr, pc, (uptr)m);
   1097   }
   1098   return res;
   1099 }
   1100 
   1101 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
   1102   SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
   1103   int res = REAL(pthread_spin_trylock)(m);
   1104   if (res == 0) {
   1105     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
   1106   }
   1107   return res;
   1108 }
   1109 
   1110 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
   1111   SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
   1112   MutexUnlock(thr, pc, (uptr)m);
   1113   int res = REAL(pthread_spin_unlock)(m);
   1114   return res;
   1115 }
   1116 
   1117 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
   1118   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
   1119   int res = REAL(pthread_rwlock_init)(m, a);
   1120   if (res == 0) {
   1121     MutexCreate(thr, pc, (uptr)m, true, false, false);
   1122   }
   1123   return res;
   1124 }
   1125 
   1126 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
   1127   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
   1128   int res = REAL(pthread_rwlock_destroy)(m);
   1129   if (res == 0) {
   1130     MutexDestroy(thr, pc, (uptr)m);
   1131   }
   1132   return res;
   1133 }
   1134 
   1135 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
   1136   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
   1137   int res = REAL(pthread_rwlock_rdlock)(m);
   1138   if (res == 0) {
   1139     MutexReadLock(thr, pc, (uptr)m);
   1140   }
   1141   return res;
   1142 }
   1143 
   1144 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
   1145   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
   1146   int res = REAL(pthread_rwlock_tryrdlock)(m);
   1147   if (res == 0) {
   1148     MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true);
   1149   }
   1150   return res;
   1151 }
   1152 
   1153 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
   1154   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
   1155   int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
   1156   if (res == 0) {
   1157     MutexReadLock(thr, pc, (uptr)m);
   1158   }
   1159   return res;
   1160 }
   1161 
   1162 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
   1163   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
   1164   int res = REAL(pthread_rwlock_wrlock)(m);
   1165   if (res == 0) {
   1166     MutexLock(thr, pc, (uptr)m);
   1167   }
   1168   return res;
   1169 }
   1170 
   1171 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
   1172   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
   1173   int res = REAL(pthread_rwlock_trywrlock)(m);
   1174   if (res == 0) {
   1175     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
   1176   }
   1177   return res;
   1178 }
   1179 
   1180 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
   1181   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
   1182   int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
   1183   if (res == 0) {
   1184     MutexLock(thr, pc, (uptr)m);
   1185   }
   1186   return res;
   1187 }
   1188 
   1189 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
   1190   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
   1191   MutexReadOrWriteUnlock(thr, pc, (uptr)m);
   1192   int res = REAL(pthread_rwlock_unlock)(m);
   1193   return res;
   1194 }
   1195 
   1196 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
   1197   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
   1198   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
   1199   int res = REAL(pthread_barrier_init)(b, a, count);
   1200   return res;
   1201 }
   1202 
   1203 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
   1204   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
   1205   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
   1206   int res = REAL(pthread_barrier_destroy)(b);
   1207   return res;
   1208 }
   1209 
   1210 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
   1211   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
   1212   Release(thr, pc, (uptr)b);
   1213   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
   1214   int res = REAL(pthread_barrier_wait)(b);
   1215   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
   1216   if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
   1217     Acquire(thr, pc, (uptr)b);
   1218   }
   1219   return res;
   1220 }
   1221 
   1222 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
   1223   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
   1224   if (o == 0 || f == 0)
   1225     return EINVAL;
   1226   atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o);
   1227   u32 v = atomic_load(a, memory_order_acquire);
   1228   if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
   1229                                                memory_order_relaxed)) {
   1230     (*f)();
   1231     if (!thr->in_ignored_lib)
   1232       Release(thr, pc, (uptr)o);
   1233     atomic_store(a, 2, memory_order_release);
   1234   } else {
   1235     while (v != 2) {
   1236       pthread_yield();
   1237       v = atomic_load(a, memory_order_acquire);
   1238     }
   1239     if (!thr->in_ignored_lib)
   1240       Acquire(thr, pc, (uptr)o);
   1241   }
   1242   return 0;
   1243 }
   1244 
   1245 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) {
   1246   SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value);
   1247   int res = REAL(sem_init)(s, pshared, value);
   1248   return res;
   1249 }
   1250 
   1251 TSAN_INTERCEPTOR(int, sem_destroy, void *s) {
   1252   SCOPED_TSAN_INTERCEPTOR(sem_destroy, s);
   1253   int res = REAL(sem_destroy)(s);
   1254   return res;
   1255 }
   1256 
   1257 TSAN_INTERCEPTOR(int, sem_wait, void *s) {
   1258   SCOPED_TSAN_INTERCEPTOR(sem_wait, s);
   1259   int res = BLOCK_REAL(sem_wait)(s);
   1260   if (res == 0) {
   1261     Acquire(thr, pc, (uptr)s);
   1262   }
   1263   return res;
   1264 }
   1265 
   1266 TSAN_INTERCEPTOR(int, sem_trywait, void *s) {
   1267   SCOPED_TSAN_INTERCEPTOR(sem_trywait, s);
   1268   int res = BLOCK_REAL(sem_trywait)(s);
   1269   if (res == 0) {
   1270     Acquire(thr, pc, (uptr)s);
   1271   }
   1272   return res;
   1273 }
   1274 
   1275 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) {
   1276   SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime);
   1277   int res = BLOCK_REAL(sem_timedwait)(s, abstime);
   1278   if (res == 0) {
   1279     Acquire(thr, pc, (uptr)s);
   1280   }
   1281   return res;
   1282 }
   1283 
   1284 TSAN_INTERCEPTOR(int, sem_post, void *s) {
   1285   SCOPED_TSAN_INTERCEPTOR(sem_post, s);
   1286   Release(thr, pc, (uptr)s);
   1287   int res = REAL(sem_post)(s);
   1288   return res;
   1289 }
   1290 
   1291 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) {
   1292   SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval);
   1293   int res = REAL(sem_getvalue)(s, sval);
   1294   if (res == 0) {
   1295     Acquire(thr, pc, (uptr)s);
   1296   }
   1297   return res;
   1298 }
   1299 
   1300 TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
   1301   SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf);
   1302   return REAL(__xstat)(version, path, buf);
   1303 }
   1304 
   1305 TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) {
   1306   SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf);
   1307   return REAL(__xstat)(0, path, buf);
   1308 }
   1309 
   1310 TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) {
   1311   SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf);
   1312   return REAL(__xstat64)(version, path, buf);
   1313 }
   1314 
   1315 TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) {
   1316   SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf);
   1317   return REAL(__xstat64)(0, path, buf);
   1318 }
   1319 
   1320 TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) {
   1321   SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf);
   1322   return REAL(__lxstat)(version, path, buf);
   1323 }
   1324 
   1325 TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) {
   1326   SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf);
   1327   return REAL(__lxstat)(0, path, buf);
   1328 }
   1329 
   1330 TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
   1331   SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf);
   1332   return REAL(__lxstat64)(version, path, buf);
   1333 }
   1334 
   1335 TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) {
   1336   SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf);
   1337   return REAL(__lxstat64)(0, path, buf);
   1338 }
   1339 
   1340 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
   1341   SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
   1342   if (fd > 0)
   1343     FdAccess(thr, pc, fd);
   1344   return REAL(__fxstat)(version, fd, buf);
   1345 }
   1346 
   1347 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
   1348   SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
   1349   if (fd > 0)
   1350     FdAccess(thr, pc, fd);
   1351   return REAL(__fxstat)(0, fd, buf);
   1352 }
   1353 
   1354 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
   1355   SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
   1356   if (fd > 0)
   1357     FdAccess(thr, pc, fd);
   1358   return REAL(__fxstat64)(version, fd, buf);
   1359 }
   1360 
   1361 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
   1362   SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
   1363   if (fd > 0)
   1364     FdAccess(thr, pc, fd);
   1365   return REAL(__fxstat64)(0, fd, buf);
   1366 }
   1367 
   1368 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
   1369   SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
   1370   int fd = REAL(open)(name, flags, mode);
   1371   if (fd >= 0)
   1372     FdFileCreate(thr, pc, fd);
   1373   return fd;
   1374 }
   1375 
   1376 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
   1377   SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
   1378   int fd = REAL(open64)(name, flags, mode);
   1379   if (fd >= 0)
   1380     FdFileCreate(thr, pc, fd);
   1381   return fd;
   1382 }
   1383 
   1384 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
   1385   SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
   1386   int fd = REAL(creat)(name, mode);
   1387   if (fd >= 0)
   1388     FdFileCreate(thr, pc, fd);
   1389   return fd;
   1390 }
   1391 
   1392 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
   1393   SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
   1394   int fd = REAL(creat64)(name, mode);
   1395   if (fd >= 0)
   1396     FdFileCreate(thr, pc, fd);
   1397   return fd;
   1398 }
   1399 
   1400 TSAN_INTERCEPTOR(int, dup, int oldfd) {
   1401   SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
   1402   int newfd = REAL(dup)(oldfd);
   1403   if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
   1404     FdDup(thr, pc, oldfd, newfd);
   1405   return newfd;
   1406 }
   1407 
   1408 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
   1409   SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
   1410   int newfd2 = REAL(dup2)(oldfd, newfd);
   1411   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
   1412     FdDup(thr, pc, oldfd, newfd2);
   1413   return newfd2;
   1414 }
   1415 
   1416 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
   1417   SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
   1418   int newfd2 = REAL(dup3)(oldfd, newfd, flags);
   1419   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
   1420     FdDup(thr, pc, oldfd, newfd2);
   1421   return newfd2;
   1422 }
   1423 
   1424 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
   1425   SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
   1426   int fd = REAL(eventfd)(initval, flags);
   1427   if (fd >= 0)
   1428     FdEventCreate(thr, pc, fd);
   1429   return fd;
   1430 }
   1431 
   1432 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
   1433   SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
   1434   if (fd >= 0)
   1435     FdClose(thr, pc, fd);
   1436   fd = REAL(signalfd)(fd, mask, flags);
   1437   if (fd >= 0)
   1438     FdSignalCreate(thr, pc, fd);
   1439   return fd;
   1440 }
   1441 
   1442 TSAN_INTERCEPTOR(int, inotify_init, int fake) {
   1443   SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
   1444   int fd = REAL(inotify_init)(fake);
   1445   if (fd >= 0)
   1446     FdInotifyCreate(thr, pc, fd);
   1447   return fd;
   1448 }
   1449 
   1450 TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
   1451   SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
   1452   int fd = REAL(inotify_init1)(flags);
   1453   if (fd >= 0)
   1454     FdInotifyCreate(thr, pc, fd);
   1455   return fd;
   1456 }
   1457 
   1458 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
   1459   SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
   1460   int fd = REAL(socket)(domain, type, protocol);
   1461   if (fd >= 0)
   1462     FdSocketCreate(thr, pc, fd);
   1463   return fd;
   1464 }
   1465 
   1466 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
   1467   SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
   1468   int res = REAL(socketpair)(domain, type, protocol, fd);
   1469   if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
   1470     FdPipeCreate(thr, pc, fd[0], fd[1]);
   1471   return res;
   1472 }
   1473 
   1474 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
   1475   SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
   1476   FdSocketConnecting(thr, pc, fd);
   1477   int res = REAL(connect)(fd, addr, addrlen);
   1478   if (res == 0 && fd >= 0)
   1479     FdSocketConnect(thr, pc, fd);
   1480   return res;
   1481 }
   1482 
   1483 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
   1484   SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
   1485   int res = REAL(bind)(fd, addr, addrlen);
   1486   if (fd > 0 && res == 0)
   1487     FdAccess(thr, pc, fd);
   1488   return res;
   1489 }
   1490 
   1491 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
   1492   SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
   1493   int res = REAL(listen)(fd, backlog);
   1494   if (fd > 0 && res == 0)
   1495     FdAccess(thr, pc, fd);
   1496   return res;
   1497 }
   1498 
   1499 TSAN_INTERCEPTOR(int, epoll_create, int size) {
   1500   SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
   1501   int fd = REAL(epoll_create)(size);
   1502   if (fd >= 0)
   1503     FdPollCreate(thr, pc, fd);
   1504   return fd;
   1505 }
   1506 
   1507 TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
   1508   SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
   1509   int fd = REAL(epoll_create1)(flags);
   1510   if (fd >= 0)
   1511     FdPollCreate(thr, pc, fd);
   1512   return fd;
   1513 }
   1514 
   1515 TSAN_INTERCEPTOR(int, close, int fd) {
   1516   SCOPED_TSAN_INTERCEPTOR(close, fd);
   1517   if (fd >= 0)
   1518     FdClose(thr, pc, fd);
   1519   return REAL(close)(fd);
   1520 }
   1521 
   1522 TSAN_INTERCEPTOR(int, __close, int fd) {
   1523   SCOPED_TSAN_INTERCEPTOR(__close, fd);
   1524   if (fd >= 0)
   1525     FdClose(thr, pc, fd);
   1526   return REAL(__close)(fd);
   1527 }
   1528 
   1529 // glibc guts
   1530 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
   1531   SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
   1532   int fds[64];
   1533   int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
   1534   for (int i = 0; i < cnt; i++) {
   1535     if (fds[i] > 0)
   1536       FdClose(thr, pc, fds[i]);
   1537   }
   1538   REAL(__res_iclose)(state, free_addr);
   1539 }
   1540 
   1541 TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
   1542   SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
   1543   int res = REAL(pipe)(pipefd);
   1544   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
   1545     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
   1546   return res;
   1547 }
   1548 
   1549 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
   1550   SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
   1551   int res = REAL(pipe2)(pipefd, flags);
   1552   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
   1553     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
   1554   return res;
   1555 }
   1556 
   1557 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) {
   1558   SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags);
   1559   if (fd >= 0) {
   1560     FdAccess(thr, pc, fd);
   1561     FdRelease(thr, pc, fd);
   1562   }
   1563   int res = REAL(send)(fd, buf, len, flags);
   1564   return res;
   1565 }
   1566 
   1567 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) {
   1568   SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags);
   1569   if (fd >= 0) {
   1570     FdAccess(thr, pc, fd);
   1571     FdRelease(thr, pc, fd);
   1572   }
   1573   int res = REAL(sendmsg)(fd, msg, flags);
   1574   return res;
   1575 }
   1576 
   1577 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) {
   1578   SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags);
   1579   if (fd >= 0)
   1580     FdAccess(thr, pc, fd);
   1581   int res = REAL(recv)(fd, buf, len, flags);
   1582   if (res >= 0 && fd >= 0) {
   1583     FdAcquire(thr, pc, fd);
   1584   }
   1585   return res;
   1586 }
   1587 
   1588 TSAN_INTERCEPTOR(int, unlink, char *path) {
   1589   SCOPED_TSAN_INTERCEPTOR(unlink, path);
   1590   Release(thr, pc, File2addr(path));
   1591   int res = REAL(unlink)(path);
   1592   return res;
   1593 }
   1594 
   1595 TSAN_INTERCEPTOR(void*, tmpfile, int fake) {
   1596   SCOPED_TSAN_INTERCEPTOR(tmpfile, fake);
   1597   void *res = REAL(tmpfile)(fake);
   1598   if (res) {
   1599     int fd = fileno_unlocked(res);
   1600     if (fd >= 0)
   1601       FdFileCreate(thr, pc, fd);
   1602   }
   1603   return res;
   1604 }
   1605 
   1606 TSAN_INTERCEPTOR(void*, tmpfile64, int fake) {
   1607   SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake);
   1608   void *res = REAL(tmpfile64)(fake);
   1609   if (res) {
   1610     int fd = fileno_unlocked(res);
   1611     if (fd >= 0)
   1612       FdFileCreate(thr, pc, fd);
   1613   }
   1614   return res;
   1615 }
   1616 
   1617 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
   1618   // libc file streams can call user-supplied functions, see fopencookie.
   1619   {
   1620     SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
   1621     MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
   1622   }
   1623   return REAL(fread)(ptr, size, nmemb, f);
   1624 }
   1625 
   1626 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
   1627   // libc file streams can call user-supplied functions, see fopencookie.
   1628   {
   1629     SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
   1630     MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
   1631   }
   1632   return REAL(fwrite)(p, size, nmemb, f);
   1633 }
   1634 
   1635 TSAN_INTERCEPTOR(void, abort, int fake) {
   1636   SCOPED_TSAN_INTERCEPTOR(abort, fake);
   1637   REAL(fflush)(0);
   1638   REAL(abort)(fake);
   1639 }
   1640 
   1641 TSAN_INTERCEPTOR(int, puts, const char *s) {
   1642   SCOPED_TSAN_INTERCEPTOR(puts, s);
   1643   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false);
   1644   return REAL(puts)(s);
   1645 }
   1646 
   1647 TSAN_INTERCEPTOR(int, rmdir, char *path) {
   1648   SCOPED_TSAN_INTERCEPTOR(rmdir, path);
   1649   Release(thr, pc, Dir2addr(path));
   1650   int res = REAL(rmdir)(path);
   1651   return res;
   1652 }
   1653 
   1654 TSAN_INTERCEPTOR(void*, opendir, char *path) {
   1655   SCOPED_TSAN_INTERCEPTOR(opendir, path);
   1656   void *res = REAL(opendir)(path);
   1657   if (res != 0)
   1658     Acquire(thr, pc, Dir2addr(path));
   1659   return res;
   1660 }
   1661 
   1662 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
   1663   SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
   1664   if (epfd >= 0)
   1665     FdAccess(thr, pc, epfd);
   1666   if (epfd >= 0 && fd >= 0)
   1667     FdAccess(thr, pc, fd);
   1668   if (op == EPOLL_CTL_ADD && epfd >= 0)
   1669     FdRelease(thr, pc, epfd);
   1670   int res = REAL(epoll_ctl)(epfd, op, fd, ev);
   1671   return res;
   1672 }
   1673 
   1674 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
   1675   SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
   1676   if (epfd >= 0)
   1677     FdAccess(thr, pc, epfd);
   1678   int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
   1679   if (res > 0 && epfd >= 0)
   1680     FdAcquire(thr, pc, epfd);
   1681   return res;
   1682 }
   1683 
   1684 namespace __tsan {
   1685 
   1686 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool sigact,
   1687     int sig, my_siginfo_t *info, void *uctx) {
   1688   // Ensure that the handler does not spoil errno.
   1689   const int saved_errno = errno;
   1690   errno = 99;
   1691   // Need to remember pc before the call, because the handler can reset it.
   1692   uptr pc = sigact ?
   1693      (uptr)sigactions[sig].sa_sigaction :
   1694      (uptr)sigactions[sig].sa_handler;
   1695   pc += 1;  // return address is expected, OutputReport() will undo this
   1696   if (sigact)
   1697     sigactions[sig].sa_sigaction(sig, info, uctx);
   1698   else
   1699     sigactions[sig].sa_handler(sig);
   1700   // We do not detect errno spoiling for SIGTERM,
   1701   // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
   1702   // tsan reports false positive in such case.
   1703   // It's difficult to properly detect this situation (reraise),
   1704   // because in async signal processing case (when handler is called directly
   1705   // from rtl_generic_sighandler) we have not yet received the reraised
   1706   // signal; and it looks too fragile to intercept all ways to reraise a signal.
   1707   if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
   1708     __tsan::StackTrace stack;
   1709     stack.ObtainCurrent(thr, pc);
   1710     ThreadRegistryLock l(ctx->thread_registry);
   1711     ScopedReport rep(ReportTypeErrnoInSignal);
   1712     if (!IsFiredSuppression(ctx, rep, stack)) {
   1713       rep.AddStack(&stack, true);
   1714       OutputReport(thr, rep);
   1715     }
   1716   }
   1717   errno = saved_errno;
   1718 }
   1719 
   1720 void ProcessPendingSignals(ThreadState *thr) {
   1721   SignalContext *sctx = SigCtx(thr);
   1722   if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler)
   1723     return;
   1724   thr->in_signal_handler = true;
   1725   sctx->pending_signal_count = 0;
   1726   // These are too big for stack.
   1727   static THREADLOCAL __sanitizer_sigset_t emptyset, oldset;
   1728   REAL(sigfillset)(&emptyset);
   1729   pthread_sigmask(SIG_SETMASK, &emptyset, &oldset);
   1730   for (int sig = 0; sig < kSigCount; sig++) {
   1731     SignalDesc *signal = &sctx->pending_signals[sig];
   1732     if (signal->armed) {
   1733       signal->armed = false;
   1734       if (sigactions[sig].sa_handler != SIG_DFL
   1735           && sigactions[sig].sa_handler != SIG_IGN) {
   1736         CallUserSignalHandler(thr, false, signal->sigaction,
   1737             sig, &signal->siginfo, &signal->ctx);
   1738       }
   1739     }
   1740   }
   1741   pthread_sigmask(SIG_SETMASK, &oldset, 0);
   1742   CHECK_EQ(thr->in_signal_handler, true);
   1743   thr->in_signal_handler = false;
   1744 }
   1745 
   1746 }  // namespace __tsan
   1747 
   1748 static bool is_sync_signal(SignalContext *sctx, int sig) {
   1749   return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL ||
   1750       sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
   1751       // If we are sending signal to ourselves, we must process it now.
   1752       (sctx && sig == sctx->int_signal_send);
   1753 }
   1754 
   1755 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
   1756     my_siginfo_t *info, void *ctx) {
   1757   ThreadState *thr = cur_thread();
   1758   SignalContext *sctx = SigCtx(thr);
   1759   if (sig < 0 || sig >= kSigCount) {
   1760     VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
   1761     return;
   1762   }
   1763   // Don't mess with synchronous signals.
   1764   const bool sync = is_sync_signal(sctx, sig);
   1765   if (sync ||
   1766       // If we are in blocking function, we can safely process it now
   1767       // (but check if we are in a recursive interceptor,
   1768       // i.e. pthread_join()->munmap()).
   1769       (sctx && sctx->in_blocking_func == 1)) {
   1770     CHECK_EQ(thr->in_signal_handler, false);
   1771     thr->in_signal_handler = true;
   1772     if (sctx && sctx->in_blocking_func == 1) {
   1773       // We ignore interceptors in blocking functions,
   1774       // temporary enbled them again while we are calling user function.
   1775       int const i = thr->ignore_interceptors;
   1776       thr->ignore_interceptors = 0;
   1777       CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
   1778       thr->ignore_interceptors = i;
   1779     } else {
   1780       CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
   1781     }
   1782     CHECK_EQ(thr->in_signal_handler, true);
   1783     thr->in_signal_handler = false;
   1784     return;
   1785   }
   1786 
   1787   if (sctx == 0)
   1788     return;
   1789   SignalDesc *signal = &sctx->pending_signals[sig];
   1790   if (signal->armed == false) {
   1791     signal->armed = true;
   1792     signal->sigaction = sigact;
   1793     if (info)
   1794       internal_memcpy(&signal->siginfo, info, sizeof(*info));
   1795     if (ctx)
   1796       internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
   1797     sctx->pending_signal_count++;
   1798   }
   1799 }
   1800 
   1801 static void rtl_sighandler(int sig) {
   1802   rtl_generic_sighandler(false, sig, 0, 0);
   1803 }
   1804 
   1805 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
   1806   rtl_generic_sighandler(true, sig, info, ctx);
   1807 }
   1808 
   1809 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
   1810   SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old);
   1811   if (old)
   1812     internal_memcpy(old, &sigactions[sig], sizeof(*old));
   1813   if (act == 0)
   1814     return 0;
   1815   internal_memcpy(&sigactions[sig], act, sizeof(*act));
   1816   sigaction_t newact;
   1817   internal_memcpy(&newact, act, sizeof(newact));
   1818   REAL(sigfillset)(&newact.sa_mask);
   1819   if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
   1820     if (newact.sa_flags & SA_SIGINFO)
   1821       newact.sa_sigaction = rtl_sigaction;
   1822     else
   1823       newact.sa_handler = rtl_sighandler;
   1824   }
   1825   int res = REAL(sigaction)(sig, &newact, 0);
   1826   return res;
   1827 }
   1828 
   1829 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
   1830   sigaction_t act;
   1831   act.sa_handler = h;
   1832   REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask));
   1833   act.sa_flags = 0;
   1834   sigaction_t old;
   1835   int res = sigaction(sig, &act, &old);
   1836   if (res)
   1837     return SIG_ERR;
   1838   return old.sa_handler;
   1839 }
   1840 
   1841 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
   1842   SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
   1843   return REAL(sigsuspend)(mask);
   1844 }
   1845 
   1846 TSAN_INTERCEPTOR(int, raise, int sig) {
   1847   SCOPED_TSAN_INTERCEPTOR(raise, sig);
   1848   SignalContext *sctx = SigCtx(thr);
   1849   CHECK_NE(sctx, 0);
   1850   int prev = sctx->int_signal_send;
   1851   sctx->int_signal_send = sig;
   1852   int res = REAL(raise)(sig);
   1853   CHECK_EQ(sctx->int_signal_send, sig);
   1854   sctx->int_signal_send = prev;
   1855   return res;
   1856 }
   1857 
   1858 TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
   1859   SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
   1860   SignalContext *sctx = SigCtx(thr);
   1861   CHECK_NE(sctx, 0);
   1862   int prev = sctx->int_signal_send;
   1863   if (pid == (int)internal_getpid()) {
   1864     sctx->int_signal_send = sig;
   1865   }
   1866   int res = REAL(kill)(pid, sig);
   1867   if (pid == (int)internal_getpid()) {
   1868     CHECK_EQ(sctx->int_signal_send, sig);
   1869     sctx->int_signal_send = prev;
   1870   }
   1871   return res;
   1872 }
   1873 
   1874 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
   1875   SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
   1876   SignalContext *sctx = SigCtx(thr);
   1877   CHECK_NE(sctx, 0);
   1878   int prev = sctx->int_signal_send;
   1879   if (tid == pthread_self()) {
   1880     sctx->int_signal_send = sig;
   1881   }
   1882   int res = REAL(pthread_kill)(tid, sig);
   1883   if (tid == pthread_self()) {
   1884     CHECK_EQ(sctx->int_signal_send, sig);
   1885     sctx->int_signal_send = prev;
   1886   }
   1887   return res;
   1888 }
   1889 
   1890 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
   1891   SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
   1892   // It's intercepted merely to process pending signals.
   1893   return REAL(gettimeofday)(tv, tz);
   1894 }
   1895 
   1896 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
   1897     void *hints, void *rv) {
   1898   SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
   1899   // We miss atomic synchronization in getaddrinfo,
   1900   // and can report false race between malloc and free
   1901   // inside of getaddrinfo. So ignore memory accesses.
   1902   ThreadIgnoreBegin(thr, pc);
   1903   int res = REAL(getaddrinfo)(node, service, hints, rv);
   1904   ThreadIgnoreEnd(thr, pc);
   1905   return res;
   1906 }
   1907 
   1908 // Linux kernel has a bug that leads to kernel deadlock if a process
   1909 // maps TBs of memory and then calls mlock().
   1910 static void MlockIsUnsupported() {
   1911   static atomic_uint8_t printed;
   1912   if (atomic_exchange(&printed, 1, memory_order_relaxed))
   1913     return;
   1914   VPrintf(1, "INFO: ThreadSanitizer ignores mlock/munlock[all]\n");
   1915 }
   1916 
   1917 TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) {
   1918   MlockIsUnsupported();
   1919   return 0;
   1920 }
   1921 
   1922 TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) {
   1923   MlockIsUnsupported();
   1924   return 0;
   1925 }
   1926 
   1927 TSAN_INTERCEPTOR(int, mlockall, int flags) {
   1928   MlockIsUnsupported();
   1929   return 0;
   1930 }
   1931 
   1932 TSAN_INTERCEPTOR(int, munlockall, void) {
   1933   MlockIsUnsupported();
   1934   return 0;
   1935 }
   1936 
   1937 TSAN_INTERCEPTOR(int, fork, int fake) {
   1938   if (cur_thread()->in_symbolizer)
   1939     return REAL(fork)(fake);
   1940   SCOPED_INTERCEPTOR_RAW(fork, fake);
   1941   ForkBefore(thr, pc);
   1942   int pid = REAL(fork)(fake);
   1943   if (pid == 0) {
   1944     // child
   1945     ForkChildAfter(thr, pc);
   1946     FdOnFork(thr, pc);
   1947   } else if (pid > 0) {
   1948     // parent
   1949     ForkParentAfter(thr, pc);
   1950   } else {
   1951     // error
   1952     ForkParentAfter(thr, pc);
   1953   }
   1954   return pid;
   1955 }
   1956 
   1957 TSAN_INTERCEPTOR(int, vfork, int fake) {
   1958   // Some programs (e.g. openjdk) call close for all file descriptors
   1959   // in the child process. Under tsan it leads to false positives, because
   1960   // address space is shared, so the parent process also thinks that
   1961   // the descriptors are closed (while they are actually not).
   1962   // This leads to false positives due to missed synchronization.
   1963   // Strictly saying this is undefined behavior, because vfork child is not
   1964   // allowed to call any functions other than exec/exit. But this is what
   1965   // openjdk does, so we want to handle it.
   1966   // We could disable interceptors in the child process. But it's not possible
   1967   // to simply intercept and wrap vfork, because vfork child is not allowed
   1968   // to return from the function that calls vfork, and that's exactly what
   1969   // we would do. So this would require some assembly trickery as well.
   1970   // Instead we simply turn vfork into fork.
   1971   return WRAP(fork)(fake);
   1972 }
   1973 
   1974 static int OnExit(ThreadState *thr) {
   1975   int status = Finalize(thr);
   1976   REAL(fflush)(0);
   1977   return status;
   1978 }
   1979 
   1980 struct TsanInterceptorContext {
   1981   ThreadState *thr;
   1982   const uptr caller_pc;
   1983   const uptr pc;
   1984 };
   1985 
   1986 static void HandleRecvmsg(ThreadState *thr, uptr pc,
   1987     __sanitizer_msghdr *msg) {
   1988   int fds[64];
   1989   int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
   1990   for (int i = 0; i < cnt; i++)
   1991     FdEventCreate(thr, pc, fds[i]);
   1992 }
   1993 
   1994 #include "sanitizer_common/sanitizer_platform_interceptors.h"
   1995 // Causes interceptor recursion (getaddrinfo() and fopen())
   1996 #undef SANITIZER_INTERCEPT_GETADDRINFO
   1997 
   1998 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
   1999 
   2000 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
   2001   MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
   2002                     ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
   2003                     true)
   2004 
   2005 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
   2006   MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
   2007                     ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
   2008                     false)
   2009 
   2010 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)      \
   2011   SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);         \
   2012   TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
   2013   ctx = (void *)&_ctx;                                \
   2014   (void) ctx;
   2015 
   2016 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
   2017   SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);              \
   2018   TsanInterceptorContext _ctx = {thr, caller_pc, pc};     \
   2019   ctx = (void *)&_ctx;                                    \
   2020   (void) ctx;
   2021 
   2022 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
   2023   Acquire(thr, pc, File2addr(path));                  \
   2024   if (file) {                                         \
   2025     int fd = fileno_unlocked(file);                   \
   2026     if (fd >= 0) FdFileCreate(thr, pc, fd);           \
   2027   }
   2028 
   2029 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
   2030   if (file) {                                    \
   2031     int fd = fileno_unlocked(file);              \
   2032     if (fd >= 0) FdClose(thr, pc, fd);           \
   2033   }
   2034 
   2035 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res)  \
   2036   libignore()->OnLibraryLoaded(filename)
   2037 
   2038 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
   2039   libignore()->OnLibraryUnloaded()
   2040 
   2041 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
   2042   FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
   2043 
   2044 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
   2045   FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
   2046 
   2047 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
   2048   FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
   2049 
   2050 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
   2051   FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
   2052 
   2053 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
   2054   ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
   2055 
   2056 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
   2057   __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
   2058 
   2059 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
   2060 
   2061 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
   2062   OnExit(((TsanInterceptorContext *) ctx)->thr)
   2063 
   2064 #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
   2065   MutexLock(((TsanInterceptorContext *)ctx)->thr, \
   2066             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
   2067 
   2068 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
   2069   MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
   2070             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
   2071 
   2072 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
   2073   MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
   2074             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
   2075 
   2076 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
   2077   HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
   2078       ((TsanInterceptorContext *)ctx)->pc, msg)
   2079 
   2080 #include "sanitizer_common/sanitizer_common_interceptors.inc"
   2081 
   2082 #define TSAN_SYSCALL() \
   2083   ThreadState *thr = cur_thread(); \
   2084   if (thr->ignore_interceptors) \
   2085     return; \
   2086   ScopedSyscall scoped_syscall(thr) \
   2087 /**/
   2088 
   2089 struct ScopedSyscall {
   2090   ThreadState *thr;
   2091 
   2092   explicit ScopedSyscall(ThreadState *thr)
   2093       : thr(thr) {
   2094     Initialize(thr);
   2095   }
   2096 
   2097   ~ScopedSyscall() {
   2098     ProcessPendingSignals(thr);
   2099   }
   2100 };
   2101 
   2102 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
   2103   TSAN_SYSCALL();
   2104   MemoryAccessRange(thr, pc, p, s, write);
   2105 }
   2106 
   2107 static void syscall_acquire(uptr pc, uptr addr) {
   2108   TSAN_SYSCALL();
   2109   Acquire(thr, pc, addr);
   2110   DPrintf("syscall_acquire(%p)\n", addr);
   2111 }
   2112 
   2113 static void syscall_release(uptr pc, uptr addr) {
   2114   TSAN_SYSCALL();
   2115   DPrintf("syscall_release(%p)\n", addr);
   2116   Release(thr, pc, addr);
   2117 }
   2118 
   2119 static void syscall_fd_close(uptr pc, int fd) {
   2120   TSAN_SYSCALL();
   2121   FdClose(thr, pc, fd);
   2122 }
   2123 
   2124 static USED void syscall_fd_acquire(uptr pc, int fd) {
   2125   TSAN_SYSCALL();
   2126   FdAcquire(thr, pc, fd);
   2127   DPrintf("syscall_fd_acquire(%p)\n", fd);
   2128 }
   2129 
   2130 static USED void syscall_fd_release(uptr pc, int fd) {
   2131   TSAN_SYSCALL();
   2132   DPrintf("syscall_fd_release(%p)\n", fd);
   2133   FdRelease(thr, pc, fd);
   2134 }
   2135 
   2136 static void syscall_pre_fork(uptr pc) {
   2137   TSAN_SYSCALL();
   2138   ForkBefore(thr, pc);
   2139 }
   2140 
   2141 static void syscall_post_fork(uptr pc, int pid) {
   2142   TSAN_SYSCALL();
   2143   if (pid == 0) {
   2144     // child
   2145     ForkChildAfter(thr, pc);
   2146     FdOnFork(thr, pc);
   2147   } else if (pid > 0) {
   2148     // parent
   2149     ForkParentAfter(thr, pc);
   2150   } else {
   2151     // error
   2152     ForkParentAfter(thr, pc);
   2153   }
   2154 }
   2155 
   2156 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
   2157   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
   2158 
   2159 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
   2160   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
   2161 
   2162 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
   2163   do {                                       \
   2164     (void)(p);                               \
   2165     (void)(s);                               \
   2166   } while (false)
   2167 
   2168 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
   2169   do {                                        \
   2170     (void)(p);                                \
   2171     (void)(s);                                \
   2172   } while (false)
   2173 
   2174 #define COMMON_SYSCALL_ACQUIRE(addr) \
   2175     syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
   2176 
   2177 #define COMMON_SYSCALL_RELEASE(addr) \
   2178     syscall_release(GET_CALLER_PC(), (uptr)(addr))
   2179 
   2180 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
   2181 
   2182 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
   2183 
   2184 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
   2185 
   2186 #define COMMON_SYSCALL_PRE_FORK() \
   2187   syscall_pre_fork(GET_CALLER_PC())
   2188 
   2189 #define COMMON_SYSCALL_POST_FORK(res) \
   2190   syscall_post_fork(GET_CALLER_PC(), res)
   2191 
   2192 #include "sanitizer_common/sanitizer_common_syscalls.inc"
   2193 
   2194 namespace __tsan {
   2195 
   2196 static void finalize(void *arg) {
   2197   ThreadState *thr = cur_thread();
   2198   uptr pc = 0;
   2199   atexit_ctx->exit(thr, pc);
   2200   int status = Finalize(thr);
   2201   // Make sure the output is not lost.
   2202   // Flushing all the streams here may freeze the process if a child thread is
   2203   // performing file stream operations at the same time.
   2204   REAL(fflush)(stdout);
   2205   REAL(fflush)(stderr);
   2206   if (status)
   2207     REAL(_exit)(status);
   2208 }
   2209 
   2210 static void unreachable() {
   2211   Report("FATAL: ThreadSanitizer: unreachable called\n");
   2212   Die();
   2213 }
   2214 
   2215 void InitializeInterceptors() {
   2216   // We need to setup it early, because functions like dlsym() can call it.
   2217   REAL(memset) = internal_memset;
   2218   REAL(memcpy) = internal_memcpy;
   2219   REAL(memcmp) = internal_memcmp;
   2220 
   2221   // Instruct libc malloc to consume less memory.
   2222   mallopt(1, 0);  // M_MXFAST
   2223   mallopt(-3, 32*1024);  // M_MMAP_THRESHOLD
   2224 
   2225   InitializeCommonInterceptors();
   2226 
   2227   // We can not use TSAN_INTERCEPT to get setjmp addr,
   2228   // because it does &setjmp and setjmp is not present in some versions of libc.
   2229   using __interception::GetRealFunctionAddress;
   2230   GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0);
   2231   GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
   2232   GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0);
   2233   GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
   2234 
   2235   TSAN_INTERCEPT(longjmp);
   2236   TSAN_INTERCEPT(siglongjmp);
   2237 
   2238   TSAN_INTERCEPT(malloc);
   2239   TSAN_INTERCEPT(__libc_memalign);
   2240   TSAN_INTERCEPT(calloc);
   2241   TSAN_INTERCEPT(realloc);
   2242   TSAN_INTERCEPT(free);
   2243   TSAN_INTERCEPT(cfree);
   2244   TSAN_INTERCEPT(mmap);
   2245   TSAN_INTERCEPT(mmap64);
   2246   TSAN_INTERCEPT(munmap);
   2247   TSAN_INTERCEPT(memalign);
   2248   TSAN_INTERCEPT(valloc);
   2249   TSAN_INTERCEPT(pvalloc);
   2250   TSAN_INTERCEPT(posix_memalign);
   2251 
   2252   TSAN_INTERCEPT(strlen);
   2253   TSAN_INTERCEPT(memset);
   2254   TSAN_INTERCEPT(memcpy);
   2255   TSAN_INTERCEPT(memmove);
   2256   TSAN_INTERCEPT(memcmp);
   2257   TSAN_INTERCEPT(strchr);
   2258   TSAN_INTERCEPT(strchrnul);
   2259   TSAN_INTERCEPT(strrchr);
   2260   TSAN_INTERCEPT(strcpy);  // NOLINT
   2261   TSAN_INTERCEPT(strncpy);
   2262   TSAN_INTERCEPT(strstr);
   2263   TSAN_INTERCEPT(strdup);
   2264 
   2265   TSAN_INTERCEPT(pthread_create);
   2266   TSAN_INTERCEPT(pthread_join);
   2267   TSAN_INTERCEPT(pthread_detach);
   2268 
   2269   TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2");
   2270   TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2");
   2271   TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
   2272   TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2");
   2273   TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
   2274   TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2");
   2275 
   2276   TSAN_INTERCEPT(pthread_mutex_init);
   2277   TSAN_INTERCEPT(pthread_mutex_destroy);
   2278   TSAN_INTERCEPT(pthread_mutex_trylock);
   2279   TSAN_INTERCEPT(pthread_mutex_timedlock);
   2280 
   2281   TSAN_INTERCEPT(pthread_spin_init);
   2282   TSAN_INTERCEPT(pthread_spin_destroy);
   2283   TSAN_INTERCEPT(pthread_spin_lock);
   2284   TSAN_INTERCEPT(pthread_spin_trylock);
   2285   TSAN_INTERCEPT(pthread_spin_unlock);
   2286 
   2287   TSAN_INTERCEPT(pthread_rwlock_init);
   2288   TSAN_INTERCEPT(pthread_rwlock_destroy);
   2289   TSAN_INTERCEPT(pthread_rwlock_rdlock);
   2290   TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
   2291   TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
   2292   TSAN_INTERCEPT(pthread_rwlock_wrlock);
   2293   TSAN_INTERCEPT(pthread_rwlock_trywrlock);
   2294   TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
   2295   TSAN_INTERCEPT(pthread_rwlock_unlock);
   2296 
   2297   TSAN_INTERCEPT(pthread_barrier_init);
   2298   TSAN_INTERCEPT(pthread_barrier_destroy);
   2299   TSAN_INTERCEPT(pthread_barrier_wait);
   2300 
   2301   TSAN_INTERCEPT(pthread_once);
   2302 
   2303   TSAN_INTERCEPT(sem_init);
   2304   TSAN_INTERCEPT(sem_destroy);
   2305   TSAN_INTERCEPT(sem_wait);
   2306   TSAN_INTERCEPT(sem_trywait);
   2307   TSAN_INTERCEPT(sem_timedwait);
   2308   TSAN_INTERCEPT(sem_post);
   2309   TSAN_INTERCEPT(sem_getvalue);
   2310 
   2311   TSAN_INTERCEPT(stat);
   2312   TSAN_INTERCEPT(__xstat);
   2313   TSAN_INTERCEPT(stat64);
   2314   TSAN_INTERCEPT(__xstat64);
   2315   TSAN_INTERCEPT(lstat);
   2316   TSAN_INTERCEPT(__lxstat);
   2317   TSAN_INTERCEPT(lstat64);
   2318   TSAN_INTERCEPT(__lxstat64);
   2319   TSAN_INTERCEPT(fstat);
   2320   TSAN_INTERCEPT(__fxstat);
   2321   TSAN_INTERCEPT(fstat64);
   2322   TSAN_INTERCEPT(__fxstat64);
   2323   TSAN_INTERCEPT(open);
   2324   TSAN_INTERCEPT(open64);
   2325   TSAN_INTERCEPT(creat);
   2326   TSAN_INTERCEPT(creat64);
   2327   TSAN_INTERCEPT(dup);
   2328   TSAN_INTERCEPT(dup2);
   2329   TSAN_INTERCEPT(dup3);
   2330   TSAN_INTERCEPT(eventfd);
   2331   TSAN_INTERCEPT(signalfd);
   2332   TSAN_INTERCEPT(inotify_init);
   2333   TSAN_INTERCEPT(inotify_init1);
   2334   TSAN_INTERCEPT(socket);
   2335   TSAN_INTERCEPT(socketpair);
   2336   TSAN_INTERCEPT(connect);
   2337   TSAN_INTERCEPT(bind);
   2338   TSAN_INTERCEPT(listen);
   2339   TSAN_INTERCEPT(epoll_create);
   2340   TSAN_INTERCEPT(epoll_create1);
   2341   TSAN_INTERCEPT(close);
   2342   TSAN_INTERCEPT(__close);
   2343   TSAN_INTERCEPT(__res_iclose);
   2344   TSAN_INTERCEPT(pipe);
   2345   TSAN_INTERCEPT(pipe2);
   2346 
   2347   TSAN_INTERCEPT(send);
   2348   TSAN_INTERCEPT(sendmsg);
   2349   TSAN_INTERCEPT(recv);
   2350 
   2351   TSAN_INTERCEPT(unlink);
   2352   TSAN_INTERCEPT(tmpfile);
   2353   TSAN_INTERCEPT(tmpfile64);
   2354   TSAN_INTERCEPT(fread);
   2355   TSAN_INTERCEPT(fwrite);
   2356   TSAN_INTERCEPT(abort);
   2357   TSAN_INTERCEPT(puts);
   2358   TSAN_INTERCEPT(rmdir);
   2359   TSAN_INTERCEPT(opendir);
   2360 
   2361   TSAN_INTERCEPT(epoll_ctl);
   2362   TSAN_INTERCEPT(epoll_wait);
   2363 
   2364   TSAN_INTERCEPT(sigaction);
   2365   TSAN_INTERCEPT(signal);
   2366   TSAN_INTERCEPT(sigsuspend);
   2367   TSAN_INTERCEPT(raise);
   2368   TSAN_INTERCEPT(kill);
   2369   TSAN_INTERCEPT(pthread_kill);
   2370   TSAN_INTERCEPT(sleep);
   2371   TSAN_INTERCEPT(usleep);
   2372   TSAN_INTERCEPT(nanosleep);
   2373   TSAN_INTERCEPT(gettimeofday);
   2374   TSAN_INTERCEPT(getaddrinfo);
   2375 
   2376   TSAN_INTERCEPT(mlock);
   2377   TSAN_INTERCEPT(munlock);
   2378   TSAN_INTERCEPT(mlockall);
   2379   TSAN_INTERCEPT(munlockall);
   2380 
   2381   TSAN_INTERCEPT(fork);
   2382   TSAN_INTERCEPT(vfork);
   2383   TSAN_INTERCEPT(on_exit);
   2384   TSAN_INTERCEPT(__cxa_atexit);
   2385   TSAN_INTERCEPT(_exit);
   2386 
   2387   // Need to setup it, because interceptors check that the function is resolved.
   2388   // But atexit is emitted directly into the module, so can't be resolved.
   2389   REAL(atexit) = (int(*)(void(*)()))unreachable;
   2390   atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext)))
   2391       AtExitContext();
   2392 
   2393   if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
   2394     Printf("ThreadSanitizer: failed to setup atexit callback\n");
   2395     Die();
   2396   }
   2397 
   2398   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
   2399     Printf("ThreadSanitizer: failed to create thread key\n");
   2400     Die();
   2401   }
   2402 
   2403   FdInit();
   2404 }
   2405 
   2406 void *internal_start_thread(void(*func)(void *arg), void *arg) {
   2407   // Start the thread with signals blocked, otherwise it can steal user signals.
   2408   __sanitizer_sigset_t set, old;
   2409   internal_sigfillset(&set);
   2410   internal_sigprocmask(SIG_SETMASK, &set, &old);
   2411   void *th;
   2412   REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
   2413   internal_sigprocmask(SIG_SETMASK, &old, 0);
   2414   return th;
   2415 }
   2416 
   2417 void internal_join_thread(void *th) {
   2418   REAL(pthread_join)(th, 0);
   2419 }
   2420 
   2421 }  // namespace __tsan
   2422