Home | History | Annotate | Download | only in rtl 
      1 //===-- tsan_rtl.cc -------------------------------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file is a part of ThreadSanitizer (TSan), a race detector.
     11 //
     12 // Main file (entry points) for the TSan run-time.
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "sanitizer_common/sanitizer_atomic.h"
     16 #include "sanitizer_common/sanitizer_common.h"
     17 #include "sanitizer_common/sanitizer_libc.h"
     18 #include "sanitizer_common/sanitizer_stackdepot.h"
     19 #include "sanitizer_common/sanitizer_placement_new.h"
     20 #include "sanitizer_common/sanitizer_symbolizer.h"
     21 #include "tsan_defs.h"
     22 #include "tsan_platform.h"
     23 #include "tsan_rtl.h"
     24 #include "tsan_mman.h"
     25 #include "tsan_suppressions.h"
     26 
     27 volatile int __tsan_resumed = 0;
     28 
     29 extern "C" void __tsan_resume() {
     30   __tsan_resumed = 1;
     31 }
     32 
     33 namespace __tsan {
     34 
     35 #ifndef TSAN_GO
     36 THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
     37 #endif
     38 static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
     39 
     40 static Context *ctx;
     41 Context *CTX() {
     42   return ctx;
     43 }
     44 
     45 Context::Context()
     46   : initialized()
     47   , report_mtx(MutexTypeReport, StatMtxReport)
     48   , nreported()
     49   , nmissed_expected()
     50   , thread_mtx(MutexTypeThreads, StatMtxThreads)
     51   , racy_stacks(MBlockRacyStacks)
     52   , racy_addresses(MBlockRacyAddresses) {
     53 }
     54 
     55 // The objects are allocated in TLS, so one may rely on zero-initialization.
     56 ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
     57                          uptr stk_addr, uptr stk_size,
     58                          uptr tls_addr, uptr tls_size)
     59   : fast_state(tid, epoch)
     60   // Do not touch these, rely on zero initialization,
     61   // they may be accessed before the ctor.
     62   // , fast_ignore_reads()
     63   // , fast_ignore_writes()
     64   // , in_rtl()
     65   , shadow_stack_pos(&shadow_stack[0])
     66   , tid(tid)
     67   , unique_id(unique_id)
     68   , stk_addr(stk_addr)
     69   , stk_size(stk_size)
     70   , tls_addr(tls_addr)
     71   , tls_size(tls_size) {
     72 }
     73 
     74 ThreadContext::ThreadContext(int tid)
     75   : tid(tid)
     76   , unique_id()
     77   , user_id()
     78   , thr()
     79   , status(ThreadStatusInvalid)
     80   , detached()
     81   , reuse_count()
     82   , epoch0()
     83   , epoch1()
     84   , dead_info()
     85   , dead_next() {
     86 }
     87 
     88 static void WriteMemoryProfile(char *buf, uptr buf_size, int num) {
     89   uptr shadow = GetShadowMemoryConsumption();
     90 
     91   int nthread = 0;
     92   int nlivethread = 0;
     93   uptr threadmem = 0;
     94   {
     95     Lock l(&ctx->thread_mtx);
     96     for (unsigned i = 0; i < kMaxTid; i++) {
     97       ThreadContext *tctx = ctx->threads[i];
     98       if (tctx == 0)
     99         continue;
    100       nthread += 1;
    101       threadmem += sizeof(ThreadContext);
    102       if (tctx->status != ThreadStatusRunning)
    103         continue;
    104       nlivethread += 1;
    105       threadmem += sizeof(ThreadState);
    106     }
    107   }
    108 
    109   uptr nsync = 0;
    110   uptr syncmem = CTX()->synctab.GetMemoryConsumption(&nsync);
    111 
    112   internal_snprintf(buf, buf_size, "%d: shadow=%zuMB"
    113                                    " thread=%zuMB(total=%d/live=%d)"
    114                                    " sync=%zuMB(cnt=%zu)\n",
    115     num,
    116     shadow >> 20,
    117     threadmem >> 20, nthread, nlivethread,
    118     syncmem >> 20, nsync);
    119 }
    120 
    121 static void MemoryProfileThread(void *arg) {
    122   ScopedInRtl in_rtl;
    123   fd_t fd = (fd_t)(uptr)arg;
    124   for (int i = 0; ; i++) {
    125     InternalScopedBuffer<char> buf(4096);
    126     WriteMemoryProfile(buf.data(), buf.size(), i);
    127     internal_write(fd, buf.data(), internal_strlen(buf.data()));
    128     SleepForSeconds(1);
    129   }
    130 }
    131 
    132 static void InitializeMemoryProfile() {
    133   if (flags()->profile_memory == 0 || flags()->profile_memory[0] == 0)
    134     return;
    135   InternalScopedBuffer<char> filename(4096);
    136   internal_snprintf(filename.data(), filename.size(), "%s.%d",
    137       flags()->profile_memory, GetPid());
    138   fd_t fd = internal_open(filename.data(), true);
    139   if (fd == kInvalidFd) {
    140     TsanPrintf("Failed to open memory profile file '%s'\n", &filename[0]);
    141     Die();
    142   }
    143   internal_start_thread(&MemoryProfileThread, (void*)(uptr)fd);
    144 }
    145 
    146 static void MemoryFlushThread(void *arg) {
    147   ScopedInRtl in_rtl;
    148   for (int i = 0; ; i++) {
    149     SleepForMillis(flags()->flush_memory_ms);
    150     FlushShadowMemory();
    151   }
    152 }
    153 
    154 static void InitializeMemoryFlush() {
    155   if (flags()->flush_memory_ms == 0)
    156     return;
    157   if (flags()->flush_memory_ms < 100)
    158     flags()->flush_memory_ms = 100;
    159   internal_start_thread(&MemoryFlushThread, 0);
    160 }
    161 
    162 void Initialize(ThreadState *thr) {
    163   // Thread safe because done before all threads exist.
    164   static bool is_initialized = false;
    165   if (is_initialized)
    166     return;
    167   is_initialized = true;
    168   ScopedInRtl in_rtl;
    169 #ifndef TSAN_GO
    170   InitializeAllocator();
    171 #endif
    172   InitializeInterceptors();
    173   const char *env = InitializePlatform();
    174   InitializeMutex();
    175   InitializeDynamicAnnotations();
    176   ctx = new(ctx_placeholder) Context;
    177   InitializeShadowMemory();
    178   ctx->dead_list_size = 0;
    179   ctx->dead_list_head = 0;
    180   ctx->dead_list_tail = 0;
    181   InitializeFlags(&ctx->flags, env);
    182   InitializeSuppressions();
    183   InitializeMemoryProfile();
    184   InitializeMemoryFlush();
    185 
    186   const char *external_symbolizer = flags()->external_symbolizer_path;
    187   if (external_symbolizer != 0 && external_symbolizer[0] != '\0') {
    188     InitializeExternalSymbolizer(external_symbolizer);
    189   }
    190 
    191   if (ctx->flags.verbosity)
    192     TsanPrintf("***** Running under ThreadSanitizer v2 (pid %d) *****\n",
    193                GetPid());
    194 
    195   // Initialize thread 0.
    196   ctx->thread_seq = 0;
    197   int tid = ThreadCreate(thr, 0, 0, true);
    198   CHECK_EQ(tid, 0);
    199   ThreadStart(thr, tid);
    200   CHECK_EQ(thr->in_rtl, 1);
    201   ctx->initialized = true;
    202 
    203   if (flags()->stop_on_start) {
    204     TsanPrintf("ThreadSanitizer is suspended at startup (pid %d)."
    205            " Call __tsan_resume().\n",
    206            GetPid());
    207     while (__tsan_resumed == 0);
    208   }
    209 }
    210 
    211 int Finalize(ThreadState *thr) {
    212   ScopedInRtl in_rtl;
    213   Context *ctx = __tsan::ctx;
    214   bool failed = false;
    215 
    216   ThreadFinalize(thr);
    217 
    218   if (ctx->nreported) {
    219     failed = true;
    220     TsanPrintf("ThreadSanitizer: reported %d warnings\n", ctx->nreported);
    221   }
    222 
    223   if (ctx->nmissed_expected) {
    224     failed = true;
    225     TsanPrintf("ThreadSanitizer: missed %d expected races\n",
    226         ctx->nmissed_expected);
    227   }
    228 
    229   StatOutput(ctx->stat);
    230   return failed ? flags()->exitcode : 0;
    231 }
    232 
    233 #ifndef TSAN_GO
    234 u32 CurrentStackId(ThreadState *thr, uptr pc) {
    235   if (thr->shadow_stack_pos == 0)  // May happen during bootstrap.
    236     return 0;
    237   if (pc) {
    238     thr->shadow_stack_pos[0] = pc;
    239     thr->shadow_stack_pos++;
    240   }
    241   u32 id = StackDepotPut(thr->shadow_stack,
    242                          thr->shadow_stack_pos - thr->shadow_stack);
    243   if (pc)
    244     thr->shadow_stack_pos--;
    245   return id;
    246 }
    247 #endif
    248 
    249 void TraceSwitch(ThreadState *thr) {
    250   thr->nomalloc++;
    251   ScopedInRtl in_rtl;
    252   Lock l(&thr->trace.mtx);
    253   unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % kTraceParts;
    254   TraceHeader *hdr = &thr->trace.headers[trace];
    255   hdr->epoch0 = thr->fast_state.epoch();
    256   hdr->stack0.ObtainCurrent(thr, 0);
    257   thr->nomalloc--;
    258 }
    259 
    260 #ifndef TSAN_GO
    261 extern "C" void __tsan_trace_switch() {
    262   TraceSwitch(cur_thread());
    263 }
    264 
    265 extern "C" void __tsan_report_race() {
    266   ReportRace(cur_thread());
    267 }
    268 #endif
    269 
    270 ALWAYS_INLINE
    271 static Shadow LoadShadow(u64 *p) {
    272   u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
    273   return Shadow(raw);
    274 }
    275 
    276 ALWAYS_INLINE
    277 static void StoreShadow(u64 *sp, u64 s) {
    278   atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
    279 }
    280 
    281 ALWAYS_INLINE
    282 static void StoreIfNotYetStored(u64 *sp, u64 *s) {
    283   StoreShadow(sp, *s);
    284   *s = 0;
    285 }
    286 
    287 static inline void HandleRace(ThreadState *thr, u64 *shadow_mem,
    288                               Shadow cur, Shadow old) {
    289   thr->racy_state[0] = cur.raw();
    290   thr->racy_state[1] = old.raw();
    291   thr->racy_shadow_addr = shadow_mem;
    292 #ifndef TSAN_GO
    293   HACKY_CALL(__tsan_report_race);
    294 #else
    295   ReportRace(thr);
    296 #endif
    297 }
    298 
    299 static inline bool BothReads(Shadow s, int kAccessIsWrite) {
    300   return !kAccessIsWrite && !s.is_write();
    301 }
    302 
    303 static inline bool OldIsRWStronger(Shadow old, int kAccessIsWrite) {
    304   return old.is_write() || !kAccessIsWrite;
    305 }
    306 
    307 static inline bool OldIsRWWeaker(Shadow old, int kAccessIsWrite) {
    308   return !old.is_write() || kAccessIsWrite;
    309 }
    310 
    311 static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) {
    312   return old.epoch() >= thr->fast_synch_epoch;
    313 }
    314 
    315 static inline bool HappensBefore(Shadow old, ThreadState *thr) {
    316   return thr->clock.get(old.tid()) >= old.epoch();
    317 }
    318 
    319 ALWAYS_INLINE
    320 void MemoryAccessImpl(ThreadState *thr, uptr addr,
    321     int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state,
    322     u64 *shadow_mem, Shadow cur) {
    323   StatInc(thr, StatMop);
    324   StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
    325   StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
    326 
    327   // This potentially can live in an MMX/SSE scratch register.
    328   // The required intrinsics are:
    329   // __m128i _mm_move_epi64(__m128i*);
    330   // _mm_storel_epi64(u64*, __m128i);
    331   u64 store_word = cur.raw();
    332 
    333   // scan all the shadow values and dispatch to 4 categories:
    334   // same, replace, candidate and race (see comments below).
    335   // we consider only 3 cases regarding access sizes:
    336   // equal, intersect and not intersect. initially I considered
    337   // larger and smaller as well, it allowed to replace some
    338   // 'candidates' with 'same' or 'replace', but I think
    339   // it's just not worth it (performance- and complexity-wise).
    340 
    341   Shadow old(0);
    342   if (kShadowCnt == 1) {
    343     int idx = 0;
    344 #include "tsan_update_shadow_word_inl.h"
    345   } else if (kShadowCnt == 2) {
    346     int idx = 0;
    347 #include "tsan_update_shadow_word_inl.h"
    348     idx = 1;
    349 #include "tsan_update_shadow_word_inl.h"
    350   } else if (kShadowCnt == 4) {
    351     int idx = 0;
    352 #include "tsan_update_shadow_word_inl.h"
    353     idx = 1;
    354 #include "tsan_update_shadow_word_inl.h"
    355     idx = 2;
    356 #include "tsan_update_shadow_word_inl.h"
    357     idx = 3;
    358 #include "tsan_update_shadow_word_inl.h"
    359   } else if (kShadowCnt == 8) {
    360     int idx = 0;
    361 #include "tsan_update_shadow_word_inl.h"
    362     idx = 1;
    363 #include "tsan_update_shadow_word_inl.h"
    364     idx = 2;
    365 #include "tsan_update_shadow_word_inl.h"
    366     idx = 3;
    367 #include "tsan_update_shadow_word_inl.h"
    368     idx = 4;
    369 #include "tsan_update_shadow_word_inl.h"
    370     idx = 5;
    371 #include "tsan_update_shadow_word_inl.h"
    372     idx = 6;
    373 #include "tsan_update_shadow_word_inl.h"
    374     idx = 7;
    375 #include "tsan_update_shadow_word_inl.h"
    376   } else {
    377     CHECK(false);
    378   }
    379 
    380   // we did not find any races and had already stored
    381   // the current access info, so we are done
    382   if (LIKELY(store_word == 0))
    383     return;
    384   // choose a random candidate slot and replace it
    385   StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
    386   StatInc(thr, StatShadowReplace);
    387   return;
    388  RACE:
    389   HandleRace(thr, shadow_mem, cur, old);
    390   return;
    391 }
    392 
    393 ALWAYS_INLINE
    394 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
    395     int kAccessSizeLog, bool kAccessIsWrite) {
    396   u64 *shadow_mem = (u64*)MemToShadow(addr);
    397   DPrintf2("#%d: tsan::OnMemoryAccess: @%p %p size=%d"
    398       " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
    399       (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
    400       (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
    401       (uptr)shadow_mem[0], (uptr)shadow_mem[1],
    402       (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
    403 #if TSAN_DEBUG
    404   if (!IsAppMem(addr)) {
    405     TsanPrintf("Access to non app mem %zx\n", addr);
    406     DCHECK(IsAppMem(addr));
    407   }
    408   if (!IsShadowMem((uptr)shadow_mem)) {
    409     TsanPrintf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
    410     DCHECK(IsShadowMem((uptr)shadow_mem));
    411   }
    412 #endif
    413 
    414   FastState fast_state = thr->fast_state;
    415   if (fast_state.GetIgnoreBit())
    416     return;
    417   fast_state.IncrementEpoch();
    418   thr->fast_state = fast_state;
    419   Shadow cur(fast_state);
    420   cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
    421   cur.SetWrite(kAccessIsWrite);
    422 
    423   // We must not store to the trace if we do not store to the shadow.
    424   // That is, this call must be moved somewhere below.
    425   TraceAddEvent(thr, fast_state.epoch(), EventTypeMop, pc);
    426 
    427   MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, fast_state,
    428       shadow_mem, cur);
    429 }
    430 
    431 static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
    432                            u64 val) {
    433   if (size == 0)
    434     return;
    435   // FIXME: fix me.
    436   uptr offset = addr % kShadowCell;
    437   if (offset) {
    438     offset = kShadowCell - offset;
    439     if (size <= offset)
    440       return;
    441     addr += offset;
    442     size -= offset;
    443   }
    444   DCHECK_EQ(addr % 8, 0);
    445   // If a user passes some insane arguments (memset(0)),
    446   // let it just crash as usual.
    447   if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
    448     return;
    449   (void)thr;
    450   (void)pc;
    451   // Some programs mmap like hundreds of GBs but actually used a small part.
    452   // So, it's better to report a false positive on the memory
    453   // then to hang here senselessly.
    454   const uptr kMaxResetSize = 1024*1024*1024;
    455   if (size > kMaxResetSize)
    456     size = kMaxResetSize;
    457   size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
    458   u64 *p = (u64*)MemToShadow(addr);
    459   CHECK(IsShadowMem((uptr)p));
    460   CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
    461   // FIXME: may overwrite a part outside the region
    462   for (uptr i = 0; i < size * kShadowCnt / kShadowCell;) {
    463     p[i++] = val;
    464     for (uptr j = 1; j < kShadowCnt; j++)
    465       p[i++] = 0;
    466   }
    467 }
    468 
    469 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
    470   MemoryRangeSet(thr, pc, addr, size, 0);
    471 }
    472 
    473 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
    474   MemoryAccessRange(thr, pc, addr, size, true);
    475   Shadow s(thr->fast_state);
    476   s.MarkAsFreed();
    477   s.SetWrite(true);
    478   s.SetAddr0AndSizeLog(0, 3);
    479   MemoryRangeSet(thr, pc, addr, size, s.raw());
    480 }
    481 
    482 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
    483   Shadow s(thr->fast_state);
    484   s.SetWrite(true);
    485   s.SetAddr0AndSizeLog(0, 3);
    486   MemoryRangeSet(thr, pc, addr, size, s.raw());
    487 }
    488 
    489 void FuncEntry(ThreadState *thr, uptr pc) {
    490   DCHECK_EQ(thr->in_rtl, 0);
    491   StatInc(thr, StatFuncEnter);
    492   DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
    493   thr->fast_state.IncrementEpoch();
    494   TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncEnter, pc);
    495 
    496   // Shadow stack maintenance can be replaced with
    497   // stack unwinding during trace switch (which presumably must be faster).
    498   DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]);
    499 #ifndef TSAN_GO
    500   DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
    501 #else
    502   if (thr->shadow_stack_pos == thr->shadow_stack_end) {
    503     const int sz = thr->shadow_stack_end - thr->shadow_stack;
    504     const int newsz = 2 * sz;
    505     uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack,
    506         newsz * sizeof(uptr));
    507     internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
    508     internal_free(thr->shadow_stack);
    509     thr->shadow_stack = newstack;
    510     thr->shadow_stack_pos = newstack + sz;
    511     thr->shadow_stack_end = newstack + newsz;
    512   }
    513 #endif
    514   thr->shadow_stack_pos[0] = pc;
    515   thr->shadow_stack_pos++;
    516 }
    517 
    518 void FuncExit(ThreadState *thr) {
    519   DCHECK_EQ(thr->in_rtl, 0);
    520   StatInc(thr, StatFuncExit);
    521   DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
    522   thr->fast_state.IncrementEpoch();
    523   TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncExit, 0);
    524 
    525   DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]);
    526 #ifndef TSAN_GO
    527   DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]);
    528 #endif
    529   thr->shadow_stack_pos--;
    530 }
    531 
    532 void IgnoreCtl(ThreadState *thr, bool write, bool begin) {
    533   DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin);
    534   thr->ignore_reads_and_writes += begin ? 1 : -1;
    535   CHECK_GE(thr->ignore_reads_and_writes, 0);
    536   if (thr->ignore_reads_and_writes)
    537     thr->fast_state.SetIgnoreBit();
    538   else
    539     thr->fast_state.ClearIgnoreBit();
    540 }
    541 
    542 bool MD5Hash::operator==(const MD5Hash &other) const {
    543   return hash[0] == other.hash[0] && hash[1] == other.hash[1];
    544 }
    545 
    546 #if TSAN_DEBUG
    547 void build_consistency_debug() {}
    548 #else
    549 void build_consistency_release() {}
    550 #endif
    551 
    552 #if TSAN_COLLECT_STATS
    553 void build_consistency_stats() {}
    554 #else
    555 void build_consistency_nostats() {}
    556 #endif
    557 
    558 #if TSAN_SHADOW_COUNT == 1
    559 void build_consistency_shadow1() {}
    560 #elif TSAN_SHADOW_COUNT == 2
    561 void build_consistency_shadow2() {}
    562 #elif TSAN_SHADOW_COUNT == 4
    563 void build_consistency_shadow4() {}
    564 #else
    565 void build_consistency_shadow8() {}
    566 #endif
    567 
    568 }  // namespace __tsan
    569 
    570 #ifndef TSAN_GO
    571 // Must be included in this file to make sure everything is inlined.
    572 #include "tsan_interface_inl.h"
    573 #endif
    574