1 //===-- tsan_rtl.cc -------------------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer (TSan), a race detector. 11 // 12 // Main file (entry points) for the TSan run-time. 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_common/sanitizer_atomic.h" 16 #include "sanitizer_common/sanitizer_common.h" 17 #include "sanitizer_common/sanitizer_libc.h" 18 #include "sanitizer_common/sanitizer_stackdepot.h" 19 #include "sanitizer_common/sanitizer_placement_new.h" 20 #include "sanitizer_common/sanitizer_symbolizer.h" 21 #include "tsan_defs.h" 22 #include "tsan_platform.h" 23 #include "tsan_rtl.h" 24 #include "tsan_mman.h" 25 #include "tsan_suppressions.h" 26 27 volatile int __tsan_resumed = 0; 28 29 extern "C" void __tsan_resume() { 30 __tsan_resumed = 1; 31 } 32 33 namespace __tsan { 34 35 #ifndef TSAN_GO 36 THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); 37 #endif 38 static char ctx_placeholder[sizeof(Context)] ALIGNED(64); 39 40 static Context *ctx; 41 Context *CTX() { 42 return ctx; 43 } 44 45 Context::Context() 46 : initialized() 47 , report_mtx(MutexTypeReport, StatMtxReport) 48 , nreported() 49 , nmissed_expected() 50 , thread_mtx(MutexTypeThreads, StatMtxThreads) 51 , racy_stacks(MBlockRacyStacks) 52 , racy_addresses(MBlockRacyAddresses) { 53 } 54 55 // The objects are allocated in TLS, so one may rely on zero-initialization. 56 ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, 57 uptr stk_addr, uptr stk_size, 58 uptr tls_addr, uptr tls_size) 59 : fast_state(tid, epoch) 60 // Do not touch these, rely on zero initialization, 61 // they may be accessed before the ctor. 62 // , fast_ignore_reads() 63 // , fast_ignore_writes() 64 // , in_rtl() 65 , shadow_stack_pos(&shadow_stack[0]) 66 , tid(tid) 67 , unique_id(unique_id) 68 , stk_addr(stk_addr) 69 , stk_size(stk_size) 70 , tls_addr(tls_addr) 71 , tls_size(tls_size) { 72 } 73 74 ThreadContext::ThreadContext(int tid) 75 : tid(tid) 76 , unique_id() 77 , user_id() 78 , thr() 79 , status(ThreadStatusInvalid) 80 , detached() 81 , reuse_count() 82 , epoch0() 83 , epoch1() 84 , dead_info() 85 , dead_next() { 86 } 87 88 static void WriteMemoryProfile(char *buf, uptr buf_size, int num) { 89 uptr shadow = GetShadowMemoryConsumption(); 90 91 int nthread = 0; 92 int nlivethread = 0; 93 uptr threadmem = 0; 94 { 95 Lock l(&ctx->thread_mtx); 96 for (unsigned i = 0; i < kMaxTid; i++) { 97 ThreadContext *tctx = ctx->threads[i]; 98 if (tctx == 0) 99 continue; 100 nthread += 1; 101 threadmem += sizeof(ThreadContext); 102 if (tctx->status != ThreadStatusRunning) 103 continue; 104 nlivethread += 1; 105 threadmem += sizeof(ThreadState); 106 } 107 } 108 109 uptr nsync = 0; 110 uptr syncmem = CTX()->synctab.GetMemoryConsumption(&nsync); 111 112 internal_snprintf(buf, buf_size, "%d: shadow=%zuMB" 113 " thread=%zuMB(total=%d/live=%d)" 114 " sync=%zuMB(cnt=%zu)\n", 115 num, 116 shadow >> 20, 117 threadmem >> 20, nthread, nlivethread, 118 syncmem >> 20, nsync); 119 } 120 121 static void MemoryProfileThread(void *arg) { 122 ScopedInRtl in_rtl; 123 fd_t fd = (fd_t)(uptr)arg; 124 for (int i = 0; ; i++) { 125 InternalScopedBuffer<char> buf(4096); 126 WriteMemoryProfile(buf.data(), buf.size(), i); 127 internal_write(fd, buf.data(), internal_strlen(buf.data())); 128 SleepForSeconds(1); 129 } 130 } 131 132 static void InitializeMemoryProfile() { 133 if (flags()->profile_memory == 0 || flags()->profile_memory[0] == 0) 134 return; 135 InternalScopedBuffer<char> filename(4096); 136 internal_snprintf(filename.data(), filename.size(), "%s.%d", 137 flags()->profile_memory, GetPid()); 138 fd_t fd = internal_open(filename.data(), true); 139 if (fd == kInvalidFd) { 140 TsanPrintf("Failed to open memory profile file '%s'\n", &filename[0]); 141 Die(); 142 } 143 internal_start_thread(&MemoryProfileThread, (void*)(uptr)fd); 144 } 145 146 static void MemoryFlushThread(void *arg) { 147 ScopedInRtl in_rtl; 148 for (int i = 0; ; i++) { 149 SleepForMillis(flags()->flush_memory_ms); 150 FlushShadowMemory(); 151 } 152 } 153 154 static void InitializeMemoryFlush() { 155 if (flags()->flush_memory_ms == 0) 156 return; 157 if (flags()->flush_memory_ms < 100) 158 flags()->flush_memory_ms = 100; 159 internal_start_thread(&MemoryFlushThread, 0); 160 } 161 162 void Initialize(ThreadState *thr) { 163 // Thread safe because done before all threads exist. 164 static bool is_initialized = false; 165 if (is_initialized) 166 return; 167 is_initialized = true; 168 ScopedInRtl in_rtl; 169 #ifndef TSAN_GO 170 InitializeAllocator(); 171 #endif 172 InitializeInterceptors(); 173 const char *env = InitializePlatform(); 174 InitializeMutex(); 175 InitializeDynamicAnnotations(); 176 ctx = new(ctx_placeholder) Context; 177 InitializeShadowMemory(); 178 ctx->dead_list_size = 0; 179 ctx->dead_list_head = 0; 180 ctx->dead_list_tail = 0; 181 InitializeFlags(&ctx->flags, env); 182 InitializeSuppressions(); 183 InitializeMemoryProfile(); 184 InitializeMemoryFlush(); 185 186 const char *external_symbolizer = flags()->external_symbolizer_path; 187 if (external_symbolizer != 0 && external_symbolizer[0] != '\0') { 188 InitializeExternalSymbolizer(external_symbolizer); 189 } 190 191 if (ctx->flags.verbosity) 192 TsanPrintf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", 193 GetPid()); 194 195 // Initialize thread 0. 196 ctx->thread_seq = 0; 197 int tid = ThreadCreate(thr, 0, 0, true); 198 CHECK_EQ(tid, 0); 199 ThreadStart(thr, tid); 200 CHECK_EQ(thr->in_rtl, 1); 201 ctx->initialized = true; 202 203 if (flags()->stop_on_start) { 204 TsanPrintf("ThreadSanitizer is suspended at startup (pid %d)." 205 " Call __tsan_resume().\n", 206 GetPid()); 207 while (__tsan_resumed == 0); 208 } 209 } 210 211 int Finalize(ThreadState *thr) { 212 ScopedInRtl in_rtl; 213 Context *ctx = __tsan::ctx; 214 bool failed = false; 215 216 ThreadFinalize(thr); 217 218 if (ctx->nreported) { 219 failed = true; 220 TsanPrintf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); 221 } 222 223 if (ctx->nmissed_expected) { 224 failed = true; 225 TsanPrintf("ThreadSanitizer: missed %d expected races\n", 226 ctx->nmissed_expected); 227 } 228 229 StatOutput(ctx->stat); 230 return failed ? flags()->exitcode : 0; 231 } 232 233 #ifndef TSAN_GO 234 u32 CurrentStackId(ThreadState *thr, uptr pc) { 235 if (thr->shadow_stack_pos == 0) // May happen during bootstrap. 236 return 0; 237 if (pc) { 238 thr->shadow_stack_pos[0] = pc; 239 thr->shadow_stack_pos++; 240 } 241 u32 id = StackDepotPut(thr->shadow_stack, 242 thr->shadow_stack_pos - thr->shadow_stack); 243 if (pc) 244 thr->shadow_stack_pos--; 245 return id; 246 } 247 #endif 248 249 void TraceSwitch(ThreadState *thr) { 250 thr->nomalloc++; 251 ScopedInRtl in_rtl; 252 Lock l(&thr->trace.mtx); 253 unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % kTraceParts; 254 TraceHeader *hdr = &thr->trace.headers[trace]; 255 hdr->epoch0 = thr->fast_state.epoch(); 256 hdr->stack0.ObtainCurrent(thr, 0); 257 thr->nomalloc--; 258 } 259 260 #ifndef TSAN_GO 261 extern "C" void __tsan_trace_switch() { 262 TraceSwitch(cur_thread()); 263 } 264 265 extern "C" void __tsan_report_race() { 266 ReportRace(cur_thread()); 267 } 268 #endif 269 270 ALWAYS_INLINE 271 static Shadow LoadShadow(u64 *p) { 272 u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); 273 return Shadow(raw); 274 } 275 276 ALWAYS_INLINE 277 static void StoreShadow(u64 *sp, u64 s) { 278 atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); 279 } 280 281 ALWAYS_INLINE 282 static void StoreIfNotYetStored(u64 *sp, u64 *s) { 283 StoreShadow(sp, *s); 284 *s = 0; 285 } 286 287 static inline void HandleRace(ThreadState *thr, u64 *shadow_mem, 288 Shadow cur, Shadow old) { 289 thr->racy_state[0] = cur.raw(); 290 thr->racy_state[1] = old.raw(); 291 thr->racy_shadow_addr = shadow_mem; 292 #ifndef TSAN_GO 293 HACKY_CALL(__tsan_report_race); 294 #else 295 ReportRace(thr); 296 #endif 297 } 298 299 static inline bool BothReads(Shadow s, int kAccessIsWrite) { 300 return !kAccessIsWrite && !s.is_write(); 301 } 302 303 static inline bool OldIsRWStronger(Shadow old, int kAccessIsWrite) { 304 return old.is_write() || !kAccessIsWrite; 305 } 306 307 static inline bool OldIsRWWeaker(Shadow old, int kAccessIsWrite) { 308 return !old.is_write() || kAccessIsWrite; 309 } 310 311 static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) { 312 return old.epoch() >= thr->fast_synch_epoch; 313 } 314 315 static inline bool HappensBefore(Shadow old, ThreadState *thr) { 316 return thr->clock.get(old.tid()) >= old.epoch(); 317 } 318 319 ALWAYS_INLINE 320 void MemoryAccessImpl(ThreadState *thr, uptr addr, 321 int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state, 322 u64 *shadow_mem, Shadow cur) { 323 StatInc(thr, StatMop); 324 StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); 325 StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); 326 327 // This potentially can live in an MMX/SSE scratch register. 328 // The required intrinsics are: 329 // __m128i _mm_move_epi64(__m128i*); 330 // _mm_storel_epi64(u64*, __m128i); 331 u64 store_word = cur.raw(); 332 333 // scan all the shadow values and dispatch to 4 categories: 334 // same, replace, candidate and race (see comments below). 335 // we consider only 3 cases regarding access sizes: 336 // equal, intersect and not intersect. initially I considered 337 // larger and smaller as well, it allowed to replace some 338 // 'candidates' with 'same' or 'replace', but I think 339 // it's just not worth it (performance- and complexity-wise). 340 341 Shadow old(0); 342 if (kShadowCnt == 1) { 343 int idx = 0; 344 #include "tsan_update_shadow_word_inl.h" 345 } else if (kShadowCnt == 2) { 346 int idx = 0; 347 #include "tsan_update_shadow_word_inl.h" 348 idx = 1; 349 #include "tsan_update_shadow_word_inl.h" 350 } else if (kShadowCnt == 4) { 351 int idx = 0; 352 #include "tsan_update_shadow_word_inl.h" 353 idx = 1; 354 #include "tsan_update_shadow_word_inl.h" 355 idx = 2; 356 #include "tsan_update_shadow_word_inl.h" 357 idx = 3; 358 #include "tsan_update_shadow_word_inl.h" 359 } else if (kShadowCnt == 8) { 360 int idx = 0; 361 #include "tsan_update_shadow_word_inl.h" 362 idx = 1; 363 #include "tsan_update_shadow_word_inl.h" 364 idx = 2; 365 #include "tsan_update_shadow_word_inl.h" 366 idx = 3; 367 #include "tsan_update_shadow_word_inl.h" 368 idx = 4; 369 #include "tsan_update_shadow_word_inl.h" 370 idx = 5; 371 #include "tsan_update_shadow_word_inl.h" 372 idx = 6; 373 #include "tsan_update_shadow_word_inl.h" 374 idx = 7; 375 #include "tsan_update_shadow_word_inl.h" 376 } else { 377 CHECK(false); 378 } 379 380 // we did not find any races and had already stored 381 // the current access info, so we are done 382 if (LIKELY(store_word == 0)) 383 return; 384 // choose a random candidate slot and replace it 385 StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); 386 StatInc(thr, StatShadowReplace); 387 return; 388 RACE: 389 HandleRace(thr, shadow_mem, cur, old); 390 return; 391 } 392 393 ALWAYS_INLINE 394 void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, 395 int kAccessSizeLog, bool kAccessIsWrite) { 396 u64 *shadow_mem = (u64*)MemToShadow(addr); 397 DPrintf2("#%d: tsan::OnMemoryAccess: @%p %p size=%d" 398 " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", 399 (int)thr->fast_state.tid(), (void*)pc, (void*)addr, 400 (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, 401 (uptr)shadow_mem[0], (uptr)shadow_mem[1], 402 (uptr)shadow_mem[2], (uptr)shadow_mem[3]); 403 #if TSAN_DEBUG 404 if (!IsAppMem(addr)) { 405 TsanPrintf("Access to non app mem %zx\n", addr); 406 DCHECK(IsAppMem(addr)); 407 } 408 if (!IsShadowMem((uptr)shadow_mem)) { 409 TsanPrintf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); 410 DCHECK(IsShadowMem((uptr)shadow_mem)); 411 } 412 #endif 413 414 FastState fast_state = thr->fast_state; 415 if (fast_state.GetIgnoreBit()) 416 return; 417 fast_state.IncrementEpoch(); 418 thr->fast_state = fast_state; 419 Shadow cur(fast_state); 420 cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); 421 cur.SetWrite(kAccessIsWrite); 422 423 // We must not store to the trace if we do not store to the shadow. 424 // That is, this call must be moved somewhere below. 425 TraceAddEvent(thr, fast_state.epoch(), EventTypeMop, pc); 426 427 MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, fast_state, 428 shadow_mem, cur); 429 } 430 431 static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, 432 u64 val) { 433 if (size == 0) 434 return; 435 // FIXME: fix me. 436 uptr offset = addr % kShadowCell; 437 if (offset) { 438 offset = kShadowCell - offset; 439 if (size <= offset) 440 return; 441 addr += offset; 442 size -= offset; 443 } 444 DCHECK_EQ(addr % 8, 0); 445 // If a user passes some insane arguments (memset(0)), 446 // let it just crash as usual. 447 if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) 448 return; 449 (void)thr; 450 (void)pc; 451 // Some programs mmap like hundreds of GBs but actually used a small part. 452 // So, it's better to report a false positive on the memory 453 // then to hang here senselessly. 454 const uptr kMaxResetSize = 1024*1024*1024; 455 if (size > kMaxResetSize) 456 size = kMaxResetSize; 457 size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); 458 u64 *p = (u64*)MemToShadow(addr); 459 CHECK(IsShadowMem((uptr)p)); 460 CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); 461 // FIXME: may overwrite a part outside the region 462 for (uptr i = 0; i < size * kShadowCnt / kShadowCell;) { 463 p[i++] = val; 464 for (uptr j = 1; j < kShadowCnt; j++) 465 p[i++] = 0; 466 } 467 } 468 469 void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { 470 MemoryRangeSet(thr, pc, addr, size, 0); 471 } 472 473 void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { 474 MemoryAccessRange(thr, pc, addr, size, true); 475 Shadow s(thr->fast_state); 476 s.MarkAsFreed(); 477 s.SetWrite(true); 478 s.SetAddr0AndSizeLog(0, 3); 479 MemoryRangeSet(thr, pc, addr, size, s.raw()); 480 } 481 482 void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { 483 Shadow s(thr->fast_state); 484 s.SetWrite(true); 485 s.SetAddr0AndSizeLog(0, 3); 486 MemoryRangeSet(thr, pc, addr, size, s.raw()); 487 } 488 489 void FuncEntry(ThreadState *thr, uptr pc) { 490 DCHECK_EQ(thr->in_rtl, 0); 491 StatInc(thr, StatFuncEnter); 492 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); 493 thr->fast_state.IncrementEpoch(); 494 TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncEnter, pc); 495 496 // Shadow stack maintenance can be replaced with 497 // stack unwinding during trace switch (which presumably must be faster). 498 DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]); 499 #ifndef TSAN_GO 500 DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); 501 #else 502 if (thr->shadow_stack_pos == thr->shadow_stack_end) { 503 const int sz = thr->shadow_stack_end - thr->shadow_stack; 504 const int newsz = 2 * sz; 505 uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack, 506 newsz * sizeof(uptr)); 507 internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr)); 508 internal_free(thr->shadow_stack); 509 thr->shadow_stack = newstack; 510 thr->shadow_stack_pos = newstack + sz; 511 thr->shadow_stack_end = newstack + newsz; 512 } 513 #endif 514 thr->shadow_stack_pos[0] = pc; 515 thr->shadow_stack_pos++; 516 } 517 518 void FuncExit(ThreadState *thr) { 519 DCHECK_EQ(thr->in_rtl, 0); 520 StatInc(thr, StatFuncExit); 521 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); 522 thr->fast_state.IncrementEpoch(); 523 TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncExit, 0); 524 525 DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]); 526 #ifndef TSAN_GO 527 DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); 528 #endif 529 thr->shadow_stack_pos--; 530 } 531 532 void IgnoreCtl(ThreadState *thr, bool write, bool begin) { 533 DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); 534 thr->ignore_reads_and_writes += begin ? 1 : -1; 535 CHECK_GE(thr->ignore_reads_and_writes, 0); 536 if (thr->ignore_reads_and_writes) 537 thr->fast_state.SetIgnoreBit(); 538 else 539 thr->fast_state.ClearIgnoreBit(); 540 } 541 542 bool MD5Hash::operator==(const MD5Hash &other) const { 543 return hash[0] == other.hash[0] && hash[1] == other.hash[1]; 544 } 545 546 #if TSAN_DEBUG 547 void build_consistency_debug() {} 548 #else 549 void build_consistency_release() {} 550 #endif 551 552 #if TSAN_COLLECT_STATS 553 void build_consistency_stats() {} 554 #else 555 void build_consistency_nostats() {} 556 #endif 557 558 #if TSAN_SHADOW_COUNT == 1 559 void build_consistency_shadow1() {} 560 #elif TSAN_SHADOW_COUNT == 2 561 void build_consistency_shadow2() {} 562 #elif TSAN_SHADOW_COUNT == 4 563 void build_consistency_shadow4() {} 564 #else 565 void build_consistency_shadow8() {} 566 #endif 567 568 } // namespace __tsan 569 570 #ifndef TSAN_GO 571 // Must be included in this file to make sure everything is inlined. 572 #include "tsan_interface_inl.h" 573 #endif 574