1 //===-- tsan_platform_linux.cc --------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is a part of ThreadSanitizer (TSan), a race detector. 11 // 12 // Linux- and FreeBSD-specific code. 13 //===----------------------------------------------------------------------===// 14 15 16 #include "sanitizer_common/sanitizer_platform.h" 17 #if SANITIZER_LINUX || SANITIZER_FREEBSD 18 19 #include "sanitizer_common/sanitizer_common.h" 20 #include "sanitizer_common/sanitizer_libc.h" 21 #include "sanitizer_common/sanitizer_linux.h" 22 #include "sanitizer_common/sanitizer_platform_limits_posix.h" 23 #include "sanitizer_common/sanitizer_posix.h" 24 #include "sanitizer_common/sanitizer_procmaps.h" 25 #include "sanitizer_common/sanitizer_stoptheworld.h" 26 #include "sanitizer_common/sanitizer_stackdepot.h" 27 #include "tsan_platform.h" 28 #include "tsan_rtl.h" 29 #include "tsan_flags.h" 30 31 #include <fcntl.h> 32 #include <pthread.h> 33 #include <signal.h> 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <stdarg.h> 38 #include <sys/mman.h> 39 #if SANITIZER_LINUX 40 #include <sys/personality.h> 41 #endif 42 #include <sys/syscall.h> 43 #include <sys/socket.h> 44 #include <sys/time.h> 45 #include <sys/types.h> 46 #include <sys/resource.h> 47 #include <sys/stat.h> 48 #include <unistd.h> 49 #include <errno.h> 50 #include <sched.h> 51 #include <dlfcn.h> 52 #if SANITIZER_LINUX 53 #define __need_res_state 54 #include <resolv.h> 55 #endif 56 57 #ifdef sa_handler 58 # undef sa_handler 59 #endif 60 61 #ifdef sa_sigaction 62 # undef sa_sigaction 63 #endif 64 65 #if SANITIZER_FREEBSD 66 extern "C" void *__libc_stack_end; 67 void *__libc_stack_end = 0; 68 #endif 69 70 namespace __tsan { 71 72 #ifdef TSAN_RUNTIME_VMA 73 // Runtime detected VMA size. 74 uptr vmaSize; 75 #endif 76 77 enum { 78 MemTotal = 0, 79 MemShadow = 1, 80 MemMeta = 2, 81 MemFile = 3, 82 MemMmap = 4, 83 MemTrace = 5, 84 MemHeap = 6, 85 MemOther = 7, 86 MemCount = 8, 87 }; 88 89 void FillProfileCallback(uptr p, uptr rss, bool file, 90 uptr *mem, uptr stats_size) { 91 mem[MemTotal] += rss; 92 if (p >= ShadowBeg() && p < ShadowEnd()) 93 mem[MemShadow] += rss; 94 else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) 95 mem[MemMeta] += rss; 96 #ifndef SANITIZER_GO 97 else if (p >= HeapMemBeg() && p < HeapMemEnd()) 98 mem[MemHeap] += rss; 99 else if (p >= LoAppMemBeg() && p < LoAppMemEnd()) 100 mem[file ? MemFile : MemMmap] += rss; 101 else if (p >= HiAppMemBeg() && p < HiAppMemEnd()) 102 mem[file ? MemFile : MemMmap] += rss; 103 #else 104 else if (p >= AppMemBeg() && p < AppMemEnd()) 105 mem[file ? MemFile : MemMmap] += rss; 106 #endif 107 else if (p >= TraceMemBeg() && p < TraceMemEnd()) 108 mem[MemTrace] += rss; 109 else 110 mem[MemOther] += rss; 111 } 112 113 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { 114 uptr mem[MemCount]; 115 internal_memset(mem, 0, sizeof(mem[0]) * MemCount); 116 __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); 117 StackDepotStats *stacks = StackDepotGetStats(); 118 internal_snprintf(buf, buf_size, 119 "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd" 120 " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n", 121 mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20, 122 mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20, 123 mem[MemHeap] >> 20, mem[MemOther] >> 20, 124 stacks->allocated >> 20, stacks->n_uniq_ids, 125 nlive, nthread); 126 } 127 128 #if SANITIZER_LINUX 129 void FlushShadowMemoryCallback( 130 const SuspendedThreadsList &suspended_threads_list, 131 void *argument) { 132 FlushUnneededShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg()); 133 } 134 #endif 135 136 void FlushShadowMemory() { 137 #if SANITIZER_LINUX 138 StopTheWorld(FlushShadowMemoryCallback, 0); 139 #endif 140 } 141 142 #ifndef SANITIZER_GO 143 // Mark shadow for .rodata sections with the special kShadowRodata marker. 144 // Accesses to .rodata can't race, so this saves time, memory and trace space. 145 static void MapRodata() { 146 // First create temp file. 147 const char *tmpdir = GetEnv("TMPDIR"); 148 if (tmpdir == 0) 149 tmpdir = GetEnv("TEST_TMPDIR"); 150 #ifdef P_tmpdir 151 if (tmpdir == 0) 152 tmpdir = P_tmpdir; 153 #endif 154 if (tmpdir == 0) 155 return; 156 char name[256]; 157 internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", 158 tmpdir, (int)internal_getpid()); 159 uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); 160 if (internal_iserror(openrv)) 161 return; 162 internal_unlink(name); // Unlink it now, so that we can reuse the buffer. 163 fd_t fd = openrv; 164 // Fill the file with kShadowRodata. 165 const uptr kMarkerSize = 512 * 1024 / sizeof(u64); 166 InternalScopedBuffer<u64> marker(kMarkerSize); 167 // volatile to prevent insertion of memset 168 for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) 169 *p = kShadowRodata; 170 internal_write(fd, marker.data(), marker.size()); 171 // Map the file into memory. 172 uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE, 173 MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); 174 if (internal_iserror(page)) { 175 internal_close(fd); 176 return; 177 } 178 // Map the file into shadow of .rodata sections. 179 MemoryMappingLayout proc_maps(/*cache_enabled*/true); 180 uptr start, end, offset, prot; 181 // Reusing the buffer 'name'. 182 while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) { 183 if (name[0] != 0 && name[0] != '[' 184 && (prot & MemoryMappingLayout::kProtectionRead) 185 && (prot & MemoryMappingLayout::kProtectionExecute) 186 && !(prot & MemoryMappingLayout::kProtectionWrite) 187 && IsAppMem(start)) { 188 // Assume it's .rodata 189 char *shadow_start = (char*)MemToShadow(start); 190 char *shadow_end = (char*)MemToShadow(end); 191 for (char *p = shadow_start; p < shadow_end; p += marker.size()) { 192 internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p), 193 PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); 194 } 195 } 196 } 197 internal_close(fd); 198 } 199 200 void InitializeShadowMemoryPlatform() { 201 MapRodata(); 202 } 203 204 #endif // #ifndef SANITIZER_GO 205 206 void InitializePlatformEarly() { 207 #ifdef TSAN_RUNTIME_VMA 208 vmaSize = 209 (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); 210 #if defined(__aarch64__) 211 if (vmaSize != 39 && vmaSize != 42) { 212 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 213 Printf("FATAL: Found %d - Supported 39 and 42\n", vmaSize); 214 Die(); 215 } 216 #elif defined(__powerpc64__) 217 if (vmaSize != 44 && vmaSize != 46) { 218 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); 219 Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize); 220 Die(); 221 } 222 #endif 223 #endif 224 } 225 226 void InitializePlatform() { 227 DisableCoreDumperIfNecessary(); 228 229 // Go maps shadow memory lazily and works fine with limited address space. 230 // Unlimited stack is not a problem as well, because the executable 231 // is not compiled with -pie. 232 if (kCppMode) { 233 bool reexec = false; 234 // TSan doesn't play well with unlimited stack size (as stack 235 // overlaps with shadow memory). If we detect unlimited stack size, 236 // we re-exec the program with limited stack size as a best effort. 237 if (StackSizeIsUnlimited()) { 238 const uptr kMaxStackSize = 32 * 1024 * 1024; 239 VReport(1, "Program is run with unlimited stack size, which wouldn't " 240 "work with ThreadSanitizer.\n" 241 "Re-execing with stack size limited to %zd bytes.\n", 242 kMaxStackSize); 243 SetStackSizeLimitInBytes(kMaxStackSize); 244 reexec = true; 245 } 246 247 if (!AddressSpaceIsUnlimited()) { 248 Report("WARNING: Program is run with limited virtual address space," 249 " which wouldn't work with ThreadSanitizer.\n"); 250 Report("Re-execing with unlimited virtual address space.\n"); 251 SetAddressSpaceUnlimited(); 252 reexec = true; 253 } 254 #if SANITIZER_LINUX && defined(__aarch64__) 255 // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in 256 // linux kernel, the random gap between stack and mapped area is increased 257 // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover 258 // this big range, we should disable randomized virtual space on aarch64. 259 int old_personality = personality(0xffffffff); 260 if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { 261 VReport(1, "WARNING: Program is run with randomized virtual address " 262 "space, which wouldn't work with ThreadSanitizer.\n" 263 "Re-execing with fixed virtual address space.\n"); 264 CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); 265 reexec = true; 266 } 267 #endif 268 if (reexec) 269 ReExec(); 270 } 271 272 #ifndef SANITIZER_GO 273 CheckAndProtect(); 274 InitTlsSize(); 275 #endif 276 } 277 278 #ifndef SANITIZER_GO 279 // Extract file descriptors passed to glibc internal __res_iclose function. 280 // This is required to properly "close" the fds, because we do not see internal 281 // closes within glibc. The code is a pure hack. 282 int ExtractResolvFDs(void *state, int *fds, int nfd) { 283 #if SANITIZER_LINUX && !SANITIZER_ANDROID 284 int cnt = 0; 285 __res_state *statp = (__res_state*)state; 286 for (int i = 0; i < MAXNS && cnt < nfd; i++) { 287 if (statp->_u._ext.nsaddrs[i] && statp->_u._ext.nssocks[i] != -1) 288 fds[cnt++] = statp->_u._ext.nssocks[i]; 289 } 290 return cnt; 291 #else 292 return 0; 293 #endif 294 } 295 296 // Extract file descriptors passed via UNIX domain sockets. 297 // This is requried to properly handle "open" of these fds. 298 // see 'man recvmsg' and 'man 3 cmsg'. 299 int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { 300 int res = 0; 301 msghdr *msg = (msghdr*)msgp; 302 struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); 303 for (; cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { 304 if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) 305 continue; 306 int n = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(fds[0]); 307 for (int i = 0; i < n; i++) { 308 fds[res++] = ((int*)CMSG_DATA(cmsg))[i]; 309 if (res == nfd) 310 return res; 311 } 312 } 313 return res; 314 } 315 316 // Note: this function runs with async signals enabled, 317 // so it must not touch any tsan state. 318 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, 319 void *abstime), void *c, void *m, void *abstime, 320 void(*cleanup)(void *arg), void *arg) { 321 // pthread_cleanup_push/pop are hardcore macros mess. 322 // We can't intercept nor call them w/o including pthread.h. 323 int res; 324 pthread_cleanup_push(cleanup, arg); 325 res = fn(c, m, abstime); 326 pthread_cleanup_pop(0); 327 return res; 328 } 329 #endif 330 331 #ifndef SANITIZER_GO 332 void ReplaceSystemMalloc() { } 333 #endif 334 335 #ifndef SANITIZER_GO 336 #if SANITIZER_ANDROID 337 338 #if defined(__aarch64__) 339 # define __get_tls() \ 340 ({ void** __val; __asm__("mrs %0, tpidr_el0" : "=r"(__val)); __val; }) 341 #elif defined(__x86_64__) 342 # define __get_tls() \ 343 ({ void** __val; __asm__("mov %%fs:0, %0" : "=r"(__val)); __val; }) 344 #else 345 #error unsupported architecture 346 #endif 347 348 // On Android, __thread is not supported. So we store the pointer to ThreadState 349 // in TLS_SLOT_TSAN, which is the tls slot allocated by Android bionic for tsan. 350 static const int TLS_SLOT_TSAN = 8; 351 // On Android, one thread can call intercepted functions after 352 // DestroyThreadState(), so add a fake thread state for "dead" threads. 353 static ThreadState *dead_thread_state = nullptr; 354 355 ThreadState *cur_thread() { 356 ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN]; 357 if (thr == nullptr) { 358 __sanitizer_sigset_t emptyset; 359 internal_sigfillset(&emptyset); 360 __sanitizer_sigset_t oldset; 361 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 362 thr = reinterpret_cast<ThreadState*>(__get_tls()[TLS_SLOT_TSAN]); 363 if (thr == nullptr) { 364 thr = reinterpret_cast<ThreadState*>(MmapOrDie(sizeof(ThreadState), 365 "ThreadState")); 366 __get_tls()[TLS_SLOT_TSAN] = thr; 367 if (dead_thread_state == nullptr) { 368 dead_thread_state = reinterpret_cast<ThreadState*>( 369 MmapOrDie(sizeof(ThreadState), "ThreadState")); 370 dead_thread_state->fast_state.SetIgnoreBit(); 371 dead_thread_state->ignore_interceptors = 1; 372 dead_thread_state->is_dead = true; 373 *const_cast<int*>(&dead_thread_state->tid) = -1; 374 CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState), 375 PROT_READ)); 376 } 377 } 378 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 379 } 380 return thr; 381 } 382 383 void cur_thread_finalize() { 384 __sanitizer_sigset_t emptyset; 385 internal_sigfillset(&emptyset); 386 __sanitizer_sigset_t oldset; 387 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &emptyset, &oldset)); 388 ThreadState* thr = (ThreadState*)__get_tls()[TLS_SLOT_TSAN]; 389 if (thr != dead_thread_state) { 390 __get_tls()[TLS_SLOT_TSAN] = dead_thread_state; 391 UnmapOrDie(thr, sizeof(ThreadState)); 392 } 393 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr)); 394 } 395 #endif // SANITIZER_ANDROID 396 #endif // ifndef SANITIZER_GO 397 398 } // namespace __tsan 399 400 #endif // SANITIZER_LINUX || SANITIZER_FREEBSD 401
