1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 //go:cgo_export_dynamic runtime.end _end 10 //go:cgo_export_dynamic runtime.etext _etext 11 //go:cgo_export_dynamic runtime.edata _edata 12 13 //go:cgo_import_dynamic libc____errno ___errno "libc.so" 14 //go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" 15 //go:cgo_import_dynamic libc_close close "libc.so" 16 //go:cgo_import_dynamic libc_exit exit "libc.so" 17 //go:cgo_import_dynamic libc_fstat fstat "libc.so" 18 //go:cgo_import_dynamic libc_getcontext getcontext "libc.so" 19 //go:cgo_import_dynamic libc_getrlimit getrlimit "libc.so" 20 //go:cgo_import_dynamic libc_kill kill "libc.so" 21 //go:cgo_import_dynamic libc_madvise madvise "libc.so" 22 //go:cgo_import_dynamic libc_malloc malloc "libc.so" 23 //go:cgo_import_dynamic libc_mmap mmap "libc.so" 24 //go:cgo_import_dynamic libc_munmap munmap "libc.so" 25 //go:cgo_import_dynamic libc_open open "libc.so" 26 //go:cgo_import_dynamic libc_pthread_attr_destroy pthread_attr_destroy "libc.so" 27 //go:cgo_import_dynamic libc_pthread_attr_getstack pthread_attr_getstack "libc.so" 28 //go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "libc.so" 29 //go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "libc.so" 30 //go:cgo_import_dynamic libc_pthread_attr_setstack pthread_attr_setstack "libc.so" 31 //go:cgo_import_dynamic libc_pthread_create pthread_create "libc.so" 32 //go:cgo_import_dynamic libc_raise raise "libc.so" 33 //go:cgo_import_dynamic libc_read read "libc.so" 34 //go:cgo_import_dynamic libc_select select "libc.so" 35 //go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" 36 //go:cgo_import_dynamic libc_sem_init sem_init "libc.so" 37 //go:cgo_import_dynamic libc_sem_post sem_post "libc.so" 38 //go:cgo_import_dynamic libc_sem_reltimedwait_np sem_reltimedwait_np "libc.so" 39 //go:cgo_import_dynamic libc_sem_wait sem_wait "libc.so" 40 //go:cgo_import_dynamic libc_setitimer setitimer "libc.so" 41 //go:cgo_import_dynamic libc_sigaction sigaction "libc.so" 42 //go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" 43 //go:cgo_import_dynamic libc_sigprocmask sigprocmask "libc.so" 44 //go:cgo_import_dynamic libc_sysconf sysconf "libc.so" 45 //go:cgo_import_dynamic libc_usleep usleep "libc.so" 46 //go:cgo_import_dynamic libc_write write "libc.so" 47 48 //go:linkname libc____errno libc____errno 49 //go:linkname libc_clock_gettime libc_clock_gettime 50 //go:linkname libc_close libc_close 51 //go:linkname libc_exit libc_exit 52 //go:linkname libc_fstat libc_fstat 53 //go:linkname libc_getcontext libc_getcontext 54 //go:linkname libc_getrlimit libc_getrlimit 55 //go:linkname libc_kill libc_kill 56 //go:linkname libc_madvise libc_madvise 57 //go:linkname libc_malloc libc_malloc 58 //go:linkname libc_mmap libc_mmap 59 //go:linkname libc_munmap libc_munmap 60 //go:linkname libc_open libc_open 61 //go:linkname libc_pthread_attr_destroy libc_pthread_attr_destroy 62 //go:linkname libc_pthread_attr_getstack libc_pthread_attr_getstack 63 //go:linkname libc_pthread_attr_init libc_pthread_attr_init 64 //go:linkname libc_pthread_attr_setdetachstate libc_pthread_attr_setdetachstate 65 //go:linkname libc_pthread_attr_setstack libc_pthread_attr_setstack 66 //go:linkname libc_pthread_create libc_pthread_create 67 //go:linkname libc_raise libc_raise 68 //go:linkname libc_read libc_read 69 //go:linkname libc_select libc_select 70 //go:linkname libc_sched_yield libc_sched_yield 71 //go:linkname libc_sem_init libc_sem_init 72 //go:linkname libc_sem_post libc_sem_post 73 //go:linkname libc_sem_reltimedwait_np libc_sem_reltimedwait_np 74 //go:linkname libc_sem_wait libc_sem_wait 75 //go:linkname libc_setitimer libc_setitimer 76 //go:linkname libc_sigaction libc_sigaction 77 //go:linkname libc_sigaltstack libc_sigaltstack 78 //go:linkname libc_sigprocmask libc_sigprocmask 79 //go:linkname libc_sysconf libc_sysconf 80 //go:linkname libc_usleep libc_usleep 81 //go:linkname libc_write libc_write 82 83 var ( 84 libc____errno, 85 libc_clock_gettime, 86 libc_close, 87 libc_exit, 88 libc_fstat, 89 libc_getcontext, 90 libc_getrlimit, 91 libc_kill, 92 libc_madvise, 93 libc_malloc, 94 libc_mmap, 95 libc_munmap, 96 libc_open, 97 libc_pthread_attr_destroy, 98 libc_pthread_attr_getstack, 99 libc_pthread_attr_init, 100 libc_pthread_attr_setdetachstate, 101 libc_pthread_attr_setstack, 102 libc_pthread_create, 103 libc_raise, 104 libc_read, 105 libc_sched_yield, 106 libc_select, 107 libc_sem_init, 108 libc_sem_post, 109 libc_sem_reltimedwait_np, 110 libc_sem_wait, 111 libc_setitimer, 112 libc_sigaction, 113 libc_sigaltstack, 114 libc_sigprocmask, 115 libc_sysconf, 116 libc_usleep, 117 libc_write libcFunc 118 ) 119 120 var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} 121 122 func getncpu() int32 { 123 n := int32(sysconf(__SC_NPROCESSORS_ONLN)) 124 if n < 1 { 125 return 1 126 } 127 return n 128 } 129 130 func osinit() { 131 ncpu = getncpu() 132 } 133 134 func tstart_sysvicall() 135 136 // May run with m.p==nil, so write barriers are not allowed. 137 //go:nowritebarrier 138 func newosproc(mp *m, _ unsafe.Pointer) { 139 var ( 140 attr pthreadattr 141 oset sigset 142 tid pthread 143 ret int32 144 size uint64 145 ) 146 147 if pthread_attr_init(&attr) != 0 { 148 throw("pthread_attr_init") 149 } 150 if pthread_attr_setstack(&attr, 0, 0x200000) != 0 { 151 throw("pthread_attr_setstack") 152 } 153 if pthread_attr_getstack(&attr, unsafe.Pointer(&mp.g0.stack.hi), &size) != 0 { 154 throw("pthread_attr_getstack") 155 } 156 mp.g0.stack.lo = mp.g0.stack.hi - uintptr(size) 157 if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { 158 throw("pthread_attr_setdetachstate") 159 } 160 161 // Disable signals during create, so that the new thread starts 162 // with signals disabled. It will enable them in minit. 163 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 164 ret = pthread_create(&tid, &attr, funcPC(tstart_sysvicall), unsafe.Pointer(mp)) 165 sigprocmask(_SIG_SETMASK, &oset, nil) 166 if ret != 0 { 167 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n") 168 throw("newosproc") 169 } 170 } 171 172 var urandom_dev = []byte("/dev/urandom\x00") 173 174 //go:nosplit 175 func getRandomData(r []byte) { 176 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 177 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 178 closefd(fd) 179 extendRandom(r, int(n)) 180 } 181 182 func goenvs() { 183 goenvs_unix() 184 } 185 186 // Called to initialize a new m (including the bootstrap m). 187 // Called on the parent thread (main thread in case of bootstrap), can allocate memory. 188 func mpreinit(mp *m) { 189 mp.gsignal = malg(32 * 1024) 190 mp.gsignal.m = mp 191 } 192 193 func miniterrno() 194 195 func msigsave(mp *m) { 196 smask := (*sigset)(unsafe.Pointer(&mp.sigmask)) 197 if unsafe.Sizeof(*smask) > unsafe.Sizeof(mp.sigmask) { 198 throw("insufficient storage for signal mask") 199 } 200 sigprocmask(_SIG_SETMASK, nil, smask) 201 } 202 203 // Called to initialize a new m (including the bootstrap m). 204 // Called on the new thread, can not allocate memory. 205 func minit() { 206 _g_ := getg() 207 asmcgocall(unsafe.Pointer(funcPC(miniterrno)), unsafe.Pointer(&libc____errno)) 208 // Initialize signal handling 209 signalstack(&_g_.m.gsignal.stack) 210 211 // restore signal mask from m.sigmask and unblock essential signals 212 nmask := *(*sigset)(unsafe.Pointer(&_g_.m.sigmask)) 213 for i := range sigtable { 214 if sigtable[i].flags&_SigUnblock != 0 { 215 nmask.__sigbits[(i-1)/32] &^= 1 << ((uint32(i) - 1) & 31) 216 } 217 } 218 sigprocmask(_SIG_SETMASK, &nmask, nil) 219 } 220 221 // Called from dropm to undo the effect of an minit. 222 func unminit() { 223 _g_ := getg() 224 smask := (*sigset)(unsafe.Pointer(&_g_.m.sigmask)) 225 sigprocmask(_SIG_SETMASK, smask, nil) 226 227 signalstack(nil) 228 } 229 230 func memlimit() uintptr { 231 /* 232 TODO: Convert to Go when something actually uses the result. 233 Rlimit rl; 234 extern byte runtimetext[], runtimeend[]; 235 uintptr used; 236 237 if(runtimegetrlimit(RLIMIT_AS, &rl) != 0) 238 return 0; 239 if(rl.rlim_cur >= 0x7fffffff) 240 return 0; 241 242 // Estimate our VM footprint excluding the heap. 243 // Not an exact science: use size of binary plus 244 // some room for thread stacks. 245 used = runtimeend - runtimetext + (64<<20); 246 if(used >= rl.rlim_cur) 247 return 0; 248 249 // If there's not at least 16 MB left, we're probably 250 // not going to be able to do much. Treat as no limit. 251 rl.rlim_cur -= used; 252 if(rl.rlim_cur < (16<<20)) 253 return 0; 254 255 return rl.rlim_cur - used; 256 */ 257 258 return 0 259 } 260 261 func sigtramp() 262 263 func setsig(i int32, fn uintptr, restart bool) { 264 var sa sigactiont 265 266 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK 267 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK 268 if restart { 269 sa.sa_flags |= _SA_RESTART 270 } 271 sa.sa_mask = sigset_all 272 if fn == funcPC(sighandler) { 273 fn = funcPC(sigtramp) 274 } 275 *((*uintptr)(unsafe.Pointer(&sa._funcptr))) = fn 276 sigaction(i, &sa, nil) 277 } 278 279 func setsigstack(i int32) { 280 throw("setsigstack") 281 } 282 283 func getsig(i int32) uintptr { 284 var sa sigactiont 285 sigaction(i, nil, &sa) 286 if *((*uintptr)(unsafe.Pointer(&sa._funcptr))) == funcPC(sigtramp) { 287 return funcPC(sighandler) 288 } 289 return *((*uintptr)(unsafe.Pointer(&sa._funcptr))) 290 } 291 292 func signalstack(s *stack) { 293 var st sigaltstackt 294 if s == nil { 295 st.ss_flags = _SS_DISABLE 296 } else { 297 st.ss_sp = (*byte)(unsafe.Pointer(s.lo)) 298 st.ss_size = uint64(s.hi - s.lo) 299 st.ss_flags = 0 300 } 301 sigaltstack(&st, nil) 302 } 303 304 func updatesigmask(m sigmask) { 305 var mask sigset 306 copy(mask.__sigbits[:], m[:]) 307 sigprocmask(_SIG_SETMASK, &mask, nil) 308 } 309 310 func unblocksig(sig int32) { 311 var mask sigset 312 mask.__sigbits[(sig-1)/32] |= 1 << ((uint32(sig) - 1) & 31) 313 sigprocmask(_SIG_UNBLOCK, &mask, nil) 314 } 315 316 //go:nosplit 317 func semacreate() uintptr { 318 var sem *semt 319 _g_ := getg() 320 321 // Call libc's malloc rather than malloc. This will 322 // allocate space on the C heap. We can't call malloc 323 // here because it could cause a deadlock. 324 _g_.m.libcall.fn = uintptr(unsafe.Pointer(&libc_malloc)) 325 _g_.m.libcall.n = 1 326 memclr(unsafe.Pointer(&_g_.m.scratch), uintptr(len(_g_.m.scratch.v))) 327 _g_.m.scratch.v[0] = unsafe.Sizeof(*sem) 328 _g_.m.libcall.args = uintptr(unsafe.Pointer(&_g_.m.scratch)) 329 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_g_.m.libcall)) 330 sem = (*semt)(unsafe.Pointer(_g_.m.libcall.r1)) 331 if sem_init(sem, 0, 0) != 0 { 332 throw("sem_init") 333 } 334 return uintptr(unsafe.Pointer(sem)) 335 } 336 337 //go:nosplit 338 func semasleep(ns int64) int32 { 339 _m_ := getg().m 340 if ns >= 0 { 341 _m_.ts.tv_sec = ns / 1000000000 342 _m_.ts.tv_nsec = ns % 1000000000 343 344 _m_.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_reltimedwait_np)) 345 _m_.libcall.n = 2 346 memclr(unsafe.Pointer(&_m_.scratch), uintptr(len(_m_.scratch.v))) 347 _m_.scratch.v[0] = _m_.waitsema 348 _m_.scratch.v[1] = uintptr(unsafe.Pointer(&_m_.ts)) 349 _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) 350 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) 351 if *_m_.perrno != 0 { 352 if *_m_.perrno == _ETIMEDOUT || *_m_.perrno == _EAGAIN || *_m_.perrno == _EINTR { 353 return -1 354 } 355 throw("sem_reltimedwait_np") 356 } 357 return 0 358 } 359 for { 360 _m_.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_wait)) 361 _m_.libcall.n = 1 362 memclr(unsafe.Pointer(&_m_.scratch), uintptr(len(_m_.scratch.v))) 363 _m_.scratch.v[0] = _m_.waitsema 364 _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) 365 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) 366 if _m_.libcall.r1 == 0 { 367 break 368 } 369 if *_m_.perrno == _EINTR { 370 continue 371 } 372 throw("sem_wait") 373 } 374 return 0 375 } 376 377 //go:nosplit 378 func semawakeup(mp *m) { 379 if sem_post((*semt)(unsafe.Pointer(mp.waitsema))) != 0 { 380 throw("sem_post") 381 } 382 } 383 384 //go:nosplit 385 func closefd(fd int32) int32 { 386 return int32(sysvicall1(&libc_close, uintptr(fd))) 387 } 388 389 //go:nosplit 390 func exit(r int32) { 391 sysvicall1(&libc_exit, uintptr(r)) 392 } 393 394 //go:nosplit 395 func getcontext(context *ucontext) /* int32 */ { 396 sysvicall1(&libc_getcontext, uintptr(unsafe.Pointer(context))) 397 } 398 399 //go:nosplit 400 func madvise(addr unsafe.Pointer, n uintptr, flags int32) { 401 sysvicall3(&libc_madvise, uintptr(addr), uintptr(n), uintptr(flags)) 402 } 403 404 //go:nosplit 405 func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer { 406 return unsafe.Pointer(sysvicall6(&libc_mmap, uintptr(addr), uintptr(n), uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off))) 407 } 408 409 //go:nosplit 410 func munmap(addr unsafe.Pointer, n uintptr) { 411 sysvicall2(&libc_munmap, uintptr(addr), uintptr(n)) 412 } 413 414 func nanotime1() 415 416 //go:nosplit 417 func nanotime() int64 { 418 return int64(sysvicall0((*libcFunc)(unsafe.Pointer(funcPC(nanotime1))))) 419 } 420 421 //go:nosplit 422 func open(path *byte, mode, perm int32) int32 { 423 return int32(sysvicall3(&libc_open, uintptr(unsafe.Pointer(path)), uintptr(mode), uintptr(perm))) 424 } 425 426 func pthread_attr_destroy(attr *pthreadattr) int32 { 427 return int32(sysvicall1(&libc_pthread_attr_destroy, uintptr(unsafe.Pointer(attr)))) 428 } 429 430 func pthread_attr_getstack(attr *pthreadattr, addr unsafe.Pointer, size *uint64) int32 { 431 return int32(sysvicall3(&libc_pthread_attr_getstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(unsafe.Pointer(size)))) 432 } 433 434 func pthread_attr_init(attr *pthreadattr) int32 { 435 return int32(sysvicall1(&libc_pthread_attr_init, uintptr(unsafe.Pointer(attr)))) 436 } 437 438 func pthread_attr_setdetachstate(attr *pthreadattr, state int32) int32 { 439 return int32(sysvicall2(&libc_pthread_attr_setdetachstate, uintptr(unsafe.Pointer(attr)), uintptr(state))) 440 } 441 442 func pthread_attr_setstack(attr *pthreadattr, addr uintptr, size uint64) int32 { 443 return int32(sysvicall3(&libc_pthread_attr_setstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(size))) 444 } 445 446 func pthread_create(thread *pthread, attr *pthreadattr, fn uintptr, arg unsafe.Pointer) int32 { 447 return int32(sysvicall4(&libc_pthread_create, uintptr(unsafe.Pointer(thread)), uintptr(unsafe.Pointer(attr)), uintptr(fn), uintptr(arg))) 448 } 449 450 func raise(sig int32) /* int32 */ { 451 sysvicall1(&libc_raise, uintptr(sig)) 452 } 453 454 func raiseproc(sig int32) /* int32 */ { 455 pid := sysvicall0(&libc_getpid) 456 sysvicall2(&libc_kill, pid, uintptr(sig)) 457 } 458 459 //go:nosplit 460 func read(fd int32, buf unsafe.Pointer, nbyte int32) int32 { 461 return int32(sysvicall3(&libc_read, uintptr(fd), uintptr(buf), uintptr(nbyte))) 462 } 463 464 //go:nosplit 465 func sem_init(sem *semt, pshared int32, value uint32) int32 { 466 return int32(sysvicall3(&libc_sem_init, uintptr(unsafe.Pointer(sem)), uintptr(pshared), uintptr(value))) 467 } 468 469 //go:nosplit 470 func sem_post(sem *semt) int32 { 471 return int32(sysvicall1(&libc_sem_post, uintptr(unsafe.Pointer(sem)))) 472 } 473 474 //go:nosplit 475 func sem_reltimedwait_np(sem *semt, timeout *timespec) int32 { 476 return int32(sysvicall2(&libc_sem_reltimedwait_np, uintptr(unsafe.Pointer(sem)), uintptr(unsafe.Pointer(timeout)))) 477 } 478 479 //go:nosplit 480 func sem_wait(sem *semt) int32 { 481 return int32(sysvicall1(&libc_sem_wait, uintptr(unsafe.Pointer(sem)))) 482 } 483 484 func setitimer(which int32, value *itimerval, ovalue *itimerval) /* int32 */ { 485 sysvicall3(&libc_setitimer, uintptr(which), uintptr(unsafe.Pointer(value)), uintptr(unsafe.Pointer(ovalue))) 486 } 487 488 func sigaction(sig int32, act *sigactiont, oact *sigactiont) /* int32 */ { 489 sysvicall3(&libc_sigaction, uintptr(sig), uintptr(unsafe.Pointer(act)), uintptr(unsafe.Pointer(oact))) 490 } 491 492 func sigaltstack(ss *sigaltstackt, oss *sigaltstackt) /* int32 */ { 493 sysvicall2(&libc_sigaltstack, uintptr(unsafe.Pointer(ss)), uintptr(unsafe.Pointer(oss))) 494 } 495 496 func sigprocmask(how int32, set *sigset, oset *sigset) /* int32 */ { 497 sysvicall3(&libc_sigprocmask, uintptr(how), uintptr(unsafe.Pointer(set)), uintptr(unsafe.Pointer(oset))) 498 } 499 500 func sysconf(name int32) int64 { 501 return int64(sysvicall1(&libc_sysconf, uintptr(name))) 502 } 503 504 func usleep1(uint32) 505 506 //go:nosplit 507 func usleep(s uint32) { 508 usleep1(s) 509 } 510 511 //go:nosplit 512 func write(fd uintptr, buf unsafe.Pointer, nbyte int32) int32 { 513 return int32(sysvicall3(&libc_write, uintptr(fd), uintptr(buf), uintptr(nbyte))) 514 } 515 516 func osyield1() 517 518 //go:nosplit 519 func osyield() { 520 _g_ := getg() 521 522 // Check the validity of m because we might be called in cgo callback 523 // path early enough where there isn't a m available yet. 524 if _g_ != nil && _g_.m != nil { 525 sysvicall0(&libc_sched_yield) 526 return 527 } 528 osyield1() 529 } 530
