1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 //go:cgo_export_dynamic runtime.end _end 10 //go:cgo_export_dynamic runtime.etext _etext 11 //go:cgo_export_dynamic runtime.edata _edata 12 13 //go:cgo_import_dynamic libc____errno ___errno "libc.so" 14 //go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" 15 //go:cgo_import_dynamic libc_close close "libc.so" 16 //go:cgo_import_dynamic libc_exit exit "libc.so" 17 //go:cgo_import_dynamic libc_fstat fstat "libc.so" 18 //go:cgo_import_dynamic libc_getcontext getcontext "libc.so" 19 //go:cgo_import_dynamic libc_getrlimit getrlimit "libc.so" 20 //go:cgo_import_dynamic libc_kill kill "libc.so" 21 //go:cgo_import_dynamic libc_madvise madvise "libc.so" 22 //go:cgo_import_dynamic libc_malloc malloc "libc.so" 23 //go:cgo_import_dynamic libc_mmap mmap "libc.so" 24 //go:cgo_import_dynamic libc_munmap munmap "libc.so" 25 //go:cgo_import_dynamic libc_open open "libc.so" 26 //go:cgo_import_dynamic libc_pthread_attr_destroy pthread_attr_destroy "libc.so" 27 //go:cgo_import_dynamic libc_pthread_attr_getstack pthread_attr_getstack "libc.so" 28 //go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "libc.so" 29 //go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "libc.so" 30 //go:cgo_import_dynamic libc_pthread_attr_setstack pthread_attr_setstack "libc.so" 31 //go:cgo_import_dynamic libc_pthread_create pthread_create "libc.so" 32 //go:cgo_import_dynamic libc_raise raise "libc.so" 33 //go:cgo_import_dynamic libc_read read "libc.so" 34 //go:cgo_import_dynamic libc_select select "libc.so" 35 //go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" 36 //go:cgo_import_dynamic libc_sem_init sem_init "libc.so" 37 //go:cgo_import_dynamic libc_sem_post sem_post "libc.so" 38 //go:cgo_import_dynamic libc_sem_reltimedwait_np sem_reltimedwait_np "libc.so" 39 //go:cgo_import_dynamic libc_sem_wait sem_wait "libc.so" 40 //go:cgo_import_dynamic libc_setitimer setitimer "libc.so" 41 //go:cgo_import_dynamic libc_sigaction sigaction "libc.so" 42 //go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" 43 //go:cgo_import_dynamic libc_sigprocmask sigprocmask "libc.so" 44 //go:cgo_import_dynamic libc_sysconf sysconf "libc.so" 45 //go:cgo_import_dynamic libc_usleep usleep "libc.so" 46 //go:cgo_import_dynamic libc_write write "libc.so" 47 48 //go:linkname libc____errno libc____errno 49 //go:linkname libc_clock_gettime libc_clock_gettime 50 //go:linkname libc_close libc_close 51 //go:linkname libc_exit libc_exit 52 //go:linkname libc_fstat libc_fstat 53 //go:linkname libc_getcontext libc_getcontext 54 //go:linkname libc_getrlimit libc_getrlimit 55 //go:linkname libc_kill libc_kill 56 //go:linkname libc_madvise libc_madvise 57 //go:linkname libc_malloc libc_malloc 58 //go:linkname libc_mmap libc_mmap 59 //go:linkname libc_munmap libc_munmap 60 //go:linkname libc_open libc_open 61 //go:linkname libc_pthread_attr_destroy libc_pthread_attr_destroy 62 //go:linkname libc_pthread_attr_getstack libc_pthread_attr_getstack 63 //go:linkname libc_pthread_attr_init libc_pthread_attr_init 64 //go:linkname libc_pthread_attr_setdetachstate libc_pthread_attr_setdetachstate 65 //go:linkname libc_pthread_attr_setstack libc_pthread_attr_setstack 66 //go:linkname libc_pthread_create libc_pthread_create 67 //go:linkname libc_raise libc_raise 68 //go:linkname libc_read libc_read 69 //go:linkname libc_select libc_select 70 //go:linkname libc_sched_yield libc_sched_yield 71 //go:linkname libc_sem_init libc_sem_init 72 //go:linkname libc_sem_post libc_sem_post 73 //go:linkname libc_sem_reltimedwait_np libc_sem_reltimedwait_np 74 //go:linkname libc_sem_wait libc_sem_wait 75 //go:linkname libc_setitimer libc_setitimer 76 //go:linkname libc_sigaction libc_sigaction 77 //go:linkname libc_sigaltstack libc_sigaltstack 78 //go:linkname libc_sigprocmask libc_sigprocmask 79 //go:linkname libc_sysconf libc_sysconf 80 //go:linkname libc_usleep libc_usleep 81 //go:linkname libc_write libc_write 82 83 var ( 84 libc____errno, 85 libc_clock_gettime, 86 libc_close, 87 libc_exit, 88 libc_fstat, 89 libc_getcontext, 90 libc_getrlimit, 91 libc_kill, 92 libc_madvise, 93 libc_malloc, 94 libc_mmap, 95 libc_munmap, 96 libc_open, 97 libc_pthread_attr_destroy, 98 libc_pthread_attr_getstack, 99 libc_pthread_attr_init, 100 libc_pthread_attr_setdetachstate, 101 libc_pthread_attr_setstack, 102 libc_pthread_create, 103 libc_raise, 104 libc_read, 105 libc_sched_yield, 106 libc_select, 107 libc_sem_init, 108 libc_sem_post, 109 libc_sem_reltimedwait_np, 110 libc_sem_wait, 111 libc_setitimer, 112 libc_sigaction, 113 libc_sigaltstack, 114 libc_sigprocmask, 115 libc_sysconf, 116 libc_usleep, 117 libc_write libcFunc 118 ) 119 120 var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}} 121 122 func getncpu() int32 { 123 n := int32(sysconf(__SC_NPROCESSORS_ONLN)) 124 if n < 1 { 125 return 1 126 } 127 return n 128 } 129 130 func getPageSize() uintptr { 131 n := int32(sysconf(__SC_PAGESIZE)) 132 if n <= 0 { 133 return 0 134 } 135 return uintptr(n) 136 } 137 138 func osinit() { 139 ncpu = getncpu() 140 physPageSize = getPageSize() 141 } 142 143 func tstart_sysvicall(newm *m) uint32 144 145 // May run with m.p==nil, so write barriers are not allowed. 146 //go:nowritebarrier 147 func newosproc(mp *m, _ unsafe.Pointer) { 148 var ( 149 attr pthreadattr 150 oset sigset 151 tid pthread 152 ret int32 153 size uint64 154 ) 155 156 if pthread_attr_init(&attr) != 0 { 157 throw("pthread_attr_init") 158 } 159 if pthread_attr_setstack(&attr, 0, 0x200000) != 0 { 160 throw("pthread_attr_setstack") 161 } 162 if pthread_attr_getstack(&attr, unsafe.Pointer(&mp.g0.stack.hi), &size) != 0 { 163 throw("pthread_attr_getstack") 164 } 165 mp.g0.stack.lo = mp.g0.stack.hi - uintptr(size) 166 if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { 167 throw("pthread_attr_setdetachstate") 168 } 169 170 // Disable signals during create, so that the new thread starts 171 // with signals disabled. It will enable them in minit. 172 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 173 ret = pthread_create(&tid, &attr, funcPC(tstart_sysvicall), unsafe.Pointer(mp)) 174 sigprocmask(_SIG_SETMASK, &oset, nil) 175 if ret != 0 { 176 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n") 177 if ret == -_EAGAIN { 178 println("runtime: may need to increase max user processes (ulimit -u)") 179 } 180 throw("newosproc") 181 } 182 } 183 184 var urandom_dev = []byte("/dev/urandom\x00") 185 186 //go:nosplit 187 func getRandomData(r []byte) { 188 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 189 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 190 closefd(fd) 191 extendRandom(r, int(n)) 192 } 193 194 func goenvs() { 195 goenvs_unix() 196 } 197 198 // Called to initialize a new m (including the bootstrap m). 199 // Called on the parent thread (main thread in case of bootstrap), can allocate memory. 200 func mpreinit(mp *m) { 201 mp.gsignal = malg(32 * 1024) 202 mp.gsignal.m = mp 203 } 204 205 func miniterrno() 206 207 // Called to initialize a new m (including the bootstrap m). 208 // Called on the new thread, cannot allocate memory. 209 func minit() { 210 asmcgocall(unsafe.Pointer(funcPC(miniterrno)), unsafe.Pointer(&libc____errno)) 211 212 minitSignals() 213 } 214 215 // Called from dropm to undo the effect of an minit. 216 func unminit() { 217 unminitSignals() 218 } 219 220 func memlimit() uintptr { 221 /* 222 TODO: Convert to Go when something actually uses the result. 223 Rlimit rl; 224 extern byte runtimetext[], runtimeend[]; 225 uintptr used; 226 227 if(runtimegetrlimit(RLIMIT_AS, &rl) != 0) 228 return 0; 229 if(rl.rlim_cur >= 0x7fffffff) 230 return 0; 231 232 // Estimate our VM footprint excluding the heap. 233 // Not an exact science: use size of binary plus 234 // some room for thread stacks. 235 used = runtimeend - runtimetext + (64<<20); 236 if(used >= rl.rlim_cur) 237 return 0; 238 239 // If there's not at least 16 MB left, we're probably 240 // not going to be able to do much. Treat as no limit. 241 rl.rlim_cur -= used; 242 if(rl.rlim_cur < (16<<20)) 243 return 0; 244 245 return rl.rlim_cur - used; 246 */ 247 248 return 0 249 } 250 251 func sigtramp() 252 253 //go:nosplit 254 //go:nowritebarrierrec 255 func setsig(i uint32, fn uintptr) { 256 var sa sigactiont 257 258 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTART 259 sa.sa_mask = sigset_all 260 if fn == funcPC(sighandler) { 261 fn = funcPC(sigtramp) 262 } 263 *((*uintptr)(unsafe.Pointer(&sa._funcptr))) = fn 264 sigaction(i, &sa, nil) 265 } 266 267 //go:nosplit 268 //go:nowritebarrierrec 269 func setsigstack(i uint32) { 270 var sa sigactiont 271 sigaction(i, nil, &sa) 272 if sa.sa_flags&_SA_ONSTACK != 0 { 273 return 274 } 275 sa.sa_flags |= _SA_ONSTACK 276 sigaction(i, &sa, nil) 277 } 278 279 //go:nosplit 280 //go:nowritebarrierrec 281 func getsig(i uint32) uintptr { 282 var sa sigactiont 283 sigaction(i, nil, &sa) 284 return *((*uintptr)(unsafe.Pointer(&sa._funcptr))) 285 } 286 287 // setSignaltstackSP sets the ss_sp field of a stackt. 288 //go:nosplit 289 func setSignalstackSP(s *stackt, sp uintptr) { 290 *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp 291 } 292 293 //go:nosplit 294 //go:nowritebarrierrec 295 func sigaddset(mask *sigset, i int) { 296 mask.__sigbits[(i-1)/32] |= 1 << ((uint32(i) - 1) & 31) 297 } 298 299 func sigdelset(mask *sigset, i int) { 300 mask.__sigbits[(i-1)/32] &^= 1 << ((uint32(i) - 1) & 31) 301 } 302 303 func (c *sigctxt) fixsigcode(sig uint32) { 304 } 305 306 //go:nosplit 307 func semacreate(mp *m) { 308 if mp.waitsema != 0 { 309 return 310 } 311 312 var sem *semt 313 _g_ := getg() 314 315 // Call libc's malloc rather than malloc. This will 316 // allocate space on the C heap. We can't call malloc 317 // here because it could cause a deadlock. 318 _g_.m.libcall.fn = uintptr(unsafe.Pointer(&libc_malloc)) 319 _g_.m.libcall.n = 1 320 _g_.m.scratch = mscratch{} 321 _g_.m.scratch.v[0] = unsafe.Sizeof(*sem) 322 _g_.m.libcall.args = uintptr(unsafe.Pointer(&_g_.m.scratch)) 323 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_g_.m.libcall)) 324 sem = (*semt)(unsafe.Pointer(_g_.m.libcall.r1)) 325 if sem_init(sem, 0, 0) != 0 { 326 throw("sem_init") 327 } 328 mp.waitsema = uintptr(unsafe.Pointer(sem)) 329 } 330 331 //go:nosplit 332 func semasleep(ns int64) int32 { 333 _m_ := getg().m 334 if ns >= 0 { 335 _m_.ts.tv_sec = ns / 1000000000 336 _m_.ts.tv_nsec = ns % 1000000000 337 338 _m_.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_reltimedwait_np)) 339 _m_.libcall.n = 2 340 _m_.scratch = mscratch{} 341 _m_.scratch.v[0] = _m_.waitsema 342 _m_.scratch.v[1] = uintptr(unsafe.Pointer(&_m_.ts)) 343 _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) 344 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) 345 if *_m_.perrno != 0 { 346 if *_m_.perrno == _ETIMEDOUT || *_m_.perrno == _EAGAIN || *_m_.perrno == _EINTR { 347 return -1 348 } 349 throw("sem_reltimedwait_np") 350 } 351 return 0 352 } 353 for { 354 _m_.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_wait)) 355 _m_.libcall.n = 1 356 _m_.scratch = mscratch{} 357 _m_.scratch.v[0] = _m_.waitsema 358 _m_.libcall.args = uintptr(unsafe.Pointer(&_m_.scratch)) 359 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&_m_.libcall)) 360 if _m_.libcall.r1 == 0 { 361 break 362 } 363 if *_m_.perrno == _EINTR { 364 continue 365 } 366 throw("sem_wait") 367 } 368 return 0 369 } 370 371 //go:nosplit 372 func semawakeup(mp *m) { 373 if sem_post((*semt)(unsafe.Pointer(mp.waitsema))) != 0 { 374 throw("sem_post") 375 } 376 } 377 378 //go:nosplit 379 func closefd(fd int32) int32 { 380 return int32(sysvicall1(&libc_close, uintptr(fd))) 381 } 382 383 //go:nosplit 384 func exit(r int32) { 385 sysvicall1(&libc_exit, uintptr(r)) 386 } 387 388 //go:nosplit 389 func getcontext(context *ucontext) /* int32 */ { 390 sysvicall1(&libc_getcontext, uintptr(unsafe.Pointer(context))) 391 } 392 393 //go:nosplit 394 func madvise(addr unsafe.Pointer, n uintptr, flags int32) { 395 sysvicall3(&libc_madvise, uintptr(addr), uintptr(n), uintptr(flags)) 396 } 397 398 //go:nosplit 399 func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer { 400 p, err := doMmap(uintptr(addr), n, uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off)) 401 if p == ^uintptr(0) { 402 return unsafe.Pointer(err) 403 } 404 return unsafe.Pointer(p) 405 } 406 407 //go:nosplit 408 func doMmap(addr, n, prot, flags, fd, off uintptr) (uintptr, uintptr) { 409 var libcall libcall 410 libcall.fn = uintptr(unsafe.Pointer(&libc_mmap)) 411 libcall.n = 6 412 libcall.args = uintptr(noescape(unsafe.Pointer(&addr))) 413 asmcgocall(unsafe.Pointer(&asmsysvicall6), unsafe.Pointer(&libcall)) 414 return libcall.r1, libcall.err 415 } 416 417 //go:nosplit 418 func munmap(addr unsafe.Pointer, n uintptr) { 419 sysvicall2(&libc_munmap, uintptr(addr), uintptr(n)) 420 } 421 422 func nanotime1() 423 424 //go:nosplit 425 func nanotime() int64 { 426 return int64(sysvicall0((*libcFunc)(unsafe.Pointer(funcPC(nanotime1))))) 427 } 428 429 //go:nosplit 430 func open(path *byte, mode, perm int32) int32 { 431 return int32(sysvicall3(&libc_open, uintptr(unsafe.Pointer(path)), uintptr(mode), uintptr(perm))) 432 } 433 434 func pthread_attr_destroy(attr *pthreadattr) int32 { 435 return int32(sysvicall1(&libc_pthread_attr_destroy, uintptr(unsafe.Pointer(attr)))) 436 } 437 438 func pthread_attr_getstack(attr *pthreadattr, addr unsafe.Pointer, size *uint64) int32 { 439 return int32(sysvicall3(&libc_pthread_attr_getstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(unsafe.Pointer(size)))) 440 } 441 442 func pthread_attr_init(attr *pthreadattr) int32 { 443 return int32(sysvicall1(&libc_pthread_attr_init, uintptr(unsafe.Pointer(attr)))) 444 } 445 446 func pthread_attr_setdetachstate(attr *pthreadattr, state int32) int32 { 447 return int32(sysvicall2(&libc_pthread_attr_setdetachstate, uintptr(unsafe.Pointer(attr)), uintptr(state))) 448 } 449 450 func pthread_attr_setstack(attr *pthreadattr, addr uintptr, size uint64) int32 { 451 return int32(sysvicall3(&libc_pthread_attr_setstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(size))) 452 } 453 454 func pthread_create(thread *pthread, attr *pthreadattr, fn uintptr, arg unsafe.Pointer) int32 { 455 return int32(sysvicall4(&libc_pthread_create, uintptr(unsafe.Pointer(thread)), uintptr(unsafe.Pointer(attr)), uintptr(fn), uintptr(arg))) 456 } 457 458 //go:nosplit 459 //go:nowritebarrierrec 460 func raise(sig uint32) /* int32 */ { 461 sysvicall1(&libc_raise, uintptr(sig)) 462 } 463 464 func raiseproc(sig uint32) /* int32 */ { 465 pid := sysvicall0(&libc_getpid) 466 sysvicall2(&libc_kill, pid, uintptr(sig)) 467 } 468 469 //go:nosplit 470 func read(fd int32, buf unsafe.Pointer, nbyte int32) int32 { 471 return int32(sysvicall3(&libc_read, uintptr(fd), uintptr(buf), uintptr(nbyte))) 472 } 473 474 //go:nosplit 475 func sem_init(sem *semt, pshared int32, value uint32) int32 { 476 return int32(sysvicall3(&libc_sem_init, uintptr(unsafe.Pointer(sem)), uintptr(pshared), uintptr(value))) 477 } 478 479 //go:nosplit 480 func sem_post(sem *semt) int32 { 481 return int32(sysvicall1(&libc_sem_post, uintptr(unsafe.Pointer(sem)))) 482 } 483 484 //go:nosplit 485 func sem_reltimedwait_np(sem *semt, timeout *timespec) int32 { 486 return int32(sysvicall2(&libc_sem_reltimedwait_np, uintptr(unsafe.Pointer(sem)), uintptr(unsafe.Pointer(timeout)))) 487 } 488 489 //go:nosplit 490 func sem_wait(sem *semt) int32 { 491 return int32(sysvicall1(&libc_sem_wait, uintptr(unsafe.Pointer(sem)))) 492 } 493 494 func setitimer(which int32, value *itimerval, ovalue *itimerval) /* int32 */ { 495 sysvicall3(&libc_setitimer, uintptr(which), uintptr(unsafe.Pointer(value)), uintptr(unsafe.Pointer(ovalue))) 496 } 497 498 //go:nosplit 499 //go:nowritebarrierrec 500 func sigaction(sig uint32, act *sigactiont, oact *sigactiont) /* int32 */ { 501 sysvicall3(&libc_sigaction, uintptr(sig), uintptr(unsafe.Pointer(act)), uintptr(unsafe.Pointer(oact))) 502 } 503 504 //go:nosplit 505 //go:nowritebarrierrec 506 func sigaltstack(ss *stackt, oss *stackt) /* int32 */ { 507 sysvicall2(&libc_sigaltstack, uintptr(unsafe.Pointer(ss)), uintptr(unsafe.Pointer(oss))) 508 } 509 510 //go:nosplit 511 //go:nowritebarrierrec 512 func sigprocmask(how int32, set *sigset, oset *sigset) /* int32 */ { 513 sysvicall3(&libc_sigprocmask, uintptr(how), uintptr(unsafe.Pointer(set)), uintptr(unsafe.Pointer(oset))) 514 } 515 516 func sysconf(name int32) int64 { 517 return int64(sysvicall1(&libc_sysconf, uintptr(name))) 518 } 519 520 func usleep1(usec uint32) 521 522 //go:nosplit 523 func usleep(s uint32) { 524 usleep1(s) 525 } 526 527 //go:nosplit 528 func write(fd uintptr, buf unsafe.Pointer, nbyte int32) int32 { 529 return int32(sysvicall3(&libc_write, uintptr(fd), uintptr(buf), uintptr(nbyte))) 530 } 531 532 func osyield1() 533 534 //go:nosplit 535 func osyield() { 536 _g_ := getg() 537 538 // Check the validity of m because we might be called in cgo callback 539 // path early enough where there isn't a m available yet. 540 if _g_ != nil && _g_.m != nil { 541 sysvicall0(&libc_sched_yield) 542 return 543 } 544 osyield1() 545 } 546
