1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "runtime/internal/sys" 10 "unsafe" 11 ) 12 13 var buildVersion = sys.TheVersion 14 15 // Goroutine scheduler 16 // The scheduler's job is to distribute ready-to-run goroutines over worker threads. 17 // 18 // The main concepts are: 19 // G - goroutine. 20 // M - worker thread, or machine. 21 // P - processor, a resource that is required to execute Go code. 22 // M must have an associated P to execute Go code, however it can be 23 // blocked or in a syscall w/o an associated P. 24 // 25 // Design doc at https://golang.org/s/go11sched. 26 27 // Worker thread parking/unparking. 28 // We need to balance between keeping enough running worker threads to utilize 29 // available hardware parallelism and parking excessive running worker threads 30 // to conserve CPU resources and power. This is not simple for two reasons: 31 // (1) scheduler state is intentionally distributed (in particular, per-P work 32 // queues), so it is not possible to compute global predicates on fast paths; 33 // (2) for optimal thread management we would need to know the future (don't park 34 // a worker thread when a new goroutine will be readied in near future). 35 // 36 // Three rejected approaches that would work badly: 37 // 1. Centralize all scheduler state (would inhibit scalability). 38 // 2. Direct goroutine handoff. That is, when we ready a new goroutine and there 39 // is a spare P, unpark a thread and handoff it the thread and the goroutine. 40 // This would lead to thread state thrashing, as the thread that readied the 41 // goroutine can be out of work the very next moment, we will need to park it. 42 // Also, it would destroy locality of computation as we want to preserve 43 // dependent goroutines on the same thread; and introduce additional latency. 44 // 3. Unpark an additional thread whenever we ready a goroutine and there is an 45 // idle P, but don't do handoff. This would lead to excessive thread parking/ 46 // unparking as the additional threads will instantly park without discovering 47 // any work to do. 48 // 49 // The current approach: 50 // We unpark an additional thread when we ready a goroutine if (1) there is an 51 // idle P and there are no "spinning" worker threads. A worker thread is considered 52 // spinning if it is out of local work and did not find work in global run queue/ 53 // netpoller; the spinning state is denoted in m.spinning and in sched.nmspinning. 54 // Threads unparked this way are also considered spinning; we don't do goroutine 55 // handoff so such threads are out of work initially. Spinning threads do some 56 // spinning looking for work in per-P run queues before parking. If a spinning 57 // thread finds work it takes itself out of the spinning state and proceeds to 58 // execution. If it does not find work it takes itself out of the spinning state 59 // and then parks. 60 // If there is at least one spinning thread (sched.nmspinning>1), we don't unpark 61 // new threads when readying goroutines. To compensate for that, if the last spinning 62 // thread finds work and stops spinning, it must unpark a new spinning thread. 63 // This approach smooths out unjustified spikes of thread unparking, 64 // but at the same time guarantees eventual maximal CPU parallelism utilization. 65 // 66 // The main implementation complication is that we need to be very careful during 67 // spinning->non-spinning thread transition. This transition can race with submission 68 // of a new goroutine, and either one part or another needs to unpark another worker 69 // thread. If they both fail to do that, we can end up with semi-persistent CPU 70 // underutilization. The general pattern for goroutine readying is: submit a goroutine 71 // to local work queue, #StoreLoad-style memory barrier, check sched.nmspinning. 72 // The general pattern for spinning->non-spinning transition is: decrement nmspinning, 73 // #StoreLoad-style memory barrier, check all per-P work queues for new work. 74 // Note that all this complexity does not apply to global run queue as we are not 75 // sloppy about thread unparking when submitting to global queue. Also see comments 76 // for nmspinning manipulation. 77 78 var ( 79 m0 m 80 g0 g 81 raceprocctx0 uintptr 82 ) 83 84 //go:linkname runtime_init runtime.init 85 func runtime_init() 86 87 //go:linkname main_init main.init 88 func main_init() 89 90 // main_init_done is a signal used by cgocallbackg that initialization 91 // has been completed. It is made before _cgo_notify_runtime_init_done, 92 // so all cgo calls can rely on it existing. When main_init is complete, 93 // it is closed, meaning cgocallbackg can reliably receive from it. 94 var main_init_done chan bool 95 96 //go:linkname main_main main.main 97 func main_main() 98 99 // runtimeInitTime is the nanotime() at which the runtime started. 100 var runtimeInitTime int64 101 102 // Value to use for signal mask for newly created M's. 103 var initSigmask sigset 104 105 // The main goroutine. 106 func main() { 107 g := getg() 108 109 // Racectx of m0->g0 is used only as the parent of the main goroutine. 110 // It must not be used for anything else. 111 g.m.g0.racectx = 0 112 113 // Max stack size is 1 GB on 64-bit, 250 MB on 32-bit. 114 // Using decimal instead of binary GB and MB because 115 // they look nicer in the stack overflow failure message. 116 if sys.PtrSize == 8 { 117 maxstacksize = 1000000000 118 } else { 119 maxstacksize = 250000000 120 } 121 122 // Record when the world started. 123 runtimeInitTime = nanotime() 124 125 systemstack(func() { 126 newm(sysmon, nil) 127 }) 128 129 // Lock the main goroutine onto this, the main OS thread, 130 // during initialization. Most programs won't care, but a few 131 // do require certain calls to be made by the main thread. 132 // Those can arrange for main.main to run in the main thread 133 // by calling runtime.LockOSThread during initialization 134 // to preserve the lock. 135 lockOSThread() 136 137 if g.m != &m0 { 138 throw("runtime.main not on m0") 139 } 140 141 runtime_init() // must be before defer 142 143 // Defer unlock so that runtime.Goexit during init does the unlock too. 144 needUnlock := true 145 defer func() { 146 if needUnlock { 147 unlockOSThread() 148 } 149 }() 150 151 gcenable() 152 153 main_init_done = make(chan bool) 154 if iscgo { 155 if _cgo_thread_start == nil { 156 throw("_cgo_thread_start missing") 157 } 158 if GOOS != "windows" { 159 if _cgo_setenv == nil { 160 throw("_cgo_setenv missing") 161 } 162 if _cgo_unsetenv == nil { 163 throw("_cgo_unsetenv missing") 164 } 165 } 166 if _cgo_notify_runtime_init_done == nil { 167 throw("_cgo_notify_runtime_init_done missing") 168 } 169 cgocall(_cgo_notify_runtime_init_done, nil) 170 } 171 172 fn := main_init // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime 173 fn() 174 close(main_init_done) 175 176 needUnlock = false 177 unlockOSThread() 178 179 if isarchive || islibrary { 180 // A program compiled with -buildmode=c-archive or c-shared 181 // has a main, but it is not executed. 182 return 183 } 184 fn = main_main // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime 185 fn() 186 if raceenabled { 187 racefini() 188 } 189 190 // Make racy client program work: if panicking on 191 // another goroutine at the same time as main returns, 192 // let the other goroutine finish printing the panic trace. 193 // Once it does, it will exit. See issue 3934. 194 if panicking != 0 { 195 gopark(nil, nil, "panicwait", traceEvGoStop, 1) 196 } 197 198 exit(0) 199 for { 200 var x *int32 201 *x = 0 202 } 203 } 204 205 // os_beforeExit is called from os.Exit(0). 206 //go:linkname os_beforeExit os.runtime_beforeExit 207 func os_beforeExit() { 208 if raceenabled { 209 racefini() 210 } 211 } 212 213 // start forcegc helper goroutine 214 func init() { 215 go forcegchelper() 216 } 217 218 func forcegchelper() { 219 forcegc.g = getg() 220 for { 221 lock(&forcegc.lock) 222 if forcegc.idle != 0 { 223 throw("forcegc: phase error") 224 } 225 atomic.Store(&forcegc.idle, 1) 226 goparkunlock(&forcegc.lock, "force gc (idle)", traceEvGoBlock, 1) 227 // this goroutine is explicitly resumed by sysmon 228 if debug.gctrace > 0 { 229 println("GC forced") 230 } 231 gcStart(gcBackgroundMode, true) 232 } 233 } 234 235 //go:nosplit 236 237 // Gosched yields the processor, allowing other goroutines to run. It does not 238 // suspend the current goroutine, so execution resumes automatically. 239 func Gosched() { 240 mcall(gosched_m) 241 } 242 243 var alwaysFalse bool 244 245 // goschedguarded does nothing, but is written in a way that guarantees a preemption check in its prologue. 246 // Calls to this function are inserted by the compiler in otherwise uninterruptible loops (see insertLoopReschedChecks). 247 func goschedguarded() { 248 if alwaysFalse { 249 goschedguarded() 250 } 251 } 252 253 // Puts the current goroutine into a waiting state and calls unlockf. 254 // If unlockf returns false, the goroutine is resumed. 255 // unlockf must not access this G's stack, as it may be moved between 256 // the call to gopark and the call to unlockf. 257 func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string, traceEv byte, traceskip int) { 258 mp := acquirem() 259 gp := mp.curg 260 status := readgstatus(gp) 261 if status != _Grunning && status != _Gscanrunning { 262 throw("gopark: bad g status") 263 } 264 mp.waitlock = lock 265 mp.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf)) 266 gp.waitreason = reason 267 mp.waittraceev = traceEv 268 mp.waittraceskip = traceskip 269 releasem(mp) 270 // can't do anything that might move the G between Ms here. 271 mcall(park_m) 272 } 273 274 // Puts the current goroutine into a waiting state and unlocks the lock. 275 // The goroutine can be made runnable again by calling goready(gp). 276 func goparkunlock(lock *mutex, reason string, traceEv byte, traceskip int) { 277 gopark(parkunlock_c, unsafe.Pointer(lock), reason, traceEv, traceskip) 278 } 279 280 func goready(gp *g, traceskip int) { 281 systemstack(func() { 282 ready(gp, traceskip, true) 283 }) 284 } 285 286 //go:nosplit 287 func acquireSudog() *sudog { 288 // Delicate dance: the semaphore implementation calls 289 // acquireSudog, acquireSudog calls new(sudog), 290 // new calls malloc, malloc can call the garbage collector, 291 // and the garbage collector calls the semaphore implementation 292 // in stopTheWorld. 293 // Break the cycle by doing acquirem/releasem around new(sudog). 294 // The acquirem/releasem increments m.locks during new(sudog), 295 // which keeps the garbage collector from being invoked. 296 mp := acquirem() 297 pp := mp.p.ptr() 298 if len(pp.sudogcache) == 0 { 299 lock(&sched.sudoglock) 300 // First, try to grab a batch from central cache. 301 for len(pp.sudogcache) < cap(pp.sudogcache)/2 && sched.sudogcache != nil { 302 s := sched.sudogcache 303 sched.sudogcache = s.next 304 s.next = nil 305 pp.sudogcache = append(pp.sudogcache, s) 306 } 307 unlock(&sched.sudoglock) 308 // If the central cache is empty, allocate a new one. 309 if len(pp.sudogcache) == 0 { 310 pp.sudogcache = append(pp.sudogcache, new(sudog)) 311 } 312 } 313 n := len(pp.sudogcache) 314 s := pp.sudogcache[n-1] 315 pp.sudogcache[n-1] = nil 316 pp.sudogcache = pp.sudogcache[:n-1] 317 if s.elem != nil { 318 throw("acquireSudog: found s.elem != nil in cache") 319 } 320 releasem(mp) 321 return s 322 } 323 324 //go:nosplit 325 func releaseSudog(s *sudog) { 326 if s.elem != nil { 327 throw("runtime: sudog with non-nil elem") 328 } 329 if s.selectdone != nil { 330 throw("runtime: sudog with non-nil selectdone") 331 } 332 if s.next != nil { 333 throw("runtime: sudog with non-nil next") 334 } 335 if s.prev != nil { 336 throw("runtime: sudog with non-nil prev") 337 } 338 if s.waitlink != nil { 339 throw("runtime: sudog with non-nil waitlink") 340 } 341 if s.c != nil { 342 throw("runtime: sudog with non-nil c") 343 } 344 gp := getg() 345 if gp.param != nil { 346 throw("runtime: releaseSudog with non-nil gp.param") 347 } 348 mp := acquirem() // avoid rescheduling to another P 349 pp := mp.p.ptr() 350 if len(pp.sudogcache) == cap(pp.sudogcache) { 351 // Transfer half of local cache to the central cache. 352 var first, last *sudog 353 for len(pp.sudogcache) > cap(pp.sudogcache)/2 { 354 n := len(pp.sudogcache) 355 p := pp.sudogcache[n-1] 356 pp.sudogcache[n-1] = nil 357 pp.sudogcache = pp.sudogcache[:n-1] 358 if first == nil { 359 first = p 360 } else { 361 last.next = p 362 } 363 last = p 364 } 365 lock(&sched.sudoglock) 366 last.next = sched.sudogcache 367 sched.sudogcache = first 368 unlock(&sched.sudoglock) 369 } 370 pp.sudogcache = append(pp.sudogcache, s) 371 releasem(mp) 372 } 373 374 // funcPC returns the entry PC of the function f. 375 // It assumes that f is a func value. Otherwise the behavior is undefined. 376 //go:nosplit 377 func funcPC(f interface{}) uintptr { 378 return **(**uintptr)(add(unsafe.Pointer(&f), sys.PtrSize)) 379 } 380 381 // called from assembly 382 func badmcall(fn func(*g)) { 383 throw("runtime: mcall called on m->g0 stack") 384 } 385 386 func badmcall2(fn func(*g)) { 387 throw("runtime: mcall function returned") 388 } 389 390 func badreflectcall() { 391 panic(plainError("arg size to reflect.call more than 1GB")) 392 } 393 394 var badmorestackg0Msg = "fatal: morestack on g0\n" 395 396 //go:nosplit 397 //go:nowritebarrierrec 398 func badmorestackg0() { 399 sp := stringStructOf(&badmorestackg0Msg) 400 write(2, sp.str, int32(sp.len)) 401 } 402 403 var badmorestackgsignalMsg = "fatal: morestack on gsignal\n" 404 405 //go:nosplit 406 //go:nowritebarrierrec 407 func badmorestackgsignal() { 408 sp := stringStructOf(&badmorestackgsignalMsg) 409 write(2, sp.str, int32(sp.len)) 410 } 411 412 //go:nosplit 413 func badctxt() { 414 throw("ctxt != 0") 415 } 416 417 func lockedOSThread() bool { 418 gp := getg() 419 return gp.lockedm != nil && gp.m.lockedg != nil 420 } 421 422 var ( 423 allgs []*g 424 allglock mutex 425 ) 426 427 func allgadd(gp *g) { 428 if readgstatus(gp) == _Gidle { 429 throw("allgadd: bad status Gidle") 430 } 431 432 lock(&allglock) 433 allgs = append(allgs, gp) 434 allglen = uintptr(len(allgs)) 435 436 // Grow GC rescan list if necessary. 437 if len(allgs) > cap(work.rescan.list) { 438 lock(&work.rescan.lock) 439 l := work.rescan.list 440 // Let append do the heavy lifting, but keep the 441 // length the same. 442 work.rescan.list = append(l[:cap(l)], 0)[:len(l)] 443 unlock(&work.rescan.lock) 444 } 445 unlock(&allglock) 446 } 447 448 const ( 449 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once. 450 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. 451 _GoidCacheBatch = 16 452 ) 453 454 // The bootstrap sequence is: 455 // 456 // call osinit 457 // call schedinit 458 // make & queue new G 459 // call runtimemstart 460 // 461 // The new G calls runtimemain. 462 func schedinit() { 463 // raceinit must be the first call to race detector. 464 // In particular, it must be done before mallocinit below calls racemapshadow. 465 _g_ := getg() 466 if raceenabled { 467 _g_.racectx, raceprocctx0 = raceinit() 468 } 469 470 sched.maxmcount = 10000 471 472 tracebackinit() 473 moduledataverify() 474 stackinit() 475 mallocinit() 476 mcommoninit(_g_.m) 477 alginit() // maps must not be used before this call 478 modulesinit() // provides activeModules 479 typelinksinit() // uses maps, activeModules 480 itabsinit() // uses activeModules 481 482 msigsave(_g_.m) 483 initSigmask = _g_.m.sigmask 484 485 goargs() 486 goenvs() 487 parsedebugvars() 488 gcinit() 489 490 sched.lastpoll = uint64(nanotime()) 491 procs := ncpu 492 if n, ok := atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 { 493 procs = n 494 } 495 if procs > _MaxGomaxprocs { 496 procs = _MaxGomaxprocs 497 } 498 if procresize(procs) != nil { 499 throw("unknown runnable goroutine during bootstrap") 500 } 501 502 if buildVersion == "" { 503 // Condition should never trigger. This code just serves 504 // to ensure runtimebuildVersion is kept in the resulting binary. 505 buildVersion = "unknown" 506 } 507 } 508 509 func dumpgstatus(gp *g) { 510 _g_ := getg() 511 print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") 512 print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n") 513 } 514 515 func checkmcount() { 516 // sched lock is held 517 if sched.mcount > sched.maxmcount { 518 print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n") 519 throw("thread exhaustion") 520 } 521 } 522 523 func mcommoninit(mp *m) { 524 _g_ := getg() 525 526 // g0 stack won't make sense for user (and is not necessary unwindable). 527 if _g_ != _g_.m.g0 { 528 callers(1, mp.createstack[:]) 529 } 530 531 mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks()) 532 if mp.fastrand == 0 { 533 mp.fastrand = 0x49f6428a 534 } 535 536 lock(&sched.lock) 537 mp.id = sched.mcount 538 sched.mcount++ 539 checkmcount() 540 mpreinit(mp) 541 if mp.gsignal != nil { 542 mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard 543 } 544 545 // Add to allm so garbage collector doesn't free g->m 546 // when it is just in a register or thread-local storage. 547 mp.alllink = allm 548 549 // NumCgoCall() iterates over allm w/o schedlock, 550 // so we need to publish it safely. 551 atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp)) 552 unlock(&sched.lock) 553 554 // Allocate memory to hold a cgo traceback if the cgo call crashes. 555 if iscgo || GOOS == "solaris" || GOOS == "windows" { 556 mp.cgoCallers = new(cgoCallers) 557 } 558 } 559 560 // Mark gp ready to run. 561 func ready(gp *g, traceskip int, next bool) { 562 if trace.enabled { 563 traceGoUnpark(gp, traceskip) 564 } 565 566 status := readgstatus(gp) 567 568 // Mark runnable. 569 _g_ := getg() 570 _g_.m.locks++ // disable preemption because it can be holding p in a local var 571 if status&^_Gscan != _Gwaiting { 572 dumpgstatus(gp) 573 throw("bad g->status in ready") 574 } 575 576 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq 577 casgstatus(gp, _Gwaiting, _Grunnable) 578 runqput(_g_.m.p.ptr(), gp, next) 579 if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 { 580 wakep() 581 } 582 _g_.m.locks-- 583 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in Case we've cleared it in newstack 584 _g_.stackguard0 = stackPreempt 585 } 586 } 587 588 func gcprocs() int32 { 589 // Figure out how many CPUs to use during GC. 590 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. 591 lock(&sched.lock) 592 n := gomaxprocs 593 if n > ncpu { 594 n = ncpu 595 } 596 if n > _MaxGcproc { 597 n = _MaxGcproc 598 } 599 if n > sched.nmidle+1 { // one M is currently running 600 n = sched.nmidle + 1 601 } 602 unlock(&sched.lock) 603 return n 604 } 605 606 func needaddgcproc() bool { 607 lock(&sched.lock) 608 n := gomaxprocs 609 if n > ncpu { 610 n = ncpu 611 } 612 if n > _MaxGcproc { 613 n = _MaxGcproc 614 } 615 n -= sched.nmidle + 1 // one M is currently running 616 unlock(&sched.lock) 617 return n > 0 618 } 619 620 func helpgc(nproc int32) { 621 _g_ := getg() 622 lock(&sched.lock) 623 pos := 0 624 for n := int32(1); n < nproc; n++ { // one M is currently running 625 if allp[pos].mcache == _g_.m.mcache { 626 pos++ 627 } 628 mp := mget() 629 if mp == nil { 630 throw("gcprocs inconsistency") 631 } 632 mp.helpgc = n 633 mp.p.set(allp[pos]) 634 mp.mcache = allp[pos].mcache 635 pos++ 636 notewakeup(&mp.park) 637 } 638 unlock(&sched.lock) 639 } 640 641 // freezeStopWait is a large value that freezetheworld sets 642 // sched.stopwait to in order to request that all Gs permanently stop. 643 const freezeStopWait = 0x7fffffff 644 645 // freezing is set to non-zero if the runtime is trying to freeze the 646 // world. 647 var freezing uint32 648 649 // Similar to stopTheWorld but best-effort and can be called several times. 650 // There is no reverse operation, used during crashing. 651 // This function must not lock any mutexes. 652 func freezetheworld() { 653 atomic.Store(&freezing, 1) 654 // stopwait and preemption requests can be lost 655 // due to races with concurrently executing threads, 656 // so try several times 657 for i := 0; i < 5; i++ { 658 // this should tell the scheduler to not start any new goroutines 659 sched.stopwait = freezeStopWait 660 atomic.Store(&sched.gcwaiting, 1) 661 // this should stop running goroutines 662 if !preemptall() { 663 break // no running goroutines 664 } 665 usleep(1000) 666 } 667 // to be sure 668 usleep(1000) 669 preemptall() 670 usleep(1000) 671 } 672 673 func isscanstatus(status uint32) bool { 674 if status == _Gscan { 675 throw("isscanstatus: Bad status Gscan") 676 } 677 return status&_Gscan == _Gscan 678 } 679 680 // All reads and writes of g's status go through readgstatus, casgstatus 681 // castogscanstatus, casfrom_Gscanstatus. 682 //go:nosplit 683 func readgstatus(gp *g) uint32 { 684 return atomic.Load(&gp.atomicstatus) 685 } 686 687 // Ownership of gcscanvalid: 688 // 689 // If gp is running (meaning status == _Grunning or _Grunning|_Gscan), 690 // then gp owns gp.gcscanvalid, and other goroutines must not modify it. 691 // 692 // Otherwise, a second goroutine can lock the scan state by setting _Gscan 693 // in the status bit and then modify gcscanvalid, and then unlock the scan state. 694 // 695 // Note that the first condition implies an exception to the second: 696 // if a second goroutine changes gp's status to _Grunning|_Gscan, 697 // that second goroutine still does not have the right to modify gcscanvalid. 698 699 // The Gscanstatuses are acting like locks and this releases them. 700 // If it proves to be a performance hit we should be able to make these 701 // simple atomic stores but for now we are going to throw if 702 // we see an inconsistent state. 703 func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { 704 success := false 705 706 // Check that transition is valid. 707 switch oldval { 708 default: 709 print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") 710 dumpgstatus(gp) 711 throw("casfrom_Gscanstatus:top gp->status is not in scan state") 712 case _Gscanrunnable, 713 _Gscanwaiting, 714 _Gscanrunning, 715 _Gscansyscall: 716 if newval == oldval&^_Gscan { 717 success = atomic.Cas(&gp.atomicstatus, oldval, newval) 718 } 719 } 720 if !success { 721 print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") 722 dumpgstatus(gp) 723 throw("casfrom_Gscanstatus: gp->status is not in scan state") 724 } 725 } 726 727 // This will return false if the gp is not in the expected status and the cas fails. 728 // This acts like a lock acquire while the casfromgstatus acts like a lock release. 729 func castogscanstatus(gp *g, oldval, newval uint32) bool { 730 switch oldval { 731 case _Grunnable, 732 _Grunning, 733 _Gwaiting, 734 _Gsyscall: 735 if newval == oldval|_Gscan { 736 return atomic.Cas(&gp.atomicstatus, oldval, newval) 737 } 738 } 739 print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n") 740 throw("castogscanstatus") 741 panic("not reached") 742 } 743 744 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus 745 // and casfrom_Gscanstatus instead. 746 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that 747 // put it in the Gscan state is finished. 748 //go:nosplit 749 func casgstatus(gp *g, oldval, newval uint32) { 750 if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { 751 systemstack(func() { 752 print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") 753 throw("casgstatus: bad incoming values") 754 }) 755 } 756 757 if oldval == _Grunning && gp.gcscanvalid { 758 // If oldvall == _Grunning, then the actual status must be 759 // _Grunning or _Grunning|_Gscan; either way, 760 // we own gp.gcscanvalid, so it's safe to read. 761 // gp.gcscanvalid must not be true when we are running. 762 print("runtime: casgstatus ", hex(oldval), "->", hex(newval), " gp.status=", hex(gp.atomicstatus), " gp.gcscanvalid=true\n") 763 throw("casgstatus") 764 } 765 766 // See http://golang.org/cl/21503 for justification of the yield delay. 767 const yieldDelay = 5 * 1000 768 var nextYield int64 769 770 // loop if gp->atomicstatus is in a scan state giving 771 // GC time to finish and change the state to oldval. 772 for i := 0; !atomic.Cas(&gp.atomicstatus, oldval, newval); i++ { 773 if oldval == _Gwaiting && gp.atomicstatus == _Grunnable { 774 systemstack(func() { 775 throw("casgstatus: waiting for Gwaiting but is Grunnable") 776 }) 777 } 778 // Help GC if needed. 779 // if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) { 780 // gp.preemptscan = false 781 // systemstack(func() { 782 // gcphasework(gp) 783 // }) 784 // } 785 // But meanwhile just yield. 786 if i == 0 { 787 nextYield = nanotime() + yieldDelay 788 } 789 if nanotime() < nextYield { 790 for x := 0; x < 10 && gp.atomicstatus != oldval; x++ { 791 procyield(1) 792 } 793 } else { 794 osyield() 795 nextYield = nanotime() + yieldDelay/2 796 } 797 } 798 if newval == _Grunning && gp.gcscanvalid { 799 // Run queueRescan on the system stack so it has more space. 800 systemstack(func() { queueRescan(gp) }) 801 } 802 } 803 804 // casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. 805 // Returns old status. Cannot call casgstatus directly, because we are racing with an 806 // async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, 807 // it might have become Grunnable by the time we get to the cas. If we called casgstatus, 808 // it would loop waiting for the status to go back to Gwaiting, which it never will. 809 //go:nosplit 810 func casgcopystack(gp *g) uint32 { 811 for { 812 oldstatus := readgstatus(gp) &^ _Gscan 813 if oldstatus != _Gwaiting && oldstatus != _Grunnable { 814 throw("copystack: bad status, not Gwaiting or Grunnable") 815 } 816 if atomic.Cas(&gp.atomicstatus, oldstatus, _Gcopystack) { 817 return oldstatus 818 } 819 } 820 } 821 822 // scang blocks until gp's stack has been scanned. 823 // It might be scanned by scang or it might be scanned by the goroutine itself. 824 // Either way, the stack scan has completed when scang returns. 825 func scang(gp *g, gcw *gcWork) { 826 // Invariant; we (the caller, markroot for a specific goroutine) own gp.gcscandone. 827 // Nothing is racing with us now, but gcscandone might be set to true left over 828 // from an earlier round of stack scanning (we scan twice per GC). 829 // We use gcscandone to record whether the scan has been done during this round. 830 // It is important that the scan happens exactly once: if called twice, 831 // the installation of stack barriers will detect the double scan and die. 832 833 gp.gcscandone = false 834 835 // See http://golang.org/cl/21503 for justification of the yield delay. 836 const yieldDelay = 10 * 1000 837 var nextYield int64 838 839 // Endeavor to get gcscandone set to true, 840 // either by doing the stack scan ourselves or by coercing gp to scan itself. 841 // gp.gcscandone can transition from false to true when we're not looking 842 // (if we asked for preemption), so any time we lock the status using 843 // castogscanstatus we have to double-check that the scan is still not done. 844 loop: 845 for i := 0; !gp.gcscandone; i++ { 846 switch s := readgstatus(gp); s { 847 default: 848 dumpgstatus(gp) 849 throw("stopg: invalid status") 850 851 case _Gdead: 852 // No stack. 853 gp.gcscandone = true 854 break loop 855 856 case _Gcopystack: 857 // Stack being switched. Go around again. 858 859 case _Grunnable, _Gsyscall, _Gwaiting: 860 // Claim goroutine by setting scan bit. 861 // Racing with execution or readying of gp. 862 // The scan bit keeps them from running 863 // the goroutine until we're done. 864 if castogscanstatus(gp, s, s|_Gscan) { 865 if !gp.gcscandone { 866 scanstack(gp, gcw) 867 gp.gcscandone = true 868 } 869 restartg(gp) 870 break loop 871 } 872 873 case _Gscanwaiting: 874 // newstack is doing a scan for us right now. Wait. 875 876 case _Grunning: 877 // Goroutine running. Try to preempt execution so it can scan itself. 878 // The preemption handler (in newstack) does the actual scan. 879 880 // Optimization: if there is already a pending preemption request 881 // (from the previous loop iteration), don't bother with the atomics. 882 if gp.preemptscan && gp.preempt && gp.stackguard0 == stackPreempt { 883 break 884 } 885 886 // Ask for preemption and self scan. 887 if castogscanstatus(gp, _Grunning, _Gscanrunning) { 888 if !gp.gcscandone { 889 gp.preemptscan = true 890 gp.preempt = true 891 gp.stackguard0 = stackPreempt 892 } 893 casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning) 894 } 895 } 896 897 if i == 0 { 898 nextYield = nanotime() + yieldDelay 899 } 900 if nanotime() < nextYield { 901 procyield(10) 902 } else { 903 osyield() 904 nextYield = nanotime() + yieldDelay/2 905 } 906 } 907 908 gp.preemptscan = false // cancel scan request if no longer needed 909 } 910 911 // The GC requests that this routine be moved from a scanmumble state to a mumble state. 912 func restartg(gp *g) { 913 s := readgstatus(gp) 914 switch s { 915 default: 916 dumpgstatus(gp) 917 throw("restartg: unexpected status") 918 919 case _Gdead: 920 // ok 921 922 case _Gscanrunnable, 923 _Gscanwaiting, 924 _Gscansyscall: 925 casfrom_Gscanstatus(gp, s, s&^_Gscan) 926 } 927 } 928 929 // stopTheWorld stops all P's from executing goroutines, interrupting 930 // all goroutines at GC safe points and records reason as the reason 931 // for the stop. On return, only the current goroutine's P is running. 932 // stopTheWorld must not be called from a system stack and the caller 933 // must not hold worldsema. The caller must call startTheWorld when 934 // other P's should resume execution. 935 // 936 // stopTheWorld is safe for multiple goroutines to call at the 937 // same time. Each will execute its own stop, and the stops will 938 // be serialized. 939 // 940 // This is also used by routines that do stack dumps. If the system is 941 // in panic or being exited, this may not reliably stop all 942 // goroutines. 943 func stopTheWorld(reason string) { 944 semacquire(&worldsema, 0) 945 getg().m.preemptoff = reason 946 systemstack(stopTheWorldWithSema) 947 } 948 949 // startTheWorld undoes the effects of stopTheWorld. 950 func startTheWorld() { 951 systemstack(startTheWorldWithSema) 952 // worldsema must be held over startTheWorldWithSema to ensure 953 // gomaxprocs cannot change while worldsema is held. 954 semrelease(&worldsema) 955 getg().m.preemptoff = "" 956 } 957 958 // Holding worldsema grants an M the right to try to stop the world 959 // and prevents gomaxprocs from changing concurrently. 960 var worldsema uint32 = 1 961 962 // stopTheWorldWithSema is the core implementation of stopTheWorld. 963 // The caller is responsible for acquiring worldsema and disabling 964 // preemption first and then should stopTheWorldWithSema on the system 965 // stack: 966 // 967 // semacquire(&worldsema, 0) 968 // m.preemptoff = "reason" 969 // systemstack(stopTheWorldWithSema) 970 // 971 // When finished, the caller must either call startTheWorld or undo 972 // these three operations separately: 973 // 974 // m.preemptoff = "" 975 // systemstack(startTheWorldWithSema) 976 // semrelease(&worldsema) 977 // 978 // It is allowed to acquire worldsema once and then execute multiple 979 // startTheWorldWithSema/stopTheWorldWithSema pairs. 980 // Other P's are able to execute between successive calls to 981 // startTheWorldWithSema and stopTheWorldWithSema. 982 // Holding worldsema causes any other goroutines invoking 983 // stopTheWorld to block. 984 func stopTheWorldWithSema() { 985 _g_ := getg() 986 987 // If we hold a lock, then we won't be able to stop another M 988 // that is blocked trying to acquire the lock. 989 if _g_.m.locks > 0 { 990 throw("stopTheWorld: holding locks") 991 } 992 993 lock(&sched.lock) 994 sched.stopwait = gomaxprocs 995 atomic.Store(&sched.gcwaiting, 1) 996 preemptall() 997 // stop current P 998 _g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic. 999 sched.stopwait-- 1000 // try to retake all P's in Psyscall status 1001 for i := 0; i < int(gomaxprocs); i++ { 1002 p := allp[i] 1003 s := p.status 1004 if s == _Psyscall && atomic.Cas(&p.status, s, _Pgcstop) { 1005 if trace.enabled { 1006 traceGoSysBlock(p) 1007 traceProcStop(p) 1008 } 1009 p.syscalltick++ 1010 sched.stopwait-- 1011 } 1012 } 1013 // stop idle P's 1014 for { 1015 p := pidleget() 1016 if p == nil { 1017 break 1018 } 1019 p.status = _Pgcstop 1020 sched.stopwait-- 1021 } 1022 wait := sched.stopwait > 0 1023 unlock(&sched.lock) 1024 1025 // wait for remaining P's to stop voluntarily 1026 if wait { 1027 for { 1028 // wait for 100us, then try to re-preempt in case of any races 1029 if notetsleep(&sched.stopnote, 100*1000) { 1030 noteclear(&sched.stopnote) 1031 break 1032 } 1033 preemptall() 1034 } 1035 } 1036 1037 // sanity checks 1038 bad := "" 1039 if sched.stopwait != 0 { 1040 bad = "stopTheWorld: not stopped (stopwait != 0)" 1041 } else { 1042 for i := 0; i < int(gomaxprocs); i++ { 1043 p := allp[i] 1044 if p.status != _Pgcstop { 1045 bad = "stopTheWorld: not stopped (status != _Pgcstop)" 1046 } 1047 } 1048 } 1049 if atomic.Load(&freezing) != 0 { 1050 // Some other thread is panicking. This can cause the 1051 // sanity checks above to fail if the panic happens in 1052 // the signal handler on a stopped thread. Either way, 1053 // we should halt this thread. 1054 lock(&deadlock) 1055 lock(&deadlock) 1056 } 1057 if bad != "" { 1058 throw(bad) 1059 } 1060 } 1061 1062 func mhelpgc() { 1063 _g_ := getg() 1064 _g_.m.helpgc = -1 1065 } 1066 1067 func startTheWorldWithSema() { 1068 _g_ := getg() 1069 1070 _g_.m.locks++ // disable preemption because it can be holding p in a local var 1071 gp := netpoll(false) // non-blocking 1072 injectglist(gp) 1073 add := needaddgcproc() 1074 lock(&sched.lock) 1075 1076 procs := gomaxprocs 1077 if newprocs != 0 { 1078 procs = newprocs 1079 newprocs = 0 1080 } 1081 p1 := procresize(procs) 1082 sched.gcwaiting = 0 1083 if sched.sysmonwait != 0 { 1084 sched.sysmonwait = 0 1085 notewakeup(&sched.sysmonnote) 1086 } 1087 unlock(&sched.lock) 1088 1089 for p1 != nil { 1090 p := p1 1091 p1 = p1.link.ptr() 1092 if p.m != 0 { 1093 mp := p.m.ptr() 1094 p.m = 0 1095 if mp.nextp != 0 { 1096 throw("startTheWorld: inconsistent mp->nextp") 1097 } 1098 mp.nextp.set(p) 1099 notewakeup(&mp.park) 1100 } else { 1101 // Start M to run P. Do not start another M below. 1102 newm(nil, p) 1103 add = false 1104 } 1105 } 1106 1107 // Wakeup an additional proc in case we have excessive runnable goroutines 1108 // in local queues or in the global queue. If we don't, the proc will park itself. 1109 // If we have lots of excessive work, resetspinning will unpark additional procs as necessary. 1110 if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 { 1111 wakep() 1112 } 1113 1114 if add { 1115 // If GC could have used another helper proc, start one now, 1116 // in the hope that it will be available next time. 1117 // It would have been even better to start it before the collection, 1118 // but doing so requires allocating memory, so it's tricky to 1119 // coordinate. This lazy approach works out in practice: 1120 // we don't mind if the first couple gc rounds don't have quite 1121 // the maximum number of procs. 1122 newm(mhelpgc, nil) 1123 } 1124 _g_.m.locks-- 1125 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 1126 _g_.stackguard0 = stackPreempt 1127 } 1128 } 1129 1130 // Called to start an M. 1131 //go:nosplit 1132 func mstart() { 1133 _g_ := getg() 1134 1135 if _g_.stack.lo == 0 { 1136 // Initialize stack bounds from system stack. 1137 // Cgo may have left stack size in stack.hi. 1138 size := _g_.stack.hi 1139 if size == 0 { 1140 size = 8192 * sys.StackGuardMultiplier 1141 } 1142 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size))) 1143 _g_.stack.lo = _g_.stack.hi - size + 1024 1144 } 1145 // Initialize stack guards so that we can start calling 1146 // both Go and C functions with stack growth prologues. 1147 _g_.stackguard0 = _g_.stack.lo + _StackGuard 1148 _g_.stackguard1 = _g_.stackguard0 1149 mstart1() 1150 } 1151 1152 func mstart1() { 1153 _g_ := getg() 1154 1155 if _g_ != _g_.m.g0 { 1156 throw("bad runtimemstart") 1157 } 1158 1159 // Record top of stack for use by mcall. 1160 // Once we call schedule we're never coming back, 1161 // so other calls can reuse this stack space. 1162 gosave(&_g_.m.g0.sched) 1163 _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used 1164 asminit() 1165 minit() 1166 1167 // Install signal handlers; after minit so that minit can 1168 // prepare the thread to be able to handle the signals. 1169 if _g_.m == &m0 { 1170 // Create an extra M for callbacks on threads not created by Go. 1171 if iscgo && !cgoHasExtraM { 1172 cgoHasExtraM = true 1173 newextram() 1174 } 1175 initsig(false) 1176 } 1177 1178 if fn := _g_.m.mstartfn; fn != nil { 1179 fn() 1180 } 1181 1182 if _g_.m.helpgc != 0 { 1183 _g_.m.helpgc = 0 1184 stopm() 1185 } else if _g_.m != &m0 { 1186 acquirep(_g_.m.nextp.ptr()) 1187 _g_.m.nextp = 0 1188 } 1189 schedule() 1190 } 1191 1192 // forEachP calls fn(p) for every P p when p reaches a GC safe point. 1193 // If a P is currently executing code, this will bring the P to a GC 1194 // safe point and execute fn on that P. If the P is not executing code 1195 // (it is idle or in a syscall), this will call fn(p) directly while 1196 // preventing the P from exiting its state. This does not ensure that 1197 // fn will run on every CPU executing Go code, but it acts as a global 1198 // memory barrier. GC uses this as a "ragged barrier." 1199 // 1200 // The caller must hold worldsema. 1201 // 1202 //go:systemstack 1203 func forEachP(fn func(*p)) { 1204 mp := acquirem() 1205 _p_ := getg().m.p.ptr() 1206 1207 lock(&sched.lock) 1208 if sched.safePointWait != 0 { 1209 throw("forEachP: sched.safePointWait != 0") 1210 } 1211 sched.safePointWait = gomaxprocs - 1 1212 sched.safePointFn = fn 1213 1214 // Ask all Ps to run the safe point function. 1215 for _, p := range allp[:gomaxprocs] { 1216 if p != _p_ { 1217 atomic.Store(&p.runSafePointFn, 1) 1218 } 1219 } 1220 preemptall() 1221 1222 // Any P entering _Pidle or _Psyscall from now on will observe 1223 // p.runSafePointFn == 1 and will call runSafePointFn when 1224 // changing its status to _Pidle/_Psyscall. 1225 1226 // Run safe point function for all idle Ps. sched.pidle will 1227 // not change because we hold sched.lock. 1228 for p := sched.pidle.ptr(); p != nil; p = p.link.ptr() { 1229 if atomic.Cas(&p.runSafePointFn, 1, 0) { 1230 fn(p) 1231 sched.safePointWait-- 1232 } 1233 } 1234 1235 wait := sched.safePointWait > 0 1236 unlock(&sched.lock) 1237 1238 // Run fn for the current P. 1239 fn(_p_) 1240 1241 // Force Ps currently in _Psyscall into _Pidle and hand them 1242 // off to induce safe point function execution. 1243 for i := 0; i < int(gomaxprocs); i++ { 1244 p := allp[i] 1245 s := p.status 1246 if s == _Psyscall && p.runSafePointFn == 1 && atomic.Cas(&p.status, s, _Pidle) { 1247 if trace.enabled { 1248 traceGoSysBlock(p) 1249 traceProcStop(p) 1250 } 1251 p.syscalltick++ 1252 handoffp(p) 1253 } 1254 } 1255 1256 // Wait for remaining Ps to run fn. 1257 if wait { 1258 for { 1259 // Wait for 100us, then try to re-preempt in 1260 // case of any races. 1261 // 1262 // Requires system stack. 1263 if notetsleep(&sched.safePointNote, 100*1000) { 1264 noteclear(&sched.safePointNote) 1265 break 1266 } 1267 preemptall() 1268 } 1269 } 1270 if sched.safePointWait != 0 { 1271 throw("forEachP: not done") 1272 } 1273 for i := 0; i < int(gomaxprocs); i++ { 1274 p := allp[i] 1275 if p.runSafePointFn != 0 { 1276 throw("forEachP: P did not run fn") 1277 } 1278 } 1279 1280 lock(&sched.lock) 1281 sched.safePointFn = nil 1282 unlock(&sched.lock) 1283 releasem(mp) 1284 } 1285 1286 // runSafePointFn runs the safe point function, if any, for this P. 1287 // This should be called like 1288 // 1289 // if getg().m.p.runSafePointFn != 0 { 1290 // runSafePointFn() 1291 // } 1292 // 1293 // runSafePointFn must be checked on any transition in to _Pidle or 1294 // _Psyscall to avoid a race where forEachP sees that the P is running 1295 // just before the P goes into _Pidle/_Psyscall and neither forEachP 1296 // nor the P run the safe-point function. 1297 func runSafePointFn() { 1298 p := getg().m.p.ptr() 1299 // Resolve the race between forEachP running the safe-point 1300 // function on this P's behalf and this P running the 1301 // safe-point function directly. 1302 if !atomic.Cas(&p.runSafePointFn, 1, 0) { 1303 return 1304 } 1305 sched.safePointFn(p) 1306 lock(&sched.lock) 1307 sched.safePointWait-- 1308 if sched.safePointWait == 0 { 1309 notewakeup(&sched.safePointNote) 1310 } 1311 unlock(&sched.lock) 1312 } 1313 1314 // When running with cgo, we call _cgo_thread_start 1315 // to start threads for us so that we can play nicely with 1316 // foreign code. 1317 var cgoThreadStart unsafe.Pointer 1318 1319 type cgothreadstart struct { 1320 g guintptr 1321 tls *uint64 1322 fn unsafe.Pointer 1323 } 1324 1325 // Allocate a new m unassociated with any thread. 1326 // Can use p for allocation context if needed. 1327 // fn is recorded as the new m's m.mstartfn. 1328 // 1329 // This function is allowed to have write barriers even if the caller 1330 // isn't because it borrows _p_. 1331 // 1332 //go:yeswritebarrierrec 1333 func allocm(_p_ *p, fn func()) *m { 1334 _g_ := getg() 1335 _g_.m.locks++ // disable GC because it can be called from sysmon 1336 if _g_.m.p == 0 { 1337 acquirep(_p_) // temporarily borrow p for mallocs in this function 1338 } 1339 mp := new(m) 1340 mp.mstartfn = fn 1341 mcommoninit(mp) 1342 1343 // In case of cgo or Solaris, pthread_create will make us a stack. 1344 // Windows and Plan 9 will layout sched stack on OS stack. 1345 if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" { 1346 mp.g0 = malg(-1) 1347 } else { 1348 mp.g0 = malg(8192 * sys.StackGuardMultiplier) 1349 } 1350 mp.g0.m = mp 1351 1352 if _p_ == _g_.m.p.ptr() { 1353 releasep() 1354 } 1355 _g_.m.locks-- 1356 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 1357 _g_.stackguard0 = stackPreempt 1358 } 1359 1360 return mp 1361 } 1362 1363 // needm is called when a cgo callback happens on a 1364 // thread without an m (a thread not created by Go). 1365 // In this case, needm is expected to find an m to use 1366 // and return with m, g initialized correctly. 1367 // Since m and g are not set now (likely nil, but see below) 1368 // needm is limited in what routines it can call. In particular 1369 // it can only call nosplit functions (textflag 7) and cannot 1370 // do any scheduling that requires an m. 1371 // 1372 // In order to avoid needing heavy lifting here, we adopt 1373 // the following strategy: there is a stack of available m's 1374 // that can be stolen. Using compare-and-swap 1375 // to pop from the stack has ABA races, so we simulate 1376 // a lock by doing an exchange (via casp) to steal the stack 1377 // head and replace the top pointer with MLOCKED (1). 1378 // This serves as a simple spin lock that we can use even 1379 // without an m. The thread that locks the stack in this way 1380 // unlocks the stack by storing a valid stack head pointer. 1381 // 1382 // In order to make sure that there is always an m structure 1383 // available to be stolen, we maintain the invariant that there 1384 // is always one more than needed. At the beginning of the 1385 // program (if cgo is in use) the list is seeded with a single m. 1386 // If needm finds that it has taken the last m off the list, its job 1387 // is - once it has installed its own m so that it can do things like 1388 // allocate memory - to create a spare m and put it on the list. 1389 // 1390 // Each of these extra m's also has a g0 and a curg that are 1391 // pressed into service as the scheduling stack and current 1392 // goroutine for the duration of the cgo callback. 1393 // 1394 // When the callback is done with the m, it calls dropm to 1395 // put the m back on the list. 1396 //go:nosplit 1397 func needm(x byte) { 1398 if iscgo && !cgoHasExtraM { 1399 // Can happen if C/C++ code calls Go from a global ctor. 1400 // Can not throw, because scheduler is not initialized yet. 1401 write(2, unsafe.Pointer(&earlycgocallback[0]), int32(len(earlycgocallback))) 1402 exit(1) 1403 } 1404 1405 // Lock extra list, take head, unlock popped list. 1406 // nilokay=false is safe here because of the invariant above, 1407 // that the extra list always contains or will soon contain 1408 // at least one m. 1409 mp := lockextra(false) 1410 1411 // Set needextram when we've just emptied the list, 1412 // so that the eventual call into cgocallbackg will 1413 // allocate a new m for the extra list. We delay the 1414 // allocation until then so that it can be done 1415 // after exitsyscall makes sure it is okay to be 1416 // running at all (that is, there's no garbage collection 1417 // running right now). 1418 mp.needextram = mp.schedlink == 0 1419 unlockextra(mp.schedlink.ptr()) 1420 1421 // Save and block signals before installing g. 1422 // Once g is installed, any incoming signals will try to execute, 1423 // but we won't have the sigaltstack settings and other data 1424 // set up appropriately until the end of minit, which will 1425 // unblock the signals. This is the same dance as when 1426 // starting a new m to run Go code via newosproc. 1427 msigsave(mp) 1428 sigblock() 1429 1430 // Install g (= m->g0) and set the stack bounds 1431 // to match the current stack. We don't actually know 1432 // how big the stack is, like we don't know how big any 1433 // scheduling stack is, but we assume there's at least 32 kB, 1434 // which is more than enough for us. 1435 setg(mp.g0) 1436 _g_ := getg() 1437 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024 1438 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024 1439 _g_.stackguard0 = _g_.stack.lo + _StackGuard 1440 1441 // Initialize this thread to use the m. 1442 asminit() 1443 minit() 1444 } 1445 1446 var earlycgocallback = []byte("fatal error: cgo callback before cgo call\n") 1447 1448 // newextram allocates m's and puts them on the extra list. 1449 // It is called with a working local m, so that it can do things 1450 // like call schedlock and allocate. 1451 func newextram() { 1452 c := atomic.Xchg(&extraMWaiters, 0) 1453 if c > 0 { 1454 for i := uint32(0); i < c; i++ { 1455 oneNewExtraM() 1456 } 1457 } else { 1458 // Make sure there is at least one extra M. 1459 mp := lockextra(true) 1460 unlockextra(mp) 1461 if mp == nil { 1462 oneNewExtraM() 1463 } 1464 } 1465 } 1466 1467 // oneNewExtraM allocates an m and puts it on the extra list. 1468 func oneNewExtraM() { 1469 // Create extra goroutine locked to extra m. 1470 // The goroutine is the context in which the cgo callback will run. 1471 // The sched.pc will never be returned to, but setting it to 1472 // goexit makes clear to the traceback routines where 1473 // the goroutine stack ends. 1474 mp := allocm(nil, nil) 1475 gp := malg(4096) 1476 gp.sched.pc = funcPC(goexit) + sys.PCQuantum 1477 gp.sched.sp = gp.stack.hi 1478 gp.sched.sp -= 4 * sys.RegSize // extra space in case of reads slightly beyond frame 1479 gp.sched.lr = 0 1480 gp.sched.g = guintptr(unsafe.Pointer(gp)) 1481 gp.syscallpc = gp.sched.pc 1482 gp.syscallsp = gp.sched.sp 1483 gp.stktopsp = gp.sched.sp 1484 gp.gcscanvalid = true // fresh G, so no dequeueRescan necessary 1485 gp.gcscandone = true 1486 gp.gcRescan = -1 1487 // malg returns status as Gidle, change to Gsyscall before adding to allg 1488 // where GC will see it. 1489 casgstatus(gp, _Gidle, _Gsyscall) 1490 gp.m = mp 1491 mp.curg = gp 1492 mp.locked = _LockInternal 1493 mp.lockedg = gp 1494 gp.lockedm = mp 1495 gp.goid = int64(atomic.Xadd64(&sched.goidgen, 1)) 1496 if raceenabled { 1497 gp.racectx = racegostart(funcPC(newextram) + sys.PCQuantum) 1498 } 1499 // put on allg for garbage collector 1500 allgadd(gp) 1501 1502 // Add m to the extra list. 1503 mnext := lockextra(true) 1504 mp.schedlink.set(mnext) 1505 unlockextra(mp) 1506 } 1507 1508 // dropm is called when a cgo callback has called needm but is now 1509 // done with the callback and returning back into the non-Go thread. 1510 // It puts the current m back onto the extra list. 1511 // 1512 // The main expense here is the call to signalstack to release the 1513 // m's signal stack, and then the call to needm on the next callback 1514 // from this thread. It is tempting to try to save the m for next time, 1515 // which would eliminate both these costs, but there might not be 1516 // a next time: the current thread (which Go does not control) might exit. 1517 // If we saved the m for that thread, there would be an m leak each time 1518 // such a thread exited. Instead, we acquire and release an m on each 1519 // call. These should typically not be scheduling operations, just a few 1520 // atomics, so the cost should be small. 1521 // 1522 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread 1523 // variable using pthread_key_create. Unlike the pthread keys we already use 1524 // on OS X, this dummy key would never be read by Go code. It would exist 1525 // only so that we could register at thread-exit-time destructor. 1526 // That destructor would put the m back onto the extra list. 1527 // This is purely a performance optimization. The current version, 1528 // in which dropm happens on each cgo call, is still correct too. 1529 // We may have to keep the current version on systems with cgo 1530 // but without pthreads, like Windows. 1531 func dropm() { 1532 // Clear m and g, and return m to the extra list. 1533 // After the call to setg we can only call nosplit functions 1534 // with no pointer manipulation. 1535 mp := getg().m 1536 1537 // Block signals before unminit. 1538 // Unminit unregisters the signal handling stack (but needs g on some systems). 1539 // Setg(nil) clears g, which is the signal handler's cue not to run Go handlers. 1540 // It's important not to try to handle a signal between those two steps. 1541 sigmask := mp.sigmask 1542 sigblock() 1543 unminit() 1544 1545 mnext := lockextra(true) 1546 mp.schedlink.set(mnext) 1547 1548 setg(nil) 1549 1550 // Commit the release of mp. 1551 unlockextra(mp) 1552 1553 msigrestore(sigmask) 1554 } 1555 1556 // A helper function for EnsureDropM. 1557 func getm() uintptr { 1558 return uintptr(unsafe.Pointer(getg().m)) 1559 } 1560 1561 var extram uintptr 1562 var extraMWaiters uint32 1563 1564 // lockextra locks the extra list and returns the list head. 1565 // The caller must unlock the list by storing a new list head 1566 // to extram. If nilokay is true, then lockextra will 1567 // return a nil list head if that's what it finds. If nilokay is false, 1568 // lockextra will keep waiting until the list head is no longer nil. 1569 //go:nosplit 1570 func lockextra(nilokay bool) *m { 1571 const locked = 1 1572 1573 incr := false 1574 for { 1575 old := atomic.Loaduintptr(&extram) 1576 if old == locked { 1577 yield := osyield 1578 yield() 1579 continue 1580 } 1581 if old == 0 && !nilokay { 1582 if !incr { 1583 // Add 1 to the number of threads 1584 // waiting for an M. 1585 // This is cleared by newextram. 1586 atomic.Xadd(&extraMWaiters, 1) 1587 incr = true 1588 } 1589 usleep(1) 1590 continue 1591 } 1592 if atomic.Casuintptr(&extram, old, locked) { 1593 return (*m)(unsafe.Pointer(old)) 1594 } 1595 yield := osyield 1596 yield() 1597 continue 1598 } 1599 } 1600 1601 //go:nosplit 1602 func unlockextra(mp *m) { 1603 atomic.Storeuintptr(&extram, uintptr(unsafe.Pointer(mp))) 1604 } 1605 1606 // Create a new m. It will start off with a call to fn, or else the scheduler. 1607 // fn needs to be static and not a heap allocated closure. 1608 // May run with m.p==nil, so write barriers are not allowed. 1609 //go:nowritebarrierrec 1610 func newm(fn func(), _p_ *p) { 1611 mp := allocm(_p_, fn) 1612 mp.nextp.set(_p_) 1613 mp.sigmask = initSigmask 1614 if iscgo { 1615 var ts cgothreadstart 1616 if _cgo_thread_start == nil { 1617 throw("_cgo_thread_start missing") 1618 } 1619 ts.g.set(mp.g0) 1620 ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) 1621 ts.fn = unsafe.Pointer(funcPC(mstart)) 1622 if msanenabled { 1623 msanwrite(unsafe.Pointer(&ts), unsafe.Sizeof(ts)) 1624 } 1625 asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) 1626 return 1627 } 1628 newosproc(mp, unsafe.Pointer(mp.g0.stack.hi)) 1629 } 1630 1631 // Stops execution of the current m until new work is available. 1632 // Returns with acquired P. 1633 func stopm() { 1634 _g_ := getg() 1635 1636 if _g_.m.locks != 0 { 1637 throw("stopm holding locks") 1638 } 1639 if _g_.m.p != 0 { 1640 throw("stopm holding p") 1641 } 1642 if _g_.m.spinning { 1643 throw("stopm spinning") 1644 } 1645 1646 retry: 1647 lock(&sched.lock) 1648 mput(_g_.m) 1649 unlock(&sched.lock) 1650 notesleep(&_g_.m.park) 1651 noteclear(&_g_.m.park) 1652 if _g_.m.helpgc != 0 { 1653 gchelper() 1654 _g_.m.helpgc = 0 1655 _g_.m.mcache = nil 1656 _g_.m.p = 0 1657 goto retry 1658 } 1659 acquirep(_g_.m.nextp.ptr()) 1660 _g_.m.nextp = 0 1661 } 1662 1663 func mspinning() { 1664 // startm's caller incremented nmspinning. Set the new M's spinning. 1665 getg().m.spinning = true 1666 } 1667 1668 // Schedules some M to run the p (creates an M if necessary). 1669 // If p==nil, tries to get an idle P, if no idle P's does nothing. 1670 // May run with m.p==nil, so write barriers are not allowed. 1671 // If spinning is set, the caller has incremented nmspinning and startm will 1672 // either decrement nmspinning or set m.spinning in the newly started M. 1673 //go:nowritebarrierrec 1674 func startm(_p_ *p, spinning bool) { 1675 lock(&sched.lock) 1676 if _p_ == nil { 1677 _p_ = pidleget() 1678 if _p_ == nil { 1679 unlock(&sched.lock) 1680 if spinning { 1681 // The caller incremented nmspinning, but there are no idle Ps, 1682 // so it's okay to just undo the increment and give up. 1683 if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { 1684 throw("startm: negative nmspinning") 1685 } 1686 } 1687 return 1688 } 1689 } 1690 mp := mget() 1691 unlock(&sched.lock) 1692 if mp == nil { 1693 var fn func() 1694 if spinning { 1695 // The caller incremented nmspinning, so set m.spinning in the new M. 1696 fn = mspinning 1697 } 1698 newm(fn, _p_) 1699 return 1700 } 1701 if mp.spinning { 1702 throw("startm: m is spinning") 1703 } 1704 if mp.nextp != 0 { 1705 throw("startm: m has p") 1706 } 1707 if spinning && !runqempty(_p_) { 1708 throw("startm: p has runnable gs") 1709 } 1710 // The caller incremented nmspinning, so set m.spinning in the new M. 1711 mp.spinning = spinning 1712 mp.nextp.set(_p_) 1713 notewakeup(&mp.park) 1714 } 1715 1716 // Hands off P from syscall or locked M. 1717 // Always runs without a P, so write barriers are not allowed. 1718 //go:nowritebarrierrec 1719 func handoffp(_p_ *p) { 1720 // handoffp must start an M in any situation where 1721 // findrunnable would return a G to run on _p_. 1722 1723 // if it has local work, start it straight away 1724 if !runqempty(_p_) || sched.runqsize != 0 { 1725 startm(_p_, false) 1726 return 1727 } 1728 // if it has GC work, start it straight away 1729 if gcBlackenEnabled != 0 && gcMarkWorkAvailable(_p_) { 1730 startm(_p_, false) 1731 return 1732 } 1733 // no local work, check that there are no spinning/idle M's, 1734 // otherwise our help is not required 1735 if atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) == 0 && atomic.Cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic 1736 startm(_p_, true) 1737 return 1738 } 1739 lock(&sched.lock) 1740 if sched.gcwaiting != 0 { 1741 _p_.status = _Pgcstop 1742 sched.stopwait-- 1743 if sched.stopwait == 0 { 1744 notewakeup(&sched.stopnote) 1745 } 1746 unlock(&sched.lock) 1747 return 1748 } 1749 if _p_.runSafePointFn != 0 && atomic.Cas(&_p_.runSafePointFn, 1, 0) { 1750 sched.safePointFn(_p_) 1751 sched.safePointWait-- 1752 if sched.safePointWait == 0 { 1753 notewakeup(&sched.safePointNote) 1754 } 1755 } 1756 if sched.runqsize != 0 { 1757 unlock(&sched.lock) 1758 startm(_p_, false) 1759 return 1760 } 1761 // If this is the last running P and nobody is polling network, 1762 // need to wakeup another M to poll network. 1763 if sched.npidle == uint32(gomaxprocs-1) && atomic.Load64(&sched.lastpoll) != 0 { 1764 unlock(&sched.lock) 1765 startm(_p_, false) 1766 return 1767 } 1768 pidleput(_p_) 1769 unlock(&sched.lock) 1770 } 1771 1772 // Tries to add one more P to execute G's. 1773 // Called when a G is made runnable (newproc, ready). 1774 func wakep() { 1775 // be conservative about spinning threads 1776 if !atomic.Cas(&sched.nmspinning, 0, 1) { 1777 return 1778 } 1779 startm(nil, true) 1780 } 1781 1782 // Stops execution of the current m that is locked to a g until the g is runnable again. 1783 // Returns with acquired P. 1784 func stoplockedm() { 1785 _g_ := getg() 1786 1787 if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m { 1788 throw("stoplockedm: inconsistent locking") 1789 } 1790 if _g_.m.p != 0 { 1791 // Schedule another M to run this p. 1792 _p_ := releasep() 1793 handoffp(_p_) 1794 } 1795 incidlelocked(1) 1796 // Wait until another thread schedules lockedg again. 1797 notesleep(&_g_.m.park) 1798 noteclear(&_g_.m.park) 1799 status := readgstatus(_g_.m.lockedg) 1800 if status&^_Gscan != _Grunnable { 1801 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n") 1802 dumpgstatus(_g_) 1803 throw("stoplockedm: not runnable") 1804 } 1805 acquirep(_g_.m.nextp.ptr()) 1806 _g_.m.nextp = 0 1807 } 1808 1809 // Schedules the locked m to run the locked gp. 1810 // May run during STW, so write barriers are not allowed. 1811 //go:nowritebarrierrec 1812 func startlockedm(gp *g) { 1813 _g_ := getg() 1814 1815 mp := gp.lockedm 1816 if mp == _g_.m { 1817 throw("startlockedm: locked to me") 1818 } 1819 if mp.nextp != 0 { 1820 throw("startlockedm: m has p") 1821 } 1822 // directly handoff current P to the locked m 1823 incidlelocked(-1) 1824 _p_ := releasep() 1825 mp.nextp.set(_p_) 1826 notewakeup(&mp.park) 1827 stopm() 1828 } 1829 1830 // Stops the current m for stopTheWorld. 1831 // Returns when the world is restarted. 1832 func gcstopm() { 1833 _g_ := getg() 1834 1835 if sched.gcwaiting == 0 { 1836 throw("gcstopm: not waiting for gc") 1837 } 1838 if _g_.m.spinning { 1839 _g_.m.spinning = false 1840 // OK to just drop nmspinning here, 1841 // startTheWorld will unpark threads as necessary. 1842 if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { 1843 throw("gcstopm: negative nmspinning") 1844 } 1845 } 1846 _p_ := releasep() 1847 lock(&sched.lock) 1848 _p_.status = _Pgcstop 1849 sched.stopwait-- 1850 if sched.stopwait == 0 { 1851 notewakeup(&sched.stopnote) 1852 } 1853 unlock(&sched.lock) 1854 stopm() 1855 } 1856 1857 // Schedules gp to run on the current M. 1858 // If inheritTime is true, gp inherits the remaining time in the 1859 // current time slice. Otherwise, it starts a new time slice. 1860 // Never returns. 1861 // 1862 // Write barriers are allowed because this is called immediately after 1863 // acquiring a P in several places. 1864 // 1865 //go:yeswritebarrierrec 1866 func execute(gp *g, inheritTime bool) { 1867 _g_ := getg() 1868 1869 casgstatus(gp, _Grunnable, _Grunning) 1870 gp.waitsince = 0 1871 gp.preempt = false 1872 gp.stackguard0 = gp.stack.lo + _StackGuard 1873 if !inheritTime { 1874 _g_.m.p.ptr().schedtick++ 1875 } 1876 _g_.m.curg = gp 1877 gp.m = _g_.m 1878 1879 // Check whether the profiler needs to be turned on or off. 1880 hz := sched.profilehz 1881 if _g_.m.profilehz != hz { 1882 resetcpuprofiler(hz) 1883 } 1884 1885 if trace.enabled { 1886 // GoSysExit has to happen when we have a P, but before GoStart. 1887 // So we emit it here. 1888 if gp.syscallsp != 0 && gp.sysblocktraced { 1889 traceGoSysExit(gp.sysexitticks) 1890 } 1891 traceGoStart() 1892 } 1893 1894 gogo(&gp.sched) 1895 } 1896 1897 // Finds a runnable goroutine to execute. 1898 // Tries to steal from other P's, get g from global queue, poll network. 1899 func findrunnable() (gp *g, inheritTime bool) { 1900 _g_ := getg() 1901 1902 // The conditions here and in handoffp must agree: if 1903 // findrunnable would return a G to run, handoffp must start 1904 // an M. 1905 1906 top: 1907 _p_ := _g_.m.p.ptr() 1908 if sched.gcwaiting != 0 { 1909 gcstopm() 1910 goto top 1911 } 1912 if _p_.runSafePointFn != 0 { 1913 runSafePointFn() 1914 } 1915 if fingwait && fingwake { 1916 if gp := wakefing(); gp != nil { 1917 ready(gp, 0, true) 1918 } 1919 } 1920 1921 // local runq 1922 if gp, inheritTime := runqget(_p_); gp != nil { 1923 return gp, inheritTime 1924 } 1925 1926 // global runq 1927 if sched.runqsize != 0 { 1928 lock(&sched.lock) 1929 gp := globrunqget(_p_, 0) 1930 unlock(&sched.lock) 1931 if gp != nil { 1932 return gp, false 1933 } 1934 } 1935 1936 // Poll network. 1937 // This netpoll is only an optimization before we resort to stealing. 1938 // We can safely skip it if there a thread blocked in netpoll already. 1939 // If there is any kind of logical race with that blocked thread 1940 // (e.g. it has already returned from netpoll, but does not set lastpoll yet), 1941 // this thread will do blocking netpoll below anyway. 1942 if netpollinited() && sched.lastpoll != 0 { 1943 if gp := netpoll(false); gp != nil { // non-blocking 1944 // netpoll returns list of goroutines linked by schedlink. 1945 injectglist(gp.schedlink.ptr()) 1946 casgstatus(gp, _Gwaiting, _Grunnable) 1947 if trace.enabled { 1948 traceGoUnpark(gp, 0) 1949 } 1950 return gp, false 1951 } 1952 } 1953 1954 // Steal work from other P's. 1955 procs := uint32(gomaxprocs) 1956 if atomic.Load(&sched.npidle) == procs-1 { 1957 // Either GOMAXPROCS=1 or everybody, except for us, is idle already. 1958 // New work can appear from returning syscall/cgocall, network or timers. 1959 // Neither of that submits to local run queues, so no point in stealing. 1960 goto stop 1961 } 1962 // If number of spinning M's >= number of busy P's, block. 1963 // This is necessary to prevent excessive CPU consumption 1964 // when GOMAXPROCS>>1 but the program parallelism is low. 1965 if !_g_.m.spinning && 2*atomic.Load(&sched.nmspinning) >= procs-atomic.Load(&sched.npidle) { 1966 goto stop 1967 } 1968 if !_g_.m.spinning { 1969 _g_.m.spinning = true 1970 atomic.Xadd(&sched.nmspinning, 1) 1971 } 1972 for i := 0; i < 4; i++ { 1973 for enum := stealOrder.start(fastrand()); !enum.done(); enum.next() { 1974 if sched.gcwaiting != 0 { 1975 goto top 1976 } 1977 stealRunNextG := i > 2 // first look for ready queues with more than 1 g 1978 if gp := runqsteal(_p_, allp[enum.position()], stealRunNextG); gp != nil { 1979 return gp, false 1980 } 1981 } 1982 } 1983 1984 stop: 1985 1986 // We have nothing to do. If we're in the GC mark phase, can 1987 // safely scan and blacken objects, and have work to do, run 1988 // idle-time marking rather than give up the P. 1989 if gcBlackenEnabled != 0 && _p_.gcBgMarkWorker != 0 && gcMarkWorkAvailable(_p_) { 1990 _p_.gcMarkWorkerMode = gcMarkWorkerIdleMode 1991 gp := _p_.gcBgMarkWorker.ptr() 1992 casgstatus(gp, _Gwaiting, _Grunnable) 1993 if trace.enabled { 1994 traceGoUnpark(gp, 0) 1995 } 1996 return gp, false 1997 } 1998 1999 // return P and block 2000 lock(&sched.lock) 2001 if sched.gcwaiting != 0 || _p_.runSafePointFn != 0 { 2002 unlock(&sched.lock) 2003 goto top 2004 } 2005 if sched.runqsize != 0 { 2006 gp := globrunqget(_p_, 0) 2007 unlock(&sched.lock) 2008 return gp, false 2009 } 2010 if releasep() != _p_ { 2011 throw("findrunnable: wrong p") 2012 } 2013 pidleput(_p_) 2014 unlock(&sched.lock) 2015 2016 // Delicate dance: thread transitions from spinning to non-spinning state, 2017 // potentially concurrently with submission of new goroutines. We must 2018 // drop nmspinning first and then check all per-P queues again (with 2019 // #StoreLoad memory barrier in between). If we do it the other way around, 2020 // another thread can submit a goroutine after we've checked all run queues 2021 // but before we drop nmspinning; as the result nobody will unpark a thread 2022 // to run the goroutine. 2023 // If we discover new work below, we need to restore m.spinning as a signal 2024 // for resetspinning to unpark a new worker thread (because there can be more 2025 // than one starving goroutine). However, if after discovering new work 2026 // we also observe no idle Ps, it is OK to just park the current thread: 2027 // the system is fully loaded so no spinning threads are required. 2028 // Also see "Worker thread parking/unparking" comment at the top of the file. 2029 wasSpinning := _g_.m.spinning 2030 if _g_.m.spinning { 2031 _g_.m.spinning = false 2032 if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { 2033 throw("findrunnable: negative nmspinning") 2034 } 2035 } 2036 2037 // check all runqueues once again 2038 for i := 0; i < int(gomaxprocs); i++ { 2039 _p_ := allp[i] 2040 if _p_ != nil && !runqempty(_p_) { 2041 lock(&sched.lock) 2042 _p_ = pidleget() 2043 unlock(&sched.lock) 2044 if _p_ != nil { 2045 acquirep(_p_) 2046 if wasSpinning { 2047 _g_.m.spinning = true 2048 atomic.Xadd(&sched.nmspinning, 1) 2049 } 2050 goto top 2051 } 2052 break 2053 } 2054 } 2055 2056 // Check for idle-priority GC work again. 2057 if gcBlackenEnabled != 0 && gcMarkWorkAvailable(nil) { 2058 lock(&sched.lock) 2059 _p_ = pidleget() 2060 if _p_ != nil && _p_.gcBgMarkWorker == 0 { 2061 pidleput(_p_) 2062 _p_ = nil 2063 } 2064 unlock(&sched.lock) 2065 if _p_ != nil { 2066 acquirep(_p_) 2067 if wasSpinning { 2068 _g_.m.spinning = true 2069 atomic.Xadd(&sched.nmspinning, 1) 2070 } 2071 // Go back to idle GC check. 2072 goto stop 2073 } 2074 } 2075 2076 // poll network 2077 if netpollinited() && atomic.Xchg64(&sched.lastpoll, 0) != 0 { 2078 if _g_.m.p != 0 { 2079 throw("findrunnable: netpoll with p") 2080 } 2081 if _g_.m.spinning { 2082 throw("findrunnable: netpoll with spinning") 2083 } 2084 gp := netpoll(true) // block until new work is available 2085 atomic.Store64(&sched.lastpoll, uint64(nanotime())) 2086 if gp != nil { 2087 lock(&sched.lock) 2088 _p_ = pidleget() 2089 unlock(&sched.lock) 2090 if _p_ != nil { 2091 acquirep(_p_) 2092 injectglist(gp.schedlink.ptr()) 2093 casgstatus(gp, _Gwaiting, _Grunnable) 2094 if trace.enabled { 2095 traceGoUnpark(gp, 0) 2096 } 2097 return gp, false 2098 } 2099 injectglist(gp) 2100 } 2101 } 2102 stopm() 2103 goto top 2104 } 2105 2106 // pollWork returns true if there is non-background work this P could 2107 // be doing. This is a fairly lightweight check to be used for 2108 // background work loops, like idle GC. It checks a subset of the 2109 // conditions checked by the actual scheduler. 2110 func pollWork() bool { 2111 if sched.runqsize != 0 { 2112 return true 2113 } 2114 p := getg().m.p.ptr() 2115 if !runqempty(p) { 2116 return true 2117 } 2118 if netpollinited() && sched.lastpoll != 0 { 2119 if gp := netpoll(false); gp != nil { 2120 injectglist(gp) 2121 return true 2122 } 2123 } 2124 return false 2125 } 2126 2127 func resetspinning() { 2128 _g_ := getg() 2129 if !_g_.m.spinning { 2130 throw("resetspinning: not a spinning m") 2131 } 2132 _g_.m.spinning = false 2133 nmspinning := atomic.Xadd(&sched.nmspinning, -1) 2134 if int32(nmspinning) < 0 { 2135 throw("findrunnable: negative nmspinning") 2136 } 2137 // M wakeup policy is deliberately somewhat conservative, so check if we 2138 // need to wakeup another P here. See "Worker thread parking/unparking" 2139 // comment at the top of the file for details. 2140 if nmspinning == 0 && atomic.Load(&sched.npidle) > 0 { 2141 wakep() 2142 } 2143 } 2144 2145 // Injects the list of runnable G's into the scheduler. 2146 // Can run concurrently with GC. 2147 func injectglist(glist *g) { 2148 if glist == nil { 2149 return 2150 } 2151 if trace.enabled { 2152 for gp := glist; gp != nil; gp = gp.schedlink.ptr() { 2153 traceGoUnpark(gp, 0) 2154 } 2155 } 2156 lock(&sched.lock) 2157 var n int 2158 for n = 0; glist != nil; n++ { 2159 gp := glist 2160 glist = gp.schedlink.ptr() 2161 casgstatus(gp, _Gwaiting, _Grunnable) 2162 globrunqput(gp) 2163 } 2164 unlock(&sched.lock) 2165 for ; n != 0 && sched.npidle != 0; n-- { 2166 startm(nil, false) 2167 } 2168 } 2169 2170 // One round of scheduler: find a runnable goroutine and execute it. 2171 // Never returns. 2172 func schedule() { 2173 _g_ := getg() 2174 2175 if _g_.m.locks != 0 { 2176 throw("schedule: holding locks") 2177 } 2178 2179 if _g_.m.lockedg != nil { 2180 stoplockedm() 2181 execute(_g_.m.lockedg, false) // Never returns. 2182 } 2183 2184 top: 2185 if sched.gcwaiting != 0 { 2186 gcstopm() 2187 goto top 2188 } 2189 if _g_.m.p.ptr().runSafePointFn != 0 { 2190 runSafePointFn() 2191 } 2192 2193 var gp *g 2194 var inheritTime bool 2195 if trace.enabled || trace.shutdown { 2196 gp = traceReader() 2197 if gp != nil { 2198 casgstatus(gp, _Gwaiting, _Grunnable) 2199 traceGoUnpark(gp, 0) 2200 } 2201 } 2202 if gp == nil && gcBlackenEnabled != 0 { 2203 gp = gcController.findRunnableGCWorker(_g_.m.p.ptr()) 2204 } 2205 if gp == nil { 2206 // Check the global runnable queue once in a while to ensure fairness. 2207 // Otherwise two goroutines can completely occupy the local runqueue 2208 // by constantly respawning each other. 2209 if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 { 2210 lock(&sched.lock) 2211 gp = globrunqget(_g_.m.p.ptr(), 1) 2212 unlock(&sched.lock) 2213 } 2214 } 2215 if gp == nil { 2216 gp, inheritTime = runqget(_g_.m.p.ptr()) 2217 if gp != nil && _g_.m.spinning { 2218 throw("schedule: spinning with local work") 2219 } 2220 } 2221 if gp == nil { 2222 gp, inheritTime = findrunnable() // blocks until work is available 2223 } 2224 2225 // This thread is going to run a goroutine and is not spinning anymore, 2226 // so if it was marked as spinning we need to reset it now and potentially 2227 // start a new spinning M. 2228 if _g_.m.spinning { 2229 resetspinning() 2230 } 2231 2232 if gp.lockedm != nil { 2233 // Hands off own p to the locked m, 2234 // then blocks waiting for a new p. 2235 startlockedm(gp) 2236 goto top 2237 } 2238 2239 execute(gp, inheritTime) 2240 } 2241 2242 // dropg removes the association between m and the current goroutine m->curg (gp for short). 2243 // Typically a caller sets gp's status away from Grunning and then 2244 // immediately calls dropg to finish the job. The caller is also responsible 2245 // for arranging that gp will be restarted using ready at an 2246 // appropriate time. After calling dropg and arranging for gp to be 2247 // readied later, the caller can do other work but eventually should 2248 // call schedule to restart the scheduling of goroutines on this m. 2249 func dropg() { 2250 _g_ := getg() 2251 2252 setMNoWB(&_g_.m.curg.m, nil) 2253 setGNoWB(&_g_.m.curg, nil) 2254 } 2255 2256 func parkunlock_c(gp *g, lock unsafe.Pointer) bool { 2257 unlock((*mutex)(lock)) 2258 return true 2259 } 2260 2261 // park continuation on g0. 2262 func park_m(gp *g) { 2263 _g_ := getg() 2264 2265 if trace.enabled { 2266 traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip, gp) 2267 } 2268 2269 casgstatus(gp, _Grunning, _Gwaiting) 2270 dropg() 2271 2272 if _g_.m.waitunlockf != nil { 2273 fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf)) 2274 ok := fn(gp, _g_.m.waitlock) 2275 _g_.m.waitunlockf = nil 2276 _g_.m.waitlock = nil 2277 if !ok { 2278 if trace.enabled { 2279 traceGoUnpark(gp, 2) 2280 } 2281 casgstatus(gp, _Gwaiting, _Grunnable) 2282 execute(gp, true) // Schedule it back, never returns. 2283 } 2284 } 2285 schedule() 2286 } 2287 2288 func goschedImpl(gp *g) { 2289 status := readgstatus(gp) 2290 if status&^_Gscan != _Grunning { 2291 dumpgstatus(gp) 2292 throw("bad g status") 2293 } 2294 casgstatus(gp, _Grunning, _Grunnable) 2295 dropg() 2296 lock(&sched.lock) 2297 globrunqput(gp) 2298 unlock(&sched.lock) 2299 2300 schedule() 2301 } 2302 2303 // Gosched continuation on g0. 2304 func gosched_m(gp *g) { 2305 if trace.enabled { 2306 traceGoSched() 2307 } 2308 goschedImpl(gp) 2309 } 2310 2311 func gopreempt_m(gp *g) { 2312 if trace.enabled { 2313 traceGoPreempt() 2314 } 2315 goschedImpl(gp) 2316 } 2317 2318 // Finishes execution of the current goroutine. 2319 func goexit1() { 2320 if raceenabled { 2321 racegoend() 2322 } 2323 if trace.enabled { 2324 traceGoEnd() 2325 } 2326 mcall(goexit0) 2327 } 2328 2329 // goexit continuation on g0. 2330 func goexit0(gp *g) { 2331 _g_ := getg() 2332 2333 casgstatus(gp, _Grunning, _Gdead) 2334 if isSystemGoroutine(gp) { 2335 atomic.Xadd(&sched.ngsys, -1) 2336 } 2337 gp.m = nil 2338 gp.lockedm = nil 2339 _g_.m.lockedg = nil 2340 gp.paniconfault = false 2341 gp._defer = nil // should be true already but just in case. 2342 gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data. 2343 gp.writebuf = nil 2344 gp.waitreason = "" 2345 gp.param = nil 2346 2347 // Note that gp's stack scan is now "valid" because it has no 2348 // stack. We could dequeueRescan, but that takes a lock and 2349 // isn't really necessary. 2350 gp.gcscanvalid = true 2351 dropg() 2352 2353 if _g_.m.locked&^_LockExternal != 0 { 2354 print("invalid m->locked = ", _g_.m.locked, "\n") 2355 throw("internal lockOSThread error") 2356 } 2357 _g_.m.locked = 0 2358 gfput(_g_.m.p.ptr(), gp) 2359 schedule() 2360 } 2361 2362 // save updates getg().sched to refer to pc and sp so that a following 2363 // gogo will restore pc and sp. 2364 // 2365 // save must not have write barriers because invoking a write barrier 2366 // can clobber getg().sched. 2367 // 2368 //go:nosplit 2369 //go:nowritebarrierrec 2370 func save(pc, sp uintptr) { 2371 _g_ := getg() 2372 2373 _g_.sched.pc = pc 2374 _g_.sched.sp = sp 2375 _g_.sched.lr = 0 2376 _g_.sched.ret = 0 2377 _g_.sched.g = guintptr(unsafe.Pointer(_g_)) 2378 // We need to ensure ctxt is zero, but can't have a write 2379 // barrier here. However, it should always already be zero. 2380 // Assert that. 2381 if _g_.sched.ctxt != nil { 2382 badctxt() 2383 } 2384 } 2385 2386 // The goroutine g is about to enter a system call. 2387 // Record that it's not using the cpu anymore. 2388 // This is called only from the go syscall library and cgocall, 2389 // not from the low-level system calls used by the runtime. 2390 // 2391 // Entersyscall cannot split the stack: the gosave must 2392 // make g->sched refer to the caller's stack segment, because 2393 // entersyscall is going to return immediately after. 2394 // 2395 // Nothing entersyscall calls can split the stack either. 2396 // We cannot safely move the stack during an active call to syscall, 2397 // because we do not know which of the uintptr arguments are 2398 // really pointers (back into the stack). 2399 // In practice, this means that we make the fast path run through 2400 // entersyscall doing no-split things, and the slow path has to use systemstack 2401 // to run bigger things on the system stack. 2402 // 2403 // reentersyscall is the entry point used by cgo callbacks, where explicitly 2404 // saved SP and PC are restored. This is needed when exitsyscall will be called 2405 // from a function further up in the call stack than the parent, as g->syscallsp 2406 // must always point to a valid stack frame. entersyscall below is the normal 2407 // entry point for syscalls, which obtains the SP and PC from the caller. 2408 // 2409 // Syscall tracing: 2410 // At the start of a syscall we emit traceGoSysCall to capture the stack trace. 2411 // If the syscall does not block, that is it, we do not emit any other events. 2412 // If the syscall blocks (that is, P is retaken), retaker emits traceGoSysBlock; 2413 // when syscall returns we emit traceGoSysExit and when the goroutine starts running 2414 // (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart. 2415 // To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock, 2416 // we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick), 2417 // whoever emits traceGoSysBlock increments p.syscalltick afterwards; 2418 // and we wait for the increment before emitting traceGoSysExit. 2419 // Note that the increment is done even if tracing is not enabled, 2420 // because tracing can be enabled in the middle of syscall. We don't want the wait to hang. 2421 // 2422 //go:nosplit 2423 func reentersyscall(pc, sp uintptr) { 2424 _g_ := getg() 2425 2426 // Disable preemption because during this function g is in Gsyscall status, 2427 // but can have inconsistent g->sched, do not let GC observe it. 2428 _g_.m.locks++ 2429 2430 // Entersyscall must not call any function that might split/grow the stack. 2431 // (See details in comment above.) 2432 // Catch calls that might, by replacing the stack guard with something that 2433 // will trip any stack check and leaving a flag to tell newstack to die. 2434 _g_.stackguard0 = stackPreempt 2435 _g_.throwsplit = true 2436 2437 // Leave SP around for GC and traceback. 2438 save(pc, sp) 2439 _g_.syscallsp = sp 2440 _g_.syscallpc = pc 2441 casgstatus(_g_, _Grunning, _Gsyscall) 2442 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 2443 systemstack(func() { 2444 print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 2445 throw("entersyscall") 2446 }) 2447 } 2448 2449 if trace.enabled { 2450 systemstack(traceGoSysCall) 2451 // systemstack itself clobbers g.sched.{pc,sp} and we might 2452 // need them later when the G is genuinely blocked in a 2453 // syscall 2454 save(pc, sp) 2455 } 2456 2457 if atomic.Load(&sched.sysmonwait) != 0 { 2458 systemstack(entersyscall_sysmon) 2459 save(pc, sp) 2460 } 2461 2462 if _g_.m.p.ptr().runSafePointFn != 0 { 2463 // runSafePointFn may stack split if run on this stack 2464 systemstack(runSafePointFn) 2465 save(pc, sp) 2466 } 2467 2468 _g_.m.syscalltick = _g_.m.p.ptr().syscalltick 2469 _g_.sysblocktraced = true 2470 _g_.m.mcache = nil 2471 _g_.m.p.ptr().m = 0 2472 atomic.Store(&_g_.m.p.ptr().status, _Psyscall) 2473 if sched.gcwaiting != 0 { 2474 systemstack(entersyscall_gcwait) 2475 save(pc, sp) 2476 } 2477 2478 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). 2479 // We set _StackGuard to StackPreempt so that first split stack check calls morestack. 2480 // Morestack detects this case and throws. 2481 _g_.stackguard0 = stackPreempt 2482 _g_.m.locks-- 2483 } 2484 2485 // Standard syscall entry used by the go syscall library and normal cgo calls. 2486 //go:nosplit 2487 func entersyscall(dummy int32) { 2488 reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) 2489 } 2490 2491 func entersyscall_sysmon() { 2492 lock(&sched.lock) 2493 if atomic.Load(&sched.sysmonwait) != 0 { 2494 atomic.Store(&sched.sysmonwait, 0) 2495 notewakeup(&sched.sysmonnote) 2496 } 2497 unlock(&sched.lock) 2498 } 2499 2500 func entersyscall_gcwait() { 2501 _g_ := getg() 2502 _p_ := _g_.m.p.ptr() 2503 2504 lock(&sched.lock) 2505 if sched.stopwait > 0 && atomic.Cas(&_p_.status, _Psyscall, _Pgcstop) { 2506 if trace.enabled { 2507 traceGoSysBlock(_p_) 2508 traceProcStop(_p_) 2509 } 2510 _p_.syscalltick++ 2511 if sched.stopwait--; sched.stopwait == 0 { 2512 notewakeup(&sched.stopnote) 2513 } 2514 } 2515 unlock(&sched.lock) 2516 } 2517 2518 // The same as entersyscall(), but with a hint that the syscall is blocking. 2519 //go:nosplit 2520 func entersyscallblock(dummy int32) { 2521 _g_ := getg() 2522 2523 _g_.m.locks++ // see comment in entersyscall 2524 _g_.throwsplit = true 2525 _g_.stackguard0 = stackPreempt // see comment in entersyscall 2526 _g_.m.syscalltick = _g_.m.p.ptr().syscalltick 2527 _g_.sysblocktraced = true 2528 _g_.m.p.ptr().syscalltick++ 2529 2530 // Leave SP around for GC and traceback. 2531 pc := getcallerpc(unsafe.Pointer(&dummy)) 2532 sp := getcallersp(unsafe.Pointer(&dummy)) 2533 save(pc, sp) 2534 _g_.syscallsp = _g_.sched.sp 2535 _g_.syscallpc = _g_.sched.pc 2536 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 2537 sp1 := sp 2538 sp2 := _g_.sched.sp 2539 sp3 := _g_.syscallsp 2540 systemstack(func() { 2541 print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 2542 throw("entersyscallblock") 2543 }) 2544 } 2545 casgstatus(_g_, _Grunning, _Gsyscall) 2546 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 2547 systemstack(func() { 2548 print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 2549 throw("entersyscallblock") 2550 }) 2551 } 2552 2553 systemstack(entersyscallblock_handoff) 2554 2555 // Resave for traceback during blocked call. 2556 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) 2557 2558 _g_.m.locks-- 2559 } 2560 2561 func entersyscallblock_handoff() { 2562 if trace.enabled { 2563 traceGoSysCall() 2564 traceGoSysBlock(getg().m.p.ptr()) 2565 } 2566 handoffp(releasep()) 2567 } 2568 2569 // The goroutine g exited its system call. 2570 // Arrange for it to run on a cpu again. 2571 // This is called only from the go syscall library, not 2572 // from the low-level system calls used by the runtime. 2573 // 2574 // Write barriers are not allowed because our P may have been stolen. 2575 // 2576 //go:nosplit 2577 //go:nowritebarrierrec 2578 func exitsyscall(dummy int32) { 2579 _g_ := getg() 2580 2581 _g_.m.locks++ // see comment in entersyscall 2582 if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp { 2583 // throw calls print which may try to grow the stack, 2584 // but throwsplit == true so the stack can not be grown; 2585 // use systemstack to avoid that possible problem. 2586 systemstack(func() { 2587 throw("exitsyscall: syscall frame is no longer valid") 2588 }) 2589 } 2590 2591 _g_.waitsince = 0 2592 oldp := _g_.m.p.ptr() 2593 if exitsyscallfast() { 2594 if _g_.m.mcache == nil { 2595 throw("lost mcache") 2596 } 2597 if trace.enabled { 2598 if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { 2599 systemstack(traceGoStart) 2600 } 2601 } 2602 // There's a cpu for us, so we can run. 2603 _g_.m.p.ptr().syscalltick++ 2604 // We need to cas the status and scan before resuming... 2605 casgstatus(_g_, _Gsyscall, _Grunning) 2606 2607 // Garbage collector isn't running (since we are), 2608 // so okay to clear syscallsp. 2609 _g_.syscallsp = 0 2610 _g_.m.locks-- 2611 if _g_.preempt { 2612 // restore the preemption request in case we've cleared it in newstack 2613 _g_.stackguard0 = stackPreempt 2614 } else { 2615 // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock 2616 _g_.stackguard0 = _g_.stack.lo + _StackGuard 2617 } 2618 _g_.throwsplit = false 2619 return 2620 } 2621 2622 _g_.sysexitticks = 0 2623 if trace.enabled { 2624 // Wait till traceGoSysBlock event is emitted. 2625 // This ensures consistency of the trace (the goroutine is started after it is blocked). 2626 for oldp != nil && oldp.syscalltick == _g_.m.syscalltick { 2627 osyield() 2628 } 2629 // We can't trace syscall exit right now because we don't have a P. 2630 // Tracing code can invoke write barriers that cannot run without a P. 2631 // So instead we remember the syscall exit time and emit the event 2632 // in execute when we have a P. 2633 _g_.sysexitticks = cputicks() 2634 } 2635 2636 _g_.m.locks-- 2637 2638 // Call the scheduler. 2639 mcall(exitsyscall0) 2640 2641 if _g_.m.mcache == nil { 2642 throw("lost mcache") 2643 } 2644 2645 // Scheduler returned, so we're allowed to run now. 2646 // Delete the syscallsp information that we left for 2647 // the garbage collector during the system call. 2648 // Must wait until now because until gosched returns 2649 // we don't know for sure that the garbage collector 2650 // is not running. 2651 _g_.syscallsp = 0 2652 _g_.m.p.ptr().syscalltick++ 2653 _g_.throwsplit = false 2654 } 2655 2656 //go:nosplit 2657 func exitsyscallfast() bool { 2658 _g_ := getg() 2659 2660 // Freezetheworld sets stopwait but does not retake P's. 2661 if sched.stopwait == freezeStopWait { 2662 _g_.m.mcache = nil 2663 _g_.m.p = 0 2664 return false 2665 } 2666 2667 // Try to re-acquire the last P. 2668 if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && atomic.Cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) { 2669 // There's a cpu for us, so we can run. 2670 exitsyscallfast_reacquired() 2671 return true 2672 } 2673 2674 // Try to get any other idle P. 2675 oldp := _g_.m.p.ptr() 2676 _g_.m.mcache = nil 2677 _g_.m.p = 0 2678 if sched.pidle != 0 { 2679 var ok bool 2680 systemstack(func() { 2681 ok = exitsyscallfast_pidle() 2682 if ok && trace.enabled { 2683 if oldp != nil { 2684 // Wait till traceGoSysBlock event is emitted. 2685 // This ensures consistency of the trace (the goroutine is started after it is blocked). 2686 for oldp.syscalltick == _g_.m.syscalltick { 2687 osyield() 2688 } 2689 } 2690 traceGoSysExit(0) 2691 } 2692 }) 2693 if ok { 2694 return true 2695 } 2696 } 2697 return false 2698 } 2699 2700 // exitsyscallfast_reacquired is the exitsyscall path on which this G 2701 // has successfully reacquired the P it was running on before the 2702 // syscall. 2703 // 2704 // This function is allowed to have write barriers because exitsyscall 2705 // has acquired a P at this point. 2706 // 2707 //go:yeswritebarrierrec 2708 //go:nosplit 2709 func exitsyscallfast_reacquired() { 2710 _g_ := getg() 2711 _g_.m.mcache = _g_.m.p.ptr().mcache 2712 _g_.m.p.ptr().m.set(_g_.m) 2713 if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { 2714 if trace.enabled { 2715 // The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed). 2716 // traceGoSysBlock for this syscall was already emitted, 2717 // but here we effectively retake the p from the new syscall running on the same p. 2718 systemstack(func() { 2719 // Denote blocking of the new syscall. 2720 traceGoSysBlock(_g_.m.p.ptr()) 2721 // Denote completion of the current syscall. 2722 traceGoSysExit(0) 2723 }) 2724 } 2725 _g_.m.p.ptr().syscalltick++ 2726 } 2727 } 2728 2729 func exitsyscallfast_pidle() bool { 2730 lock(&sched.lock) 2731 _p_ := pidleget() 2732 if _p_ != nil && atomic.Load(&sched.sysmonwait) != 0 { 2733 atomic.Store(&sched.sysmonwait, 0) 2734 notewakeup(&sched.sysmonnote) 2735 } 2736 unlock(&sched.lock) 2737 if _p_ != nil { 2738 acquirep(_p_) 2739 return true 2740 } 2741 return false 2742 } 2743 2744 // exitsyscall slow path on g0. 2745 // Failed to acquire P, enqueue gp as runnable. 2746 // 2747 //go:nowritebarrierrec 2748 func exitsyscall0(gp *g) { 2749 _g_ := getg() 2750 2751 casgstatus(gp, _Gsyscall, _Grunnable) 2752 dropg() 2753 lock(&sched.lock) 2754 _p_ := pidleget() 2755 if _p_ == nil { 2756 globrunqput(gp) 2757 } else if atomic.Load(&sched.sysmonwait) != 0 { 2758 atomic.Store(&sched.sysmonwait, 0) 2759 notewakeup(&sched.sysmonnote) 2760 } 2761 unlock(&sched.lock) 2762 if _p_ != nil { 2763 acquirep(_p_) 2764 execute(gp, false) // Never returns. 2765 } 2766 if _g_.m.lockedg != nil { 2767 // Wait until another thread schedules gp and so m again. 2768 stoplockedm() 2769 execute(gp, false) // Never returns. 2770 } 2771 stopm() 2772 schedule() // Never returns. 2773 } 2774 2775 func beforefork() { 2776 gp := getg().m.curg 2777 2778 // Fork can hang if preempted with signals frequently enough (see issue 5517). 2779 // Ensure that we stay on the same M where we disable profiling. 2780 gp.m.locks++ 2781 if gp.m.profilehz != 0 { 2782 resetcpuprofiler(0) 2783 } 2784 2785 // This function is called before fork in syscall package. 2786 // Code between fork and exec must not allocate memory nor even try to grow stack. 2787 // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack. 2788 // runtime_AfterFork will undo this in parent process, but not in child. 2789 gp.stackguard0 = stackFork 2790 } 2791 2792 // Called from syscall package before fork. 2793 //go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork 2794 //go:nosplit 2795 func syscall_runtime_BeforeFork() { 2796 systemstack(beforefork) 2797 } 2798 2799 func afterfork() { 2800 gp := getg().m.curg 2801 2802 // See the comment in beforefork. 2803 gp.stackguard0 = gp.stack.lo + _StackGuard 2804 2805 hz := sched.profilehz 2806 if hz != 0 { 2807 resetcpuprofiler(hz) 2808 } 2809 gp.m.locks-- 2810 } 2811 2812 // Called from syscall package after fork in parent. 2813 //go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork 2814 //go:nosplit 2815 func syscall_runtime_AfterFork() { 2816 systemstack(afterfork) 2817 } 2818 2819 // Allocate a new g, with a stack big enough for stacksize bytes. 2820 func malg(stacksize int32) *g { 2821 newg := new(g) 2822 if stacksize >= 0 { 2823 stacksize = round2(_StackSystem + stacksize) 2824 systemstack(func() { 2825 newg.stack, newg.stkbar = stackalloc(uint32(stacksize)) 2826 }) 2827 newg.stackguard0 = newg.stack.lo + _StackGuard 2828 newg.stackguard1 = ^uintptr(0) 2829 newg.stackAlloc = uintptr(stacksize) 2830 } 2831 return newg 2832 } 2833 2834 // Create a new g running fn with siz bytes of arguments. 2835 // Put it on the queue of g's waiting to run. 2836 // The compiler turns a go statement into a call to this. 2837 // Cannot split the stack because it assumes that the arguments 2838 // are available sequentially after &fn; they would not be 2839 // copied if a stack split occurred. 2840 //go:nosplit 2841 func newproc(siz int32, fn *funcval) { 2842 argp := add(unsafe.Pointer(&fn), sys.PtrSize) 2843 pc := getcallerpc(unsafe.Pointer(&siz)) 2844 systemstack(func() { 2845 newproc1(fn, (*uint8)(argp), siz, 0, pc) 2846 }) 2847 } 2848 2849 // Create a new g running fn with narg bytes of arguments starting 2850 // at argp and returning nret bytes of results. callerpc is the 2851 // address of the go statement that created this. The new g is put 2852 // on the queue of g's waiting to run. 2853 func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g { 2854 _g_ := getg() 2855 2856 if fn == nil { 2857 _g_.m.throwing = -1 // do not dump full stacks 2858 throw("go of nil func value") 2859 } 2860 _g_.m.locks++ // disable preemption because it can be holding p in a local var 2861 siz := narg + nret 2862 siz = (siz + 7) &^ 7 2863 2864 // We could allocate a larger initial stack if necessary. 2865 // Not worth it: this is almost always an error. 2866 // 4*sizeof(uintreg): extra space added below 2867 // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). 2868 if siz >= _StackMin-4*sys.RegSize-sys.RegSize { 2869 throw("newproc: function arguments too large for new goroutine") 2870 } 2871 2872 _p_ := _g_.m.p.ptr() 2873 newg := gfget(_p_) 2874 if newg == nil { 2875 newg = malg(_StackMin) 2876 casgstatus(newg, _Gidle, _Gdead) 2877 newg.gcRescan = -1 2878 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. 2879 } 2880 if newg.stack.hi == 0 { 2881 throw("newproc1: newg missing stack") 2882 } 2883 2884 if readgstatus(newg) != _Gdead { 2885 throw("newproc1: new g is not Gdead") 2886 } 2887 2888 totalSize := 4*sys.RegSize + uintptr(siz) + sys.MinFrameSize // extra space in case of reads slightly beyond frame 2889 totalSize += -totalSize & (sys.SpAlign - 1) // align to spAlign 2890 sp := newg.stack.hi - totalSize 2891 spArg := sp 2892 if usesLR { 2893 // caller's LR 2894 *(*uintptr)(unsafe.Pointer(sp)) = 0 2895 prepGoExitFrame(sp) 2896 spArg += sys.MinFrameSize 2897 } 2898 if narg > 0 { 2899 memmove(unsafe.Pointer(spArg), unsafe.Pointer(argp), uintptr(narg)) 2900 // This is a stack-to-stack copy. If write barriers 2901 // are enabled and the source stack is grey (the 2902 // destination is always black), then perform a 2903 // barrier copy. We do this *after* the memmove 2904 // because the destination stack may have garbage on 2905 // it. 2906 if writeBarrier.needed && !_g_.m.curg.gcscandone { 2907 f := findfunc(fn.fn) 2908 stkmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps)) 2909 // We're in the prologue, so it's always stack map index 0. 2910 bv := stackmapdata(stkmap, 0) 2911 bulkBarrierBitmap(spArg, spArg, uintptr(narg), 0, bv.bytedata) 2912 } 2913 } 2914 2915 memclrNoHeapPointers(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched)) 2916 newg.sched.sp = sp 2917 newg.stktopsp = sp 2918 newg.sched.pc = funcPC(goexit) + sys.PCQuantum // +PCQuantum so that previous instruction is in same function 2919 newg.sched.g = guintptr(unsafe.Pointer(newg)) 2920 gostartcallfn(&newg.sched, fn) 2921 newg.gopc = callerpc 2922 newg.startpc = fn.fn 2923 if isSystemGoroutine(newg) { 2924 atomic.Xadd(&sched.ngsys, +1) 2925 } 2926 // The stack is dirty from the argument frame, so queue it for 2927 // scanning. Do this before setting it to runnable so we still 2928 // own the G. If we're recycling a G, it may already be on the 2929 // rescan list. 2930 if newg.gcRescan == -1 { 2931 queueRescan(newg) 2932 } else { 2933 // The recycled G is already on the rescan list. Just 2934 // mark the stack dirty. 2935 newg.gcscanvalid = false 2936 } 2937 casgstatus(newg, _Gdead, _Grunnable) 2938 2939 if _p_.goidcache == _p_.goidcacheend { 2940 // Sched.goidgen is the last allocated id, 2941 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. 2942 // At startup sched.goidgen=0, so main goroutine receives goid=1. 2943 _p_.goidcache = atomic.Xadd64(&sched.goidgen, _GoidCacheBatch) 2944 _p_.goidcache -= _GoidCacheBatch - 1 2945 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch 2946 } 2947 newg.goid = int64(_p_.goidcache) 2948 _p_.goidcache++ 2949 if raceenabled { 2950 newg.racectx = racegostart(callerpc) 2951 } 2952 if trace.enabled { 2953 traceGoCreate(newg, newg.startpc) 2954 } 2955 runqput(_p_, newg, true) 2956 2957 if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 && runtimeInitTime != 0 { 2958 wakep() 2959 } 2960 _g_.m.locks-- 2961 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 2962 _g_.stackguard0 = stackPreempt 2963 } 2964 return newg 2965 } 2966 2967 // Put on gfree list. 2968 // If local list is too long, transfer a batch to the global list. 2969 func gfput(_p_ *p, gp *g) { 2970 if readgstatus(gp) != _Gdead { 2971 throw("gfput: bad status (not Gdead)") 2972 } 2973 2974 stksize := gp.stackAlloc 2975 2976 if stksize != _FixedStack { 2977 // non-standard stack size - free it. 2978 stackfree(gp.stack, gp.stackAlloc) 2979 gp.stack.lo = 0 2980 gp.stack.hi = 0 2981 gp.stackguard0 = 0 2982 gp.stkbar = nil 2983 gp.stkbarPos = 0 2984 } else { 2985 // Reset stack barriers. 2986 gp.stkbar = gp.stkbar[:0] 2987 gp.stkbarPos = 0 2988 } 2989 2990 gp.schedlink.set(_p_.gfree) 2991 _p_.gfree = gp 2992 _p_.gfreecnt++ 2993 if _p_.gfreecnt >= 64 { 2994 lock(&sched.gflock) 2995 for _p_.gfreecnt >= 32 { 2996 _p_.gfreecnt-- 2997 gp = _p_.gfree 2998 _p_.gfree = gp.schedlink.ptr() 2999 if gp.stack.lo == 0 { 3000 gp.schedlink.set(sched.gfreeNoStack) 3001 sched.gfreeNoStack = gp 3002 } else { 3003 gp.schedlink.set(sched.gfreeStack) 3004 sched.gfreeStack = gp 3005 } 3006 sched.ngfree++ 3007 } 3008 unlock(&sched.gflock) 3009 } 3010 } 3011 3012 // Get from gfree list. 3013 // If local list is empty, grab a batch from global list. 3014 func gfget(_p_ *p) *g { 3015 retry: 3016 gp := _p_.gfree 3017 if gp == nil && (sched.gfreeStack != nil || sched.gfreeNoStack != nil) { 3018 lock(&sched.gflock) 3019 for _p_.gfreecnt < 32 { 3020 if sched.gfreeStack != nil { 3021 // Prefer Gs with stacks. 3022 gp = sched.gfreeStack 3023 sched.gfreeStack = gp.schedlink.ptr() 3024 } else if sched.gfreeNoStack != nil { 3025 gp = sched.gfreeNoStack 3026 sched.gfreeNoStack = gp.schedlink.ptr() 3027 } else { 3028 break 3029 } 3030 _p_.gfreecnt++ 3031 sched.ngfree-- 3032 gp.schedlink.set(_p_.gfree) 3033 _p_.gfree = gp 3034 } 3035 unlock(&sched.gflock) 3036 goto retry 3037 } 3038 if gp != nil { 3039 _p_.gfree = gp.schedlink.ptr() 3040 _p_.gfreecnt-- 3041 if gp.stack.lo == 0 { 3042 // Stack was deallocated in gfput. Allocate a new one. 3043 systemstack(func() { 3044 gp.stack, gp.stkbar = stackalloc(_FixedStack) 3045 }) 3046 gp.stackguard0 = gp.stack.lo + _StackGuard 3047 gp.stackAlloc = _FixedStack 3048 } else { 3049 if raceenabled { 3050 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc) 3051 } 3052 if msanenabled { 3053 msanmalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc) 3054 } 3055 } 3056 } 3057 return gp 3058 } 3059 3060 // Purge all cached G's from gfree list to the global list. 3061 func gfpurge(_p_ *p) { 3062 lock(&sched.gflock) 3063 for _p_.gfreecnt != 0 { 3064 _p_.gfreecnt-- 3065 gp := _p_.gfree 3066 _p_.gfree = gp.schedlink.ptr() 3067 if gp.stack.lo == 0 { 3068 gp.schedlink.set(sched.gfreeNoStack) 3069 sched.gfreeNoStack = gp 3070 } else { 3071 gp.schedlink.set(sched.gfreeStack) 3072 sched.gfreeStack = gp 3073 } 3074 sched.ngfree++ 3075 } 3076 unlock(&sched.gflock) 3077 } 3078 3079 // Breakpoint executes a breakpoint trap. 3080 func Breakpoint() { 3081 breakpoint() 3082 } 3083 3084 // dolockOSThread is called by LockOSThread and lockOSThread below 3085 // after they modify m.locked. Do not allow preemption during this call, 3086 // or else the m might be different in this function than in the caller. 3087 //go:nosplit 3088 func dolockOSThread() { 3089 _g_ := getg() 3090 _g_.m.lockedg = _g_ 3091 _g_.lockedm = _g_.m 3092 } 3093 3094 //go:nosplit 3095 3096 // LockOSThread wires the calling goroutine to its current operating system thread. 3097 // Until the calling goroutine exits or calls UnlockOSThread, it will always 3098 // execute in that thread, and no other goroutine can. 3099 func LockOSThread() { 3100 getg().m.locked |= _LockExternal 3101 dolockOSThread() 3102 } 3103 3104 //go:nosplit 3105 func lockOSThread() { 3106 getg().m.locked += _LockInternal 3107 dolockOSThread() 3108 } 3109 3110 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below 3111 // after they update m->locked. Do not allow preemption during this call, 3112 // or else the m might be in different in this function than in the caller. 3113 //go:nosplit 3114 func dounlockOSThread() { 3115 _g_ := getg() 3116 if _g_.m.locked != 0 { 3117 return 3118 } 3119 _g_.m.lockedg = nil 3120 _g_.lockedm = nil 3121 } 3122 3123 //go:nosplit 3124 3125 // UnlockOSThread unwires the calling goroutine from its fixed operating system thread. 3126 // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op. 3127 func UnlockOSThread() { 3128 getg().m.locked &^= _LockExternal 3129 dounlockOSThread() 3130 } 3131 3132 //go:nosplit 3133 func unlockOSThread() { 3134 _g_ := getg() 3135 if _g_.m.locked < _LockInternal { 3136 systemstack(badunlockosthread) 3137 } 3138 _g_.m.locked -= _LockInternal 3139 dounlockOSThread() 3140 } 3141 3142 func badunlockosthread() { 3143 throw("runtime: internal error: misuse of lockOSThread/unlockOSThread") 3144 } 3145 3146 func gcount() int32 { 3147 n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys)) 3148 for i := 0; ; i++ { 3149 _p_ := allp[i] 3150 if _p_ == nil { 3151 break 3152 } 3153 n -= _p_.gfreecnt 3154 } 3155 3156 // All these variables can be changed concurrently, so the result can be inconsistent. 3157 // But at least the current goroutine is running. 3158 if n < 1 { 3159 n = 1 3160 } 3161 return n 3162 } 3163 3164 func mcount() int32 { 3165 return sched.mcount 3166 } 3167 3168 var prof struct { 3169 lock uint32 3170 hz int32 3171 } 3172 3173 func _System() { _System() } 3174 func _ExternalCode() { _ExternalCode() } 3175 func _GC() { _GC() } 3176 3177 // Called if we receive a SIGPROF signal. 3178 // Called by the signal handler, may run during STW. 3179 //go:nowritebarrierrec 3180 func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { 3181 if prof.hz == 0 { 3182 return 3183 } 3184 3185 // Profiling runs concurrently with GC, so it must not allocate. 3186 // Set a trap in case the code does allocate. 3187 // Note that on windows, one thread takes profiles of all the 3188 // other threads, so mp is usually not getg().m. 3189 // In fact mp may not even be stopped. 3190 // See golang.org/issue/17165. 3191 getg().m.mallocing++ 3192 3193 // Define that a "user g" is a user-created goroutine, and a "system g" 3194 // is one that is m->g0 or m->gsignal. 3195 // 3196 // We might be interrupted for profiling halfway through a 3197 // goroutine switch. The switch involves updating three (or four) values: 3198 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, 3199 // because once it gets updated the new g is running. 3200 // 3201 // When switching from a user g to a system g, LR is not considered live, 3202 // so the update only affects g, SP, and PC. Since PC must be last, there 3203 // the possible partial transitions in ordinary execution are (1) g alone is updated, 3204 // (2) both g and SP are updated, and (3) SP alone is updated. 3205 // If SP or g alone is updated, we can detect the partial transition by checking 3206 // whether the SP is within g's stack bounds. (We could also require that SP 3207 // be changed only after g, but the stack bounds check is needed by other 3208 // cases, so there is no need to impose an additional requirement.) 3209 // 3210 // There is one exceptional transition to a system g, not in ordinary execution. 3211 // When a signal arrives, the operating system starts the signal handler running 3212 // with an updated PC and SP. The g is updated last, at the beginning of the 3213 // handler. There are two reasons this is okay. First, until g is updated the 3214 // g and SP do not match, so the stack bounds check detects the partial transition. 3215 // Second, signal handlers currently run with signals disabled, so a profiling 3216 // signal cannot arrive during the handler. 3217 // 3218 // When switching from a system g to a user g, there are three possibilities. 3219 // 3220 // First, it may be that the g switch has no PC update, because the SP 3221 // either corresponds to a user g throughout (as in asmcgocall) 3222 // or because it has been arranged to look like a user g frame 3223 // (as in cgocallback_gofunc). In this case, since the entire 3224 // transition is a g+SP update, a partial transition updating just one of 3225 // those will be detected by the stack bounds check. 3226 // 3227 // Second, when returning from a signal handler, the PC and SP updates 3228 // are performed by the operating system in an atomic update, so the g 3229 // update must be done before them. The stack bounds check detects 3230 // the partial transition here, and (again) signal handlers run with signals 3231 // disabled, so a profiling signal cannot arrive then anyway. 3232 // 3233 // Third, the common case: it may be that the switch updates g, SP, and PC 3234 // separately. If the PC is within any of the functions that does this, 3235 // we don't ask for a traceback. C.F. the function setsSP for more about this. 3236 // 3237 // There is another apparently viable approach, recorded here in case 3238 // the "PC within setsSP function" check turns out not to be usable. 3239 // It would be possible to delay the update of either g or SP until immediately 3240 // before the PC update instruction. Then, because of the stack bounds check, 3241 // the only problematic interrupt point is just before that PC update instruction, 3242 // and the sigprof handler can detect that instruction and simulate stepping past 3243 // it in order to reach a consistent state. On ARM, the update of g must be made 3244 // in two places (in R10 and also in a TLS slot), so the delayed update would 3245 // need to be the SP update. The sigprof handler must read the instruction at 3246 // the current PC and if it was the known instruction (for example, JMP BX or 3247 // MOV R2, PC), use that other register in place of the PC value. 3248 // The biggest drawback to this solution is that it requires that we can tell 3249 // whether it's safe to read from the memory pointed at by PC. 3250 // In a correct program, we can test PC == nil and otherwise read, 3251 // but if a profiling signal happens at the instant that a program executes 3252 // a bad jump (before the program manages to handle the resulting fault) 3253 // the profiling handler could fault trying to read nonexistent memory. 3254 // 3255 // To recap, there are no constraints on the assembly being used for the 3256 // transition. We simply require that g and SP match and that the PC is not 3257 // in gogo. 3258 traceback := true 3259 if gp == nil || sp < gp.stack.lo || gp.stack.hi < sp || setsSP(pc) { 3260 traceback = false 3261 } 3262 var stk [maxCPUProfStack]uintptr 3263 var haveStackLock *g 3264 n := 0 3265 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { 3266 cgoOff := 0 3267 // Check cgoCallersUse to make sure that we are not 3268 // interrupting other code that is fiddling with 3269 // cgoCallers. We are running in a signal handler 3270 // with all signals blocked, so we don't have to worry 3271 // about any other code interrupting us. 3272 if atomic.Load(&mp.cgoCallersUse) == 0 && mp.cgoCallers != nil && mp.cgoCallers[0] != 0 { 3273 for cgoOff < len(mp.cgoCallers) && mp.cgoCallers[cgoOff] != 0 { 3274 cgoOff++ 3275 } 3276 copy(stk[:], mp.cgoCallers[:cgoOff]) 3277 mp.cgoCallers[0] = 0 3278 } 3279 3280 // Collect Go stack that leads to the cgo call. 3281 if gcTryLockStackBarriers(mp.curg) { 3282 haveStackLock = mp.curg 3283 n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[cgoOff], len(stk)-cgoOff, nil, nil, 0) 3284 } 3285 } else if traceback { 3286 var flags uint = _TraceTrap 3287 if gp.m.curg != nil && gcTryLockStackBarriers(gp.m.curg) { 3288 // It's safe to traceback the user stack. 3289 haveStackLock = gp.m.curg 3290 flags |= _TraceJumpStack 3291 } 3292 // Traceback is safe if we're on the system stack (if 3293 // necessary, flags will stop it before switching to 3294 // the user stack), or if we locked the user stack. 3295 if gp != gp.m.curg || haveStackLock != nil { 3296 n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, flags) 3297 } 3298 } 3299 if haveStackLock != nil { 3300 gcUnlockStackBarriers(haveStackLock) 3301 } 3302 3303 if n <= 0 { 3304 // Normal traceback is impossible or has failed. 3305 // See if it falls into several common cases. 3306 n = 0 3307 if GOOS == "windows" && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { 3308 // Libcall, i.e. runtime syscall on windows. 3309 // Collect Go stack that leads to the call. 3310 if gcTryLockStackBarriers(mp.libcallg.ptr()) { 3311 n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0) 3312 gcUnlockStackBarriers(mp.libcallg.ptr()) 3313 } 3314 } 3315 if n == 0 { 3316 // If all of the above has failed, account it against abstract "System" or "GC". 3317 n = 2 3318 // "ExternalCode" is better than "etext". 3319 if pc > firstmoduledata.etext { 3320 pc = funcPC(_ExternalCode) + sys.PCQuantum 3321 } 3322 stk[0] = pc 3323 if mp.preemptoff != "" || mp.helpgc != 0 { 3324 stk[1] = funcPC(_GC) + sys.PCQuantum 3325 } else { 3326 stk[1] = funcPC(_System) + sys.PCQuantum 3327 } 3328 } 3329 } 3330 3331 if prof.hz != 0 { 3332 // Simple cas-lock to coordinate with setcpuprofilerate. 3333 for !atomic.Cas(&prof.lock, 0, 1) { 3334 osyield() 3335 } 3336 if prof.hz != 0 { 3337 cpuprof.add(stk[:n]) 3338 } 3339 atomic.Store(&prof.lock, 0) 3340 } 3341 getg().m.mallocing-- 3342 } 3343 3344 // If the signal handler receives a SIGPROF signal on a non-Go thread, 3345 // it tries to collect a traceback into sigprofCallers. 3346 // sigprofCallersUse is set to non-zero while sigprofCallers holds a traceback. 3347 var sigprofCallers cgoCallers 3348 var sigprofCallersUse uint32 3349 3350 // sigprofNonGo is called if we receive a SIGPROF signal on a non-Go thread, 3351 // and the signal handler collected a stack trace in sigprofCallers. 3352 // When this is called, sigprofCallersUse will be non-zero. 3353 // g is nil, and what we can do is very limited. 3354 //go:nosplit 3355 //go:nowritebarrierrec 3356 func sigprofNonGo() { 3357 if prof.hz != 0 { 3358 n := 0 3359 for n < len(sigprofCallers) && sigprofCallers[n] != 0 { 3360 n++ 3361 } 3362 3363 // Simple cas-lock to coordinate with setcpuprofilerate. 3364 for !atomic.Cas(&prof.lock, 0, 1) { 3365 osyield() 3366 } 3367 if prof.hz != 0 { 3368 cpuprof.addNonGo(sigprofCallers[:n]) 3369 } 3370 atomic.Store(&prof.lock, 0) 3371 } 3372 3373 atomic.Store(&sigprofCallersUse, 0) 3374 } 3375 3376 // sigprofNonGoPC is called when a profiling signal arrived on a 3377 // non-Go thread and we have a single PC value, not a stack trace. 3378 // g is nil, and what we can do is very limited. 3379 //go:nosplit 3380 //go:nowritebarrierrec 3381 func sigprofNonGoPC(pc uintptr) { 3382 if prof.hz != 0 { 3383 pc := []uintptr{ 3384 pc, 3385 funcPC(_ExternalCode) + sys.PCQuantum, 3386 } 3387 3388 // Simple cas-lock to coordinate with setcpuprofilerate. 3389 for !atomic.Cas(&prof.lock, 0, 1) { 3390 osyield() 3391 } 3392 if prof.hz != 0 { 3393 cpuprof.addNonGo(pc) 3394 } 3395 atomic.Store(&prof.lock, 0) 3396 } 3397 } 3398 3399 // Reports whether a function will set the SP 3400 // to an absolute value. Important that 3401 // we don't traceback when these are at the bottom 3402 // of the stack since we can't be sure that we will 3403 // find the caller. 3404 // 3405 // If the function is not on the bottom of the stack 3406 // we assume that it will have set it up so that traceback will be consistent, 3407 // either by being a traceback terminating function 3408 // or putting one on the stack at the right offset. 3409 func setsSP(pc uintptr) bool { 3410 f := findfunc(pc) 3411 if f == nil { 3412 // couldn't find the function for this PC, 3413 // so assume the worst and stop traceback 3414 return true 3415 } 3416 switch f.entry { 3417 case gogoPC, systemstackPC, mcallPC, morestackPC: 3418 return true 3419 } 3420 return false 3421 } 3422 3423 // Arrange to call fn with a traceback hz times a second. 3424 func setcpuprofilerate_m(hz int32) { 3425 // Force sane arguments. 3426 if hz < 0 { 3427 hz = 0 3428 } 3429 3430 // Disable preemption, otherwise we can be rescheduled to another thread 3431 // that has profiling enabled. 3432 _g_ := getg() 3433 _g_.m.locks++ 3434 3435 // Stop profiler on this thread so that it is safe to lock prof. 3436 // if a profiling signal came in while we had prof locked, 3437 // it would deadlock. 3438 resetcpuprofiler(0) 3439 3440 for !atomic.Cas(&prof.lock, 0, 1) { 3441 osyield() 3442 } 3443 prof.hz = hz 3444 atomic.Store(&prof.lock, 0) 3445 3446 lock(&sched.lock) 3447 sched.profilehz = hz 3448 unlock(&sched.lock) 3449 3450 if hz != 0 { 3451 resetcpuprofiler(hz) 3452 } 3453 3454 _g_.m.locks-- 3455 } 3456 3457 // Change number of processors. The world is stopped, sched is locked. 3458 // gcworkbufs are not being modified by either the GC or 3459 // the write barrier code. 3460 // Returns list of Ps with local work, they need to be scheduled by the caller. 3461 func procresize(nprocs int32) *p { 3462 old := gomaxprocs 3463 if old < 0 || old > _MaxGomaxprocs || nprocs <= 0 || nprocs > _MaxGomaxprocs { 3464 throw("procresize: invalid arg") 3465 } 3466 if trace.enabled { 3467 traceGomaxprocs(nprocs) 3468 } 3469 3470 // update statistics 3471 now := nanotime() 3472 if sched.procresizetime != 0 { 3473 sched.totaltime += int64(old) * (now - sched.procresizetime) 3474 } 3475 sched.procresizetime = now 3476 3477 // initialize new P's 3478 for i := int32(0); i < nprocs; i++ { 3479 pp := allp[i] 3480 if pp == nil { 3481 pp = new(p) 3482 pp.id = i 3483 pp.status = _Pgcstop 3484 pp.sudogcache = pp.sudogbuf[:0] 3485 for i := range pp.deferpool { 3486 pp.deferpool[i] = pp.deferpoolbuf[i][:0] 3487 } 3488 atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp)) 3489 } 3490 if pp.mcache == nil { 3491 if old == 0 && i == 0 { 3492 if getg().m.mcache == nil { 3493 throw("missing mcache?") 3494 } 3495 pp.mcache = getg().m.mcache // bootstrap 3496 } else { 3497 pp.mcache = allocmcache() 3498 } 3499 } 3500 if raceenabled && pp.racectx == 0 { 3501 if old == 0 && i == 0 { 3502 pp.racectx = raceprocctx0 3503 raceprocctx0 = 0 // bootstrap 3504 } else { 3505 pp.racectx = raceproccreate() 3506 } 3507 } 3508 } 3509 3510 // free unused P's 3511 for i := nprocs; i < old; i++ { 3512 p := allp[i] 3513 if trace.enabled { 3514 if p == getg().m.p.ptr() { 3515 // moving to p[0], pretend that we were descheduled 3516 // and then scheduled again to keep the trace sane. 3517 traceGoSched() 3518 traceProcStop(p) 3519 } 3520 } 3521 // move all runnable goroutines to the global queue 3522 for p.runqhead != p.runqtail { 3523 // pop from tail of local queue 3524 p.runqtail-- 3525 gp := p.runq[p.runqtail%uint32(len(p.runq))].ptr() 3526 // push onto head of global queue 3527 globrunqputhead(gp) 3528 } 3529 if p.runnext != 0 { 3530 globrunqputhead(p.runnext.ptr()) 3531 p.runnext = 0 3532 } 3533 // if there's a background worker, make it runnable and put 3534 // it on the global queue so it can clean itself up 3535 if gp := p.gcBgMarkWorker.ptr(); gp != nil { 3536 casgstatus(gp, _Gwaiting, _Grunnable) 3537 if trace.enabled { 3538 traceGoUnpark(gp, 0) 3539 } 3540 globrunqput(gp) 3541 // This assignment doesn't race because the 3542 // world is stopped. 3543 p.gcBgMarkWorker.set(nil) 3544 } 3545 for i := range p.sudogbuf { 3546 p.sudogbuf[i] = nil 3547 } 3548 p.sudogcache = p.sudogbuf[:0] 3549 for i := range p.deferpool { 3550 for j := range p.deferpoolbuf[i] { 3551 p.deferpoolbuf[i][j] = nil 3552 } 3553 p.deferpool[i] = p.deferpoolbuf[i][:0] 3554 } 3555 freemcache(p.mcache) 3556 p.mcache = nil 3557 gfpurge(p) 3558 traceProcFree(p) 3559 if raceenabled { 3560 raceprocdestroy(p.racectx) 3561 p.racectx = 0 3562 } 3563 p.status = _Pdead 3564 // can't free P itself because it can be referenced by an M in syscall 3565 } 3566 3567 _g_ := getg() 3568 if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs { 3569 // continue to use the current P 3570 _g_.m.p.ptr().status = _Prunning 3571 } else { 3572 // release the current P and acquire allp[0] 3573 if _g_.m.p != 0 { 3574 _g_.m.p.ptr().m = 0 3575 } 3576 _g_.m.p = 0 3577 _g_.m.mcache = nil 3578 p := allp[0] 3579 p.m = 0 3580 p.status = _Pidle 3581 acquirep(p) 3582 if trace.enabled { 3583 traceGoStart() 3584 } 3585 } 3586 var runnablePs *p 3587 for i := nprocs - 1; i >= 0; i-- { 3588 p := allp[i] 3589 if _g_.m.p.ptr() == p { 3590 continue 3591 } 3592 p.status = _Pidle 3593 if runqempty(p) { 3594 pidleput(p) 3595 } else { 3596 p.m.set(mget()) 3597 p.link.set(runnablePs) 3598 runnablePs = p 3599 } 3600 } 3601 stealOrder.reset(uint32(nprocs)) 3602 var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32 3603 atomic.Store((*uint32)(unsafe.Pointer(int32p)), uint32(nprocs)) 3604 return runnablePs 3605 } 3606 3607 // Associate p and the current m. 3608 // 3609 // This function is allowed to have write barriers even if the caller 3610 // isn't because it immediately acquires _p_. 3611 // 3612 //go:yeswritebarrierrec 3613 func acquirep(_p_ *p) { 3614 // Do the part that isn't allowed to have write barriers. 3615 acquirep1(_p_) 3616 3617 // have p; write barriers now allowed 3618 _g_ := getg() 3619 _g_.m.mcache = _p_.mcache 3620 3621 if trace.enabled { 3622 traceProcStart() 3623 } 3624 } 3625 3626 // acquirep1 is the first step of acquirep, which actually acquires 3627 // _p_. This is broken out so we can disallow write barriers for this 3628 // part, since we don't yet have a P. 3629 // 3630 //go:nowritebarrierrec 3631 func acquirep1(_p_ *p) { 3632 _g_ := getg() 3633 3634 if _g_.m.p != 0 || _g_.m.mcache != nil { 3635 throw("acquirep: already in go") 3636 } 3637 if _p_.m != 0 || _p_.status != _Pidle { 3638 id := int32(0) 3639 if _p_.m != 0 { 3640 id = _p_.m.ptr().id 3641 } 3642 print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") 3643 throw("acquirep: invalid p state") 3644 } 3645 _g_.m.p.set(_p_) 3646 _p_.m.set(_g_.m) 3647 _p_.status = _Prunning 3648 } 3649 3650 // Disassociate p and the current m. 3651 func releasep() *p { 3652 _g_ := getg() 3653 3654 if _g_.m.p == 0 || _g_.m.mcache == nil { 3655 throw("releasep: invalid arg") 3656 } 3657 _p_ := _g_.m.p.ptr() 3658 if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { 3659 print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") 3660 throw("releasep: invalid p state") 3661 } 3662 if trace.enabled { 3663 traceProcStop(_g_.m.p.ptr()) 3664 } 3665 _g_.m.p = 0 3666 _g_.m.mcache = nil 3667 _p_.m = 0 3668 _p_.status = _Pidle 3669 return _p_ 3670 } 3671 3672 func incidlelocked(v int32) { 3673 lock(&sched.lock) 3674 sched.nmidlelocked += v 3675 if v > 0 { 3676 checkdead() 3677 } 3678 unlock(&sched.lock) 3679 } 3680 3681 // Check for deadlock situation. 3682 // The check is based on number of running M's, if 0 -> deadlock. 3683 func checkdead() { 3684 // For -buildmode=c-shared or -buildmode=c-archive it's OK if 3685 // there are no running goroutines. The calling program is 3686 // assumed to be running. 3687 if islibrary || isarchive { 3688 return 3689 } 3690 3691 // If we are dying because of a signal caught on an already idle thread, 3692 // freezetheworld will cause all running threads to block. 3693 // And runtime will essentially enter into deadlock state, 3694 // except that there is a thread that will call exit soon. 3695 if panicking > 0 { 3696 return 3697 } 3698 3699 // -1 for sysmon 3700 run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1 3701 if run > 0 { 3702 return 3703 } 3704 if run < 0 { 3705 print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n") 3706 throw("checkdead: inconsistent counts") 3707 } 3708 3709 grunning := 0 3710 lock(&allglock) 3711 for i := 0; i < len(allgs); i++ { 3712 gp := allgs[i] 3713 if isSystemGoroutine(gp) { 3714 continue 3715 } 3716 s := readgstatus(gp) 3717 switch s &^ _Gscan { 3718 case _Gwaiting: 3719 grunning++ 3720 case _Grunnable, 3721 _Grunning, 3722 _Gsyscall: 3723 unlock(&allglock) 3724 print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n") 3725 throw("checkdead: runnable g") 3726 } 3727 } 3728 unlock(&allglock) 3729 if grunning == 0 { // possible if main goroutine calls runtimeGoexit() 3730 throw("no goroutines (main called runtime.Goexit) - deadlock!") 3731 } 3732 3733 // Maybe jump time forward for playground. 3734 gp := timejump() 3735 if gp != nil { 3736 casgstatus(gp, _Gwaiting, _Grunnable) 3737 globrunqput(gp) 3738 _p_ := pidleget() 3739 if _p_ == nil { 3740 throw("checkdead: no p for timer") 3741 } 3742 mp := mget() 3743 if mp == nil { 3744 // There should always be a free M since 3745 // nothing is running. 3746 throw("checkdead: no m for timer") 3747 } 3748 mp.nextp.set(_p_) 3749 notewakeup(&mp.park) 3750 return 3751 } 3752 3753 getg().m.throwing = -1 // do not dump full stacks 3754 throw("all goroutines are asleep - deadlock!") 3755 } 3756 3757 // forcegcperiod is the maximum time in nanoseconds between garbage 3758 // collections. If we go this long without a garbage collection, one 3759 // is forced to run. 3760 // 3761 // This is a variable for testing purposes. It normally doesn't change. 3762 var forcegcperiod int64 = 2 * 60 * 1e9 3763 3764 // Always runs without a P, so write barriers are not allowed. 3765 // 3766 //go:nowritebarrierrec 3767 func sysmon() { 3768 // If a heap span goes unused for 5 minutes after a garbage collection, 3769 // we hand it back to the operating system. 3770 scavengelimit := int64(5 * 60 * 1e9) 3771 3772 if debug.scavenge > 0 { 3773 // Scavenge-a-lot for testing. 3774 forcegcperiod = 10 * 1e6 3775 scavengelimit = 20 * 1e6 3776 } 3777 3778 lastscavenge := nanotime() 3779 nscavenge := 0 3780 3781 lasttrace := int64(0) 3782 idle := 0 // how many cycles in succession we had not wokeup somebody 3783 delay := uint32(0) 3784 for { 3785 if idle == 0 { // start with 20us sleep... 3786 delay = 20 3787 } else if idle > 50 { // start doubling the sleep after 1ms... 3788 delay *= 2 3789 } 3790 if delay > 10*1000 { // up to 10ms 3791 delay = 10 * 1000 3792 } 3793 usleep(delay) 3794 if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs)) { 3795 lock(&sched.lock) 3796 if atomic.Load(&sched.gcwaiting) != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs) { 3797 atomic.Store(&sched.sysmonwait, 1) 3798 unlock(&sched.lock) 3799 // Make wake-up period small enough 3800 // for the sampling to be correct. 3801 maxsleep := forcegcperiod / 2 3802 if scavengelimit < forcegcperiod { 3803 maxsleep = scavengelimit / 2 3804 } 3805 notetsleep(&sched.sysmonnote, maxsleep) 3806 lock(&sched.lock) 3807 atomic.Store(&sched.sysmonwait, 0) 3808 noteclear(&sched.sysmonnote) 3809 idle = 0 3810 delay = 20 3811 } 3812 unlock(&sched.lock) 3813 } 3814 // poll network if not polled for more than 10ms 3815 lastpoll := int64(atomic.Load64(&sched.lastpoll)) 3816 now := nanotime() 3817 unixnow := unixnanotime() 3818 if lastpoll != 0 && lastpoll+10*1000*1000 < now { 3819 atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now)) 3820 gp := netpoll(false) // non-blocking - returns list of goroutines 3821 if gp != nil { 3822 // Need to decrement number of idle locked M's 3823 // (pretending that one more is running) before injectglist. 3824 // Otherwise it can lead to the following situation: 3825 // injectglist grabs all P's but before it starts M's to run the P's, 3826 // another M returns from syscall, finishes running its G, 3827 // observes that there is no work to do and no other running M's 3828 // and reports deadlock. 3829 incidlelocked(-1) 3830 injectglist(gp) 3831 incidlelocked(1) 3832 } 3833 } 3834 // retake P's blocked in syscalls 3835 // and preempt long running G's 3836 if retake(now) != 0 { 3837 idle = 0 3838 } else { 3839 idle++ 3840 } 3841 // check if we need to force a GC 3842 lastgc := int64(atomic.Load64(&memstats.last_gc)) 3843 if gcphase == _GCoff && lastgc != 0 && unixnow-lastgc > forcegcperiod && atomic.Load(&forcegc.idle) != 0 { 3844 lock(&forcegc.lock) 3845 forcegc.idle = 0 3846 forcegc.g.schedlink = 0 3847 injectglist(forcegc.g) 3848 unlock(&forcegc.lock) 3849 } 3850 // scavenge heap once in a while 3851 if lastscavenge+scavengelimit/2 < now { 3852 mheap_.scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit)) 3853 lastscavenge = now 3854 nscavenge++ 3855 } 3856 if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace)*1000000 <= now { 3857 lasttrace = now 3858 schedtrace(debug.scheddetail > 0) 3859 } 3860 } 3861 } 3862 3863 var pdesc [_MaxGomaxprocs]struct { 3864 schedtick uint32 3865 schedwhen int64 3866 syscalltick uint32 3867 syscallwhen int64 3868 } 3869 3870 // forcePreemptNS is the time slice given to a G before it is 3871 // preempted. 3872 const forcePreemptNS = 10 * 1000 * 1000 // 10ms 3873 3874 func retake(now int64) uint32 { 3875 n := 0 3876 for i := int32(0); i < gomaxprocs; i++ { 3877 _p_ := allp[i] 3878 if _p_ == nil { 3879 continue 3880 } 3881 pd := &pdesc[i] 3882 s := _p_.status 3883 if s == _Psyscall { 3884 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). 3885 t := int64(_p_.syscalltick) 3886 if int64(pd.syscalltick) != t { 3887 pd.syscalltick = uint32(t) 3888 pd.syscallwhen = now 3889 continue 3890 } 3891 // On the one hand we don't want to retake Ps if there is no other work to do, 3892 // but on the other hand we want to retake them eventually 3893 // because they can prevent the sysmon thread from deep sleep. 3894 if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now { 3895 continue 3896 } 3897 // Need to decrement number of idle locked M's 3898 // (pretending that one more is running) before the CAS. 3899 // Otherwise the M from which we retake can exit the syscall, 3900 // increment nmidle and report deadlock. 3901 incidlelocked(-1) 3902 if atomic.Cas(&_p_.status, s, _Pidle) { 3903 if trace.enabled { 3904 traceGoSysBlock(_p_) 3905 traceProcStop(_p_) 3906 } 3907 n++ 3908 _p_.syscalltick++ 3909 handoffp(_p_) 3910 } 3911 incidlelocked(1) 3912 } else if s == _Prunning { 3913 // Preempt G if it's running for too long. 3914 t := int64(_p_.schedtick) 3915 if int64(pd.schedtick) != t { 3916 pd.schedtick = uint32(t) 3917 pd.schedwhen = now 3918 continue 3919 } 3920 if pd.schedwhen+forcePreemptNS > now { 3921 continue 3922 } 3923 preemptone(_p_) 3924 } 3925 } 3926 return uint32(n) 3927 } 3928 3929 // Tell all goroutines that they have been preempted and they should stop. 3930 // This function is purely best-effort. It can fail to inform a goroutine if a 3931 // processor just started running it. 3932 // No locks need to be held. 3933 // Returns true if preemption request was issued to at least one goroutine. 3934 func preemptall() bool { 3935 res := false 3936 for i := int32(0); i < gomaxprocs; i++ { 3937 _p_ := allp[i] 3938 if _p_ == nil || _p_.status != _Prunning { 3939 continue 3940 } 3941 if preemptone(_p_) { 3942 res = true 3943 } 3944 } 3945 return res 3946 } 3947 3948 // Tell the goroutine running on processor P to stop. 3949 // This function is purely best-effort. It can incorrectly fail to inform the 3950 // goroutine. It can send inform the wrong goroutine. Even if it informs the 3951 // correct goroutine, that goroutine might ignore the request if it is 3952 // simultaneously executing newstack. 3953 // No lock needs to be held. 3954 // Returns true if preemption request was issued. 3955 // The actual preemption will happen at some point in the future 3956 // and will be indicated by the gp->status no longer being 3957 // Grunning 3958 func preemptone(_p_ *p) bool { 3959 mp := _p_.m.ptr() 3960 if mp == nil || mp == getg().m { 3961 return false 3962 } 3963 gp := mp.curg 3964 if gp == nil || gp == mp.g0 { 3965 return false 3966 } 3967 3968 gp.preempt = true 3969 3970 // Every call in a go routine checks for stack overflow by 3971 // comparing the current stack pointer to gp->stackguard0. 3972 // Setting gp->stackguard0 to StackPreempt folds 3973 // preemption into the normal stack overflow check. 3974 gp.stackguard0 = stackPreempt 3975 return true 3976 } 3977 3978 var starttime int64 3979 3980 func schedtrace(detailed bool) { 3981 now := nanotime() 3982 if starttime == 0 { 3983 starttime = now 3984 } 3985 3986 lock(&sched.lock) 3987 print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize) 3988 if detailed { 3989 print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n") 3990 } 3991 // We must be careful while reading data from P's, M's and G's. 3992 // Even if we hold schedlock, most data can be changed concurrently. 3993 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. 3994 for i := int32(0); i < gomaxprocs; i++ { 3995 _p_ := allp[i] 3996 if _p_ == nil { 3997 continue 3998 } 3999 mp := _p_.m.ptr() 4000 h := atomic.Load(&_p_.runqhead) 4001 t := atomic.Load(&_p_.runqtail) 4002 if detailed { 4003 id := int32(-1) 4004 if mp != nil { 4005 id = mp.id 4006 } 4007 print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n") 4008 } else { 4009 // In non-detailed mode format lengths of per-P run queues as: 4010 // [len1 len2 len3 len4] 4011 print(" ") 4012 if i == 0 { 4013 print("[") 4014 } 4015 print(t - h) 4016 if i == gomaxprocs-1 { 4017 print("]\n") 4018 } 4019 } 4020 } 4021 4022 if !detailed { 4023 unlock(&sched.lock) 4024 return 4025 } 4026 4027 for mp := allm; mp != nil; mp = mp.alllink { 4028 _p_ := mp.p.ptr() 4029 gp := mp.curg 4030 lockedg := mp.lockedg 4031 id1 := int32(-1) 4032 if _p_ != nil { 4033 id1 = _p_.id 4034 } 4035 id2 := int64(-1) 4036 if gp != nil { 4037 id2 = gp.goid 4038 } 4039 id3 := int64(-1) 4040 if lockedg != nil { 4041 id3 = lockedg.goid 4042 } 4043 print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n") 4044 } 4045 4046 lock(&allglock) 4047 for gi := 0; gi < len(allgs); gi++ { 4048 gp := allgs[gi] 4049 mp := gp.m 4050 lockedm := gp.lockedm 4051 id1 := int32(-1) 4052 if mp != nil { 4053 id1 = mp.id 4054 } 4055 id2 := int32(-1) 4056 if lockedm != nil { 4057 id2 = lockedm.id 4058 } 4059 print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n") 4060 } 4061 unlock(&allglock) 4062 unlock(&sched.lock) 4063 } 4064 4065 // Put mp on midle list. 4066 // Sched must be locked. 4067 // May run during STW, so write barriers are not allowed. 4068 //go:nowritebarrierrec 4069 func mput(mp *m) { 4070 mp.schedlink = sched.midle 4071 sched.midle.set(mp) 4072 sched.nmidle++ 4073 checkdead() 4074 } 4075 4076 // Try to get an m from midle list. 4077 // Sched must be locked. 4078 // May run during STW, so write barriers are not allowed. 4079 //go:nowritebarrierrec 4080 func mget() *m { 4081 mp := sched.midle.ptr() 4082 if mp != nil { 4083 sched.midle = mp.schedlink 4084 sched.nmidle-- 4085 } 4086 return mp 4087 } 4088 4089 // Put gp on the global runnable queue. 4090 // Sched must be locked. 4091 // May run during STW, so write barriers are not allowed. 4092 //go:nowritebarrierrec 4093 func globrunqput(gp *g) { 4094 gp.schedlink = 0 4095 if sched.runqtail != 0 { 4096 sched.runqtail.ptr().schedlink.set(gp) 4097 } else { 4098 sched.runqhead.set(gp) 4099 } 4100 sched.runqtail.set(gp) 4101 sched.runqsize++ 4102 } 4103 4104 // Put gp at the head of the global runnable queue. 4105 // Sched must be locked. 4106 // May run during STW, so write barriers are not allowed. 4107 //go:nowritebarrierrec 4108 func globrunqputhead(gp *g) { 4109 gp.schedlink = sched.runqhead 4110 sched.runqhead.set(gp) 4111 if sched.runqtail == 0 { 4112 sched.runqtail.set(gp) 4113 } 4114 sched.runqsize++ 4115 } 4116 4117 // Put a batch of runnable goroutines on the global runnable queue. 4118 // Sched must be locked. 4119 func globrunqputbatch(ghead *g, gtail *g, n int32) { 4120 gtail.schedlink = 0 4121 if sched.runqtail != 0 { 4122 sched.runqtail.ptr().schedlink.set(ghead) 4123 } else { 4124 sched.runqhead.set(ghead) 4125 } 4126 sched.runqtail.set(gtail) 4127 sched.runqsize += n 4128 } 4129 4130 // Try get a batch of G's from the global runnable queue. 4131 // Sched must be locked. 4132 func globrunqget(_p_ *p, max int32) *g { 4133 if sched.runqsize == 0 { 4134 return nil 4135 } 4136 4137 n := sched.runqsize/gomaxprocs + 1 4138 if n > sched.runqsize { 4139 n = sched.runqsize 4140 } 4141 if max > 0 && n > max { 4142 n = max 4143 } 4144 if n > int32(len(_p_.runq))/2 { 4145 n = int32(len(_p_.runq)) / 2 4146 } 4147 4148 sched.runqsize -= n 4149 if sched.runqsize == 0 { 4150 sched.runqtail = 0 4151 } 4152 4153 gp := sched.runqhead.ptr() 4154 sched.runqhead = gp.schedlink 4155 n-- 4156 for ; n > 0; n-- { 4157 gp1 := sched.runqhead.ptr() 4158 sched.runqhead = gp1.schedlink 4159 runqput(_p_, gp1, false) 4160 } 4161 return gp 4162 } 4163 4164 // Put p to on _Pidle list. 4165 // Sched must be locked. 4166 // May run during STW, so write barriers are not allowed. 4167 //go:nowritebarrierrec 4168 func pidleput(_p_ *p) { 4169 if !runqempty(_p_) { 4170 throw("pidleput: P has non-empty run queue") 4171 } 4172 _p_.link = sched.pidle 4173 sched.pidle.set(_p_) 4174 atomic.Xadd(&sched.npidle, 1) // TODO: fast atomic 4175 } 4176 4177 // Try get a p from _Pidle list. 4178 // Sched must be locked. 4179 // May run during STW, so write barriers are not allowed. 4180 //go:nowritebarrierrec 4181 func pidleget() *p { 4182 _p_ := sched.pidle.ptr() 4183 if _p_ != nil { 4184 sched.pidle = _p_.link 4185 atomic.Xadd(&sched.npidle, -1) // TODO: fast atomic 4186 } 4187 return _p_ 4188 } 4189 4190 // runqempty returns true if _p_ has no Gs on its local run queue. 4191 // It never returns true spuriously. 4192 func runqempty(_p_ *p) bool { 4193 // Defend against a race where 1) _p_ has G1 in runqnext but runqhead == runqtail, 4194 // 2) runqput on _p_ kicks G1 to the runq, 3) runqget on _p_ empties runqnext. 4195 // Simply observing that runqhead == runqtail and then observing that runqnext == nil 4196 // does not mean the queue is empty. 4197 for { 4198 head := atomic.Load(&_p_.runqhead) 4199 tail := atomic.Load(&_p_.runqtail) 4200 runnext := atomic.Loaduintptr((*uintptr)(unsafe.Pointer(&_p_.runnext))) 4201 if tail == atomic.Load(&_p_.runqtail) { 4202 return head == tail && runnext == 0 4203 } 4204 } 4205 } 4206 4207 // To shake out latent assumptions about scheduling order, 4208 // we introduce some randomness into scheduling decisions 4209 // when running with the race detector. 4210 // The need for this was made obvious by changing the 4211 // (deterministic) scheduling order in Go 1.5 and breaking 4212 // many poorly-written tests. 4213 // With the randomness here, as long as the tests pass 4214 // consistently with -race, they shouldn't have latent scheduling 4215 // assumptions. 4216 const randomizeScheduler = raceenabled 4217 4218 // runqput tries to put g on the local runnable queue. 4219 // If next if false, runqput adds g to the tail of the runnable queue. 4220 // If next is true, runqput puts g in the _p_.runnext slot. 4221 // If the run queue is full, runnext puts g on the global queue. 4222 // Executed only by the owner P. 4223 func runqput(_p_ *p, gp *g, next bool) { 4224 if randomizeScheduler && next && fastrand()%2 == 0 { 4225 next = false 4226 } 4227 4228 if next { 4229 retryNext: 4230 oldnext := _p_.runnext 4231 if !_p_.runnext.cas(oldnext, guintptr(unsafe.Pointer(gp))) { 4232 goto retryNext 4233 } 4234 if oldnext == 0 { 4235 return 4236 } 4237 // Kick the old runnext out to the regular run queue. 4238 gp = oldnext.ptr() 4239 } 4240 4241 retry: 4242 h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers 4243 t := _p_.runqtail 4244 if t-h < uint32(len(_p_.runq)) { 4245 _p_.runq[t%uint32(len(_p_.runq))].set(gp) 4246 atomic.Store(&_p_.runqtail, t+1) // store-release, makes the item available for consumption 4247 return 4248 } 4249 if runqputslow(_p_, gp, h, t) { 4250 return 4251 } 4252 // the queue is not full, now the put above must succeed 4253 goto retry 4254 } 4255 4256 // Put g and a batch of work from local runnable queue on global queue. 4257 // Executed only by the owner P. 4258 func runqputslow(_p_ *p, gp *g, h, t uint32) bool { 4259 var batch [len(_p_.runq)/2 + 1]*g 4260 4261 // First, grab a batch from local queue. 4262 n := t - h 4263 n = n / 2 4264 if n != uint32(len(_p_.runq)/2) { 4265 throw("runqputslow: queue is not full") 4266 } 4267 for i := uint32(0); i < n; i++ { 4268 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))].ptr() 4269 } 4270 if !atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume 4271 return false 4272 } 4273 batch[n] = gp 4274 4275 if randomizeScheduler { 4276 for i := uint32(1); i <= n; i++ { 4277 j := fastrand() % (i + 1) 4278 batch[i], batch[j] = batch[j], batch[i] 4279 } 4280 } 4281 4282 // Link the goroutines. 4283 for i := uint32(0); i < n; i++ { 4284 batch[i].schedlink.set(batch[i+1]) 4285 } 4286 4287 // Now put the batch on global queue. 4288 lock(&sched.lock) 4289 globrunqputbatch(batch[0], batch[n], int32(n+1)) 4290 unlock(&sched.lock) 4291 return true 4292 } 4293 4294 // Get g from local runnable queue. 4295 // If inheritTime is true, gp should inherit the remaining time in the 4296 // current time slice. Otherwise, it should start a new time slice. 4297 // Executed only by the owner P. 4298 func runqget(_p_ *p) (gp *g, inheritTime bool) { 4299 // If there's a runnext, it's the next G to run. 4300 for { 4301 next := _p_.runnext 4302 if next == 0 { 4303 break 4304 } 4305 if _p_.runnext.cas(next, 0) { 4306 return next.ptr(), true 4307 } 4308 } 4309 4310 for { 4311 h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers 4312 t := _p_.runqtail 4313 if t == h { 4314 return nil, false 4315 } 4316 gp := _p_.runq[h%uint32(len(_p_.runq))].ptr() 4317 if atomic.Cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume 4318 return gp, false 4319 } 4320 } 4321 } 4322 4323 // Grabs a batch of goroutines from _p_'s runnable queue into batch. 4324 // Batch is a ring buffer starting at batchHead. 4325 // Returns number of grabbed goroutines. 4326 // Can be executed by any P. 4327 func runqgrab(_p_ *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 { 4328 for { 4329 h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers 4330 t := atomic.Load(&_p_.runqtail) // load-acquire, synchronize with the producer 4331 n := t - h 4332 n = n - n/2 4333 if n == 0 { 4334 if stealRunNextG { 4335 // Try to steal from _p_.runnext. 4336 if next := _p_.runnext; next != 0 { 4337 // Sleep to ensure that _p_ isn't about to run the g we 4338 // are about to steal. 4339 // The important use case here is when the g running on _p_ 4340 // ready()s another g and then almost immediately blocks. 4341 // Instead of stealing runnext in this window, back off 4342 // to give _p_ a chance to schedule runnext. This will avoid 4343 // thrashing gs between different Ps. 4344 // A sync chan send/recv takes ~50ns as of time of writing, 4345 // so 3us gives ~50x overshoot. 4346 if GOOS != "windows" { 4347 usleep(3) 4348 } else { 4349 // On windows system timer granularity is 1-15ms, 4350 // which is way too much for this optimization. 4351 // So just yield. 4352 osyield() 4353 } 4354 if !_p_.runnext.cas(next, 0) { 4355 continue 4356 } 4357 batch[batchHead%uint32(len(batch))] = next 4358 return 1 4359 } 4360 } 4361 return 0 4362 } 4363 if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t 4364 continue 4365 } 4366 for i := uint32(0); i < n; i++ { 4367 g := _p_.runq[(h+i)%uint32(len(_p_.runq))] 4368 batch[(batchHead+i)%uint32(len(batch))] = g 4369 } 4370 if atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume 4371 return n 4372 } 4373 } 4374 } 4375 4376 // Steal half of elements from local runnable queue of p2 4377 // and put onto local runnable queue of p. 4378 // Returns one of the stolen elements (or nil if failed). 4379 func runqsteal(_p_, p2 *p, stealRunNextG bool) *g { 4380 t := _p_.runqtail 4381 n := runqgrab(p2, &_p_.runq, t, stealRunNextG) 4382 if n == 0 { 4383 return nil 4384 } 4385 n-- 4386 gp := _p_.runq[(t+n)%uint32(len(_p_.runq))].ptr() 4387 if n == 0 { 4388 return gp 4389 } 4390 h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers 4391 if t-h+n >= uint32(len(_p_.runq)) { 4392 throw("runqsteal: runq overflow") 4393 } 4394 atomic.Store(&_p_.runqtail, t+n) // store-release, makes the item available for consumption 4395 return gp 4396 } 4397 4398 //go:linkname setMaxThreads runtime/debug.setMaxThreads 4399 func setMaxThreads(in int) (out int) { 4400 lock(&sched.lock) 4401 out = int(sched.maxmcount) 4402 if in > 0x7fffffff { // MaxInt32 4403 sched.maxmcount = 0x7fffffff 4404 } else { 4405 sched.maxmcount = int32(in) 4406 } 4407 checkmcount() 4408 unlock(&sched.lock) 4409 return 4410 } 4411 4412 func haveexperiment(name string) bool { 4413 if name == "framepointer" { 4414 return framepointer_enabled // set by linker 4415 } 4416 x := sys.Goexperiment 4417 for x != "" { 4418 xname := "" 4419 i := index(x, ",") 4420 if i < 0 { 4421 xname, x = x, "" 4422 } else { 4423 xname, x = x[:i], x[i+1:] 4424 } 4425 if xname == name { 4426 return true 4427 } 4428 if len(xname) > 2 && xname[:2] == "no" && xname[2:] == name { 4429 return false 4430 } 4431 } 4432 return false 4433 } 4434 4435 //go:nosplit 4436 func procPin() int { 4437 _g_ := getg() 4438 mp := _g_.m 4439 4440 mp.locks++ 4441 return int(mp.p.ptr().id) 4442 } 4443 4444 //go:nosplit 4445 func procUnpin() { 4446 _g_ := getg() 4447 _g_.m.locks-- 4448 } 4449 4450 //go:linkname sync_runtime_procPin sync.runtime_procPin 4451 //go:nosplit 4452 func sync_runtime_procPin() int { 4453 return procPin() 4454 } 4455 4456 //go:linkname sync_runtime_procUnpin sync.runtime_procUnpin 4457 //go:nosplit 4458 func sync_runtime_procUnpin() { 4459 procUnpin() 4460 } 4461 4462 //go:linkname sync_atomic_runtime_procPin sync/atomic.runtime_procPin 4463 //go:nosplit 4464 func sync_atomic_runtime_procPin() int { 4465 return procPin() 4466 } 4467 4468 //go:linkname sync_atomic_runtime_procUnpin sync/atomic.runtime_procUnpin 4469 //go:nosplit 4470 func sync_atomic_runtime_procUnpin() { 4471 procUnpin() 4472 } 4473 4474 // Active spinning for sync.Mutex. 4475 //go:linkname sync_runtime_canSpin sync.runtime_canSpin 4476 //go:nosplit 4477 func sync_runtime_canSpin(i int) bool { 4478 // sync.Mutex is cooperative, so we are conservative with spinning. 4479 // Spin only few times and only if running on a multicore machine and 4480 // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. 4481 // As opposed to runtime mutex we don't do passive spinning here, 4482 // because there can be work on global runq on on other Ps. 4483 if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 { 4484 return false 4485 } 4486 if p := getg().m.p.ptr(); !runqempty(p) { 4487 return false 4488 } 4489 return true 4490 } 4491 4492 //go:linkname sync_runtime_doSpin sync.runtime_doSpin 4493 //go:nosplit 4494 func sync_runtime_doSpin() { 4495 procyield(active_spin_cnt) 4496 } 4497 4498 var stealOrder randomOrder 4499 4500 // randomOrder/randomEnum are helper types for randomized work stealing. 4501 // They allow to enumerate all Ps in different pseudo-random orders without repetitions. 4502 // The algorithm is based on the fact that if we have X such that X and GOMAXPROCS 4503 // are coprime, then a sequences of (i + X) % GOMAXPROCS gives the required enumeration. 4504 type randomOrder struct { 4505 count uint32 4506 coprimes []uint32 4507 } 4508 4509 type randomEnum struct { 4510 i uint32 4511 count uint32 4512 pos uint32 4513 inc uint32 4514 } 4515 4516 func (ord *randomOrder) reset(count uint32) { 4517 ord.count = count 4518 ord.coprimes = ord.coprimes[:0] 4519 for i := uint32(1); i <= count; i++ { 4520 if gcd(i, count) == 1 { 4521 ord.coprimes = append(ord.coprimes, i) 4522 } 4523 } 4524 } 4525 4526 func (ord *randomOrder) start(i uint32) randomEnum { 4527 return randomEnum{ 4528 count: ord.count, 4529 pos: i % ord.count, 4530 inc: ord.coprimes[i%uint32(len(ord.coprimes))], 4531 } 4532 } 4533 4534 func (enum *randomEnum) done() bool { 4535 return enum.i == enum.count 4536 } 4537 4538 func (enum *randomEnum) next() { 4539 enum.i++ 4540 enum.pos = (enum.pos + enum.inc) % enum.count 4541 } 4542 4543 func (enum *randomEnum) position() uint32 { 4544 return enum.pos 4545 } 4546 4547 func gcd(a, b uint32) uint32 { 4548 for b != 0 { 4549 a, b = b, a%b 4550 } 4551 return a 4552 } 4553