1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build ppc64 ppc64le 6 7 #include "go_asm.h" 8 #include "go_tls.h" 9 #include "funcdata.h" 10 #include "textflag.h" 11 12 TEXT runtimert0_go(SB),NOSPLIT,$0 13 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer 14 15 // initialize essential registers 16 BL runtimereginit(SB) 17 18 SUB $24, R1 19 MOVW R3, 8(R1) // argc 20 MOVD R4, 16(R1) // argv 21 22 // create istack out of the given (operating system) stack. 23 // _cgo_init may update stackguard. 24 MOVD $runtimeg0(SB), g 25 MOVD $(-64*1024), R31 26 ADD R31, R1, R3 27 MOVD R3, g_stackguard0(g) 28 MOVD R3, g_stackguard1(g) 29 MOVD R3, (g_stack+stack_lo)(g) 30 MOVD R1, (g_stack+stack_hi)(g) 31 32 // if there is a _cgo_init, call it using the gcc ABI. 33 MOVD _cgo_init(SB), R12 34 CMP R0, R12 35 BEQ nocgo 36 MOVD R12, CTR // r12 = "global function entry point" 37 MOVD R13, R5 // arg 2: TLS base pointer 38 MOVD $setg_gcc<>(SB), R4 // arg 1: setg 39 MOVD g, R3 // arg 0: G 40 // C functions expect 32 bytes of space on caller stack frame 41 // and a 16-byte aligned R1 42 MOVD R1, R14 // save current stack 43 SUB $32, R1 // reserve 32 bytes 44 RLDCR $0, R1, $~15, R1 // 16-byte align 45 BL (CTR) // may clobber R0, R3-R12 46 MOVD R14, R1 // restore stack 47 XOR R0, R0 // fix R0 48 49 nocgo: 50 // update stackguard after _cgo_init 51 MOVD (g_stack+stack_lo)(g), R3 52 ADD $const__StackGuard, R3 53 MOVD R3, g_stackguard0(g) 54 MOVD R3, g_stackguard1(g) 55 56 // set the per-goroutine and per-mach "registers" 57 MOVD $runtimem0(SB), R3 58 59 // save m->g0 = g0 60 MOVD g, m_g0(R3) 61 // save m0 to g0->m 62 MOVD R3, g_m(g) 63 64 BL runtimecheck(SB) 65 66 // args are already prepared 67 BL runtimeargs(SB) 68 BL runtimeosinit(SB) 69 BL runtimeschedinit(SB) 70 71 // create a new goroutine to start program 72 MOVD $runtimemainPC(SB), R3 // entry 73 MOVDU R3, -8(R1) 74 MOVDU R0, -8(R1) 75 MOVDU R0, -8(R1) 76 BL runtimenewproc(SB) 77 ADD $24, R1 78 79 // start this M 80 BL runtimemstart(SB) 81 82 MOVD R0, 1(R0) 83 RET 84 85 DATA runtimemainPC+0(SB)/8,$runtimemain(SB) 86 GLOBL runtimemainPC(SB),RODATA,$8 87 88 TEXT runtimebreakpoint(SB),NOSPLIT,$-8-0 89 MOVD R0, 2(R0) // TODO: TD 90 RET 91 92 TEXT runtimeasminit(SB),NOSPLIT,$-8-0 93 RET 94 95 TEXT _cgo_reginit(SB),NOSPLIT,$-8-0 96 // crosscall_ppc64 and crosscall2 need to reginit, but can't 97 // get at the 'runtime.reginit' symbol. 98 BR runtimereginit(SB) 99 100 TEXT runtimereginit(SB),NOSPLIT,$-8-0 101 // set R0 to zero, it's expected by the toolchain 102 XOR R0, R0 103 // initialize essential FP registers 104 FMOVD $4503601774854144.0, F27 105 FMOVD $0.5, F29 106 FSUB F29, F29, F28 107 FADD F29, F29, F30 108 FADD F30, F30, F31 109 RET 110 111 /* 112 * go-routine 113 */ 114 115 // void gosave(Gobuf*) 116 // save state in Gobuf; setjmp 117 TEXT runtimegosave(SB), NOSPLIT, $-8-8 118 MOVD buf+0(FP), R3 119 MOVD R1, gobuf_sp(R3) 120 MOVD LR, R31 121 MOVD R31, gobuf_pc(R3) 122 MOVD g, gobuf_g(R3) 123 MOVD R0, gobuf_lr(R3) 124 MOVD R0, gobuf_ret(R3) 125 MOVD R0, gobuf_ctxt(R3) 126 RET 127 128 // void gogo(Gobuf*) 129 // restore state from Gobuf; longjmp 130 TEXT runtimegogo(SB), NOSPLIT, $-8-8 131 MOVD buf+0(FP), R5 132 MOVD gobuf_g(R5), g // make sure g is not nil 133 BL runtimesave_g(SB) 134 135 MOVD 0(g), R4 136 MOVD gobuf_sp(R5), R1 137 MOVD gobuf_lr(R5), R31 138 MOVD R31, LR 139 MOVD gobuf_ret(R5), R3 140 MOVD gobuf_ctxt(R5), R11 141 MOVD R0, gobuf_sp(R5) 142 MOVD R0, gobuf_ret(R5) 143 MOVD R0, gobuf_lr(R5) 144 MOVD R0, gobuf_ctxt(R5) 145 CMP R0, R0 // set condition codes for == test, needed by stack split 146 MOVD gobuf_pc(R5), R31 147 MOVD R31, CTR 148 BR (CTR) 149 150 // void mcall(fn func(*g)) 151 // Switch to m->g0's stack, call fn(g). 152 // Fn must never return. It should gogo(&g->sched) 153 // to keep running g. 154 TEXT runtimemcall(SB), NOSPLIT, $-8-8 155 // Save caller state in g->sched 156 MOVD R1, (g_sched+gobuf_sp)(g) 157 MOVD LR, R31 158 MOVD R31, (g_sched+gobuf_pc)(g) 159 MOVD R0, (g_sched+gobuf_lr)(g) 160 MOVD g, (g_sched+gobuf_g)(g) 161 162 // Switch to m->g0 & its stack, call fn. 163 MOVD g, R3 164 MOVD g_m(g), R8 165 MOVD m_g0(R8), g 166 BL runtimesave_g(SB) 167 CMP g, R3 168 BNE 2(PC) 169 BR runtimebadmcall(SB) 170 MOVD fn+0(FP), R11 // context 171 MOVD 0(R11), R4 // code pointer 172 MOVD R4, CTR 173 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp 174 MOVDU R3, -8(R1) 175 MOVDU R0, -8(R1) 176 BL (CTR) 177 BR runtimebadmcall2(SB) 178 179 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 180 // of the G stack. We need to distinguish the routine that 181 // lives at the bottom of the G stack from the one that lives 182 // at the top of the system stack because the one at the top of 183 // the system stack terminates the stack walk (see topofstack()). 184 TEXT runtimesystemstack_switch(SB), NOSPLIT, $0-0 185 UNDEF 186 BL (LR) // make sure this function is not leaf 187 RET 188 189 // func systemstack(fn func()) 190 TEXT runtimesystemstack(SB), NOSPLIT, $0-8 191 MOVD fn+0(FP), R3 // R3 = fn 192 MOVD R3, R11 // context 193 MOVD g_m(g), R4 // R4 = m 194 195 MOVD m_gsignal(R4), R5 // R5 = gsignal 196 CMP g, R5 197 BEQ noswitch 198 199 MOVD m_g0(R4), R5 // R5 = g0 200 CMP g, R5 201 BEQ noswitch 202 203 MOVD m_curg(R4), R6 204 CMP g, R6 205 BEQ switch 206 207 // Bad: g is not gsignal, not g0, not curg. What is it? 208 // Hide call from linker nosplit analysis. 209 MOVD $runtimebadsystemstack(SB), R3 210 MOVD R3, CTR 211 BL (CTR) 212 213 switch: 214 // save our state in g->sched. Pretend to 215 // be systemstack_switch if the G stack is scanned. 216 MOVD $runtimesystemstack_switch(SB), R6 217 ADD $8, R6 // get past prologue 218 MOVD R6, (g_sched+gobuf_pc)(g) 219 MOVD R1, (g_sched+gobuf_sp)(g) 220 MOVD R0, (g_sched+gobuf_lr)(g) 221 MOVD g, (g_sched+gobuf_g)(g) 222 223 // switch to g0 224 MOVD R5, g 225 BL runtimesave_g(SB) 226 MOVD (g_sched+gobuf_sp)(g), R3 227 // make it look like mstart called systemstack on g0, to stop traceback 228 SUB $8, R3 229 MOVD $runtimemstart(SB), R4 230 MOVD R4, 0(R3) 231 MOVD R3, R1 232 233 // call target function 234 MOVD 0(R11), R3 // code pointer 235 MOVD R3, CTR 236 BL (CTR) 237 238 // switch back to g 239 MOVD g_m(g), R3 240 MOVD m_curg(R3), g 241 BL runtimesave_g(SB) 242 MOVD (g_sched+gobuf_sp)(g), R1 243 MOVD R0, (g_sched+gobuf_sp)(g) 244 RET 245 246 noswitch: 247 // already on m stack, just call directly 248 MOVD 0(R11), R3 // code pointer 249 MOVD R3, CTR 250 BL (CTR) 251 RET 252 253 /* 254 * support for morestack 255 */ 256 257 // Called during function prolog when more stack is needed. 258 // Caller has already loaded: 259 // R3: framesize, R4: argsize, R5: LR 260 // 261 // The traceback routines see morestack on a g0 as being 262 // the top of a stack (for example, morestack calling newstack 263 // calling the scheduler calling newm calling gc), so we must 264 // record an argument size. For that purpose, it has no arguments. 265 TEXT runtimemorestack(SB),NOSPLIT,$-8-0 266 // Cannot grow scheduler stack (m->g0). 267 MOVD g_m(g), R7 268 MOVD m_g0(R7), R8 269 CMP g, R8 270 BNE 2(PC) 271 BL runtimeabort(SB) 272 273 // Cannot grow signal stack (m->gsignal). 274 MOVD m_gsignal(R7), R8 275 CMP g, R8 276 BNE 2(PC) 277 BL runtimeabort(SB) 278 279 // Called from f. 280 // Set g->sched to context in f. 281 MOVD R11, (g_sched+gobuf_ctxt)(g) 282 MOVD R1, (g_sched+gobuf_sp)(g) 283 MOVD LR, R8 284 MOVD R8, (g_sched+gobuf_pc)(g) 285 MOVD R5, (g_sched+gobuf_lr)(g) 286 287 // Called from f. 288 // Set m->morebuf to f's caller. 289 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 290 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 291 MOVD g, (m_morebuf+gobuf_g)(R7) 292 293 // Call newstack on m->g0's stack. 294 MOVD m_g0(R7), g 295 BL runtimesave_g(SB) 296 MOVD (g_sched+gobuf_sp)(g), R1 297 BL runtimenewstack(SB) 298 299 // Not reached, but make sure the return PC from the call to newstack 300 // is still in this function, and not the beginning of the next. 301 UNDEF 302 303 TEXT runtimemorestack_noctxt(SB),NOSPLIT,$-8-0 304 MOVD R0, R11 305 BR runtimemorestack(SB) 306 307 TEXT runtimestackBarrier(SB),NOSPLIT,$0 308 // We came here via a RET to an overwritten LR. 309 // R3 may be live. Other registers are available. 310 311 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 312 MOVD (g_stkbar+slice_array)(g), R4 313 MOVD g_stkbarPos(g), R5 314 MOVD $stkbar__size, R6 315 MULLD R5, R6 316 ADD R4, R6 317 MOVD stkbar_savedLRVal(R6), R6 318 // Record that this stack barrier was hit. 319 ADD $1, R5 320 MOVD R5, g_stkbarPos(g) 321 // Jump to the original return PC. 322 MOVD R6, CTR 323 BR (CTR) 324 325 // reflectcall: call a function with the given argument list 326 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 327 // we don't have variable-sized frames, so we use a small number 328 // of constant-sized-frame functions to encode a few bits of size in the pc. 329 // Caution: ugly multiline assembly macros in your future! 330 331 #define DISPATCH(NAME,MAXSIZE) \ 332 MOVD $MAXSIZE, R31; \ 333 CMP R3, R31; \ 334 BGT 4(PC); \ 335 MOVD $NAME(SB), R31; \ 336 MOVD R31, CTR; \ 337 BR (CTR) 338 // Note: can't just "BR NAME(SB)" - bad inlining results. 339 340 TEXT reflectcall(SB), NOSPLIT, $0-0 341 BR reflectcall(SB) 342 343 TEXT reflectcall(SB), NOSPLIT, $-8-32 344 MOVWZ argsize+24(FP), R3 345 // NOTE(rsc): No call16, because CALLFN needs four words 346 // of argument space to invoke callwritebarrier. 347 DISPATCH(runtimecall32, 32) 348 DISPATCH(runtimecall64, 64) 349 DISPATCH(runtimecall128, 128) 350 DISPATCH(runtimecall256, 256) 351 DISPATCH(runtimecall512, 512) 352 DISPATCH(runtimecall1024, 1024) 353 DISPATCH(runtimecall2048, 2048) 354 DISPATCH(runtimecall4096, 4096) 355 DISPATCH(runtimecall8192, 8192) 356 DISPATCH(runtimecall16384, 16384) 357 DISPATCH(runtimecall32768, 32768) 358 DISPATCH(runtimecall65536, 65536) 359 DISPATCH(runtimecall131072, 131072) 360 DISPATCH(runtimecall262144, 262144) 361 DISPATCH(runtimecall524288, 524288) 362 DISPATCH(runtimecall1048576, 1048576) 363 DISPATCH(runtimecall2097152, 2097152) 364 DISPATCH(runtimecall4194304, 4194304) 365 DISPATCH(runtimecall8388608, 8388608) 366 DISPATCH(runtimecall16777216, 16777216) 367 DISPATCH(runtimecall33554432, 33554432) 368 DISPATCH(runtimecall67108864, 67108864) 369 DISPATCH(runtimecall134217728, 134217728) 370 DISPATCH(runtimecall268435456, 268435456) 371 DISPATCH(runtimecall536870912, 536870912) 372 DISPATCH(runtimecall1073741824, 1073741824) 373 MOVD $runtimebadreflectcall(SB), R31 374 MOVD R31, CTR 375 BR (CTR) 376 377 #define CALLFN(NAME,MAXSIZE) \ 378 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ 379 NO_LOCAL_POINTERS; \ 380 /* copy arguments to stack */ \ 381 MOVD arg+16(FP), R3; \ 382 MOVWZ argsize+24(FP), R4; \ 383 MOVD R1, R5; \ 384 ADD $(8-1), R5; \ 385 SUB $1, R3; \ 386 ADD R5, R4; \ 387 CMP R5, R4; \ 388 BEQ 4(PC); \ 389 MOVBZU 1(R3), R6; \ 390 MOVBZU R6, 1(R5); \ 391 BR -4(PC); \ 392 /* call function */ \ 393 MOVD f+8(FP), R11; \ 394 MOVD (R11), R31; \ 395 MOVD R31, CTR; \ 396 PCDATA $PCDATA_StackMapIndex, $0; \ 397 BL (CTR); \ 398 /* copy return values back */ \ 399 MOVD arg+16(FP), R3; \ 400 MOVWZ n+24(FP), R4; \ 401 MOVWZ retoffset+28(FP), R6; \ 402 MOVD R1, R5; \ 403 ADD R6, R5; \ 404 ADD R6, R3; \ 405 SUB R6, R4; \ 406 ADD $(8-1), R5; \ 407 SUB $1, R3; \ 408 ADD R5, R4; \ 409 loop: \ 410 CMP R5, R4; \ 411 BEQ end; \ 412 MOVBZU 1(R5), R6; \ 413 MOVBZU R6, 1(R3); \ 414 BR loop; \ 415 end: \ 416 /* execute write barrier updates */ \ 417 MOVD argtype+0(FP), R7; \ 418 MOVD arg+16(FP), R3; \ 419 MOVWZ n+24(FP), R4; \ 420 MOVWZ retoffset+28(FP), R6; \ 421 MOVD R7, 8(R1); \ 422 MOVD R3, 16(R1); \ 423 MOVD R4, 24(R1); \ 424 MOVD R6, 32(R1); \ 425 BL runtimecallwritebarrier(SB); \ 426 RET 427 428 CALLFN(call16, 16) 429 CALLFN(call32, 32) 430 CALLFN(call64, 64) 431 CALLFN(call128, 128) 432 CALLFN(call256, 256) 433 CALLFN(call512, 512) 434 CALLFN(call1024, 1024) 435 CALLFN(call2048, 2048) 436 CALLFN(call4096, 4096) 437 CALLFN(call8192, 8192) 438 CALLFN(call16384, 16384) 439 CALLFN(call32768, 32768) 440 CALLFN(call65536, 65536) 441 CALLFN(call131072, 131072) 442 CALLFN(call262144, 262144) 443 CALLFN(call524288, 524288) 444 CALLFN(call1048576, 1048576) 445 CALLFN(call2097152, 2097152) 446 CALLFN(call4194304, 4194304) 447 CALLFN(call8388608, 8388608) 448 CALLFN(call16777216, 16777216) 449 CALLFN(call33554432, 33554432) 450 CALLFN(call67108864, 67108864) 451 CALLFN(call134217728, 134217728) 452 CALLFN(call268435456, 268435456) 453 CALLFN(call536870912, 536870912) 454 CALLFN(call1073741824, 1073741824) 455 456 // bool cas(uint32 *ptr, uint32 old, uint32 new) 457 // Atomically: 458 // if(*val == old){ 459 // *val = new; 460 // return 1; 461 // } else 462 // return 0; 463 TEXT runtimecas(SB), NOSPLIT, $0-17 464 MOVD ptr+0(FP), R3 465 MOVWZ old+8(FP), R4 466 MOVWZ new+12(FP), R5 467 cas_again: 468 SYNC 469 LWAR (R3), R6 470 CMPW R6, R4 471 BNE cas_fail 472 STWCCC R5, (R3) 473 BNE cas_again 474 MOVD $1, R3 475 SYNC 476 ISYNC 477 MOVB R3, ret+16(FP) 478 RET 479 cas_fail: 480 MOVD $0, R3 481 BR -5(PC) 482 483 // bool runtimecas64(uint64 *ptr, uint64 old, uint64 new) 484 // Atomically: 485 // if(*val == *old){ 486 // *val = new; 487 // return 1; 488 // } else { 489 // return 0; 490 // } 491 TEXT runtimecas64(SB), NOSPLIT, $0-25 492 MOVD ptr+0(FP), R3 493 MOVD old+8(FP), R4 494 MOVD new+16(FP), R5 495 cas64_again: 496 SYNC 497 LDAR (R3), R6 498 CMP R6, R4 499 BNE cas64_fail 500 STDCCC R5, (R3) 501 BNE cas64_again 502 MOVD $1, R3 503 SYNC 504 ISYNC 505 MOVB R3, ret+24(FP) 506 RET 507 cas64_fail: 508 MOVD $0, R3 509 BR -5(PC) 510 511 TEXT runtimecasuintptr(SB), NOSPLIT, $0-25 512 BR runtimecas64(SB) 513 514 TEXT runtimeatomicloaduintptr(SB), NOSPLIT, $-8-16 515 BR runtimeatomicload64(SB) 516 517 TEXT runtimeatomicloaduint(SB), NOSPLIT, $-8-16 518 BR runtimeatomicload64(SB) 519 520 TEXT runtimeatomicstoreuintptr(SB), NOSPLIT, $0-16 521 BR runtimeatomicstore64(SB) 522 523 // bool casp(void **val, void *old, void *new) 524 // Atomically: 525 // if(*val == old){ 526 // *val = new; 527 // return 1; 528 // } else 529 // return 0; 530 TEXT runtimecasp1(SB), NOSPLIT, $0-25 531 BR runtimecas64(SB) 532 533 // uint32 xadd(uint32 volatile *ptr, int32 delta) 534 // Atomically: 535 // *val += delta; 536 // return *val; 537 TEXT runtimexadd(SB), NOSPLIT, $0-20 538 MOVD ptr+0(FP), R4 539 MOVW delta+8(FP), R5 540 SYNC 541 LWAR (R4), R3 542 ADD R5, R3 543 STWCCC R3, (R4) 544 BNE -4(PC) 545 SYNC 546 ISYNC 547 MOVW R3, ret+16(FP) 548 RET 549 550 TEXT runtimexadd64(SB), NOSPLIT, $0-24 551 MOVD ptr+0(FP), R4 552 MOVD delta+8(FP), R5 553 SYNC 554 LDAR (R4), R3 555 ADD R5, R3 556 STDCCC R3, (R4) 557 BNE -4(PC) 558 SYNC 559 ISYNC 560 MOVD R3, ret+16(FP) 561 RET 562 563 TEXT runtimexchg(SB), NOSPLIT, $0-20 564 MOVD ptr+0(FP), R4 565 MOVW new+8(FP), R5 566 SYNC 567 LWAR (R4), R3 568 STWCCC R5, (R4) 569 BNE -3(PC) 570 SYNC 571 ISYNC 572 MOVW R3, ret+16(FP) 573 RET 574 575 TEXT runtimexchg64(SB), NOSPLIT, $0-24 576 MOVD ptr+0(FP), R4 577 MOVD new+8(FP), R5 578 SYNC 579 LDAR (R4), R3 580 STDCCC R5, (R4) 581 BNE -3(PC) 582 SYNC 583 ISYNC 584 MOVD R3, ret+16(FP) 585 RET 586 587 TEXT runtimexchgp1(SB), NOSPLIT, $0-24 588 BR runtimexchg64(SB) 589 590 TEXT runtimexchguintptr(SB), NOSPLIT, $0-24 591 BR runtimexchg64(SB) 592 593 TEXT runtimeprocyield(SB),NOSPLIT,$0-0 594 RET 595 596 TEXT runtimeatomicstorep1(SB), NOSPLIT, $0-16 597 BR runtimeatomicstore64(SB) 598 599 TEXT runtimeatomicstore(SB), NOSPLIT, $0-12 600 MOVD ptr+0(FP), R3 601 MOVW val+8(FP), R4 602 SYNC 603 MOVW R4, 0(R3) 604 RET 605 606 TEXT runtimeatomicstore64(SB), NOSPLIT, $0-16 607 MOVD ptr+0(FP), R3 608 MOVD val+8(FP), R4 609 SYNC 610 MOVD R4, 0(R3) 611 RET 612 613 // void runtimeatomicor8(byte volatile*, byte); 614 TEXT runtimeatomicor8(SB), NOSPLIT, $0-9 615 MOVD ptr+0(FP), R3 616 MOVBZ val+8(FP), R4 617 // Align ptr down to 4 bytes so we can use 32-bit load/store. 618 // R5 = (R3 << 0) & ~3 619 RLDCR $0, R3, $~3, R5 620 // Compute val shift. 621 #ifdef GOARCH_ppc64 622 // Big endian. ptr = ptr ^ 3 623 XOR $3, R3 624 #endif 625 // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) 626 RLDC $3, R3, $(3*8), R6 627 // Shift val for aligned ptr. R4 = val << R6 628 SLD R6, R4, R4 629 630 again: 631 SYNC 632 LWAR (R5), R6 633 OR R4, R6 634 STWCCC R6, (R5) 635 BNE again 636 SYNC 637 ISYNC 638 RET 639 640 // void runtimeatomicand8(byte volatile*, byte); 641 TEXT runtimeatomicand8(SB), NOSPLIT, $0-9 642 MOVD ptr+0(FP), R3 643 MOVBZ val+8(FP), R4 644 // Align ptr down to 4 bytes so we can use 32-bit load/store. 645 // R5 = (R3 << 0) & ~3 646 RLDCR $0, R3, $~3, R5 647 // Compute val shift. 648 #ifdef GOARCH_ppc64 649 // Big endian. ptr = ptr ^ 3 650 XOR $3, R3 651 #endif 652 // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) 653 RLDC $3, R3, $(3*8), R6 654 // Shift val for aligned ptr. R4 = val << R6 | ^(0xFF << R6) 655 MOVD $0xFF, R7 656 SLD R6, R4 657 SLD R6, R7 658 XOR $-1, R7 659 OR R7, R4 660 again: 661 SYNC 662 LWAR (R5), R6 663 AND R4, R6 664 STWCCC R6, (R5) 665 BNE again 666 SYNC 667 ISYNC 668 RET 669 670 // void jmpdefer(fv, sp); 671 // called from deferreturn. 672 // 1. grab stored LR for caller 673 // 2. sub 4 bytes to get back to BL deferreturn 674 // 3. BR to fn 675 TEXT runtimejmpdefer(SB), NOSPLIT, $-8-16 676 MOVD 0(R1), R31 677 SUB $4, R31 678 MOVD R31, LR 679 680 MOVD fv+0(FP), R11 681 MOVD argp+8(FP), R1 682 SUB $8, R1 683 MOVD 0(R11), R3 684 MOVD R3, CTR 685 BR (CTR) 686 687 // Save state of caller into g->sched. Smashes R31. 688 TEXT gosave<>(SB),NOSPLIT,$-8 689 MOVD LR, R31 690 MOVD R31, (g_sched+gobuf_pc)(g) 691 MOVD R1, (g_sched+gobuf_sp)(g) 692 MOVD R0, (g_sched+gobuf_lr)(g) 693 MOVD R0, (g_sched+gobuf_ret)(g) 694 MOVD R0, (g_sched+gobuf_ctxt)(g) 695 RET 696 697 // func asmcgocall(fn, arg unsafe.Pointer) int32 698 // Call fn(arg) on the scheduler stack, 699 // aligned appropriately for the gcc ABI. 700 // See cgocall.go for more details. 701 TEXT asmcgocall(SB),NOSPLIT,$0-20 702 MOVD fn+0(FP), R3 703 MOVD arg+8(FP), R4 704 705 MOVD R1, R2 // save original stack pointer 706 MOVD g, R5 707 708 // Figure out if we need to switch to m->g0 stack. 709 // We get called to create new OS threads too, and those 710 // come in on the m->g0 stack already. 711 MOVD g_m(g), R6 712 MOVD m_g0(R6), R6 713 CMP R6, g 714 BEQ g0 715 BL gosave<>(SB) 716 MOVD R6, g 717 BL runtimesave_g(SB) 718 MOVD (g_sched+gobuf_sp)(g), R1 719 720 // Now on a scheduling stack (a pthread-created stack). 721 g0: 722 // Save room for two of our pointers, plus 32 bytes of callee 723 // save area that lives on the caller stack. 724 SUB $48, R1 725 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI 726 MOVD R5, 40(R1) // save old g on stack 727 MOVD (g_stack+stack_hi)(R5), R5 728 SUB R2, R5 729 MOVD R5, 32(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 730 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?) 731 // This is a "global call", so put the global entry point in r12 732 MOVD R3, R12 733 MOVD R12, CTR 734 MOVD R4, R3 // arg in r3 735 BL (CTR) 736 737 // C code can clobber R0, so set it back to 0. F27-F31 are 738 // callee save, so we don't need to recover those. 739 XOR R0, R0 740 // Restore g, stack pointer. R3 is errno, so don't touch it 741 MOVD 40(R1), g 742 BL runtimesave_g(SB) 743 MOVD (g_stack+stack_hi)(g), R5 744 MOVD 32(R1), R6 745 SUB R6, R5 746 MOVD R5, R1 747 748 MOVW R3, ret+16(FP) 749 RET 750 751 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize) 752 // Turn the fn into a Go func (by taking its address) and call 753 // cgocallback_gofunc. 754 TEXT runtimecgocallback(SB),NOSPLIT,$24-24 755 MOVD $fn+0(FP), R3 756 MOVD R3, 8(R1) 757 MOVD frame+8(FP), R3 758 MOVD R3, 16(R1) 759 MOVD framesize+16(FP), R3 760 MOVD R3, 24(R1) 761 MOVD $runtimecgocallback_gofunc(SB), R3 762 MOVD R3, CTR 763 BL (CTR) 764 RET 765 766 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize) 767 // See cgocall.go for more details. 768 TEXT cgocallback_gofunc(SB),NOSPLIT,$16-24 769 NO_LOCAL_POINTERS 770 771 // Load m and g from thread-local storage. 772 MOVB runtimeiscgo(SB), R3 773 CMP R3, $0 774 BEQ nocgo 775 BL runtimeload_g(SB) 776 nocgo: 777 778 // If g is nil, Go did not create the current thread. 779 // Call needm to obtain one for temporary use. 780 // In this case, we're running on the thread stack, so there's 781 // lots of space, but the linker doesn't know. Hide the call from 782 // the linker analysis by using an indirect call. 783 CMP g, $0 784 BNE havem 785 MOVD g, savedm-8(SP) // g is zero, so is m. 786 MOVD $runtimeneedm(SB), R3 787 MOVD R3, CTR 788 BL (CTR) 789 790 // Set m->sched.sp = SP, so that if a panic happens 791 // during the function we are about to execute, it will 792 // have a valid SP to run on the g0 stack. 793 // The next few lines (after the havem label) 794 // will save this SP onto the stack and then write 795 // the same SP back to m->sched.sp. That seems redundant, 796 // but if an unrecovered panic happens, unwindm will 797 // restore the g->sched.sp from the stack location 798 // and then systemstack will try to use it. If we don't set it here, 799 // that restored SP will be uninitialized (typically 0) and 800 // will not be usable. 801 MOVD g_m(g), R3 802 MOVD m_g0(R3), R3 803 MOVD R1, (g_sched+gobuf_sp)(R3) 804 805 havem: 806 MOVD g_m(g), R8 807 MOVD R8, savedm-8(SP) 808 // Now there's a valid m, and we're running on its m->g0. 809 // Save current m->g0->sched.sp on stack and then set it to SP. 810 // Save current sp in m->g0->sched.sp in preparation for 811 // switch back to m->curg stack. 812 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). 813 MOVD m_g0(R8), R3 814 MOVD (g_sched+gobuf_sp)(R3), R4 815 MOVD R4, savedsp-16(SP) 816 MOVD R1, (g_sched+gobuf_sp)(R3) 817 818 // Switch to m->curg stack and call runtime.cgocallbackg. 819 // Because we are taking over the execution of m->curg 820 // but *not* resuming what had been running, we need to 821 // save that information (m->curg->sched) so we can restore it. 822 // We can restore m->curg->sched.sp easily, because calling 823 // runtime.cgocallbackg leaves SP unchanged upon return. 824 // To save m->curg->sched.pc, we push it onto the stack. 825 // This has the added benefit that it looks to the traceback 826 // routine like cgocallbackg is going to return to that 827 // PC (because the frame we allocate below has the same 828 // size as cgocallback_gofunc's frame declared above) 829 // so that the traceback will seamlessly trace back into 830 // the earlier calls. 831 // 832 // In the new goroutine, -16(SP) and -8(SP) are unused. 833 MOVD m_curg(R8), g 834 BL runtimesave_g(SB) 835 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 836 MOVD (g_sched+gobuf_pc)(g), R5 837 MOVD R5, -24(R4) 838 MOVD $-24(R4), R1 839 BL runtimecgocallbackg(SB) 840 841 // Restore g->sched (== m->curg->sched) from saved values. 842 MOVD 0(R1), R5 843 MOVD R5, (g_sched+gobuf_pc)(g) 844 MOVD $24(R1), R4 845 MOVD R4, (g_sched+gobuf_sp)(g) 846 847 // Switch back to m->g0's stack and restore m->g0->sched.sp. 848 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 849 // so we do not have to restore it.) 850 MOVD g_m(g), R8 851 MOVD m_g0(R8), g 852 BL runtimesave_g(SB) 853 MOVD (g_sched+gobuf_sp)(g), R1 854 MOVD savedsp-16(SP), R4 855 MOVD R4, (g_sched+gobuf_sp)(g) 856 857 // If the m on entry was nil, we called needm above to borrow an m 858 // for the duration of the call. Since the call is over, return it with dropm. 859 MOVD savedm-8(SP), R6 860 CMP R6, $0 861 BNE droppedm 862 MOVD $runtimedropm(SB), R3 863 MOVD R3, CTR 864 BL (CTR) 865 droppedm: 866 867 // Done! 868 RET 869 870 // void setg(G*); set g. for use by needm. 871 TEXT runtimesetg(SB), NOSPLIT, $0-8 872 MOVD gg+0(FP), g 873 // This only happens if iscgo, so jump straight to save_g 874 BL runtimesave_g(SB) 875 RET 876 877 // void setg_gcc(G*); set g in C TLS. 878 // Must obey the gcc calling convention. 879 TEXT setg_gcc<>(SB),NOSPLIT,$-8-0 880 // The standard prologue clobbers R31, which is callee-save in 881 // the C ABI, so we have to use $-8-0 and save LR ourselves. 882 MOVD LR, R4 883 // Also save g and R31, since they're callee-save in C ABI 884 MOVD R31, R5 885 MOVD g, R6 886 887 MOVD R3, g 888 BL runtimesave_g(SB) 889 890 MOVD R6, g 891 MOVD R5, R31 892 MOVD R4, LR 893 RET 894 895 TEXT runtimegetcallerpc(SB),NOSPLIT,$8-16 896 MOVD 16(R1), R3 // LR saved by caller 897 MOVD runtimestackBarrierPC(SB), R4 898 CMP R3, R4 899 BNE nobar 900 // Get original return PC. 901 BL runtimenextBarrierPC(SB) 902 MOVD 8(R1), R3 903 nobar: 904 MOVD R3, ret+8(FP) 905 RET 906 907 TEXT runtimesetcallerpc(SB),NOSPLIT,$8-16 908 MOVD pc+8(FP), R3 909 MOVD 16(R1), R4 910 MOVD runtimestackBarrierPC(SB), R5 911 CMP R4, R5 912 BEQ setbar 913 MOVD R3, 16(R1) // set LR in caller 914 RET 915 setbar: 916 // Set the stack barrier return PC. 917 MOVD R3, 8(R1) 918 BL runtimesetNextBarrierPC(SB) 919 RET 920 921 TEXT runtimegetcallersp(SB),NOSPLIT,$0-16 922 MOVD argp+0(FP), R3 923 SUB $8, R3 924 MOVD R3, ret+8(FP) 925 RET 926 927 TEXT runtimeabort(SB),NOSPLIT,$-8-0 928 MOVW (R0), R0 929 UNDEF 930 931 #define TBRL 268 932 #define TBRU 269 /* Time base Upper/Lower */ 933 934 // int64 runtimecputicks(void) 935 TEXT runtimecputicks(SB),NOSPLIT,$0-8 936 MOVW SPR(TBRU), R4 937 MOVW SPR(TBRL), R3 938 MOVW SPR(TBRU), R5 939 CMPW R4, R5 940 BNE -4(PC) 941 SLD $32, R5 942 OR R5, R3 943 MOVD R3, ret+0(FP) 944 RET 945 946 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 947 // redirects to memhash(p, h, size) using the size 948 // stored in the closure. 949 TEXT runtimememhash_varlen(SB),NOSPLIT,$40-24 950 GO_ARGS 951 NO_LOCAL_POINTERS 952 MOVD p+0(FP), R3 953 MOVD h+8(FP), R4 954 MOVD 8(R11), R5 955 MOVD R3, 8(R1) 956 MOVD R4, 16(R1) 957 MOVD R5, 24(R1) 958 BL runtimememhash(SB) 959 MOVD 32(R1), R3 960 MOVD R3, ret+16(FP) 961 RET 962 963 // AES hashing not implemented for ppc64 964 TEXT runtimeaeshash(SB),NOSPLIT,$-8-0 965 MOVW (R0), R1 966 TEXT runtimeaeshash32(SB),NOSPLIT,$-8-0 967 MOVW (R0), R1 968 TEXT runtimeaeshash64(SB),NOSPLIT,$-8-0 969 MOVW (R0), R1 970 TEXT runtimeaeshashstr(SB),NOSPLIT,$-8-0 971 MOVW (R0), R1 972 973 TEXT runtimememeq(SB),NOSPLIT,$-8-25 974 MOVD a+0(FP), R3 975 MOVD b+8(FP), R4 976 MOVD size+16(FP), R5 977 SUB $1, R3 978 SUB $1, R4 979 ADD R3, R5, R8 980 loop: 981 CMP R3, R8 982 BNE test 983 MOVD $1, R3 984 MOVB R3, ret+24(FP) 985 RET 986 test: 987 MOVBZU 1(R3), R6 988 MOVBZU 1(R4), R7 989 CMP R6, R7 990 BEQ loop 991 992 MOVB R0, ret+24(FP) 993 RET 994 995 // memequal_varlen(a, b unsafe.Pointer) bool 996 TEXT runtimememequal_varlen(SB),NOSPLIT,$40-17 997 MOVD a+0(FP), R3 998 MOVD b+8(FP), R4 999 CMP R3, R4 1000 BEQ eq 1001 MOVD 8(R11), R5 // compiler stores size at offset 8 in the closure 1002 MOVD R3, 8(R1) 1003 MOVD R4, 16(R1) 1004 MOVD R5, 24(R1) 1005 BL runtimememeq(SB) 1006 MOVBZ 32(R1), R3 1007 MOVB R3, ret+16(FP) 1008 RET 1009 eq: 1010 MOVD $1, R3 1011 MOVB R3, ret+16(FP) 1012 RET 1013 1014 // eqstring tests whether two strings are equal. 1015 // The compiler guarantees that strings passed 1016 // to eqstring have equal length. 1017 // See runtime_test.go:eqstring_generic for 1018 // equivalent Go code. 1019 TEXT runtimeeqstring(SB),NOSPLIT,$0-33 1020 MOVD s1str+0(FP), R3 1021 MOVD s2str+16(FP), R4 1022 MOVD $1, R5 1023 MOVB R5, ret+32(FP) 1024 CMP R3, R4 1025 BNE 2(PC) 1026 RET 1027 MOVD s1len+8(FP), R5 1028 SUB $1, R3 1029 SUB $1, R4 1030 ADD R3, R5, R8 1031 loop: 1032 CMP R3, R8 1033 BNE 2(PC) 1034 RET 1035 MOVBZU 1(R3), R6 1036 MOVBZU 1(R4), R7 1037 CMP R6, R7 1038 BEQ loop 1039 MOVB R0, ret+32(FP) 1040 RET 1041 1042 // TODO: share code with memeq? 1043 TEXT bytesEqual(SB),NOSPLIT,$0-49 1044 MOVD a_len+8(FP), R3 1045 MOVD b_len+32(FP), R4 1046 1047 CMP R3, R4 // unequal lengths are not equal 1048 BNE noteq 1049 1050 MOVD a+0(FP), R5 1051 MOVD b+24(FP), R6 1052 SUB $1, R5 1053 SUB $1, R6 1054 ADD R5, R3 // end-1 1055 1056 loop: 1057 CMP R5, R3 1058 BEQ equal // reached the end 1059 MOVBZU 1(R5), R4 1060 MOVBZU 1(R6), R7 1061 CMP R4, R7 1062 BEQ loop 1063 1064 noteq: 1065 MOVBZ R0, ret+48(FP) 1066 RET 1067 1068 equal: 1069 MOVD $1, R3 1070 MOVBZ R3, ret+48(FP) 1071 RET 1072 1073 TEXT bytesIndexByte(SB),NOSPLIT,$0-40 1074 MOVD s+0(FP), R3 1075 MOVD s_len+8(FP), R4 1076 MOVBZ c+24(FP), R5 // byte to find 1077 MOVD R3, R6 // store base for later 1078 SUB $1, R3 1079 ADD R3, R4 // end-1 1080 1081 loop: 1082 CMP R3, R4 1083 BEQ notfound 1084 MOVBZU 1(R3), R7 1085 CMP R7, R5 1086 BNE loop 1087 1088 SUB R6, R3 // remove base 1089 MOVD R3, ret+32(FP) 1090 RET 1091 1092 notfound: 1093 MOVD $-1, R3 1094 MOVD R3, ret+32(FP) 1095 RET 1096 1097 TEXT stringsIndexByte(SB),NOSPLIT,$0-32 1098 MOVD p+0(FP), R3 1099 MOVD b_len+8(FP), R4 1100 MOVBZ c+16(FP), R5 // byte to find 1101 MOVD R3, R6 // store base for later 1102 SUB $1, R3 1103 ADD R3, R4 // end-1 1104 1105 loop: 1106 CMP R3, R4 1107 BEQ notfound 1108 MOVBZU 1(R3), R7 1109 CMP R7, R5 1110 BNE loop 1111 1112 SUB R6, R3 // remove base 1113 MOVD R3, ret+24(FP) 1114 RET 1115 1116 notfound: 1117 MOVD $-1, R3 1118 MOVD R3, ret+24(FP) 1119 RET 1120 1121 TEXT runtimefastrand1(SB), NOSPLIT, $0-4 1122 MOVD g_m(g), R4 1123 MOVWZ m_fastrand(R4), R3 1124 ADD R3, R3 1125 CMPW R3, $0 1126 BGE 2(PC) 1127 XOR $0x88888eef, R3 1128 MOVW R3, m_fastrand(R4) 1129 MOVW R3, ret+0(FP) 1130 RET 1131 1132 TEXT runtimereturn0(SB), NOSPLIT, $0 1133 MOVW $0, R3 1134 RET 1135 1136 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1137 // Must obey the gcc calling convention. 1138 TEXT _cgo_topofstack(SB),NOSPLIT,$-8 1139 // g (R30) and R31 are callee-save in the C ABI, so save them 1140 MOVD g, R4 1141 MOVD R31, R5 1142 MOVD LR, R6 1143 1144 BL runtimeload_g(SB) // clobbers g (R30), R31 1145 MOVD g_m(g), R3 1146 MOVD m_curg(R3), R3 1147 MOVD (g_stack+stack_hi)(R3), R3 1148 1149 MOVD R4, g 1150 MOVD R5, R31 1151 MOVD R6, LR 1152 RET 1153 1154 // The top-most function running on a goroutine 1155 // returns to goexit+PCQuantum. 1156 TEXT runtimegoexit(SB),NOSPLIT,$-8-0 1157 MOVD R0, R0 // NOP 1158 BL runtimegoexit1(SB) // does not return 1159 // traceback from goexit1 must hit code range of goexit 1160 MOVD R0, R0 // NOP 1161 1162 TEXT runtimeprefetcht0(SB),NOSPLIT,$0-8 1163 RET 1164 1165 TEXT runtimeprefetcht1(SB),NOSPLIT,$0-8 1166 RET 1167 1168 TEXT runtimeprefetcht2(SB),NOSPLIT,$0-8 1169 RET 1170 1171 TEXT runtimeprefetchnta(SB),NOSPLIT,$0-8 1172 RET 1173
