1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "go_asm.h" 6 #include "go_tls.h" 7 #include "funcdata.h" 8 #include "textflag.h" 9 10 // using frame size $-4 means do not save LR on stack. 11 TEXT runtimert0_go(SB),NOSPLIT,$-4 12 MOVW $0xcafebabe, R12 13 14 // copy arguments forward on an even stack 15 // use R13 instead of SP to avoid linker rewriting the offsets 16 MOVW 0(R13), R0 // argc 17 MOVW 4(R13), R1 // argv 18 SUB $64, R13 // plenty of scratch 19 AND $~7, R13 20 MOVW R0, 60(R13) // save argc, argv away 21 MOVW R1, 64(R13) 22 23 // set up g register 24 // g is R10 25 MOVW $runtimeg0(SB), g 26 MOVW $runtimem0(SB), R8 27 28 // save m->g0 = g0 29 MOVW g, m_g0(R8) 30 // save g->m = m0 31 MOVW R8, g_m(g) 32 33 // create istack out of the OS stack 34 MOVW $(-8192+104)(R13), R0 35 MOVW R0, g_stackguard0(g) 36 MOVW R0, g_stackguard1(g) 37 MOVW R0, (g_stack+stack_lo)(g) 38 MOVW R13, (g_stack+stack_hi)(g) 39 40 BL runtimeemptyfunc(SB) // fault if stack check is wrong 41 42 BL runtime_initcgo(SB) // will clobber R0-R3 43 44 // update stackguard after _cgo_init 45 MOVW (g_stack+stack_lo)(g), R0 46 ADD $const__StackGuard, R0 47 MOVW R0, g_stackguard0(g) 48 MOVW R0, g_stackguard1(g) 49 50 BL runtimecheck(SB) 51 52 // saved argc, argv 53 MOVW 60(R13), R0 54 MOVW R0, 4(R13) 55 MOVW 64(R13), R1 56 MOVW R1, 8(R13) 57 BL runtimeargs(SB) 58 BL runtimecheckgoarm(SB) 59 BL runtimeosinit(SB) 60 BL runtimeschedinit(SB) 61 62 // create a new goroutine to start program 63 MOVW $runtimemainPC(SB), R0 64 MOVW.W R0, -4(R13) 65 MOVW $8, R0 66 MOVW.W R0, -4(R13) 67 MOVW $0, R0 68 MOVW.W R0, -4(R13) // push $0 as guard 69 BL runtimenewproc(SB) 70 MOVW $12(R13), R13 // pop args and LR 71 72 // start this M 73 BL runtimemstart(SB) 74 75 MOVW $1234, R0 76 MOVW $1000, R1 77 MOVW R0, (R1) // fail hard 78 79 DATA runtimemainPC+0(SB)/4,$runtimemain(SB) 80 GLOBL runtimemainPC(SB),RODATA,$4 81 82 TEXT runtimebreakpoint(SB),NOSPLIT,$0-0 83 // gdb won't skip this breakpoint instruction automatically, 84 // so you must manually "set $pc+=4" to skip it and continue. 85 #ifdef GOOS_nacl 86 WORD $0xe125be7f // BKPT 0x5bef, NACL_INSTR_ARM_BREAKPOINT 87 #else 88 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint 89 #endif 90 RET 91 92 TEXT runtimeasminit(SB),NOSPLIT,$0-0 93 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5 94 MOVB runtimegoarm(SB), R11 95 CMP $5, R11 96 BLE 4(PC) 97 WORD $0xeef1ba10 // vmrs r11, fpscr 98 BIC $(1<<24), R11 99 WORD $0xeee1ba10 // vmsr fpscr, r11 100 RET 101 102 /* 103 * go-routine 104 */ 105 106 // void gosave(Gobuf*) 107 // save state in Gobuf; setjmp 108 TEXT runtimegosave(SB),NOSPLIT,$-4-4 109 MOVW buf+0(FP), R0 110 MOVW R13, gobuf_sp(R0) 111 MOVW LR, gobuf_pc(R0) 112 MOVW g, gobuf_g(R0) 113 MOVW $0, R11 114 MOVW R11, gobuf_lr(R0) 115 MOVW R11, gobuf_ret(R0) 116 MOVW R11, gobuf_ctxt(R0) 117 RET 118 119 // void gogo(Gobuf*) 120 // restore state from Gobuf; longjmp 121 TEXT runtimegogo(SB),NOSPLIT,$-4-4 122 MOVW buf+0(FP), R1 123 MOVW gobuf_g(R1), R0 124 BL setg<>(SB) 125 126 // NOTE: We updated g above, and we are about to update SP. 127 // Until LR and PC are also updated, the g/SP/LR/PC quadruple 128 // are out of sync and must not be used as the basis of a traceback. 129 // Sigprof skips the traceback when SP is not within g's bounds, 130 // and when the PC is inside this function, runtime.gogo. 131 // Since we are about to update SP, until we complete runtime.gogo 132 // we must not leave this function. In particular, no calls 133 // after this point: it must be straight-line code until the 134 // final B instruction. 135 // See large comment in sigprof for more details. 136 MOVW gobuf_sp(R1), R13 // restore SP==R13 137 MOVW gobuf_lr(R1), LR 138 MOVW gobuf_ret(R1), R0 139 MOVW gobuf_ctxt(R1), R7 140 MOVW $0, R11 141 MOVW R11, gobuf_sp(R1) // clear to help garbage collector 142 MOVW R11, gobuf_ret(R1) 143 MOVW R11, gobuf_lr(R1) 144 MOVW R11, gobuf_ctxt(R1) 145 MOVW gobuf_pc(R1), R11 146 CMP R11, R11 // set condition codes for == test, needed by stack split 147 B (R11) 148 149 // func mcall(fn func(*g)) 150 // Switch to m->g0's stack, call fn(g). 151 // Fn must never return. It should gogo(&g->sched) 152 // to keep running g. 153 TEXT runtimemcall(SB),NOSPLIT,$-4-4 154 // Save caller state in g->sched. 155 MOVW R13, (g_sched+gobuf_sp)(g) 156 MOVW LR, (g_sched+gobuf_pc)(g) 157 MOVW $0, R11 158 MOVW R11, (g_sched+gobuf_lr)(g) 159 MOVW g, (g_sched+gobuf_g)(g) 160 161 // Switch to m->g0 & its stack, call fn. 162 MOVW g, R1 163 MOVW g_m(g), R8 164 MOVW m_g0(R8), R0 165 BL setg<>(SB) 166 CMP g, R1 167 B.NE 2(PC) 168 B runtimebadmcall(SB) 169 MOVB runtimeiscgo(SB), R11 170 CMP $0, R11 171 BL.NE runtimesave_g(SB) 172 MOVW fn+0(FP), R0 173 MOVW (g_sched+gobuf_sp)(g), R13 174 SUB $8, R13 175 MOVW R1, 4(R13) 176 MOVW R0, R7 177 MOVW 0(R0), R0 178 BL (R0) 179 B runtimebadmcall2(SB) 180 RET 181 182 // systemstack_switch is a dummy routine that systemstack leaves at the bottom 183 // of the G stack. We need to distinguish the routine that 184 // lives at the bottom of the G stack from the one that lives 185 // at the top of the system stack because the one at the top of 186 // the system stack terminates the stack walk (see topofstack()). 187 TEXT runtimesystemstack_switch(SB),NOSPLIT,$0-0 188 MOVW $0, R0 189 BL (R0) // clobber lr to ensure push {lr} is kept 190 RET 191 192 // func systemstack(fn func()) 193 TEXT runtimesystemstack(SB),NOSPLIT,$0-4 194 MOVW fn+0(FP), R0 // R0 = fn 195 MOVW g_m(g), R1 // R1 = m 196 197 MOVW m_gsignal(R1), R2 // R2 = gsignal 198 CMP g, R2 199 B.EQ noswitch 200 201 MOVW m_g0(R1), R2 // R2 = g0 202 CMP g, R2 203 B.EQ noswitch 204 205 MOVW m_curg(R1), R3 206 CMP g, R3 207 B.EQ switch 208 209 // Bad: g is not gsignal, not g0, not curg. What is it? 210 // Hide call from linker nosplit analysis. 211 MOVW $runtimebadsystemstack(SB), R0 212 BL (R0) 213 214 switch: 215 // save our state in g->sched. Pretend to 216 // be systemstack_switch if the G stack is scanned. 217 MOVW $runtimesystemstack_switch(SB), R3 218 #ifdef GOOS_nacl 219 ADD $4, R3, R3 // get past nacl-insert bic instruction 220 #endif 221 ADD $4, R3, R3 // get past push {lr} 222 MOVW R3, (g_sched+gobuf_pc)(g) 223 MOVW R13, (g_sched+gobuf_sp)(g) 224 MOVW LR, (g_sched+gobuf_lr)(g) 225 MOVW g, (g_sched+gobuf_g)(g) 226 227 // switch to g0 228 MOVW R0, R5 229 MOVW R2, R0 230 BL setg<>(SB) 231 MOVW R5, R0 232 MOVW (g_sched+gobuf_sp)(R2), R3 233 // make it look like mstart called systemstack on g0, to stop traceback 234 SUB $4, R3, R3 235 MOVW $runtimemstart(SB), R4 236 MOVW R4, 0(R3) 237 MOVW R3, R13 238 239 // call target function 240 MOVW R0, R7 241 MOVW 0(R0), R0 242 BL (R0) 243 244 // switch back to g 245 MOVW g_m(g), R1 246 MOVW m_curg(R1), R0 247 BL setg<>(SB) 248 MOVW (g_sched+gobuf_sp)(g), R13 249 MOVW $0, R3 250 MOVW R3, (g_sched+gobuf_sp)(g) 251 RET 252 253 noswitch: 254 MOVW R0, R7 255 MOVW 0(R0), R0 256 BL (R0) 257 RET 258 259 /* 260 * support for morestack 261 */ 262 263 // Called during function prolog when more stack is needed. 264 // R1 frame size 265 // R3 prolog's LR 266 // NB. we do not save R0 because we've forced 5c to pass all arguments 267 // on the stack. 268 // using frame size $-4 means do not save LR on stack. 269 // 270 // The traceback routines see morestack on a g0 as being 271 // the top of a stack (for example, morestack calling newstack 272 // calling the scheduler calling newm calling gc), so we must 273 // record an argument size. For that purpose, it has no arguments. 274 TEXT runtimemorestack(SB),NOSPLIT,$-4-0 275 // Cannot grow scheduler stack (m->g0). 276 MOVW g_m(g), R8 277 MOVW m_g0(R8), R4 278 CMP g, R4 279 BL.EQ runtimeabort(SB) 280 281 // Cannot grow signal stack (m->gsignal). 282 MOVW m_gsignal(R8), R4 283 CMP g, R4 284 BL.EQ runtimeabort(SB) 285 286 // Called from f. 287 // Set g->sched to context in f. 288 MOVW R7, (g_sched+gobuf_ctxt)(g) 289 MOVW R13, (g_sched+gobuf_sp)(g) 290 MOVW LR, (g_sched+gobuf_pc)(g) 291 MOVW R3, (g_sched+gobuf_lr)(g) 292 293 // Called from f. 294 // Set m->morebuf to f's caller. 295 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC 296 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP 297 MOVW $4(R13), R3 // f's argument pointer 298 MOVW g, (m_morebuf+gobuf_g)(R8) 299 300 // Call newstack on m->g0's stack. 301 MOVW m_g0(R8), R0 302 BL setg<>(SB) 303 MOVW (g_sched+gobuf_sp)(g), R13 304 BL runtimenewstack(SB) 305 306 // Not reached, but make sure the return PC from the call to newstack 307 // is still in this function, and not the beginning of the next. 308 RET 309 310 TEXT runtimemorestack_noctxt(SB),NOSPLIT,$-4-0 311 MOVW $0, R7 312 B runtimemorestack(SB) 313 314 TEXT runtimestackBarrier(SB),NOSPLIT,$0 315 // We came here via a RET to an overwritten LR. 316 // R0 may be live. Other registers are available. 317 318 // Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal. 319 MOVW (g_stkbar+slice_array)(g), R4 320 MOVW g_stkbarPos(g), R5 321 MOVW $stkbar__size, R6 322 MUL R5, R6 323 ADD R4, R6 324 MOVW stkbar_savedLRVal(R6), R6 325 // Record that this stack barrier was hit. 326 ADD $1, R5 327 MOVW R5, g_stkbarPos(g) 328 // Jump to the original return PC. 329 B (R6) 330 331 // reflectcall: call a function with the given argument list 332 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32). 333 // we don't have variable-sized frames, so we use a small number 334 // of constant-sized-frame functions to encode a few bits of size in the pc. 335 // Caution: ugly multiline assembly macros in your future! 336 337 #define DISPATCH(NAME,MAXSIZE) \ 338 CMP $MAXSIZE, R0; \ 339 B.HI 3(PC); \ 340 MOVW $NAME(SB), R1; \ 341 B (R1) 342 343 TEXT reflectcall(SB), NOSPLIT, $0-0 344 B reflectcall(SB) 345 346 TEXT reflectcall(SB),NOSPLIT,$-4-20 347 MOVW argsize+12(FP), R0 348 DISPATCH(runtimecall16, 16) 349 DISPATCH(runtimecall32, 32) 350 DISPATCH(runtimecall64, 64) 351 DISPATCH(runtimecall128, 128) 352 DISPATCH(runtimecall256, 256) 353 DISPATCH(runtimecall512, 512) 354 DISPATCH(runtimecall1024, 1024) 355 DISPATCH(runtimecall2048, 2048) 356 DISPATCH(runtimecall4096, 4096) 357 DISPATCH(runtimecall8192, 8192) 358 DISPATCH(runtimecall16384, 16384) 359 DISPATCH(runtimecall32768, 32768) 360 DISPATCH(runtimecall65536, 65536) 361 DISPATCH(runtimecall131072, 131072) 362 DISPATCH(runtimecall262144, 262144) 363 DISPATCH(runtimecall524288, 524288) 364 DISPATCH(runtimecall1048576, 1048576) 365 DISPATCH(runtimecall2097152, 2097152) 366 DISPATCH(runtimecall4194304, 4194304) 367 DISPATCH(runtimecall8388608, 8388608) 368 DISPATCH(runtimecall16777216, 16777216) 369 DISPATCH(runtimecall33554432, 33554432) 370 DISPATCH(runtimecall67108864, 67108864) 371 DISPATCH(runtimecall134217728, 134217728) 372 DISPATCH(runtimecall268435456, 268435456) 373 DISPATCH(runtimecall536870912, 536870912) 374 DISPATCH(runtimecall1073741824, 1073741824) 375 MOVW $runtimebadreflectcall(SB), R1 376 B (R1) 377 378 #define CALLFN(NAME,MAXSIZE) \ 379 TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \ 380 NO_LOCAL_POINTERS; \ 381 /* copy arguments to stack */ \ 382 MOVW argptr+8(FP), R0; \ 383 MOVW argsize+12(FP), R2; \ 384 ADD $4, R13, R1; \ 385 CMP $0, R2; \ 386 B.EQ 5(PC); \ 387 MOVBU.P 1(R0), R5; \ 388 MOVBU.P R5, 1(R1); \ 389 SUB $1, R2, R2; \ 390 B -5(PC); \ 391 /* call function */ \ 392 MOVW f+4(FP), R7; \ 393 MOVW (R7), R0; \ 394 PCDATA $PCDATA_StackMapIndex, $0; \ 395 BL (R0); \ 396 /* copy return values back */ \ 397 MOVW argptr+8(FP), R0; \ 398 MOVW argsize+12(FP), R2; \ 399 MOVW retoffset+16(FP), R3; \ 400 ADD $4, R13, R1; \ 401 ADD R3, R1; \ 402 ADD R3, R0; \ 403 SUB R3, R2; \ 404 loop: \ 405 CMP $0, R2; \ 406 B.EQ end; \ 407 MOVBU.P 1(R1), R5; \ 408 MOVBU.P R5, 1(R0); \ 409 SUB $1, R2, R2; \ 410 B loop; \ 411 end: \ 412 /* execute write barrier updates */ \ 413 MOVW argtype+0(FP), R1; \ 414 MOVW argptr+8(FP), R0; \ 415 MOVW argsize+12(FP), R2; \ 416 MOVW retoffset+16(FP), R3; \ 417 MOVW R1, 4(R13); \ 418 MOVW R0, 8(R13); \ 419 MOVW R2, 12(R13); \ 420 MOVW R3, 16(R13); \ 421 BL runtimecallwritebarrier(SB); \ 422 RET 423 424 CALLFN(call16, 16) 425 CALLFN(call32, 32) 426 CALLFN(call64, 64) 427 CALLFN(call128, 128) 428 CALLFN(call256, 256) 429 CALLFN(call512, 512) 430 CALLFN(call1024, 1024) 431 CALLFN(call2048, 2048) 432 CALLFN(call4096, 4096) 433 CALLFN(call8192, 8192) 434 CALLFN(call16384, 16384) 435 CALLFN(call32768, 32768) 436 CALLFN(call65536, 65536) 437 CALLFN(call131072, 131072) 438 CALLFN(call262144, 262144) 439 CALLFN(call524288, 524288) 440 CALLFN(call1048576, 1048576) 441 CALLFN(call2097152, 2097152) 442 CALLFN(call4194304, 4194304) 443 CALLFN(call8388608, 8388608) 444 CALLFN(call16777216, 16777216) 445 CALLFN(call33554432, 33554432) 446 CALLFN(call67108864, 67108864) 447 CALLFN(call134217728, 134217728) 448 CALLFN(call268435456, 268435456) 449 CALLFN(call536870912, 536870912) 450 CALLFN(call1073741824, 1073741824) 451 452 // void jmpdefer(fn, sp); 453 // called from deferreturn. 454 // 1. grab stored LR for caller 455 // 2. sub 4 bytes to get back to BL deferreturn 456 // 3. B to fn 457 // TODO(rsc): Push things on stack and then use pop 458 // to load all registers simultaneously, so that a profiling 459 // interrupt can never see mismatched SP/LR/PC. 460 // (And double-check that pop is atomic in that way.) 461 TEXT runtimejmpdefer(SB),NOSPLIT,$0-8 462 MOVW 0(R13), LR 463 MOVW $-4(LR), LR // BL deferreturn 464 MOVW fv+0(FP), R7 465 MOVW argp+4(FP), R13 466 MOVW $-4(R13), R13 // SP is 4 below argp, due to saved LR 467 MOVW 0(R7), R1 468 B (R1) 469 470 // Save state of caller into g->sched. Smashes R11. 471 TEXT gosave<>(SB),NOSPLIT,$0 472 MOVW LR, (g_sched+gobuf_pc)(g) 473 MOVW R13, (g_sched+gobuf_sp)(g) 474 MOVW $0, R11 475 MOVW R11, (g_sched+gobuf_lr)(g) 476 MOVW R11, (g_sched+gobuf_ret)(g) 477 MOVW R11, (g_sched+gobuf_ctxt)(g) 478 RET 479 480 // func asmcgocall(fn, arg unsafe.Pointer) int32 481 // Call fn(arg) on the scheduler stack, 482 // aligned appropriately for the gcc ABI. 483 // See cgocall.go for more details. 484 TEXT asmcgocall(SB),NOSPLIT,$0-12 485 MOVW fn+0(FP), R1 486 MOVW arg+4(FP), R0 487 488 MOVW R13, R2 489 MOVW g, R4 490 491 // Figure out if we need to switch to m->g0 stack. 492 // We get called to create new OS threads too, and those 493 // come in on the m->g0 stack already. 494 MOVW g_m(g), R8 495 MOVW m_g0(R8), R3 496 CMP R3, g 497 BEQ g0 498 BL gosave<>(SB) 499 MOVW R0, R5 500 MOVW R3, R0 501 BL setg<>(SB) 502 MOVW R5, R0 503 MOVW (g_sched+gobuf_sp)(g), R13 504 505 // Now on a scheduling stack (a pthread-created stack). 506 g0: 507 SUB $24, R13 508 BIC $0x7, R13 // alignment for gcc ABI 509 MOVW R4, 20(R13) // save old g 510 MOVW (g_stack+stack_hi)(R4), R4 511 SUB R2, R4 512 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback) 513 BL (R1) 514 515 // Restore registers, g, stack pointer. 516 MOVW R0, R5 517 MOVW 20(R13), R0 518 BL setg<>(SB) 519 MOVW (g_stack+stack_hi)(g), R1 520 MOVW 16(R13), R2 521 SUB R2, R1 522 MOVW R5, R0 523 MOVW R1, R13 524 525 MOVW R0, ret+8(FP) 526 RET 527 528 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize) 529 // Turn the fn into a Go func (by taking its address) and call 530 // cgocallback_gofunc. 531 TEXT runtimecgocallback(SB),NOSPLIT,$12-12 532 MOVW $fn+0(FP), R0 533 MOVW R0, 4(R13) 534 MOVW frame+4(FP), R0 535 MOVW R0, 8(R13) 536 MOVW framesize+8(FP), R0 537 MOVW R0, 12(R13) 538 MOVW $runtimecgocallback_gofunc(SB), R0 539 BL (R0) 540 RET 541 542 // cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize) 543 // See cgocall.go for more details. 544 TEXT cgocallback_gofunc(SB),NOSPLIT,$8-12 545 NO_LOCAL_POINTERS 546 547 // Load m and g from thread-local storage. 548 MOVB runtimeiscgo(SB), R0 549 CMP $0, R0 550 BL.NE runtimeload_g(SB) 551 552 // If g is nil, Go did not create the current thread. 553 // Call needm to obtain one for temporary use. 554 // In this case, we're running on the thread stack, so there's 555 // lots of space, but the linker doesn't know. Hide the call from 556 // the linker analysis by using an indirect call. 557 CMP $0, g 558 B.NE havem 559 MOVW g, savedm-4(SP) // g is zero, so is m. 560 MOVW $runtimeneedm(SB), R0 561 BL (R0) 562 563 // Set m->sched.sp = SP, so that if a panic happens 564 // during the function we are about to execute, it will 565 // have a valid SP to run on the g0 stack. 566 // The next few lines (after the havem label) 567 // will save this SP onto the stack and then write 568 // the same SP back to m->sched.sp. That seems redundant, 569 // but if an unrecovered panic happens, unwindm will 570 // restore the g->sched.sp from the stack location 571 // and then systemstack will try to use it. If we don't set it here, 572 // that restored SP will be uninitialized (typically 0) and 573 // will not be usable. 574 MOVW g_m(g), R8 575 MOVW m_g0(R8), R3 576 MOVW R13, (g_sched+gobuf_sp)(R3) 577 578 havem: 579 MOVW g_m(g), R8 580 MOVW R8, savedm-4(SP) 581 // Now there's a valid m, and we're running on its m->g0. 582 // Save current m->g0->sched.sp on stack and then set it to SP. 583 // Save current sp in m->g0->sched.sp in preparation for 584 // switch back to m->curg stack. 585 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-8(SP). 586 MOVW m_g0(R8), R3 587 MOVW (g_sched+gobuf_sp)(R3), R4 588 MOVW R4, savedsp-8(SP) 589 MOVW R13, (g_sched+gobuf_sp)(R3) 590 591 // Switch to m->curg stack and call runtime.cgocallbackg. 592 // Because we are taking over the execution of m->curg 593 // but *not* resuming what had been running, we need to 594 // save that information (m->curg->sched) so we can restore it. 595 // We can restore m->curg->sched.sp easily, because calling 596 // runtime.cgocallbackg leaves SP unchanged upon return. 597 // To save m->curg->sched.pc, we push it onto the stack. 598 // This has the added benefit that it looks to the traceback 599 // routine like cgocallbackg is going to return to that 600 // PC (because the frame we allocate below has the same 601 // size as cgocallback_gofunc's frame declared above) 602 // so that the traceback will seamlessly trace back into 603 // the earlier calls. 604 // 605 // In the new goroutine, -8(SP) and -4(SP) are unused. 606 MOVW m_curg(R8), R0 607 BL setg<>(SB) 608 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 609 MOVW (g_sched+gobuf_pc)(g), R5 610 MOVW R5, -12(R4) 611 MOVW $-12(R4), R13 612 BL runtimecgocallbackg(SB) 613 614 // Restore g->sched (== m->curg->sched) from saved values. 615 MOVW 0(R13), R5 616 MOVW R5, (g_sched+gobuf_pc)(g) 617 MOVW $12(R13), R4 618 MOVW R4, (g_sched+gobuf_sp)(g) 619 620 // Switch back to m->g0's stack and restore m->g0->sched.sp. 621 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 622 // so we do not have to restore it.) 623 MOVW g_m(g), R8 624 MOVW m_g0(R8), R0 625 BL setg<>(SB) 626 MOVW (g_sched+gobuf_sp)(g), R13 627 MOVW savedsp-8(SP), R4 628 MOVW R4, (g_sched+gobuf_sp)(g) 629 630 // If the m on entry was nil, we called needm above to borrow an m 631 // for the duration of the call. Since the call is over, return it with dropm. 632 MOVW savedm-4(SP), R6 633 CMP $0, R6 634 B.NE 3(PC) 635 MOVW $runtimedropm(SB), R0 636 BL (R0) 637 638 // Done! 639 RET 640 641 // void setg(G*); set g. for use by needm. 642 TEXT runtimesetg(SB),NOSPLIT,$-4-4 643 MOVW gg+0(FP), R0 644 B setg<>(SB) 645 646 TEXT setg<>(SB),NOSPLIT,$-4-0 647 MOVW R0, g 648 649 // Save g to thread-local storage. 650 MOVB runtimeiscgo(SB), R0 651 CMP $0, R0 652 B.EQ 2(PC) 653 B runtimesave_g(SB) 654 655 MOVW g, R0 656 RET 657 658 TEXT runtimegetcallerpc(SB),NOSPLIT,$4-8 659 MOVW 8(R13), R0 // LR saved by caller 660 MOVW runtimestackBarrierPC(SB), R1 661 CMP R0, R1 662 BNE nobar 663 // Get original return PC. 664 BL runtimenextBarrierPC(SB) 665 MOVW 4(R13), R0 666 nobar: 667 MOVW R0, ret+4(FP) 668 RET 669 670 TEXT runtimesetcallerpc(SB),NOSPLIT,$4-8 671 MOVW pc+4(FP), R0 672 MOVW 8(R13), R1 673 MOVW runtimestackBarrierPC(SB), R2 674 CMP R1, R2 675 BEQ setbar 676 MOVW R0, 8(R13) // set LR in caller 677 RET 678 setbar: 679 // Set the stack barrier return PC. 680 MOVW R0, 4(R13) 681 BL runtimesetNextBarrierPC(SB) 682 RET 683 684 TEXT runtimegetcallersp(SB),NOSPLIT,$-4-8 685 MOVW argp+0(FP), R0 686 MOVW $-4(R0), R0 687 MOVW R0, ret+4(FP) 688 RET 689 690 TEXT runtimeemptyfunc(SB),0,$0-0 691 RET 692 693 TEXT runtimeabort(SB),NOSPLIT,$-4-0 694 MOVW $0, R0 695 MOVW (R0), R1 696 697 // bool armcas(int32 *val, int32 old, int32 new) 698 // Atomically: 699 // if(*val == old){ 700 // *val = new; 701 // return 1; 702 // }else 703 // return 0; 704 // 705 // To implement runtimecas in sys_$GOOS_arm.s 706 // using the native instructions, use: 707 // 708 // TEXT runtimecas(SB),NOSPLIT,$0 709 // B runtimearmcas(SB) 710 // 711 TEXT runtimearmcas(SB),NOSPLIT,$0-13 712 MOVW valptr+0(FP), R1 713 MOVW old+4(FP), R2 714 MOVW new+8(FP), R3 715 casl: 716 LDREX (R1), R0 717 CMP R0, R2 718 BNE casfail 719 720 MOVB runtimegoarm(SB), R11 721 CMP $7, R11 722 BLT 2(PC) 723 WORD $0xf57ff05a // dmb ishst 724 725 STREX R3, (R1), R0 726 CMP $0, R0 727 BNE casl 728 MOVW $1, R0 729 730 MOVB runtimegoarm(SB), R11 731 CMP $7, R11 732 BLT 2(PC) 733 WORD $0xf57ff05b // dmb ish 734 735 MOVB R0, ret+12(FP) 736 RET 737 casfail: 738 MOVW $0, R0 739 MOVB R0, ret+12(FP) 740 RET 741 742 TEXT runtimecasuintptr(SB),NOSPLIT,$0-13 743 B runtimecas(SB) 744 745 TEXT runtimeatomicloaduintptr(SB),NOSPLIT,$0-8 746 B runtimeatomicload(SB) 747 748 TEXT runtimeatomicloaduint(SB),NOSPLIT,$0-8 749 B runtimeatomicload(SB) 750 751 TEXT runtimeatomicstoreuintptr(SB),NOSPLIT,$0-8 752 B runtimeatomicstore(SB) 753 754 // armPublicationBarrier is a native store/store barrier for ARMv7+. 755 // On earlier ARM revisions, armPublicationBarrier is a no-op. 756 // This will not work on SMP ARMv6 machines, if any are in use. 757 // To implement publiationBarrier in sys_$GOOS_arm.s using the native 758 // instructions, use: 759 // 760 // TEXT publicationBarrier(SB),NOSPLIT,$-4-0 761 // B runtimearmPublicationBarrier(SB) 762 // 763 TEXT runtimearmPublicationBarrier(SB),NOSPLIT,$-4-0 764 MOVB runtimegoarm(SB), R11 765 CMP $7, R11 766 BLT 2(PC) 767 WORD $0xf57ff05e // DMB ST 768 RET 769 770 // AES hashing not implemented for ARM 771 TEXT runtimeaeshash(SB),NOSPLIT,$-4-0 772 MOVW $0, R0 773 MOVW (R0), R1 774 TEXT runtimeaeshash32(SB),NOSPLIT,$-4-0 775 MOVW $0, R0 776 MOVW (R0), R1 777 TEXT runtimeaeshash64(SB),NOSPLIT,$-4-0 778 MOVW $0, R0 779 MOVW (R0), R1 780 TEXT runtimeaeshashstr(SB),NOSPLIT,$-4-0 781 MOVW $0, R0 782 MOVW (R0), R1 783 784 // memhash_varlen(p unsafe.Pointer, h seed) uintptr 785 // redirects to memhash(p, h, size) using the size 786 // stored in the closure. 787 TEXT runtimememhash_varlen(SB),NOSPLIT,$16-12 788 GO_ARGS 789 NO_LOCAL_POINTERS 790 MOVW p+0(FP), R0 791 MOVW h+4(FP), R1 792 MOVW 4(R7), R2 793 MOVW R0, 4(R13) 794 MOVW R1, 8(R13) 795 MOVW R2, 12(R13) 796 BL runtimememhash(SB) 797 MOVW 16(R13), R0 798 MOVW R0, ret+8(FP) 799 RET 800 801 TEXT runtimememeq(SB),NOSPLIT,$-4-13 802 MOVW a+0(FP), R1 803 MOVW b+4(FP), R2 804 MOVW size+8(FP), R3 805 ADD R1, R3, R6 806 MOVW $1, R0 807 MOVB R0, ret+12(FP) 808 loop: 809 CMP R1, R6 810 RET.EQ 811 MOVBU.P 1(R1), R4 812 MOVBU.P 1(R2), R5 813 CMP R4, R5 814 BEQ loop 815 816 MOVW $0, R0 817 MOVB R0, ret+12(FP) 818 RET 819 820 // memequal_varlen(a, b unsafe.Pointer) bool 821 TEXT runtimememequal_varlen(SB),NOSPLIT,$16-9 822 MOVW a+0(FP), R0 823 MOVW b+4(FP), R1 824 CMP R0, R1 825 BEQ eq 826 MOVW 4(R7), R2 // compiler stores size at offset 4 in the closure 827 MOVW R0, 4(R13) 828 MOVW R1, 8(R13) 829 MOVW R2, 12(R13) 830 BL runtimememeq(SB) 831 MOVB 16(R13), R0 832 MOVB R0, ret+8(FP) 833 RET 834 eq: 835 MOVW $1, R0 836 MOVB R0, ret+8(FP) 837 RET 838 839 TEXT runtimecmpstring(SB),NOSPLIT,$-4-20 840 MOVW s1_base+0(FP), R2 841 MOVW s1_len+4(FP), R0 842 MOVW s2_base+8(FP), R3 843 MOVW s2_len+12(FP), R1 844 ADD $20, R13, R7 845 B runtimecmpbody(SB) 846 847 TEXT bytesCompare(SB),NOSPLIT,$-4-28 848 MOVW s1+0(FP), R2 849 MOVW s1+4(FP), R0 850 MOVW s2+12(FP), R3 851 MOVW s2+16(FP), R1 852 ADD $28, R13, R7 853 B runtimecmpbody(SB) 854 855 // On entry: 856 // R0 is the length of s1 857 // R1 is the length of s2 858 // R2 points to the start of s1 859 // R3 points to the start of s2 860 // R7 points to return value (-1/0/1 will be written here) 861 // 862 // On exit: 863 // R4, R5, and R6 are clobbered 864 TEXT runtimecmpbody(SB),NOSPLIT,$-4-0 865 CMP R0, R1 866 MOVW R0, R6 867 MOVW.LT R1, R6 // R6 is min(R0, R1) 868 869 ADD R2, R6 // R2 is current byte in s1, R6 is last byte in s1 to compare 870 loop: 871 CMP R2, R6 872 BEQ samebytes // all compared bytes were the same; compare lengths 873 MOVBU.P 1(R2), R4 874 MOVBU.P 1(R3), R5 875 CMP R4, R5 876 BEQ loop 877 // bytes differed 878 MOVW.LT $1, R0 879 MOVW.GT $-1, R0 880 MOVW R0, (R7) 881 RET 882 samebytes: 883 CMP R0, R1 884 MOVW.LT $1, R0 885 MOVW.GT $-1, R0 886 MOVW.EQ $0, R0 887 MOVW R0, (R7) 888 RET 889 890 // eqstring tests whether two strings are equal. 891 // The compiler guarantees that strings passed 892 // to eqstring have equal length. 893 // See runtime_test.go:eqstring_generic for 894 // equivalent Go code. 895 TEXT runtimeeqstring(SB),NOSPLIT,$-4-17 896 MOVW s1str+0(FP), R2 897 MOVW s2str+8(FP), R3 898 MOVW $1, R8 899 MOVB R8, v+16(FP) 900 CMP R2, R3 901 RET.EQ 902 MOVW s1len+4(FP), R0 903 ADD R2, R0, R6 904 loop: 905 CMP R2, R6 906 RET.EQ 907 MOVBU.P 1(R2), R4 908 MOVBU.P 1(R3), R5 909 CMP R4, R5 910 BEQ loop 911 MOVW $0, R8 912 MOVB R8, v+16(FP) 913 RET 914 915 // TODO: share code with memeq? 916 TEXT bytesEqual(SB),NOSPLIT,$0-25 917 MOVW a_len+4(FP), R1 918 MOVW b_len+16(FP), R3 919 920 CMP R1, R3 // unequal lengths are not equal 921 B.NE notequal 922 923 MOVW a+0(FP), R0 924 MOVW b+12(FP), R2 925 ADD R0, R1 // end 926 927 loop: 928 CMP R0, R1 929 B.EQ equal // reached the end 930 MOVBU.P 1(R0), R4 931 MOVBU.P 1(R2), R5 932 CMP R4, R5 933 B.EQ loop 934 935 notequal: 936 MOVW $0, R0 937 MOVBU R0, ret+24(FP) 938 RET 939 940 equal: 941 MOVW $1, R0 942 MOVBU R0, ret+24(FP) 943 RET 944 945 TEXT bytesIndexByte(SB),NOSPLIT,$0-20 946 MOVW s+0(FP), R0 947 MOVW s_len+4(FP), R1 948 MOVBU c+12(FP), R2 // byte to find 949 MOVW R0, R4 // store base for later 950 ADD R0, R1 // end 951 952 _loop: 953 CMP R0, R1 954 B.EQ _notfound 955 MOVBU.P 1(R0), R3 956 CMP R2, R3 957 B.NE _loop 958 959 SUB $1, R0 // R0 will be one beyond the position we want 960 SUB R4, R0 // remove base 961 MOVW R0, ret+16(FP) 962 RET 963 964 _notfound: 965 MOVW $-1, R0 966 MOVW R0, ret+16(FP) 967 RET 968 969 TEXT stringsIndexByte(SB),NOSPLIT,$0-16 970 MOVW s+0(FP), R0 971 MOVW s_len+4(FP), R1 972 MOVBU c+8(FP), R2 // byte to find 973 MOVW R0, R4 // store base for later 974 ADD R0, R1 // end 975 976 _sib_loop: 977 CMP R0, R1 978 B.EQ _sib_notfound 979 MOVBU.P 1(R0), R3 980 CMP R2, R3 981 B.NE _sib_loop 982 983 SUB $1, R0 // R0 will be one beyond the position we want 984 SUB R4, R0 // remove base 985 MOVW R0, ret+12(FP) 986 RET 987 988 _sib_notfound: 989 MOVW $-1, R0 990 MOVW R0, ret+12(FP) 991 RET 992 993 TEXT runtimefastrand1(SB),NOSPLIT,$-4-4 994 MOVW g_m(g), R1 995 MOVW m_fastrand(R1), R0 996 ADD.S R0, R0 997 EOR.MI $0x88888eef, R0 998 MOVW R0, m_fastrand(R1) 999 MOVW R0, ret+0(FP) 1000 RET 1001 1002 TEXT runtimereturn0(SB),NOSPLIT,$0 1003 MOVW $0, R0 1004 RET 1005 1006 TEXT runtimeprocyield(SB),NOSPLIT,$-4 1007 MOVW cycles+0(FP), R1 1008 MOVW $0, R0 1009 yieldloop: 1010 CMP R0, R1 1011 B.NE 2(PC) 1012 RET 1013 SUB $1, R1 1014 B yieldloop 1015 1016 // Called from cgo wrappers, this function returns g->m->curg.stack.hi. 1017 // Must obey the gcc calling convention. 1018 TEXT _cgo_topofstack(SB),NOSPLIT,$8 1019 // R11 and g register are clobbered by load_g. They are 1020 // callee-save in the gcc calling convention, so save them here. 1021 MOVW R11, saveR11-4(SP) 1022 MOVW g, saveG-8(SP) 1023 1024 BL runtimeload_g(SB) 1025 MOVW g_m(g), R0 1026 MOVW m_curg(R0), R0 1027 MOVW (g_stack+stack_hi)(R0), R0 1028 1029 MOVW saveG-8(SP), g 1030 MOVW saveR11-4(SP), R11 1031 RET 1032 1033 // The top-most function running on a goroutine 1034 // returns to goexit+PCQuantum. 1035 TEXT runtimegoexit(SB),NOSPLIT,$-4-0 1036 MOVW R0, R0 // NOP 1037 BL runtimegoexit1(SB) // does not return 1038 // traceback from goexit1 must hit code range of goexit 1039 MOVW R0, R0 // NOP 1040 1041 TEXT runtimeprefetcht0(SB),NOSPLIT,$0-4 1042 RET 1043 1044 TEXT runtimeprefetcht1(SB),NOSPLIT,$0-4 1045 RET 1046 1047 TEXT runtimeprefetcht2(SB),NOSPLIT,$0-4 1048 RET 1049 1050 TEXT runtimeprefetchnta(SB),NOSPLIT,$0-4 1051 RET 1052 1053 // x -> x/1000000, x%1000000, called from Go with args, results on stack. 1054 TEXT runtimeusplit(SB),NOSPLIT,$0-12 1055 MOVW x+0(FP), R0 1056 CALL runtimeusplitR0(SB) 1057 MOVW R0, q+4(FP) 1058 MOVW R1, r+8(FP) 1059 RET 1060 1061 // R0, R1 = R0/1000000, R0%1000000 1062 TEXT runtimeusplitR0(SB),NOSPLIT,$0 1063 // magic multiply to avoid software divide without available m. 1064 // see output of go tool compile -S for x/1000000. 1065 MOVW R0, R3 1066 MOVW $1125899907, R1 1067 MULLU R1, R0, (R0, R1) 1068 MOVW R0>>18, R0 1069 MOVW $1000000, R1 1070 MULU R0, R1 1071 SUB R1, R3, R1 1072 RET 1073
