1 // Copyright 2016 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package arm64 6 7 import ( 8 "math" 9 10 "cmd/compile/internal/gc" 11 "cmd/compile/internal/ssa" 12 "cmd/internal/obj" 13 "cmd/internal/obj/arm64" 14 ) 15 16 // loadByType returns the load instruction of the given type. 17 func loadByType(t ssa.Type) obj.As { 18 if t.IsFloat() { 19 switch t.Size() { 20 case 4: 21 return arm64.AFMOVS 22 case 8: 23 return arm64.AFMOVD 24 } 25 } else { 26 switch t.Size() { 27 case 1: 28 if t.IsSigned() { 29 return arm64.AMOVB 30 } else { 31 return arm64.AMOVBU 32 } 33 case 2: 34 if t.IsSigned() { 35 return arm64.AMOVH 36 } else { 37 return arm64.AMOVHU 38 } 39 case 4: 40 if t.IsSigned() { 41 return arm64.AMOVW 42 } else { 43 return arm64.AMOVWU 44 } 45 case 8: 46 return arm64.AMOVD 47 } 48 } 49 panic("bad load type") 50 } 51 52 // storeByType returns the store instruction of the given type. 53 func storeByType(t ssa.Type) obj.As { 54 if t.IsFloat() { 55 switch t.Size() { 56 case 4: 57 return arm64.AFMOVS 58 case 8: 59 return arm64.AFMOVD 60 } 61 } else { 62 switch t.Size() { 63 case 1: 64 return arm64.AMOVB 65 case 2: 66 return arm64.AMOVH 67 case 4: 68 return arm64.AMOVW 69 case 8: 70 return arm64.AMOVD 71 } 72 } 73 panic("bad store type") 74 } 75 76 // makeshift encodes a register shifted by a constant, used as an Offset in Prog 77 func makeshift(reg int16, typ int64, s int64) int64 { 78 return int64(reg&31)<<16 | typ | (s&63)<<10 79 } 80 81 // genshift generates a Prog for r = r0 op (r1 shifted by s) 82 func genshift(as obj.As, r0, r1, r int16, typ int64, s int64) *obj.Prog { 83 p := gc.Prog(as) 84 p.From.Type = obj.TYPE_SHIFT 85 p.From.Offset = makeshift(r1, typ, s) 86 p.Reg = r0 87 if r != 0 { 88 p.To.Type = obj.TYPE_REG 89 p.To.Reg = r 90 } 91 return p 92 } 93 94 func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { 95 s.SetLineno(v.Line) 96 switch v.Op { 97 case ssa.OpInitMem: 98 // memory arg needs no code 99 case ssa.OpArg: 100 // input args need no code 101 case ssa.OpSP, ssa.OpSB, ssa.OpGetG: 102 // nothing to do 103 case ssa.OpCopy, ssa.OpARM64MOVDconvert, ssa.OpARM64MOVDreg: 104 if v.Type.IsMemory() { 105 return 106 } 107 x := v.Args[0].Reg() 108 y := v.Reg() 109 if x == y { 110 return 111 } 112 as := arm64.AMOVD 113 if v.Type.IsFloat() { 114 switch v.Type.Size() { 115 case 4: 116 as = arm64.AFMOVS 117 case 8: 118 as = arm64.AFMOVD 119 default: 120 panic("bad float size") 121 } 122 } 123 p := gc.Prog(as) 124 p.From.Type = obj.TYPE_REG 125 p.From.Reg = x 126 p.To.Type = obj.TYPE_REG 127 p.To.Reg = y 128 case ssa.OpARM64MOVDnop: 129 if v.Reg() != v.Args[0].Reg() { 130 v.Fatalf("input[0] and output not in same register %s", v.LongString()) 131 } 132 // nothing to do 133 case ssa.OpLoadReg: 134 if v.Type.IsFlags() { 135 v.Fatalf("load flags not implemented: %v", v.LongString()) 136 return 137 } 138 p := gc.Prog(loadByType(v.Type)) 139 gc.AddrAuto(&p.From, v.Args[0]) 140 p.To.Type = obj.TYPE_REG 141 p.To.Reg = v.Reg() 142 case ssa.OpPhi: 143 gc.CheckLoweredPhi(v) 144 case ssa.OpStoreReg: 145 if v.Type.IsFlags() { 146 v.Fatalf("store flags not implemented: %v", v.LongString()) 147 return 148 } 149 p := gc.Prog(storeByType(v.Type)) 150 p.From.Type = obj.TYPE_REG 151 p.From.Reg = v.Args[0].Reg() 152 gc.AddrAuto(&p.To, v) 153 case ssa.OpARM64ADD, 154 ssa.OpARM64SUB, 155 ssa.OpARM64AND, 156 ssa.OpARM64OR, 157 ssa.OpARM64XOR, 158 ssa.OpARM64BIC, 159 ssa.OpARM64MUL, 160 ssa.OpARM64MULW, 161 ssa.OpARM64MULH, 162 ssa.OpARM64UMULH, 163 ssa.OpARM64MULL, 164 ssa.OpARM64UMULL, 165 ssa.OpARM64DIV, 166 ssa.OpARM64UDIV, 167 ssa.OpARM64DIVW, 168 ssa.OpARM64UDIVW, 169 ssa.OpARM64MOD, 170 ssa.OpARM64UMOD, 171 ssa.OpARM64MODW, 172 ssa.OpARM64UMODW, 173 ssa.OpARM64SLL, 174 ssa.OpARM64SRL, 175 ssa.OpARM64SRA, 176 ssa.OpARM64FADDS, 177 ssa.OpARM64FADDD, 178 ssa.OpARM64FSUBS, 179 ssa.OpARM64FSUBD, 180 ssa.OpARM64FMULS, 181 ssa.OpARM64FMULD, 182 ssa.OpARM64FDIVS, 183 ssa.OpARM64FDIVD: 184 r := v.Reg() 185 r1 := v.Args[0].Reg() 186 r2 := v.Args[1].Reg() 187 p := gc.Prog(v.Op.Asm()) 188 p.From.Type = obj.TYPE_REG 189 p.From.Reg = r2 190 p.Reg = r1 191 p.To.Type = obj.TYPE_REG 192 p.To.Reg = r 193 case ssa.OpARM64ADDconst, 194 ssa.OpARM64SUBconst, 195 ssa.OpARM64ANDconst, 196 ssa.OpARM64ORconst, 197 ssa.OpARM64XORconst, 198 ssa.OpARM64BICconst, 199 ssa.OpARM64SLLconst, 200 ssa.OpARM64SRLconst, 201 ssa.OpARM64SRAconst, 202 ssa.OpARM64RORconst, 203 ssa.OpARM64RORWconst: 204 p := gc.Prog(v.Op.Asm()) 205 p.From.Type = obj.TYPE_CONST 206 p.From.Offset = v.AuxInt 207 p.Reg = v.Args[0].Reg() 208 p.To.Type = obj.TYPE_REG 209 p.To.Reg = v.Reg() 210 case ssa.OpARM64ADDshiftLL, 211 ssa.OpARM64SUBshiftLL, 212 ssa.OpARM64ANDshiftLL, 213 ssa.OpARM64ORshiftLL, 214 ssa.OpARM64XORshiftLL, 215 ssa.OpARM64BICshiftLL: 216 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt) 217 case ssa.OpARM64ADDshiftRL, 218 ssa.OpARM64SUBshiftRL, 219 ssa.OpARM64ANDshiftRL, 220 ssa.OpARM64ORshiftRL, 221 ssa.OpARM64XORshiftRL, 222 ssa.OpARM64BICshiftRL: 223 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt) 224 case ssa.OpARM64ADDshiftRA, 225 ssa.OpARM64SUBshiftRA, 226 ssa.OpARM64ANDshiftRA, 227 ssa.OpARM64ORshiftRA, 228 ssa.OpARM64XORshiftRA, 229 ssa.OpARM64BICshiftRA: 230 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt) 231 case ssa.OpARM64MOVDconst: 232 p := gc.Prog(v.Op.Asm()) 233 p.From.Type = obj.TYPE_CONST 234 p.From.Offset = v.AuxInt 235 p.To.Type = obj.TYPE_REG 236 p.To.Reg = v.Reg() 237 case ssa.OpARM64FMOVSconst, 238 ssa.OpARM64FMOVDconst: 239 p := gc.Prog(v.Op.Asm()) 240 p.From.Type = obj.TYPE_FCONST 241 p.From.Val = math.Float64frombits(uint64(v.AuxInt)) 242 p.To.Type = obj.TYPE_REG 243 p.To.Reg = v.Reg() 244 case ssa.OpARM64CMP, 245 ssa.OpARM64CMPW, 246 ssa.OpARM64CMN, 247 ssa.OpARM64CMNW, 248 ssa.OpARM64FCMPS, 249 ssa.OpARM64FCMPD: 250 p := gc.Prog(v.Op.Asm()) 251 p.From.Type = obj.TYPE_REG 252 p.From.Reg = v.Args[1].Reg() 253 p.Reg = v.Args[0].Reg() 254 case ssa.OpARM64CMPconst, 255 ssa.OpARM64CMPWconst, 256 ssa.OpARM64CMNconst, 257 ssa.OpARM64CMNWconst: 258 p := gc.Prog(v.Op.Asm()) 259 p.From.Type = obj.TYPE_CONST 260 p.From.Offset = v.AuxInt 261 p.Reg = v.Args[0].Reg() 262 case ssa.OpARM64CMPshiftLL: 263 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt) 264 case ssa.OpARM64CMPshiftRL: 265 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt) 266 case ssa.OpARM64CMPshiftRA: 267 genshift(v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt) 268 case ssa.OpARM64MOVDaddr: 269 p := gc.Prog(arm64.AMOVD) 270 p.From.Type = obj.TYPE_ADDR 271 p.To.Type = obj.TYPE_REG 272 p.To.Reg = v.Reg() 273 274 var wantreg string 275 // MOVD $sym+off(base), R 276 // the assembler expands it as the following: 277 // - base is SP: add constant offset to SP (R13) 278 // when constant is large, tmp register (R11) may be used 279 // - base is SB: load external address from constant pool (use relocation) 280 switch v.Aux.(type) { 281 default: 282 v.Fatalf("aux is of unknown type %T", v.Aux) 283 case *ssa.ExternSymbol: 284 wantreg = "SB" 285 gc.AddAux(&p.From, v) 286 case *ssa.ArgSymbol, *ssa.AutoSymbol: 287 wantreg = "SP" 288 gc.AddAux(&p.From, v) 289 case nil: 290 // No sym, just MOVD $off(SP), R 291 wantreg = "SP" 292 p.From.Reg = arm64.REGSP 293 p.From.Offset = v.AuxInt 294 } 295 if reg := v.Args[0].RegName(); reg != wantreg { 296 v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg) 297 } 298 case ssa.OpARM64MOVBload, 299 ssa.OpARM64MOVBUload, 300 ssa.OpARM64MOVHload, 301 ssa.OpARM64MOVHUload, 302 ssa.OpARM64MOVWload, 303 ssa.OpARM64MOVWUload, 304 ssa.OpARM64MOVDload, 305 ssa.OpARM64FMOVSload, 306 ssa.OpARM64FMOVDload: 307 p := gc.Prog(v.Op.Asm()) 308 p.From.Type = obj.TYPE_MEM 309 p.From.Reg = v.Args[0].Reg() 310 gc.AddAux(&p.From, v) 311 p.To.Type = obj.TYPE_REG 312 p.To.Reg = v.Reg() 313 case ssa.OpARM64LDAR, 314 ssa.OpARM64LDARW: 315 p := gc.Prog(v.Op.Asm()) 316 p.From.Type = obj.TYPE_MEM 317 p.From.Reg = v.Args[0].Reg() 318 gc.AddAux(&p.From, v) 319 p.To.Type = obj.TYPE_REG 320 p.To.Reg = v.Reg0() 321 case ssa.OpARM64MOVBstore, 322 ssa.OpARM64MOVHstore, 323 ssa.OpARM64MOVWstore, 324 ssa.OpARM64MOVDstore, 325 ssa.OpARM64FMOVSstore, 326 ssa.OpARM64FMOVDstore, 327 ssa.OpARM64STLR, 328 ssa.OpARM64STLRW: 329 p := gc.Prog(v.Op.Asm()) 330 p.From.Type = obj.TYPE_REG 331 p.From.Reg = v.Args[1].Reg() 332 p.To.Type = obj.TYPE_MEM 333 p.To.Reg = v.Args[0].Reg() 334 gc.AddAux(&p.To, v) 335 case ssa.OpARM64MOVBstorezero, 336 ssa.OpARM64MOVHstorezero, 337 ssa.OpARM64MOVWstorezero, 338 ssa.OpARM64MOVDstorezero: 339 p := gc.Prog(v.Op.Asm()) 340 p.From.Type = obj.TYPE_REG 341 p.From.Reg = arm64.REGZERO 342 p.To.Type = obj.TYPE_MEM 343 p.To.Reg = v.Args[0].Reg() 344 gc.AddAux(&p.To, v) 345 case ssa.OpARM64LoweredAtomicExchange64, 346 ssa.OpARM64LoweredAtomicExchange32: 347 // LDAXR (Rarg0), Rout 348 // STLXR Rarg1, (Rarg0), Rtmp 349 // CBNZ Rtmp, -2(PC) 350 ld := arm64.ALDAXR 351 st := arm64.ASTLXR 352 if v.Op == ssa.OpARM64LoweredAtomicExchange32 { 353 ld = arm64.ALDAXRW 354 st = arm64.ASTLXRW 355 } 356 r0 := v.Args[0].Reg() 357 r1 := v.Args[1].Reg() 358 out := v.Reg0() 359 p := gc.Prog(ld) 360 p.From.Type = obj.TYPE_MEM 361 p.From.Reg = r0 362 p.To.Type = obj.TYPE_REG 363 p.To.Reg = out 364 p1 := gc.Prog(st) 365 p1.From.Type = obj.TYPE_REG 366 p1.From.Reg = r1 367 p1.To.Type = obj.TYPE_MEM 368 p1.To.Reg = r0 369 p1.RegTo2 = arm64.REGTMP 370 p2 := gc.Prog(arm64.ACBNZ) 371 p2.From.Type = obj.TYPE_REG 372 p2.From.Reg = arm64.REGTMP 373 p2.To.Type = obj.TYPE_BRANCH 374 gc.Patch(p2, p) 375 case ssa.OpARM64LoweredAtomicAdd64, 376 ssa.OpARM64LoweredAtomicAdd32: 377 // LDAXR (Rarg0), Rout 378 // ADD Rarg1, Rout 379 // STLXR Rout, (Rarg0), Rtmp 380 // CBNZ Rtmp, -3(PC) 381 ld := arm64.ALDAXR 382 st := arm64.ASTLXR 383 if v.Op == ssa.OpARM64LoweredAtomicAdd32 { 384 ld = arm64.ALDAXRW 385 st = arm64.ASTLXRW 386 } 387 r0 := v.Args[0].Reg() 388 r1 := v.Args[1].Reg() 389 out := v.Reg0() 390 p := gc.Prog(ld) 391 p.From.Type = obj.TYPE_MEM 392 p.From.Reg = r0 393 p.To.Type = obj.TYPE_REG 394 p.To.Reg = out 395 p1 := gc.Prog(arm64.AADD) 396 p1.From.Type = obj.TYPE_REG 397 p1.From.Reg = r1 398 p1.To.Type = obj.TYPE_REG 399 p1.To.Reg = out 400 p2 := gc.Prog(st) 401 p2.From.Type = obj.TYPE_REG 402 p2.From.Reg = out 403 p2.To.Type = obj.TYPE_MEM 404 p2.To.Reg = r0 405 p2.RegTo2 = arm64.REGTMP 406 p3 := gc.Prog(arm64.ACBNZ) 407 p3.From.Type = obj.TYPE_REG 408 p3.From.Reg = arm64.REGTMP 409 p3.To.Type = obj.TYPE_BRANCH 410 gc.Patch(p3, p) 411 case ssa.OpARM64LoweredAtomicCas64, 412 ssa.OpARM64LoweredAtomicCas32: 413 // LDAXR (Rarg0), Rtmp 414 // CMP Rarg1, Rtmp 415 // BNE 3(PC) 416 // STLXR Rarg2, (Rarg0), Rtmp 417 // CBNZ Rtmp, -4(PC) 418 // CSET EQ, Rout 419 ld := arm64.ALDAXR 420 st := arm64.ASTLXR 421 cmp := arm64.ACMP 422 if v.Op == ssa.OpARM64LoweredAtomicCas32 { 423 ld = arm64.ALDAXRW 424 st = arm64.ASTLXRW 425 cmp = arm64.ACMPW 426 } 427 r0 := v.Args[0].Reg() 428 r1 := v.Args[1].Reg() 429 r2 := v.Args[2].Reg() 430 out := v.Reg0() 431 p := gc.Prog(ld) 432 p.From.Type = obj.TYPE_MEM 433 p.From.Reg = r0 434 p.To.Type = obj.TYPE_REG 435 p.To.Reg = arm64.REGTMP 436 p1 := gc.Prog(cmp) 437 p1.From.Type = obj.TYPE_REG 438 p1.From.Reg = r1 439 p1.Reg = arm64.REGTMP 440 p2 := gc.Prog(arm64.ABNE) 441 p2.To.Type = obj.TYPE_BRANCH 442 p3 := gc.Prog(st) 443 p3.From.Type = obj.TYPE_REG 444 p3.From.Reg = r2 445 p3.To.Type = obj.TYPE_MEM 446 p3.To.Reg = r0 447 p3.RegTo2 = arm64.REGTMP 448 p4 := gc.Prog(arm64.ACBNZ) 449 p4.From.Type = obj.TYPE_REG 450 p4.From.Reg = arm64.REGTMP 451 p4.To.Type = obj.TYPE_BRANCH 452 gc.Patch(p4, p) 453 p5 := gc.Prog(arm64.ACSET) 454 p5.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 455 p5.From.Reg = arm64.COND_EQ 456 p5.To.Type = obj.TYPE_REG 457 p5.To.Reg = out 458 gc.Patch(p2, p5) 459 case ssa.OpARM64LoweredAtomicAnd8, 460 ssa.OpARM64LoweredAtomicOr8: 461 // LDAXRB (Rarg0), Rtmp 462 // AND/OR Rarg1, Rtmp 463 // STLXRB Rtmp, (Rarg0), Rtmp 464 // CBNZ Rtmp, -3(PC) 465 r0 := v.Args[0].Reg() 466 r1 := v.Args[1].Reg() 467 p := gc.Prog(arm64.ALDAXRB) 468 p.From.Type = obj.TYPE_MEM 469 p.From.Reg = r0 470 p.To.Type = obj.TYPE_REG 471 p.To.Reg = arm64.REGTMP 472 p1 := gc.Prog(v.Op.Asm()) 473 p1.From.Type = obj.TYPE_REG 474 p1.From.Reg = r1 475 p1.To.Type = obj.TYPE_REG 476 p1.To.Reg = arm64.REGTMP 477 p2 := gc.Prog(arm64.ASTLXRB) 478 p2.From.Type = obj.TYPE_REG 479 p2.From.Reg = arm64.REGTMP 480 p2.To.Type = obj.TYPE_MEM 481 p2.To.Reg = r0 482 p2.RegTo2 = arm64.REGTMP 483 p3 := gc.Prog(arm64.ACBNZ) 484 p3.From.Type = obj.TYPE_REG 485 p3.From.Reg = arm64.REGTMP 486 p3.To.Type = obj.TYPE_BRANCH 487 gc.Patch(p3, p) 488 case ssa.OpARM64MOVBreg, 489 ssa.OpARM64MOVBUreg, 490 ssa.OpARM64MOVHreg, 491 ssa.OpARM64MOVHUreg, 492 ssa.OpARM64MOVWreg, 493 ssa.OpARM64MOVWUreg: 494 a := v.Args[0] 495 for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg { 496 a = a.Args[0] 497 } 498 if a.Op == ssa.OpLoadReg { 499 t := a.Type 500 switch { 501 case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(), 502 v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(), 503 v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(), 504 v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(), 505 v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(), 506 v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned(): 507 // arg is a proper-typed load, already zero/sign-extended, don't extend again 508 if v.Reg() == v.Args[0].Reg() { 509 return 510 } 511 p := gc.Prog(arm64.AMOVD) 512 p.From.Type = obj.TYPE_REG 513 p.From.Reg = v.Args[0].Reg() 514 p.To.Type = obj.TYPE_REG 515 p.To.Reg = v.Reg() 516 return 517 default: 518 } 519 } 520 fallthrough 521 case ssa.OpARM64MVN, 522 ssa.OpARM64NEG, 523 ssa.OpARM64FNEGS, 524 ssa.OpARM64FNEGD, 525 ssa.OpARM64FSQRTD, 526 ssa.OpARM64FCVTZSSW, 527 ssa.OpARM64FCVTZSDW, 528 ssa.OpARM64FCVTZUSW, 529 ssa.OpARM64FCVTZUDW, 530 ssa.OpARM64FCVTZSS, 531 ssa.OpARM64FCVTZSD, 532 ssa.OpARM64FCVTZUS, 533 ssa.OpARM64FCVTZUD, 534 ssa.OpARM64SCVTFWS, 535 ssa.OpARM64SCVTFWD, 536 ssa.OpARM64SCVTFS, 537 ssa.OpARM64SCVTFD, 538 ssa.OpARM64UCVTFWS, 539 ssa.OpARM64UCVTFWD, 540 ssa.OpARM64UCVTFS, 541 ssa.OpARM64UCVTFD, 542 ssa.OpARM64FCVTSD, 543 ssa.OpARM64FCVTDS, 544 ssa.OpARM64REV, 545 ssa.OpARM64REVW, 546 ssa.OpARM64REV16W, 547 ssa.OpARM64RBIT, 548 ssa.OpARM64RBITW, 549 ssa.OpARM64CLZ, 550 ssa.OpARM64CLZW: 551 p := gc.Prog(v.Op.Asm()) 552 p.From.Type = obj.TYPE_REG 553 p.From.Reg = v.Args[0].Reg() 554 p.To.Type = obj.TYPE_REG 555 p.To.Reg = v.Reg() 556 case ssa.OpARM64CSELULT, 557 ssa.OpARM64CSELULT0: 558 r1 := int16(arm64.REGZERO) 559 if v.Op == ssa.OpARM64CSELULT { 560 r1 = v.Args[1].Reg() 561 } 562 p := gc.Prog(v.Op.Asm()) 563 p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 564 p.From.Reg = arm64.COND_LO 565 p.Reg = v.Args[0].Reg() 566 p.From3 = &obj.Addr{Type: obj.TYPE_REG, Reg: r1} 567 p.To.Type = obj.TYPE_REG 568 p.To.Reg = v.Reg() 569 case ssa.OpARM64DUFFZERO: 570 // runtime.duffzero expects start address - 8 in R16 571 p := gc.Prog(arm64.ASUB) 572 p.From.Type = obj.TYPE_CONST 573 p.From.Offset = 8 574 p.Reg = v.Args[0].Reg() 575 p.To.Type = obj.TYPE_REG 576 p.To.Reg = arm64.REG_R16 577 p = gc.Prog(obj.ADUFFZERO) 578 p.To.Type = obj.TYPE_MEM 579 p.To.Name = obj.NAME_EXTERN 580 p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg)) 581 p.To.Offset = v.AuxInt 582 case ssa.OpARM64LoweredZero: 583 // MOVD.P ZR, 8(R16) 584 // CMP Rarg1, R16 585 // BLE -2(PC) 586 // arg1 is the address of the last element to zero 587 p := gc.Prog(arm64.AMOVD) 588 p.Scond = arm64.C_XPOST 589 p.From.Type = obj.TYPE_REG 590 p.From.Reg = arm64.REGZERO 591 p.To.Type = obj.TYPE_MEM 592 p.To.Reg = arm64.REG_R16 593 p.To.Offset = 8 594 p2 := gc.Prog(arm64.ACMP) 595 p2.From.Type = obj.TYPE_REG 596 p2.From.Reg = v.Args[1].Reg() 597 p2.Reg = arm64.REG_R16 598 p3 := gc.Prog(arm64.ABLE) 599 p3.To.Type = obj.TYPE_BRANCH 600 gc.Patch(p3, p) 601 case ssa.OpARM64DUFFCOPY: 602 p := gc.Prog(obj.ADUFFCOPY) 603 p.To.Type = obj.TYPE_MEM 604 p.To.Name = obj.NAME_EXTERN 605 p.To.Sym = gc.Linksym(gc.Pkglookup("duffcopy", gc.Runtimepkg)) 606 p.To.Offset = v.AuxInt 607 case ssa.OpARM64LoweredMove: 608 // MOVD.P 8(R16), Rtmp 609 // MOVD.P Rtmp, 8(R17) 610 // CMP Rarg2, R16 611 // BLE -3(PC) 612 // arg2 is the address of the last element of src 613 p := gc.Prog(arm64.AMOVD) 614 p.Scond = arm64.C_XPOST 615 p.From.Type = obj.TYPE_MEM 616 p.From.Reg = arm64.REG_R16 617 p.From.Offset = 8 618 p.To.Type = obj.TYPE_REG 619 p.To.Reg = arm64.REGTMP 620 p2 := gc.Prog(arm64.AMOVD) 621 p2.Scond = arm64.C_XPOST 622 p2.From.Type = obj.TYPE_REG 623 p2.From.Reg = arm64.REGTMP 624 p2.To.Type = obj.TYPE_MEM 625 p2.To.Reg = arm64.REG_R17 626 p2.To.Offset = 8 627 p3 := gc.Prog(arm64.ACMP) 628 p3.From.Type = obj.TYPE_REG 629 p3.From.Reg = v.Args[2].Reg() 630 p3.Reg = arm64.REG_R16 631 p4 := gc.Prog(arm64.ABLE) 632 p4.To.Type = obj.TYPE_BRANCH 633 gc.Patch(p4, p) 634 case ssa.OpARM64CALLstatic: 635 if v.Aux.(*gc.Sym) == gc.Deferreturn.Sym { 636 // Deferred calls will appear to be returning to 637 // the CALL deferreturn(SB) that we are about to emit. 638 // However, the stack trace code will show the line 639 // of the instruction byte before the return PC. 640 // To avoid that being an unrelated instruction, 641 // insert an actual hardware NOP that will have the right line number. 642 // This is different from obj.ANOP, which is a virtual no-op 643 // that doesn't make it into the instruction stream. 644 ginsnop() 645 } 646 p := gc.Prog(obj.ACALL) 647 p.To.Type = obj.TYPE_MEM 648 p.To.Name = obj.NAME_EXTERN 649 p.To.Sym = gc.Linksym(v.Aux.(*gc.Sym)) 650 if gc.Maxarg < v.AuxInt { 651 gc.Maxarg = v.AuxInt 652 } 653 case ssa.OpARM64CALLclosure: 654 p := gc.Prog(obj.ACALL) 655 p.To.Type = obj.TYPE_MEM 656 p.To.Offset = 0 657 p.To.Reg = v.Args[0].Reg() 658 if gc.Maxarg < v.AuxInt { 659 gc.Maxarg = v.AuxInt 660 } 661 case ssa.OpARM64CALLdefer: 662 p := gc.Prog(obj.ACALL) 663 p.To.Type = obj.TYPE_MEM 664 p.To.Name = obj.NAME_EXTERN 665 p.To.Sym = gc.Linksym(gc.Deferproc.Sym) 666 if gc.Maxarg < v.AuxInt { 667 gc.Maxarg = v.AuxInt 668 } 669 case ssa.OpARM64CALLgo: 670 p := gc.Prog(obj.ACALL) 671 p.To.Type = obj.TYPE_MEM 672 p.To.Name = obj.NAME_EXTERN 673 p.To.Sym = gc.Linksym(gc.Newproc.Sym) 674 if gc.Maxarg < v.AuxInt { 675 gc.Maxarg = v.AuxInt 676 } 677 case ssa.OpARM64CALLinter: 678 p := gc.Prog(obj.ACALL) 679 p.To.Type = obj.TYPE_MEM 680 p.To.Offset = 0 681 p.To.Reg = v.Args[0].Reg() 682 if gc.Maxarg < v.AuxInt { 683 gc.Maxarg = v.AuxInt 684 } 685 case ssa.OpARM64LoweredNilCheck: 686 // Issue a load which will fault if arg is nil. 687 p := gc.Prog(arm64.AMOVB) 688 p.From.Type = obj.TYPE_MEM 689 p.From.Reg = v.Args[0].Reg() 690 gc.AddAux(&p.From, v) 691 p.To.Type = obj.TYPE_REG 692 p.To.Reg = arm64.REGTMP 693 if gc.Debug_checknil != 0 && v.Line > 1 { // v.Line==1 in generated wrappers 694 gc.Warnl(v.Line, "generated nil check") 695 } 696 case ssa.OpVarDef: 697 gc.Gvardef(v.Aux.(*gc.Node)) 698 case ssa.OpVarKill: 699 gc.Gvarkill(v.Aux.(*gc.Node)) 700 case ssa.OpVarLive: 701 gc.Gvarlive(v.Aux.(*gc.Node)) 702 case ssa.OpKeepAlive: 703 gc.KeepAlive(v) 704 case ssa.OpARM64Equal, 705 ssa.OpARM64NotEqual, 706 ssa.OpARM64LessThan, 707 ssa.OpARM64LessEqual, 708 ssa.OpARM64GreaterThan, 709 ssa.OpARM64GreaterEqual, 710 ssa.OpARM64LessThanU, 711 ssa.OpARM64LessEqualU, 712 ssa.OpARM64GreaterThanU, 713 ssa.OpARM64GreaterEqualU: 714 // generate boolean values using CSET 715 p := gc.Prog(arm64.ACSET) 716 p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg 717 p.From.Reg = condBits[v.Op] 718 p.To.Type = obj.TYPE_REG 719 p.To.Reg = v.Reg() 720 case ssa.OpSelect0, ssa.OpSelect1: 721 // nothing to do 722 case ssa.OpARM64LoweredGetClosurePtr: 723 // Closure pointer is R26 (arm64.REGCTXT). 724 gc.CheckLoweredGetClosurePtr(v) 725 case ssa.OpARM64FlagEQ, 726 ssa.OpARM64FlagLT_ULT, 727 ssa.OpARM64FlagLT_UGT, 728 ssa.OpARM64FlagGT_ULT, 729 ssa.OpARM64FlagGT_UGT: 730 v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString()) 731 case ssa.OpARM64InvertFlags: 732 v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString()) 733 default: 734 v.Fatalf("genValue not implemented: %s", v.LongString()) 735 } 736 } 737 738 var condBits = map[ssa.Op]int16{ 739 ssa.OpARM64Equal: arm64.COND_EQ, 740 ssa.OpARM64NotEqual: arm64.COND_NE, 741 ssa.OpARM64LessThan: arm64.COND_LT, 742 ssa.OpARM64LessThanU: arm64.COND_LO, 743 ssa.OpARM64LessEqual: arm64.COND_LE, 744 ssa.OpARM64LessEqualU: arm64.COND_LS, 745 ssa.OpARM64GreaterThan: arm64.COND_GT, 746 ssa.OpARM64GreaterThanU: arm64.COND_HI, 747 ssa.OpARM64GreaterEqual: arm64.COND_GE, 748 ssa.OpARM64GreaterEqualU: arm64.COND_HS, 749 } 750 751 var blockJump = map[ssa.BlockKind]struct { 752 asm, invasm obj.As 753 }{ 754 ssa.BlockARM64EQ: {arm64.ABEQ, arm64.ABNE}, 755 ssa.BlockARM64NE: {arm64.ABNE, arm64.ABEQ}, 756 ssa.BlockARM64LT: {arm64.ABLT, arm64.ABGE}, 757 ssa.BlockARM64GE: {arm64.ABGE, arm64.ABLT}, 758 ssa.BlockARM64LE: {arm64.ABLE, arm64.ABGT}, 759 ssa.BlockARM64GT: {arm64.ABGT, arm64.ABLE}, 760 ssa.BlockARM64ULT: {arm64.ABLO, arm64.ABHS}, 761 ssa.BlockARM64UGE: {arm64.ABHS, arm64.ABLO}, 762 ssa.BlockARM64UGT: {arm64.ABHI, arm64.ABLS}, 763 ssa.BlockARM64ULE: {arm64.ABLS, arm64.ABHI}, 764 ssa.BlockARM64Z: {arm64.ACBZ, arm64.ACBNZ}, 765 ssa.BlockARM64NZ: {arm64.ACBNZ, arm64.ACBZ}, 766 ssa.BlockARM64ZW: {arm64.ACBZW, arm64.ACBNZW}, 767 ssa.BlockARM64NZW: {arm64.ACBNZW, arm64.ACBZW}, 768 } 769 770 func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { 771 s.SetLineno(b.Line) 772 773 switch b.Kind { 774 case ssa.BlockPlain: 775 if b.Succs[0].Block() != next { 776 p := gc.Prog(obj.AJMP) 777 p.To.Type = obj.TYPE_BRANCH 778 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 779 } 780 781 case ssa.BlockDefer: 782 // defer returns in R0: 783 // 0 if we should continue executing 784 // 1 if we should jump to deferreturn call 785 p := gc.Prog(arm64.ACMP) 786 p.From.Type = obj.TYPE_CONST 787 p.From.Offset = 0 788 p.Reg = arm64.REG_R0 789 p = gc.Prog(arm64.ABNE) 790 p.To.Type = obj.TYPE_BRANCH 791 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) 792 if b.Succs[0].Block() != next { 793 p := gc.Prog(obj.AJMP) 794 p.To.Type = obj.TYPE_BRANCH 795 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 796 } 797 798 case ssa.BlockExit: 799 gc.Prog(obj.AUNDEF) // tell plive.go that we never reach here 800 801 case ssa.BlockRet: 802 gc.Prog(obj.ARET) 803 804 case ssa.BlockRetJmp: 805 p := gc.Prog(obj.ARET) 806 p.To.Type = obj.TYPE_MEM 807 p.To.Name = obj.NAME_EXTERN 808 p.To.Sym = gc.Linksym(b.Aux.(*gc.Sym)) 809 810 case ssa.BlockARM64EQ, ssa.BlockARM64NE, 811 ssa.BlockARM64LT, ssa.BlockARM64GE, 812 ssa.BlockARM64LE, ssa.BlockARM64GT, 813 ssa.BlockARM64ULT, ssa.BlockARM64UGT, 814 ssa.BlockARM64ULE, ssa.BlockARM64UGE, 815 ssa.BlockARM64Z, ssa.BlockARM64NZ, 816 ssa.BlockARM64ZW, ssa.BlockARM64NZW: 817 jmp := blockJump[b.Kind] 818 var p *obj.Prog 819 switch next { 820 case b.Succs[0].Block(): 821 p = gc.Prog(jmp.invasm) 822 p.To.Type = obj.TYPE_BRANCH 823 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()}) 824 case b.Succs[1].Block(): 825 p = gc.Prog(jmp.asm) 826 p.To.Type = obj.TYPE_BRANCH 827 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 828 default: 829 p = gc.Prog(jmp.asm) 830 p.To.Type = obj.TYPE_BRANCH 831 s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) 832 q := gc.Prog(obj.AJMP) 833 q.To.Type = obj.TYPE_BRANCH 834 s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) 835 } 836 if !b.Control.Type.IsFlags() { 837 p.From.Type = obj.TYPE_REG 838 p.From.Reg = b.Control.Reg() 839 } 840 841 default: 842 b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) 843 } 844 } 845
