1 //===- subzero/src/IceInstX86BaseImpl.h - Generic X86 instructions -*- C++ -*=// 2 // 3 // The Subzero Code Generator 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// 10 /// \file 11 /// \brief Implements the InstX86Base class and its descendants. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #ifndef SUBZERO_SRC_ICEINSTX86BASEIMPL_H 16 #define SUBZERO_SRC_ICEINSTX86BASEIMPL_H 17 18 #include "IceInstX86Base.h" 19 20 #include "IceAssemblerX86Base.h" 21 #include "IceCfg.h" 22 #include "IceCfgNode.h" 23 #include "IceDefs.h" 24 #include "IceInst.h" 25 #include "IceOperand.h" 26 #include "IceTargetLowering.h" 27 #include "IceTargetLoweringX86Base.h" 28 29 namespace Ice { 30 31 namespace X86NAMESPACE { 32 33 template <typename TraitsType> 34 const char *InstImpl<TraitsType>::InstX86Base::getWidthString(Type Ty) { 35 return Traits::TypeAttributes[Ty].WidthString; 36 } 37 38 template <typename TraitsType> 39 const char *InstImpl<TraitsType>::InstX86Base::getFldString(Type Ty) { 40 return Traits::TypeAttributes[Ty].FldString; 41 } 42 43 template <typename TraitsType> 44 typename InstImpl<TraitsType>::Cond::BrCond 45 InstImpl<TraitsType>::InstX86Base::getOppositeCondition(BrCond Cond) { 46 return Traits::InstBrAttributes[Cond].Opposite; 47 } 48 49 template <typename TraitsType> 50 InstImpl<TraitsType>::InstX86FakeRMW::InstX86FakeRMW(Cfg *Func, Operand *Data, 51 Operand *Addr, 52 InstArithmetic::OpKind Op, 53 Variable *Beacon) 54 : InstX86Base(Func, InstX86Base::FakeRMW, 3, nullptr), Op(Op) { 55 this->addSource(Data); 56 this->addSource(Addr); 57 this->addSource(Beacon); 58 } 59 60 template <typename TraitsType> 61 InstImpl<TraitsType>::InstX86GetIP::InstX86GetIP(Cfg *Func, Variable *Dest) 62 : InstX86Base(Func, InstX86Base::GetIP, 0, Dest) {} 63 64 template <typename TraitsType> 65 InstImpl<TraitsType>::InstX86Mul::InstX86Mul(Cfg *Func, Variable *Dest, 66 Variable *Source1, 67 Operand *Source2) 68 : InstX86Base(Func, InstX86Base::Mul, 2, Dest) { 69 this->addSource(Source1); 70 this->addSource(Source2); 71 } 72 73 template <typename TraitsType> 74 InstImpl<TraitsType>::InstX86Shld::InstX86Shld(Cfg *Func, Variable *Dest, 75 Variable *Source1, 76 Operand *Source2) 77 : InstX86Base(Func, InstX86Base::Shld, 3, Dest) { 78 this->addSource(Dest); 79 this->addSource(Source1); 80 this->addSource(Source2); 81 } 82 83 template <typename TraitsType> 84 InstImpl<TraitsType>::InstX86Shrd::InstX86Shrd(Cfg *Func, Variable *Dest, 85 Variable *Source1, 86 Operand *Source2) 87 : InstX86Base(Func, InstX86Base::Shrd, 3, Dest) { 88 this->addSource(Dest); 89 this->addSource(Source1); 90 this->addSource(Source2); 91 } 92 93 template <typename TraitsType> 94 InstImpl<TraitsType>::InstX86Label::InstX86Label(Cfg *Func, 95 TargetLowering *Target) 96 : InstX86Base(Func, InstX86Base::Label, 0, nullptr), 97 LabelNumber(Target->makeNextLabelNumber()) { 98 if (BuildDefs::dump()) { 99 Name = GlobalString::createWithString( 100 Func->getContext(), ".L" + Func->getFunctionName() + "$local$__" + 101 std::to_string(LabelNumber)); 102 } else { 103 Name = GlobalString::createWithoutString(Func->getContext()); 104 } 105 } 106 107 template <typename TraitsType> 108 InstImpl<TraitsType>::InstX86Br::InstX86Br(Cfg *Func, const CfgNode *TargetTrue, 109 const CfgNode *TargetFalse, 110 const InstX86Label *Label, 111 BrCond Condition, Mode Kind) 112 : InstX86Base(Func, InstX86Base::Br, 0, nullptr), Condition(Condition), 113 TargetTrue(TargetTrue), TargetFalse(TargetFalse), Label(Label), 114 Kind(Kind) {} 115 116 template <typename TraitsType> 117 bool InstImpl<TraitsType>::InstX86Br::optimizeBranch(const CfgNode *NextNode) { 118 // If there is no next block, then there can be no fallthrough to optimize. 119 if (NextNode == nullptr) 120 return false; 121 // Intra-block conditional branches can't be optimized. 122 if (Label) 123 return false; 124 // If there is no fallthrough node, such as a non-default case label for a 125 // switch instruction, then there is no opportunity to optimize. 126 if (getTargetFalse() == nullptr) 127 return false; 128 129 // Unconditional branch to the next node can be removed. 130 if (Condition == Cond::Br_None && getTargetFalse() == NextNode) { 131 assert(getTargetTrue() == nullptr); 132 this->setDeleted(); 133 return true; 134 } 135 // If the fallthrough is to the next node, set fallthrough to nullptr to 136 // indicate. 137 if (getTargetFalse() == NextNode) { 138 TargetFalse = nullptr; 139 return true; 140 } 141 // If TargetTrue is the next node, and TargetFalse is not nullptr (which was 142 // already tested above), then invert the branch condition, swap the targets, 143 // and set new fallthrough to nullptr. 144 if (getTargetTrue() == NextNode) { 145 assert(Condition != Cond::Br_None); 146 Condition = this->getOppositeCondition(Condition); 147 TargetTrue = getTargetFalse(); 148 TargetFalse = nullptr; 149 return true; 150 } 151 return false; 152 } 153 154 template <typename TraitsType> 155 bool InstImpl<TraitsType>::InstX86Br::repointEdges(CfgNode *OldNode, 156 CfgNode *NewNode) { 157 bool Found = false; 158 if (TargetFalse == OldNode) { 159 TargetFalse = NewNode; 160 Found = true; 161 } 162 if (TargetTrue == OldNode) { 163 TargetTrue = NewNode; 164 Found = true; 165 } 166 return Found; 167 } 168 169 template <typename TraitsType> 170 InstImpl<TraitsType>::InstX86Jmp::InstX86Jmp(Cfg *Func, Operand *Target) 171 : InstX86Base(Func, InstX86Base::Jmp, 1, nullptr) { 172 this->addSource(Target); 173 } 174 175 template <typename TraitsType> 176 InstImpl<TraitsType>::InstX86Call::InstX86Call(Cfg *Func, Variable *Dest, 177 Operand *CallTarget) 178 : InstX86Base(Func, InstX86Base::Call, 1, Dest) { 179 this->HasSideEffects = true; 180 this->addSource(CallTarget); 181 } 182 183 template <typename TraitsType> 184 InstImpl<TraitsType>::InstX86Movmsk::InstX86Movmsk(Cfg *Func, Variable *Dest, 185 Operand *Source) 186 : InstX86Base(Func, InstX86Base::Movmsk, 1, Dest) { 187 this->addSource(Source); 188 } 189 190 template <typename TraitsType> 191 InstImpl<TraitsType>::InstX86Cmov::InstX86Cmov(Cfg *Func, Variable *Dest, 192 Operand *Source, 193 BrCond Condition) 194 : InstX86Base(Func, InstX86Base::Cmov, 2, Dest), Condition(Condition) { 195 // The final result is either the original Dest, or Source, so mark both as 196 // sources. 197 this->addSource(Dest); 198 this->addSource(Source); 199 } 200 201 template <typename TraitsType> 202 InstImpl<TraitsType>::InstX86Cmpps::InstX86Cmpps(Cfg *Func, Variable *Dest, 203 Operand *Source, 204 CmppsCond Condition) 205 : InstX86Base(Func, InstX86Base::Cmpps, 2, Dest), Condition(Condition) { 206 this->addSource(Dest); 207 this->addSource(Source); 208 } 209 210 template <typename TraitsType> 211 InstImpl<TraitsType>::InstX86Cmpxchg::InstX86Cmpxchg(Cfg *Func, 212 Operand *DestOrAddr, 213 Variable *Eax, 214 Variable *Desired, 215 bool Locked) 216 : InstImpl<TraitsType>::InstX86BaseLockable( 217 Func, InstX86Base::Cmpxchg, 3, llvm::dyn_cast<Variable>(DestOrAddr), 218 Locked) { 219 constexpr uint16_t Encoded_rAX = 0; 220 (void)Encoded_rAX; 221 assert(Traits::getEncodedGPR(Eax->getRegNum()) == Encoded_rAX); 222 this->addSource(DestOrAddr); 223 this->addSource(Eax); 224 this->addSource(Desired); 225 } 226 227 template <typename TraitsType> 228 InstImpl<TraitsType>::InstX86Cmpxchg8b::InstX86Cmpxchg8b( 229 Cfg *Func, X86OperandMem *Addr, Variable *Edx, Variable *Eax, Variable *Ecx, 230 Variable *Ebx, bool Locked) 231 : InstImpl<TraitsType>::InstX86BaseLockable(Func, InstX86Base::Cmpxchg, 5, 232 nullptr, Locked) { 233 assert(Edx->getRegNum() == RegisterSet::Reg_edx); 234 assert(Eax->getRegNum() == RegisterSet::Reg_eax); 235 assert(Ecx->getRegNum() == RegisterSet::Reg_ecx); 236 assert(Ebx->getRegNum() == RegisterSet::Reg_ebx); 237 this->addSource(Addr); 238 this->addSource(Edx); 239 this->addSource(Eax); 240 this->addSource(Ecx); 241 this->addSource(Ebx); 242 } 243 244 template <typename TraitsType> 245 InstImpl<TraitsType>::InstX86Cvt::InstX86Cvt(Cfg *Func, Variable *Dest, 246 Operand *Source, 247 CvtVariant Variant) 248 : InstX86Base(Func, InstX86Base::Cvt, 1, Dest), Variant(Variant) { 249 this->addSource(Source); 250 } 251 252 template <typename TraitsType> 253 InstImpl<TraitsType>::InstX86Icmp::InstX86Icmp(Cfg *Func, Operand *Src0, 254 Operand *Src1) 255 : InstX86Base(Func, InstX86Base::Icmp, 2, nullptr) { 256 this->addSource(Src0); 257 this->addSource(Src1); 258 } 259 260 template <typename TraitsType> 261 InstImpl<TraitsType>::InstX86Ucomiss::InstX86Ucomiss(Cfg *Func, Operand *Src0, 262 Operand *Src1) 263 : InstX86Base(Func, InstX86Base::Ucomiss, 2, nullptr) { 264 this->addSource(Src0); 265 this->addSource(Src1); 266 } 267 268 template <typename TraitsType> 269 InstImpl<TraitsType>::InstX86UD2::InstX86UD2(Cfg *Func) 270 : InstX86Base(Func, InstX86Base::UD2, 0, nullptr) {} 271 272 template <typename TraitsType> 273 InstImpl<TraitsType>::InstX86Int3::InstX86Int3(Cfg *Func) 274 : InstX86Base(Func, InstX86Base::Int3, 0, nullptr) {} 275 276 template <typename TraitsType> 277 InstImpl<TraitsType>::InstX86Test::InstX86Test(Cfg *Func, Operand *Src1, 278 Operand *Src2) 279 : InstX86Base(Func, InstX86Base::Test, 2, nullptr) { 280 this->addSource(Src1); 281 this->addSource(Src2); 282 } 283 284 template <typename TraitsType> 285 InstImpl<TraitsType>::InstX86Mfence::InstX86Mfence(Cfg *Func) 286 : InstX86Base(Func, InstX86Base::Mfence, 0, nullptr) { 287 this->HasSideEffects = true; 288 } 289 290 template <typename TraitsType> 291 InstImpl<TraitsType>::InstX86Store::InstX86Store(Cfg *Func, Operand *Value, 292 X86Operand *Mem) 293 : InstX86Base(Func, InstX86Base::Store, 2, nullptr) { 294 this->addSource(Value); 295 this->addSource(Mem); 296 } 297 298 template <typename TraitsType> 299 InstImpl<TraitsType>::InstX86StoreP::InstX86StoreP(Cfg *Func, Variable *Value, 300 X86OperandMem *Mem) 301 : InstX86Base(Func, InstX86Base::StoreP, 2, nullptr) { 302 this->addSource(Value); 303 this->addSource(Mem); 304 } 305 306 template <typename TraitsType> 307 InstImpl<TraitsType>::InstX86StoreQ::InstX86StoreQ(Cfg *Func, Operand *Value, 308 X86OperandMem *Mem) 309 : InstX86Base(Func, InstX86Base::StoreQ, 2, nullptr) { 310 this->addSource(Value); 311 this->addSource(Mem); 312 } 313 314 template <typename TraitsType> 315 InstImpl<TraitsType>::InstX86StoreD::InstX86StoreD(Cfg *Func, Operand *Value, 316 X86OperandMem *Mem) 317 : InstX86Base(Func, InstX86Base::StoreD, 2, nullptr) { 318 this->addSource(Value); 319 this->addSource(Mem); 320 } 321 322 template <typename TraitsType> 323 InstImpl<TraitsType>::InstX86Nop::InstX86Nop(Cfg *Func, NopVariant Variant) 324 : InstX86Base(Func, InstX86Base::Nop, 0, nullptr), Variant(Variant) {} 325 326 template <typename TraitsType> 327 InstImpl<TraitsType>::InstX86Fld::InstX86Fld(Cfg *Func, Operand *Src) 328 : InstX86Base(Func, InstX86Base::Fld, 1, nullptr) { 329 this->addSource(Src); 330 } 331 332 template <typename TraitsType> 333 InstImpl<TraitsType>::InstX86Fstp::InstX86Fstp(Cfg *Func, Variable *Dest) 334 : InstX86Base(Func, InstX86Base::Fstp, 0, Dest) {} 335 336 template <typename TraitsType> 337 InstImpl<TraitsType>::InstX86Pop::InstX86Pop(Cfg *Func, Variable *Dest) 338 : InstX86Base(Func, InstX86Base::Pop, 0, Dest) { 339 // A pop instruction affects the stack pointer and so it should not be 340 // allowed to be automatically dead-code eliminated. (The corresponding push 341 // instruction doesn't need this treatment because it has no dest variable 342 // and therefore won't be dead-code eliminated.) This is needed for 343 // late-stage liveness analysis (e.g. asm-verbose mode). 344 this->HasSideEffects = true; 345 } 346 347 template <typename TraitsType> 348 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, Operand *Source) 349 : InstX86Base(Func, InstX86Base::Push, 1, nullptr) { 350 this->addSource(Source); 351 } 352 353 template <typename TraitsType> 354 InstImpl<TraitsType>::InstX86Push::InstX86Push(Cfg *Func, InstX86Label *L) 355 : InstX86Base(Func, InstX86Base::Push, 0, nullptr), Label(L) {} 356 357 template <typename TraitsType> 358 InstImpl<TraitsType>::InstX86Ret::InstX86Ret(Cfg *Func, Variable *Source) 359 : InstX86Base(Func, InstX86Base::Ret, Source ? 1 : 0, nullptr) { 360 if (Source) 361 this->addSource(Source); 362 } 363 364 template <typename TraitsType> 365 InstImpl<TraitsType>::InstX86Setcc::InstX86Setcc(Cfg *Func, Variable *Dest, 366 BrCond Cond) 367 : InstX86Base(Func, InstX86Base::Setcc, 0, Dest), Condition(Cond) {} 368 369 template <typename TraitsType> 370 InstImpl<TraitsType>::InstX86Xadd::InstX86Xadd(Cfg *Func, Operand *Dest, 371 Variable *Source, bool Locked) 372 : InstImpl<TraitsType>::InstX86BaseLockable( 373 Func, InstX86Base::Xadd, 2, llvm::dyn_cast<Variable>(Dest), Locked) { 374 this->addSource(Dest); 375 this->addSource(Source); 376 } 377 378 template <typename TraitsType> 379 InstImpl<TraitsType>::InstX86Xchg::InstX86Xchg(Cfg *Func, Operand *Dest, 380 Variable *Source) 381 : InstX86Base(Func, InstX86Base::Xchg, 2, llvm::dyn_cast<Variable>(Dest)) { 382 this->addSource(Dest); 383 this->addSource(Source); 384 } 385 386 template <typename TraitsType> 387 InstImpl<TraitsType>::InstX86IacaStart::InstX86IacaStart(Cfg *Func) 388 : InstX86Base(Func, InstX86Base::IacaStart, 0, nullptr) { 389 assert(getFlags().getAllowIacaMarks()); 390 } 391 392 template <typename TraitsType> 393 InstImpl<TraitsType>::InstX86IacaEnd::InstX86IacaEnd(Cfg *Func) 394 : InstX86Base(Func, InstX86Base::IacaEnd, 0, nullptr) { 395 assert(getFlags().getAllowIacaMarks()); 396 } 397 398 // ======================== Dump routines ======================== // 399 400 template <typename TraitsType> 401 void InstImpl<TraitsType>::InstX86Base::dump(const Cfg *Func) const { 402 if (!BuildDefs::dump()) 403 return; 404 Ostream &Str = Func->getContext()->getStrDump(); 405 Str << "[" << Traits::TargetName << "] "; 406 Inst::dump(Func); 407 } 408 409 template <typename TraitsType> 410 void InstImpl<TraitsType>::InstX86FakeRMW::dump(const Cfg *Func) const { 411 if (!BuildDefs::dump()) 412 return; 413 Ostream &Str = Func->getContext()->getStrDump(); 414 Type Ty = getData()->getType(); 415 Str << "rmw " << InstArithmetic::getOpName(getOp()) << " " << Ty << " *"; 416 getAddr()->dump(Func); 417 Str << ", "; 418 getData()->dump(Func); 419 Str << ", beacon="; 420 getBeacon()->dump(Func); 421 } 422 423 template <typename TraitsType> 424 void InstImpl<TraitsType>::InstX86GetIP::emit(const Cfg *Func) const { 425 if (!BuildDefs::dump()) 426 return; 427 const auto *Dest = this->getDest(); 428 assert(Dest->hasReg()); 429 Ostream &Str = Func->getContext()->getStrEmit(); 430 Str << "\t" 431 "call" 432 "\t"; 433 auto *Target = static_cast<TargetLowering *>(Func->getTarget()); 434 Target->emitWithoutPrefix(Target->createGetIPForRegister(Dest)); 435 } 436 437 template <typename TraitsType> 438 void InstImpl<TraitsType>::InstX86GetIP::emitIAS(const Cfg *Func) const { 439 const auto *Dest = this->getDest(); 440 Assembler *Asm = Func->getAssembler<Assembler>(); 441 assert(Dest->hasReg()); 442 Asm->call(static_cast<TargetLowering *>(Func->getTarget()) 443 ->createGetIPForRegister(Dest)); 444 } 445 446 template <typename TraitsType> 447 void InstImpl<TraitsType>::InstX86GetIP::dump(const Cfg *Func) const { 448 if (!BuildDefs::dump()) 449 return; 450 Ostream &Str = Func->getContext()->getStrDump(); 451 this->getDest()->dump(Func); 452 Str << " = call getIP"; 453 } 454 455 template <typename TraitsType> 456 void InstImpl<TraitsType>::InstX86Label::emit(const Cfg *Func) const { 457 if (!BuildDefs::dump()) 458 return; 459 Ostream &Str = Func->getContext()->getStrEmit(); 460 Str << getLabelName() << ":"; 461 } 462 463 template <typename TraitsType> 464 void InstImpl<TraitsType>::InstX86Label::emitIAS(const Cfg *Func) const { 465 Assembler *Asm = Func->getAssembler<Assembler>(); 466 Asm->bindLocalLabel(LabelNumber); 467 if (OffsetReloc != nullptr) { 468 Asm->bindRelocOffset(OffsetReloc); 469 } 470 } 471 472 template <typename TraitsType> 473 void InstImpl<TraitsType>::InstX86Label::dump(const Cfg *Func) const { 474 if (!BuildDefs::dump()) 475 return; 476 Ostream &Str = Func->getContext()->getStrDump(); 477 Str << getLabelName() << ":"; 478 } 479 480 template <typename TraitsType> 481 void InstImpl<TraitsType>::InstX86Br::emit(const Cfg *Func) const { 482 if (!BuildDefs::dump()) 483 return; 484 Ostream &Str = Func->getContext()->getStrEmit(); 485 Str << "\t"; 486 487 if (Condition == Cond::Br_None) { 488 Str << "jmp"; 489 } else { 490 Str << Traits::InstBrAttributes[Condition].EmitString; 491 } 492 493 if (Label) { 494 Str << "\t" << Label->getLabelName(); 495 } else { 496 if (Condition == Cond::Br_None) { 497 Str << "\t" << getTargetFalse()->getAsmName(); 498 } else { 499 Str << "\t" << getTargetTrue()->getAsmName(); 500 if (getTargetFalse()) { 501 Str << "\n\t" 502 "jmp\t" << getTargetFalse()->getAsmName(); 503 } 504 } 505 } 506 } 507 508 template <typename TraitsType> 509 void InstImpl<TraitsType>::InstX86Br::emitIAS(const Cfg *Func) const { 510 Assembler *Asm = Func->getAssembler<Assembler>(); 511 if (Label) { 512 auto *L = Asm->getOrCreateLocalLabel(Label->getLabelNumber()); 513 if (Condition == Cond::Br_None) { 514 Asm->jmp(L, isNear()); 515 } else { 516 Asm->j(Condition, L, isNear()); 517 } 518 } else { 519 if (Condition == Cond::Br_None) { 520 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex()); 521 assert(!getTargetTrue()); 522 Asm->jmp(L, isNear()); 523 } else { 524 auto *L = Asm->getOrCreateCfgNodeLabel(getTargetTrue()->getIndex()); 525 Asm->j(Condition, L, isNear()); 526 if (getTargetFalse()) { 527 auto *L2 = Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex()); 528 Asm->jmp(L2, isNear()); 529 } 530 } 531 } 532 } 533 534 template <typename TraitsType> 535 void InstImpl<TraitsType>::InstX86Br::dump(const Cfg *Func) const { 536 if (!BuildDefs::dump()) 537 return; 538 Ostream &Str = Func->getContext()->getStrDump(); 539 Str << "br "; 540 541 if (Condition == Cond::Br_None) { 542 if (Label) { 543 Str << "label %" << Label->getLabelName(); 544 } else { 545 Str << "label %" << getTargetFalse()->getName(); 546 } 547 return; 548 } 549 550 Str << Traits::InstBrAttributes[Condition].DisplayString; 551 if (Label) { 552 Str << ", label %" << Label->getLabelName(); 553 } else { 554 Str << ", label %" << getTargetTrue()->getName(); 555 if (getTargetFalse()) { 556 Str << ", label %" << getTargetFalse()->getName(); 557 } 558 } 559 560 Str << " // (" << (isNear() ? "near" : "far") << " jump)"; 561 } 562 563 template <typename TraitsType> 564 void InstImpl<TraitsType>::InstX86Jmp::emit(const Cfg *Func) const { 565 if (!BuildDefs::dump()) 566 return; 567 Ostream &Str = Func->getContext()->getStrEmit(); 568 assert(this->getSrcSize() == 1); 569 const Operand *Src = this->getSrc(0); 570 if (Traits::Is64Bit) { 571 if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) { 572 Str << "\t" 573 "jmp" 574 "\t" << CR->getName(); 575 return; 576 } 577 } 578 Str << "\t" 579 "jmp" 580 "\t*"; 581 getJmpTarget()->emit(Func); 582 } 583 584 template <typename TraitsType> 585 void InstImpl<TraitsType>::InstX86Jmp::emitIAS(const Cfg *Func) const { 586 // Note: Adapted (mostly copied) from 587 // InstImpl<TraitsType>::InstX86Call::emitIAS(). 588 Assembler *Asm = Func->getAssembler<Assembler>(); 589 Operand *Target = getJmpTarget(); 590 if (const auto *Var = llvm::dyn_cast<Variable>(Target)) { 591 if (Var->hasReg()) { 592 Asm->jmp(Traits::getEncodedGPR(Var->getRegNum())); 593 } else { 594 // The jmp instruction with a memory operand should be possible to 595 // encode, but it isn't a valid sandboxed instruction, and there 596 // shouldn't be a register allocation issue to jump through a scratch 597 // register, so we don't really need to bother implementing it. 598 llvm::report_fatal_error("Assembler can't jmp to memory operand"); 599 } 600 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Target)) { 601 (void)Mem; 602 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 603 llvm::report_fatal_error("Assembler can't jmp to memory operand"); 604 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Target)) { 605 Asm->jmp(CR); 606 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Target)) { 607 // NaCl trampoline calls refer to an address within the sandbox directly. 608 // This is usually only needed for non-IRT builds and otherwise not very 609 // portable or stable. Usually this is only done for "calls" and not jumps. 610 Asm->jmp(AssemblerImmediate(Imm->getValue())); 611 } else { 612 llvm::report_fatal_error("Unexpected operand type"); 613 } 614 } 615 616 template <typename TraitsType> 617 void InstImpl<TraitsType>::InstX86Jmp::dump(const Cfg *Func) const { 618 if (!BuildDefs::dump()) 619 return; 620 Ostream &Str = Func->getContext()->getStrDump(); 621 Str << "jmp "; 622 getJmpTarget()->dump(Func); 623 } 624 625 template <typename TraitsType> 626 void InstImpl<TraitsType>::InstX86Call::emit(const Cfg *Func) const { 627 if (!BuildDefs::dump()) 628 return; 629 Ostream &Str = Func->getContext()->getStrEmit(); 630 assert(this->getSrcSize() == 1); 631 Str << "\t" 632 "call\t"; 633 Operand *CallTarget = getCallTarget(); 634 auto *Target = InstX86Base::getTarget(Func); 635 if (const auto *CI = llvm::dyn_cast<ConstantInteger32>(CallTarget)) { 636 // Emit without a leading '$'. 637 Str << CI->getValue(); 638 } else if (const auto DirectCallTarget = 639 llvm::dyn_cast<ConstantRelocatable>(CallTarget)) { 640 DirectCallTarget->emitWithoutPrefix(Target); 641 } else { 642 Str << "*"; 643 CallTarget->emit(Func); 644 } 645 } 646 647 template <typename TraitsType> 648 void InstImpl<TraitsType>::InstX86Call::emitIAS(const Cfg *Func) const { 649 Assembler *Asm = Func->getAssembler<Assembler>(); 650 Operand *CallTarget = getCallTarget(); 651 auto *Target = InstX86Base::getTarget(Func); 652 if (const auto *Var = llvm::dyn_cast<Variable>(CallTarget)) { 653 if (Var->hasReg()) { 654 Asm->call(Traits::getEncodedGPR(Var->getRegNum())); 655 } else { 656 Asm->call(Target->stackVarToAsmOperand(Var)); 657 } 658 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(CallTarget)) { 659 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 660 Asm->call(Mem->toAsmAddress(Asm, Target)); 661 } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(CallTarget)) { 662 Asm->call(CR); 663 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(CallTarget)) { 664 Asm->call(AssemblerImmediate(Imm->getValue())); 665 } else { 666 llvm_unreachable("Unexpected operand type"); 667 } 668 } 669 670 template <typename TraitsType> 671 void InstImpl<TraitsType>::InstX86Call::dump(const Cfg *Func) const { 672 if (!BuildDefs::dump()) 673 return; 674 Ostream &Str = Func->getContext()->getStrDump(); 675 if (this->getDest()) { 676 this->dumpDest(Func); 677 Str << " = "; 678 } 679 Str << "call "; 680 getCallTarget()->dump(Func); 681 } 682 683 // The this->Opcode parameter needs to be char* and not std::string because of 684 // template issues. 685 template <typename TraitsType> 686 void InstImpl<TraitsType>::InstX86Base::emitTwoAddress( 687 const Cfg *Func, const char *Opcode, const char *Suffix) const { 688 if (!BuildDefs::dump()) 689 return; 690 Ostream &Str = Func->getContext()->getStrEmit(); 691 assert(getSrcSize() == 2); 692 Operand *Dest = getDest(); 693 if (Dest == nullptr) 694 Dest = getSrc(0); 695 assert(Dest == getSrc(0)); 696 Operand *Src1 = getSrc(1); 697 Str << "\t" << Opcode << Suffix 698 << InstX86Base::getWidthString(Dest->getType()) << "\t"; 699 Src1->emit(Func); 700 Str << ", "; 701 Dest->emit(Func); 702 } 703 704 template <typename TraitsType> 705 void InstImpl<TraitsType>::emitIASOpTyGPR(const Cfg *Func, Type Ty, 706 const Operand *Op, 707 const GPREmitterOneOp &Emitter) { 708 auto *Target = InstX86Base::getTarget(Func); 709 Assembler *Asm = Func->getAssembler<Assembler>(); 710 if (const auto *Var = llvm::dyn_cast<Variable>(Op)) { 711 if (Var->hasReg()) { 712 // We cheat a little and use GPRRegister even for byte operations. 713 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum()); 714 (Asm->*(Emitter.Reg))(Ty, VarReg); 715 } else { 716 Address StackAddr(Target->stackVarToAsmOperand(Var)); 717 (Asm->*(Emitter.Addr))(Ty, StackAddr); 718 } 719 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Op)) { 720 Mem->emitSegmentOverride(Asm); 721 (Asm->*(Emitter.Addr))(Ty, Mem->toAsmAddress(Asm, Target)); 722 } else { 723 llvm_unreachable("Unexpected operand type"); 724 } 725 } 726 727 template <typename TraitsType> 728 template <bool VarCanBeByte, bool SrcCanBeByte> 729 void InstImpl<TraitsType>::emitIASRegOpTyGPR(const Cfg *Func, bool IsLea, 730 Type Ty, const Variable *Var, 731 const Operand *Src, 732 const GPREmitterRegOp &Emitter) { 733 auto *Target = InstX86Base::getTarget(Func); 734 Assembler *Asm = Func->getAssembler<Assembler>(); 735 assert(Var->hasReg()); 736 // We cheat a little and use GPRRegister even for byte operations. 737 GPRRegister VarReg = VarCanBeByte ? Traits::getEncodedGPR(Var->getRegNum()) 738 : Traits::getEncodedGPR(Var->getRegNum()); 739 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 740 if (SrcVar->hasReg()) { 741 GPRRegister SrcReg = SrcCanBeByte 742 ? Traits::getEncodedGPR(SrcVar->getRegNum()) 743 : Traits::getEncodedGPR(SrcVar->getRegNum()); 744 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg); 745 } else { 746 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 747 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, SrcStackAddr); 748 } 749 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 750 Mem->emitSegmentOverride(Asm); 751 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, 752 Mem->toAsmAddress(Asm, Target, IsLea)); 753 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) { 754 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue())); 755 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) { 756 assert(Traits::Is64Bit); 757 assert(Utils::IsInt(32, Imm->getValue())); 758 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue())); 759 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) { 760 const auto FixupKind = (Reloc->getName().hasStdString() && 761 Reloc->getName().toString() == GlobalOffsetTable) 762 ? Traits::FK_GotPC 763 : Traits::TargetLowering::getAbsFixup(); 764 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc); 765 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Fixup)); 766 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Src)) { 767 (Asm->*(Emitter.GPRAddr))(Ty, VarReg, Split->toAsmAddress(Func)); 768 } else { 769 llvm_unreachable("Unexpected operand type"); 770 } 771 } 772 773 template <typename TraitsType> 774 void InstImpl<TraitsType>::emitIASAddrOpTyGPR(const Cfg *Func, Type Ty, 775 const Address &Addr, 776 const Operand *Src, 777 const GPREmitterAddrOp &Emitter) { 778 Assembler *Asm = Func->getAssembler<Assembler>(); 779 // Src can only be Reg or AssemblerImmediate. 780 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 781 assert(SrcVar->hasReg()); 782 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum()); 783 (Asm->*(Emitter.AddrGPR))(Ty, Addr, SrcReg); 784 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) { 785 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue())); 786 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) { 787 assert(Traits::Is64Bit); 788 assert(Utils::IsInt(32, Imm->getValue())); 789 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Imm->getValue())); 790 } else if (const auto *Reloc = llvm::dyn_cast<ConstantRelocatable>(Src)) { 791 const auto FixupKind = (Reloc->getName().hasStdString() && 792 Reloc->getName().toString() == GlobalOffsetTable) 793 ? Traits::FK_GotPC 794 : Traits::TargetLowering::getAbsFixup(); 795 AssemblerFixup *Fixup = Asm->createFixup(FixupKind, Reloc); 796 (Asm->*(Emitter.AddrImm))(Ty, Addr, AssemblerImmediate(Fixup)); 797 } else { 798 llvm_unreachable("Unexpected operand type"); 799 } 800 } 801 802 template <typename TraitsType> 803 void InstImpl<TraitsType>::emitIASAsAddrOpTyGPR( 804 const Cfg *Func, Type Ty, const Operand *Op0, const Operand *Op1, 805 const GPREmitterAddrOp &Emitter) { 806 auto *Target = InstX86Base::getTarget(Func); 807 if (const auto *Op0Var = llvm::dyn_cast<Variable>(Op0)) { 808 assert(!Op0Var->hasReg()); 809 Address StackAddr(Target->stackVarToAsmOperand(Op0Var)); 810 emitIASAddrOpTyGPR(Func, Ty, StackAddr, Op1, Emitter); 811 } else if (const auto *Op0Mem = llvm::dyn_cast<X86OperandMem>(Op0)) { 812 Assembler *Asm = Func->getAssembler<Assembler>(); 813 Op0Mem->emitSegmentOverride(Asm); 814 emitIASAddrOpTyGPR(Func, Ty, Op0Mem->toAsmAddress(Asm, Target), Op1, 815 Emitter); 816 } else if (const auto *Split = llvm::dyn_cast<VariableSplit>(Op0)) { 817 emitIASAddrOpTyGPR(Func, Ty, Split->toAsmAddress(Func), Op1, Emitter); 818 } else { 819 llvm_unreachable("Unexpected operand type"); 820 } 821 } 822 823 template <typename TraitsType> 824 void InstImpl<TraitsType>::emitIASGPRShift(const Cfg *Func, Type Ty, 825 const Variable *Var, 826 const Operand *Src, 827 const GPREmitterShiftOp &Emitter) { 828 Assembler *Asm = Func->getAssembler<Assembler>(); 829 // Technically, the Dest Var can be mem as well, but we only use Reg. We can 830 // extend this to check Dest if we decide to use that form. 831 assert(Var->hasReg()); 832 // We cheat a little and use GPRRegister even for byte operations. 833 GPRRegister VarReg = Traits::getEncodedGPR(Var->getRegNum()); 834 // Src must be reg == ECX or an Imm8. This is asserted by the assembler. 835 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 836 assert(SrcVar->hasReg()); 837 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar->getRegNum()); 838 (Asm->*(Emitter.GPRGPR))(Ty, VarReg, SrcReg); 839 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) { 840 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue())); 841 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger64>(Src)) { 842 assert(Traits::Is64Bit); 843 assert(Utils::IsInt(32, Imm->getValue())); 844 (Asm->*(Emitter.GPRImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue())); 845 } else { 846 llvm_unreachable("Unexpected operand type"); 847 } 848 } 849 850 template <typename TraitsType> 851 void InstImpl<TraitsType>::emitIASGPRShiftDouble( 852 const Cfg *Func, const Variable *Dest, const Operand *Src1Op, 853 const Operand *Src2Op, const GPREmitterShiftD &Emitter) { 854 Assembler *Asm = Func->getAssembler<Assembler>(); 855 // Dest can be reg or mem, but we only use the reg variant. 856 assert(Dest->hasReg()); 857 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum()); 858 // SrcVar1 must be reg. 859 const auto *SrcVar1 = llvm::cast<Variable>(Src1Op); 860 assert(SrcVar1->hasReg()); 861 GPRRegister SrcReg = Traits::getEncodedGPR(SrcVar1->getRegNum()); 862 Type Ty = SrcVar1->getType(); 863 // Src2 can be the implicit CL register or an immediate. 864 if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src2Op)) { 865 (Asm->*(Emitter.GPRGPRImm))(Ty, DestReg, SrcReg, 866 AssemblerImmediate(Imm->getValue())); 867 } else { 868 assert(llvm::cast<Variable>(Src2Op)->getRegNum() == RegisterSet::Reg_cl); 869 (Asm->*(Emitter.GPRGPR))(Ty, DestReg, SrcReg); 870 } 871 } 872 873 template <typename TraitsType> 874 void InstImpl<TraitsType>::emitIASXmmShift(const Cfg *Func, Type Ty, 875 const Variable *Var, 876 const Operand *Src, 877 const XmmEmitterShiftOp &Emitter) { 878 auto *Target = InstX86Base::getTarget(Func); 879 Assembler *Asm = Func->getAssembler<Assembler>(); 880 assert(Var->hasReg()); 881 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum()); 882 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 883 if (SrcVar->hasReg()) { 884 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum()); 885 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg); 886 } else { 887 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 888 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr); 889 } 890 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 891 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 892 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target)); 893 } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Src)) { 894 (Asm->*(Emitter.XmmImm))(Ty, VarReg, AssemblerImmediate(Imm->getValue())); 895 } else { 896 llvm_unreachable("Unexpected operand type"); 897 } 898 } 899 900 template <typename TraitsType> 901 void InstImpl<TraitsType>::emitIASRegOpTyXMM(const Cfg *Func, Type Ty, 902 const Variable *Var, 903 const Operand *Src, 904 const XmmEmitterRegOp &Emitter) { 905 auto *Target = InstX86Base::getTarget(Func); 906 Assembler *Asm = Func->getAssembler<Assembler>(); 907 assert(Var->hasReg()); 908 XmmRegister VarReg = Traits::getEncodedXmm(Var->getRegNum()); 909 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 910 if (SrcVar->hasReg()) { 911 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum()); 912 (Asm->*(Emitter.XmmXmm))(Ty, VarReg, SrcReg); 913 } else { 914 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 915 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, SrcStackAddr); 916 } 917 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 918 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 919 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, Mem->toAsmAddress(Asm, Target)); 920 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) { 921 (Asm->*(Emitter.XmmAddr))(Ty, VarReg, 922 Traits::Address::ofConstPool(Asm, Imm)); 923 } else { 924 llvm_unreachable("Unexpected operand type"); 925 } 926 } 927 928 template <typename TraitsType> 929 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT), 930 SReg_t (*srcEnc)(RegNumT)> 931 void InstImpl<TraitsType>::emitIASCastRegOp( 932 const Cfg *Func, Type DestTy, const Variable *Dest, Type SrcTy, 933 const Operand *Src, const CastEmitterRegOp<DReg_t, SReg_t> &Emitter) { 934 auto *Target = InstX86Base::getTarget(Func); 935 Assembler *Asm = Func->getAssembler<Assembler>(); 936 assert(Dest->hasReg()); 937 DReg_t DestReg = destEnc(Dest->getRegNum()); 938 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 939 if (SrcVar->hasReg()) { 940 SReg_t SrcReg = srcEnc(SrcVar->getRegNum()); 941 (Asm->*(Emitter.RegReg))(DestTy, DestReg, SrcTy, SrcReg); 942 } else { 943 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 944 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy, SrcStackAddr); 945 } 946 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 947 Mem->emitSegmentOverride(Asm); 948 (Asm->*(Emitter.RegAddr))(DestTy, DestReg, SrcTy, 949 Mem->toAsmAddress(Asm, Target)); 950 } else { 951 llvm_unreachable("Unexpected operand type"); 952 } 953 } 954 955 template <typename TraitsType> 956 template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT), 957 SReg_t (*srcEnc)(RegNumT)> 958 void InstImpl<TraitsType>::emitIASThreeOpImmOps( 959 const Cfg *Func, Type DispatchTy, const Variable *Dest, const Operand *Src0, 960 const Operand *Src1, const ThreeOpImmEmitter<DReg_t, SReg_t> Emitter) { 961 auto *Target = InstX86Base::getTarget(Func); 962 Assembler *Asm = Func->getAssembler<Assembler>(); 963 // This only handles Dest being a register, and Src1 being an immediate. 964 assert(Dest->hasReg()); 965 DReg_t DestReg = destEnc(Dest->getRegNum()); 966 AssemblerImmediate Imm(llvm::cast<ConstantInteger32>(Src1)->getValue()); 967 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src0)) { 968 if (SrcVar->hasReg()) { 969 SReg_t SrcReg = srcEnc(SrcVar->getRegNum()); 970 (Asm->*(Emitter.RegRegImm))(DispatchTy, DestReg, SrcReg, Imm); 971 } else { 972 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 973 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg, SrcStackAddr, Imm); 974 } 975 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src0)) { 976 Mem->emitSegmentOverride(Asm); 977 (Asm->*(Emitter.RegAddrImm))(DispatchTy, DestReg, 978 Mem->toAsmAddress(Asm, Target), Imm); 979 } else { 980 llvm_unreachable("Unexpected operand type"); 981 } 982 } 983 984 template <typename TraitsType> 985 void InstImpl<TraitsType>::emitIASMovlikeXMM(const Cfg *Func, 986 const Variable *Dest, 987 const Operand *Src, 988 const XmmEmitterMovOps Emitter) { 989 auto *Target = InstX86Base::getTarget(Func); 990 Assembler *Asm = Func->getAssembler<Assembler>(); 991 if (Dest->hasReg()) { 992 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum()); 993 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 994 if (SrcVar->hasReg()) { 995 (Asm->*(Emitter.XmmXmm))(DestReg, 996 Traits::getEncodedXmm(SrcVar->getRegNum())); 997 } else { 998 Address StackAddr(Target->stackVarToAsmOperand(SrcVar)); 999 (Asm->*(Emitter.XmmAddr))(DestReg, StackAddr); 1000 } 1001 } else if (const auto *SrcMem = llvm::dyn_cast<X86OperandMem>(Src)) { 1002 assert(SrcMem->getSegmentRegister() == X86OperandMem::DefaultSegment); 1003 (Asm->*(Emitter.XmmAddr))(DestReg, SrcMem->toAsmAddress(Asm, Target)); 1004 } else { 1005 llvm_unreachable("Unexpected operand type"); 1006 } 1007 } else { 1008 Address StackAddr(Target->stackVarToAsmOperand(Dest)); 1009 // Src must be a register in this case. 1010 const auto *SrcVar = llvm::cast<Variable>(Src); 1011 assert(SrcVar->hasReg()); 1012 (Asm->*(Emitter.AddrXmm))(StackAddr, 1013 Traits::getEncodedXmm(SrcVar->getRegNum())); 1014 } 1015 } 1016 1017 template <typename TraitsType> 1018 void InstImpl<TraitsType>::InstX86Movmsk::dump(const Cfg *Func) const { 1019 if (!BuildDefs::dump()) 1020 return; 1021 Ostream &Str = Func->getContext()->getStrDump(); 1022 this->dumpDest(Func); 1023 Str << " = movmsk." << this->getSrc(0)->getType() << " "; 1024 this->dumpSources(Func); 1025 } 1026 1027 template <typename TraitsType> 1028 void InstImpl<TraitsType>::InstX86Movmsk::emit(const Cfg *Func) const { 1029 if (!BuildDefs::dump()) 1030 return; 1031 Ostream &Str = Func->getContext()->getStrEmit(); 1032 assert(this->getSrcSize() == 1); 1033 Type SrcTy = this->getSrc(0)->getType(); 1034 assert(isVectorType(SrcTy)); 1035 switch (SrcTy) { 1036 case IceType_v16i8: 1037 Str << "\t" 1038 "pmovmskb" 1039 "\t"; 1040 break; 1041 case IceType_v4i32: 1042 case IceType_v4f32: 1043 Str << "\t" 1044 "movmskps" 1045 "\t"; 1046 break; 1047 default: 1048 llvm_unreachable("Unexpected operand type"); 1049 } 1050 this->getSrc(0)->emit(Func); 1051 Str << ", "; 1052 this->getDest()->emit(Func); 1053 } 1054 1055 template <typename TraitsType> 1056 void InstImpl<TraitsType>::InstX86Movmsk::emitIAS(const Cfg *Func) const { 1057 assert(this->getSrcSize() == 1); 1058 Assembler *Asm = Func->getAssembler<Assembler>(); 1059 const Variable *Dest = this->getDest(); 1060 const Variable *Src = llvm::cast<Variable>(this->getSrc(0)); 1061 const Type DestTy = Dest->getType(); 1062 (void)DestTy; 1063 const Type SrcTy = Src->getType(); 1064 assert(isVectorType(SrcTy)); 1065 assert(isScalarIntegerType(DestTy)); 1066 if (Traits::Is64Bit) { 1067 assert(DestTy == IceType_i32 || DestTy == IceType_i64); 1068 } else { 1069 assert(typeWidthInBytes(DestTy) <= 4); 1070 } 1071 XmmRegister SrcReg = Traits::getEncodedXmm(Src->getRegNum()); 1072 GPRRegister DestReg = Traits::getEncodedGPR(Dest->getRegNum()); 1073 Asm->movmsk(SrcTy, DestReg, SrcReg); 1074 } 1075 1076 template <typename TraitsType> 1077 void InstImpl<TraitsType>::InstX86Sqrt::emit(const Cfg *Func) const { 1078 if (!BuildDefs::dump()) 1079 return; 1080 Ostream &Str = Func->getContext()->getStrEmit(); 1081 assert(this->getSrcSize() == 1); 1082 Type Ty = this->getSrc(0)->getType(); 1083 assert(isScalarFloatingType(Ty)); 1084 Str << "\t" 1085 "sqrt" << Traits::TypeAttributes[Ty].SpSdString << "\t"; 1086 this->getSrc(0)->emit(Func); 1087 Str << ", "; 1088 this->getDest()->emit(Func); 1089 } 1090 1091 template <typename TraitsType> 1092 void InstImpl<TraitsType>::InstX86Div::emit(const Cfg *Func) const { 1093 if (!BuildDefs::dump()) 1094 return; 1095 Ostream &Str = Func->getContext()->getStrEmit(); 1096 assert(this->getSrcSize() == 3); 1097 Operand *Src1 = this->getSrc(1); 1098 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t"; 1099 Src1->emit(Func); 1100 } 1101 1102 template <typename TraitsType> 1103 void InstImpl<TraitsType>::InstX86Div::emitIAS(const Cfg *Func) const { 1104 assert(this->getSrcSize() == 3); 1105 const Operand *Src = this->getSrc(1); 1106 Type Ty = Src->getType(); 1107 static GPREmitterOneOp Emitter = {&Assembler::div, &Assembler::div}; 1108 emitIASOpTyGPR(Func, Ty, Src, Emitter); 1109 } 1110 1111 template <typename TraitsType> 1112 void InstImpl<TraitsType>::InstX86Idiv::emit(const Cfg *Func) const { 1113 if (!BuildDefs::dump()) 1114 return; 1115 Ostream &Str = Func->getContext()->getStrEmit(); 1116 assert(this->getSrcSize() == 3); 1117 Operand *Src1 = this->getSrc(1); 1118 Str << "\t" << this->Opcode << this->getWidthString(Src1->getType()) << "\t"; 1119 Src1->emit(Func); 1120 } 1121 1122 template <typename TraitsType> 1123 void InstImpl<TraitsType>::InstX86Idiv::emitIAS(const Cfg *Func) const { 1124 assert(this->getSrcSize() == 3); 1125 const Operand *Src = this->getSrc(1); 1126 Type Ty = Src->getType(); 1127 static const GPREmitterOneOp Emitter = {&Assembler::idiv, &Assembler::idiv}; 1128 emitIASOpTyGPR(Func, Ty, Src, Emitter); 1129 } 1130 1131 // pblendvb and blendvps take xmm0 as a final implicit argument. 1132 template <typename TraitsType> 1133 void InstImpl<TraitsType>::emitVariableBlendInst(const char *Opcode, 1134 const Inst *Instr, 1135 const Cfg *Func) { 1136 if (!BuildDefs::dump()) 1137 return; 1138 Ostream &Str = Func->getContext()->getStrEmit(); 1139 assert(Instr->getSrcSize() == 3); 1140 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() == 1141 RegisterSet::Reg_xmm0); 1142 Str << "\t" << Opcode << "\t"; 1143 Instr->getSrc(1)->emit(Func); 1144 Str << ", "; 1145 Instr->getDest()->emit(Func); 1146 } 1147 1148 template <typename TraitsType> 1149 void InstImpl<TraitsType>::emitIASVariableBlendInst( 1150 const Inst *Instr, const Cfg *Func, const XmmEmitterRegOp &Emitter) { 1151 assert(Instr->getSrcSize() == 3); 1152 assert(llvm::cast<Variable>(Instr->getSrc(2))->getRegNum() == 1153 RegisterSet::Reg_xmm0); 1154 const Variable *Dest = Instr->getDest(); 1155 const Operand *Src = Instr->getSrc(1); 1156 emitIASRegOpTyXMM(Func, Dest->getType(), Dest, Src, Emitter); 1157 } 1158 1159 template <typename TraitsType> 1160 void InstImpl<TraitsType>::InstX86Blendvps::emit(const Cfg *Func) const { 1161 if (!BuildDefs::dump()) 1162 return; 1163 emitVariableBlendInst(this->Opcode, this, Func); 1164 } 1165 1166 template <typename TraitsType> 1167 void InstImpl<TraitsType>::InstX86Blendvps::emitIAS(const Cfg *Func) const { 1168 static const XmmEmitterRegOp Emitter = {&Assembler::blendvps, 1169 &Assembler::blendvps}; 1170 emitIASVariableBlendInst(this, Func, Emitter); 1171 } 1172 1173 template <typename TraitsType> 1174 void InstImpl<TraitsType>::InstX86Pblendvb::emit(const Cfg *Func) const { 1175 if (!BuildDefs::dump()) 1176 return; 1177 emitVariableBlendInst(this->Opcode, this, Func); 1178 } 1179 1180 template <typename TraitsType> 1181 void InstImpl<TraitsType>::InstX86Pblendvb::emitIAS(const Cfg *Func) const { 1182 static const XmmEmitterRegOp Emitter = {&Assembler::pblendvb, 1183 &Assembler::pblendvb}; 1184 emitIASVariableBlendInst(this, Func, Emitter); 1185 } 1186 1187 template <typename TraitsType> 1188 void InstImpl<TraitsType>::InstX86Imul::emit(const Cfg *Func) const { 1189 if (!BuildDefs::dump()) 1190 return; 1191 Ostream &Str = Func->getContext()->getStrEmit(); 1192 assert(this->getSrcSize() == 2); 1193 Variable *Dest = this->getDest(); 1194 if (isByteSizedArithType(Dest->getType())) { 1195 // The 8-bit version of imul only allows the form "imul r/m8". 1196 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0)); 1197 (void)Src0Var; 1198 assert(Src0Var->getRegNum() == RegisterSet::Reg_al); 1199 Str << "\t" 1200 "imulb\t"; 1201 this->getSrc(1)->emit(Func); 1202 } else if (llvm::isa<Constant>(this->getSrc(1))) { 1203 Str << "\t" 1204 "imul" << this->getWidthString(Dest->getType()) << "\t"; 1205 this->getSrc(1)->emit(Func); 1206 Str << ", "; 1207 this->getSrc(0)->emit(Func); 1208 Str << ", "; 1209 Dest->emit(Func); 1210 } else { 1211 this->emitTwoAddress(Func, this->Opcode); 1212 } 1213 } 1214 1215 template <typename TraitsType> 1216 void InstImpl<TraitsType>::InstX86Imul::emitIAS(const Cfg *Func) const { 1217 assert(this->getSrcSize() == 2); 1218 const Variable *Var = this->getDest(); 1219 Type Ty = Var->getType(); 1220 const Operand *Src = this->getSrc(1); 1221 if (isByteSizedArithType(Ty)) { 1222 // The 8-bit version of imul only allows the form "imul r/m8". 1223 const auto *Src0Var = llvm::dyn_cast<Variable>(this->getSrc(0)); 1224 (void)Src0Var; 1225 assert(Src0Var->getRegNum() == RegisterSet::Reg_al); 1226 static const GPREmitterOneOp Emitter = {&Assembler::imul, &Assembler::imul}; 1227 emitIASOpTyGPR(Func, Ty, this->getSrc(1), Emitter); 1228 } else { 1229 // The two-address version is used when multiplying by a non-constant 1230 // or doing an 8-bit multiply. 1231 assert(Var == this->getSrc(0)); 1232 static const GPREmitterRegOp Emitter = {&Assembler::imul, &Assembler::imul, 1233 &Assembler::imul}; 1234 constexpr bool NotLea = false; 1235 emitIASRegOpTyGPR(Func, NotLea, Ty, Var, Src, Emitter); 1236 } 1237 } 1238 1239 template <typename TraitsType> 1240 void InstImpl<TraitsType>::InstX86ImulImm::emit(const Cfg *Func) const { 1241 if (!BuildDefs::dump()) 1242 return; 1243 Ostream &Str = Func->getContext()->getStrEmit(); 1244 assert(this->getSrcSize() == 2); 1245 Variable *Dest = this->getDest(); 1246 assert(Dest->getType() == IceType_i16 || Dest->getType() == IceType_i32); 1247 assert(llvm::isa<Constant>(this->getSrc(1))); 1248 Str << "\t" 1249 "imul" << this->getWidthString(Dest->getType()) << "\t"; 1250 this->getSrc(1)->emit(Func); 1251 Str << ", "; 1252 this->getSrc(0)->emit(Func); 1253 Str << ", "; 1254 Dest->emit(Func); 1255 } 1256 1257 template <typename TraitsType> 1258 void InstImpl<TraitsType>::InstX86ImulImm::emitIAS(const Cfg *Func) const { 1259 assert(this->getSrcSize() == 2); 1260 const Variable *Dest = this->getDest(); 1261 Type Ty = Dest->getType(); 1262 assert(llvm::isa<Constant>(this->getSrc(1))); 1263 static const ThreeOpImmEmitter<GPRRegister, GPRRegister> Emitter = { 1264 &Assembler::imul, &Assembler::imul}; 1265 emitIASThreeOpImmOps<GPRRegister, GPRRegister, Traits::getEncodedGPR, 1266 Traits::getEncodedGPR>(Func, Ty, Dest, this->getSrc(0), 1267 this->getSrc(1), Emitter); 1268 } 1269 1270 template <typename TraitsType> 1271 void InstImpl<TraitsType>::InstX86Insertps::emitIAS(const Cfg *Func) const { 1272 assert(this->getSrcSize() == 3); 1273 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1); 1274 const Variable *Dest = this->getDest(); 1275 assert(Dest == this->getSrc(0)); 1276 Type Ty = Dest->getType(); 1277 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = { 1278 &Assembler::insertps, &Assembler::insertps}; 1279 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm, 1280 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1), 1281 this->getSrc(2), Emitter); 1282 } 1283 1284 template <typename TraitsType> 1285 void InstImpl<TraitsType>::InstX86Cbwdq::emit(const Cfg *Func) const { 1286 if (!BuildDefs::dump()) 1287 return; 1288 Ostream &Str = Func->getContext()->getStrEmit(); 1289 assert(this->getSrcSize() == 1); 1290 Operand *Src0 = this->getSrc(0); 1291 const auto DestReg = this->getDest()->getRegNum(); 1292 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum(); 1293 (void)DestReg; 1294 (void)SrcReg; 1295 switch (Src0->getType()) { 1296 default: 1297 llvm_unreachable("unexpected source type!"); 1298 break; 1299 case IceType_i8: 1300 assert(SrcReg == RegisterSet::Reg_al); 1301 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah); 1302 Str << "\t" 1303 "cbtw"; 1304 break; 1305 case IceType_i16: 1306 assert(SrcReg == RegisterSet::Reg_ax); 1307 assert(DestReg == RegisterSet::Reg_dx); 1308 Str << "\t" 1309 "cwtd"; 1310 break; 1311 case IceType_i32: 1312 assert(SrcReg == RegisterSet::Reg_eax); 1313 assert(DestReg == RegisterSet::Reg_edx); 1314 Str << "\t" 1315 "cltd"; 1316 break; 1317 case IceType_i64: 1318 assert(Traits::Is64Bit); 1319 assert(SrcReg == Traits::getRaxOrDie()); 1320 assert(DestReg == Traits::getRdxOrDie()); 1321 Str << "\t" 1322 "cqo"; 1323 break; 1324 } 1325 } 1326 1327 template <typename TraitsType> 1328 void InstImpl<TraitsType>::InstX86Cbwdq::emitIAS(const Cfg *Func) const { 1329 Assembler *Asm = Func->getAssembler<Assembler>(); 1330 assert(this->getSrcSize() == 1); 1331 Operand *Src0 = this->getSrc(0); 1332 const auto DestReg = this->getDest()->getRegNum(); 1333 const auto SrcReg = llvm::cast<Variable>(Src0)->getRegNum(); 1334 (void)DestReg; 1335 (void)SrcReg; 1336 switch (Src0->getType()) { 1337 default: 1338 llvm_unreachable("unexpected source type!"); 1339 break; 1340 case IceType_i8: 1341 assert(SrcReg == RegisterSet::Reg_al); 1342 assert(DestReg == RegisterSet::Reg_ax || DestReg == RegisterSet::Reg_ah); 1343 Asm->cbw(); 1344 break; 1345 case IceType_i16: 1346 assert(SrcReg == RegisterSet::Reg_ax); 1347 assert(DestReg == RegisterSet::Reg_dx); 1348 Asm->cwd(); 1349 break; 1350 case IceType_i32: 1351 assert(SrcReg == RegisterSet::Reg_eax); 1352 assert(DestReg == RegisterSet::Reg_edx); 1353 Asm->cdq(); 1354 break; 1355 case IceType_i64: 1356 assert(Traits::Is64Bit); 1357 assert(SrcReg == Traits::getRaxOrDie()); 1358 assert(DestReg == Traits::getRdxOrDie()); 1359 Asm->cqo(); 1360 break; 1361 } 1362 } 1363 1364 template <typename TraitsType> 1365 void InstImpl<TraitsType>::InstX86Mul::emit(const Cfg *Func) const { 1366 if (!BuildDefs::dump()) 1367 return; 1368 Ostream &Str = Func->getContext()->getStrEmit(); 1369 assert(this->getSrcSize() == 2); 1370 assert(llvm::isa<Variable>(this->getSrc(0))); 1371 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() == 1372 RegisterSet::Reg_eax); 1373 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO: 1374 // allow 1375 // edx? 1376 Str << "\t" 1377 "mul" << this->getWidthString(this->getDest()->getType()) << "\t"; 1378 this->getSrc(1)->emit(Func); 1379 } 1380 1381 template <typename TraitsType> 1382 void InstImpl<TraitsType>::InstX86Mul::emitIAS(const Cfg *Func) const { 1383 assert(this->getSrcSize() == 2); 1384 assert(llvm::isa<Variable>(this->getSrc(0))); 1385 assert(llvm::cast<Variable>(this->getSrc(0))->getRegNum() == 1386 RegisterSet::Reg_eax); 1387 assert(this->getDest()->getRegNum() == RegisterSet::Reg_eax); // TODO: 1388 // allow 1389 // edx? 1390 const Operand *Src = this->getSrc(1); 1391 Type Ty = Src->getType(); 1392 static const GPREmitterOneOp Emitter = {&Assembler::mul, &Assembler::mul}; 1393 emitIASOpTyGPR(Func, Ty, Src, Emitter); 1394 } 1395 1396 template <typename TraitsType> 1397 void InstImpl<TraitsType>::InstX86Mul::dump(const Cfg *Func) const { 1398 if (!BuildDefs::dump()) 1399 return; 1400 Ostream &Str = Func->getContext()->getStrDump(); 1401 this->dumpDest(Func); 1402 Str << " = mul." << this->getDest()->getType() << " "; 1403 this->dumpSources(Func); 1404 } 1405 1406 template <typename TraitsType> 1407 void InstImpl<TraitsType>::InstX86Shld::emit(const Cfg *Func) const { 1408 if (!BuildDefs::dump()) 1409 return; 1410 Ostream &Str = Func->getContext()->getStrEmit(); 1411 Variable *Dest = this->getDest(); 1412 assert(this->getSrcSize() == 3); 1413 assert(Dest == this->getSrc(0)); 1414 Str << "\t" 1415 "shld" << this->getWidthString(Dest->getType()) << "\t"; 1416 this->getSrc(2)->emit(Func); 1417 Str << ", "; 1418 this->getSrc(1)->emit(Func); 1419 Str << ", "; 1420 Dest->emit(Func); 1421 } 1422 1423 template <typename TraitsType> 1424 void InstImpl<TraitsType>::InstX86Shld::emitIAS(const Cfg *Func) const { 1425 assert(this->getSrcSize() == 3); 1426 assert(this->getDest() == this->getSrc(0)); 1427 const Variable *Dest = this->getDest(); 1428 const Operand *Src1 = this->getSrc(1); 1429 const Operand *Src2 = this->getSrc(2); 1430 static const GPREmitterShiftD Emitter = {&Assembler::shld, &Assembler::shld}; 1431 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter); 1432 } 1433 1434 template <typename TraitsType> 1435 void InstImpl<TraitsType>::InstX86Shld::dump(const Cfg *Func) const { 1436 if (!BuildDefs::dump()) 1437 return; 1438 Ostream &Str = Func->getContext()->getStrDump(); 1439 this->dumpDest(Func); 1440 Str << " = shld." << this->getDest()->getType() << " "; 1441 this->dumpSources(Func); 1442 } 1443 1444 template <typename TraitsType> 1445 void InstImpl<TraitsType>::InstX86Shrd::emit(const Cfg *Func) const { 1446 if (!BuildDefs::dump()) 1447 return; 1448 Ostream &Str = Func->getContext()->getStrEmit(); 1449 Variable *Dest = this->getDest(); 1450 assert(this->getSrcSize() == 3); 1451 assert(Dest == this->getSrc(0)); 1452 Str << "\t" 1453 "shrd" << this->getWidthString(Dest->getType()) << "\t"; 1454 this->getSrc(2)->emit(Func); 1455 Str << ", "; 1456 this->getSrc(1)->emit(Func); 1457 Str << ", "; 1458 Dest->emit(Func); 1459 } 1460 1461 template <typename TraitsType> 1462 void InstImpl<TraitsType>::InstX86Shrd::emitIAS(const Cfg *Func) const { 1463 assert(this->getSrcSize() == 3); 1464 assert(this->getDest() == this->getSrc(0)); 1465 const Variable *Dest = this->getDest(); 1466 const Operand *Src1 = this->getSrc(1); 1467 const Operand *Src2 = this->getSrc(2); 1468 static const GPREmitterShiftD Emitter = {&Assembler::shrd, &Assembler::shrd}; 1469 emitIASGPRShiftDouble(Func, Dest, Src1, Src2, Emitter); 1470 } 1471 1472 template <typename TraitsType> 1473 void InstImpl<TraitsType>::InstX86Shrd::dump(const Cfg *Func) const { 1474 if (!BuildDefs::dump()) 1475 return; 1476 Ostream &Str = Func->getContext()->getStrDump(); 1477 this->dumpDest(Func); 1478 Str << " = shrd." << this->getDest()->getType() << " "; 1479 this->dumpSources(Func); 1480 } 1481 1482 template <typename TraitsType> 1483 void InstImpl<TraitsType>::InstX86Cmov::emit(const Cfg *Func) const { 1484 if (!BuildDefs::dump()) 1485 return; 1486 Ostream &Str = Func->getContext()->getStrEmit(); 1487 Variable *Dest = this->getDest(); 1488 Str << "\t"; 1489 assert(Condition != Cond::Br_None); 1490 assert(this->getDest()->hasReg()); 1491 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString 1492 << this->getWidthString(Dest->getType()) << "\t"; 1493 this->getSrc(1)->emit(Func); 1494 Str << ", "; 1495 Dest->emit(Func); 1496 } 1497 1498 template <typename TraitsType> 1499 void InstImpl<TraitsType>::InstX86Cmov::emitIAS(const Cfg *Func) const { 1500 assert(Condition != Cond::Br_None); 1501 assert(this->getDest()->hasReg()); 1502 assert(this->getSrcSize() == 2); 1503 Operand *Src = this->getSrc(1); 1504 Type SrcTy = Src->getType(); 1505 assert(SrcTy == IceType_i16 || SrcTy == IceType_i32 || (Traits::Is64Bit)); 1506 Assembler *Asm = Func->getAssembler<Assembler>(); 1507 auto *Target = InstX86Base::getTarget(Func); 1508 if (const auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 1509 if (SrcVar->hasReg()) { 1510 Asm->cmov(SrcTy, Condition, 1511 Traits::getEncodedGPR(this->getDest()->getRegNum()), 1512 Traits::getEncodedGPR(SrcVar->getRegNum())); 1513 } else { 1514 Asm->cmov(SrcTy, Condition, 1515 Traits::getEncodedGPR(this->getDest()->getRegNum()), 1516 Target->stackVarToAsmOperand(SrcVar)); 1517 } 1518 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 1519 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 1520 Asm->cmov(SrcTy, Condition, 1521 Traits::getEncodedGPR(this->getDest()->getRegNum()), 1522 Mem->toAsmAddress(Asm, Target)); 1523 } else { 1524 llvm_unreachable("Unexpected operand type"); 1525 } 1526 } 1527 1528 template <typename TraitsType> 1529 void InstImpl<TraitsType>::InstX86Cmov::dump(const Cfg *Func) const { 1530 if (!BuildDefs::dump()) 1531 return; 1532 Ostream &Str = Func->getContext()->getStrDump(); 1533 Str << "cmov" << Traits::InstBrAttributes[Condition].DisplayString << "."; 1534 Str << this->getDest()->getType() << " "; 1535 this->dumpDest(Func); 1536 Str << ", "; 1537 this->dumpSources(Func); 1538 } 1539 1540 template <typename TraitsType> 1541 void InstImpl<TraitsType>::InstX86Cmpps::emit(const Cfg *Func) const { 1542 if (!BuildDefs::dump()) 1543 return; 1544 Ostream &Str = Func->getContext()->getStrEmit(); 1545 assert(this->getSrcSize() == 2); 1546 assert(Condition < Cond::Cmpps_Invalid); 1547 Type DestTy = this->Dest->getType(); 1548 Str << "\t" 1549 "cmp" << Traits::InstCmppsAttributes[Condition].EmitString 1550 << Traits::TypeAttributes[DestTy].PdPsString << "\t"; 1551 this->getSrc(1)->emit(Func); 1552 Str << ", "; 1553 this->getDest()->emit(Func); 1554 } 1555 1556 template <typename TraitsType> 1557 void InstImpl<TraitsType>::InstX86Cmpps::emitIAS(const Cfg *Func) const { 1558 Assembler *Asm = Func->getAssembler<Assembler>(); 1559 assert(this->getSrcSize() == 2); 1560 assert(Condition < Cond::Cmpps_Invalid); 1561 // Assuming there isn't any load folding for cmpps, and vector constants are 1562 // not allowed in PNaCl. 1563 assert(llvm::isa<Variable>(this->getSrc(1))); 1564 auto *Target = InstX86Base::getTarget(Func); 1565 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1)); 1566 if (SrcVar->hasReg()) { 1567 Asm->cmpps(this->getDest()->getType(), 1568 Traits::getEncodedXmm(this->getDest()->getRegNum()), 1569 Traits::getEncodedXmm(SrcVar->getRegNum()), Condition); 1570 } else { 1571 Address SrcStackAddr = Target->stackVarToAsmOperand(SrcVar); 1572 Asm->cmpps(this->getDest()->getType(), 1573 Traits::getEncodedXmm(this->getDest()->getRegNum()), 1574 SrcStackAddr, Condition); 1575 } 1576 } 1577 1578 template <typename TraitsType> 1579 void InstImpl<TraitsType>::InstX86Cmpps::dump(const Cfg *Func) const { 1580 if (!BuildDefs::dump()) 1581 return; 1582 Ostream &Str = Func->getContext()->getStrDump(); 1583 assert(Condition < Cond::Cmpps_Invalid); 1584 this->dumpDest(Func); 1585 Str << " = cmp" << Traits::InstCmppsAttributes[Condition].EmitString << "ps" 1586 "\t"; 1587 this->dumpSources(Func); 1588 } 1589 1590 template <typename TraitsType> 1591 void InstImpl<TraitsType>::InstX86Cmpxchg::emit(const Cfg *Func) const { 1592 if (!BuildDefs::dump()) 1593 return; 1594 Ostream &Str = Func->getContext()->getStrEmit(); 1595 assert(this->getSrcSize() == 3); 1596 if (this->Locked) { 1597 Str << "\t" 1598 "lock"; 1599 } 1600 Str << "\t" 1601 "cmpxchg" << this->getWidthString(this->getSrc(0)->getType()) << "\t"; 1602 this->getSrc(2)->emit(Func); 1603 Str << ", "; 1604 this->getSrc(0)->emit(Func); 1605 } 1606 1607 template <typename TraitsType> 1608 void InstImpl<TraitsType>::InstX86Cmpxchg::emitIAS(const Cfg *Func) const { 1609 assert(this->getSrcSize() == 3); 1610 Assembler *Asm = Func->getAssembler<Assembler>(); 1611 Type Ty = this->getSrc(0)->getType(); 1612 auto *Target = InstX86Base::getTarget(Func); 1613 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0)); 1614 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 1615 const Address Addr = Mem->toAsmAddress(Asm, Target); 1616 const auto *VarReg = llvm::cast<Variable>(this->getSrc(2)); 1617 assert(VarReg->hasReg()); 1618 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum()); 1619 Asm->cmpxchg(Ty, Addr, Reg, this->Locked); 1620 } 1621 1622 template <typename TraitsType> 1623 void InstImpl<TraitsType>::InstX86Cmpxchg::dump(const Cfg *Func) const { 1624 if (!BuildDefs::dump()) 1625 return; 1626 Ostream &Str = Func->getContext()->getStrDump(); 1627 if (this->Locked) { 1628 Str << "lock "; 1629 } 1630 Str << "cmpxchg." << this->getSrc(0)->getType() << " "; 1631 this->dumpSources(Func); 1632 } 1633 1634 template <typename TraitsType> 1635 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emit(const Cfg *Func) const { 1636 if (!BuildDefs::dump()) 1637 return; 1638 Ostream &Str = Func->getContext()->getStrEmit(); 1639 assert(this->getSrcSize() == 5); 1640 if (this->Locked) { 1641 Str << "\t" 1642 "lock"; 1643 } 1644 Str << "\t" 1645 "cmpxchg8b\t"; 1646 this->getSrc(0)->emit(Func); 1647 } 1648 1649 template <typename TraitsType> 1650 void InstImpl<TraitsType>::InstX86Cmpxchg8b::emitIAS(const Cfg *Func) const { 1651 assert(this->getSrcSize() == 5); 1652 Assembler *Asm = Func->getAssembler<Assembler>(); 1653 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0)); 1654 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 1655 auto *Target = InstX86Base::getTarget(Func); 1656 const Address Addr = Mem->toAsmAddress(Asm, Target); 1657 Asm->cmpxchg8b(Addr, this->Locked); 1658 } 1659 1660 template <typename TraitsType> 1661 void InstImpl<TraitsType>::InstX86Cmpxchg8b::dump(const Cfg *Func) const { 1662 if (!BuildDefs::dump()) 1663 return; 1664 Ostream &Str = Func->getContext()->getStrDump(); 1665 if (this->Locked) { 1666 Str << "lock "; 1667 } 1668 Str << "cmpxchg8b "; 1669 this->dumpSources(Func); 1670 } 1671 1672 template <typename TraitsType> 1673 void InstImpl<TraitsType>::InstX86Cvt::emit(const Cfg *Func) const { 1674 if (!BuildDefs::dump()) 1675 return; 1676 Ostream &Str = Func->getContext()->getStrEmit(); 1677 assert(this->getSrcSize() == 1); 1678 Str << "\t" 1679 "cvt"; 1680 if (isTruncating()) 1681 Str << "t"; 1682 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2" 1683 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << "\t"; 1684 this->getSrc(0)->emit(Func); 1685 Str << ", "; 1686 this->getDest()->emit(Func); 1687 } 1688 1689 template <typename TraitsType> 1690 void InstImpl<TraitsType>::InstX86Cvt::emitIAS(const Cfg *Func) const { 1691 assert(this->getSrcSize() == 1); 1692 const Variable *Dest = this->getDest(); 1693 const Operand *Src = this->getSrc(0); 1694 Type DestTy = Dest->getType(); 1695 Type SrcTy = Src->getType(); 1696 switch (Variant) { 1697 case Si2ss: { 1698 assert(isScalarIntegerType(SrcTy)); 1699 if (!Traits::Is64Bit) { 1700 assert(typeWidthInBytes(SrcTy) <= 4); 1701 } else { 1702 assert(SrcTy == IceType_i32 || SrcTy == IceType_i64); 1703 } 1704 assert(isScalarFloatingType(DestTy)); 1705 static const CastEmitterRegOp<XmmRegister, GPRRegister> Emitter = { 1706 &Assembler::cvtsi2ss, &Assembler::cvtsi2ss}; 1707 emitIASCastRegOp<XmmRegister, GPRRegister, Traits::getEncodedXmm, 1708 Traits::getEncodedGPR>(Func, DestTy, Dest, SrcTy, Src, 1709 Emitter); 1710 return; 1711 } 1712 case Tss2si: { 1713 assert(isScalarFloatingType(SrcTy)); 1714 assert(isScalarIntegerType(DestTy)); 1715 if (Traits::Is64Bit) { 1716 assert(DestTy == IceType_i32 || DestTy == IceType_i64); 1717 } else { 1718 assert(typeWidthInBytes(DestTy) <= 4); 1719 } 1720 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = { 1721 &Assembler::cvttss2si, &Assembler::cvttss2si}; 1722 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR, 1723 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src, 1724 Emitter); 1725 return; 1726 } 1727 case Ss2si: { 1728 assert(isScalarFloatingType(SrcTy)); 1729 assert(isScalarIntegerType(DestTy)); 1730 if (Traits::Is64Bit) { 1731 assert(DestTy == IceType_i32 || DestTy == IceType_i64); 1732 } else { 1733 assert(typeWidthInBytes(DestTy) <= 4); 1734 } 1735 static const CastEmitterRegOp<GPRRegister, XmmRegister> Emitter = { 1736 &Assembler::cvtss2si, &Assembler::cvtss2si}; 1737 emitIASCastRegOp<GPRRegister, XmmRegister, Traits::getEncodedGPR, 1738 Traits::getEncodedXmm>(Func, DestTy, Dest, SrcTy, Src, 1739 Emitter); 1740 return; 1741 } 1742 case Float2float: { 1743 assert(isScalarFloatingType(SrcTy)); 1744 assert(isScalarFloatingType(DestTy)); 1745 assert(DestTy != SrcTy); 1746 static const XmmEmitterRegOp Emitter = {&Assembler::cvtfloat2float, 1747 &Assembler::cvtfloat2float}; 1748 emitIASRegOpTyXMM(Func, SrcTy, Dest, Src, Emitter); 1749 return; 1750 } 1751 case Dq2ps: { 1752 assert(isVectorIntegerType(SrcTy)); 1753 assert(isVectorFloatingType(DestTy)); 1754 static const XmmEmitterRegOp Emitter = {&Assembler::cvtdq2ps, 1755 &Assembler::cvtdq2ps}; 1756 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter); 1757 return; 1758 } 1759 case Tps2dq: { 1760 assert(isVectorFloatingType(SrcTy)); 1761 assert(isVectorIntegerType(DestTy)); 1762 static const XmmEmitterRegOp Emitter = {&Assembler::cvttps2dq, 1763 &Assembler::cvttps2dq}; 1764 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter); 1765 return; 1766 } 1767 case Ps2dq: { 1768 assert(isVectorFloatingType(SrcTy)); 1769 assert(isVectorIntegerType(DestTy)); 1770 static const XmmEmitterRegOp Emitter = {&Assembler::cvtps2dq, 1771 &Assembler::cvtps2dq}; 1772 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, Emitter); 1773 return; 1774 } 1775 } 1776 } 1777 1778 template <typename TraitsType> 1779 void InstImpl<TraitsType>::InstX86Cvt::dump(const Cfg *Func) const { 1780 if (!BuildDefs::dump()) 1781 return; 1782 Ostream &Str = Func->getContext()->getStrDump(); 1783 this->dumpDest(Func); 1784 Str << " = cvt"; 1785 if (isTruncating()) 1786 Str << "t"; 1787 Str << Traits::TypeAttributes[this->getSrc(0)->getType()].CvtString << "2" 1788 << Traits::TypeAttributes[this->getDest()->getType()].CvtString << " "; 1789 this->dumpSources(Func); 1790 } 1791 1792 template <typename TraitsType> 1793 void InstImpl<TraitsType>::InstX86Round::emit(const Cfg *Func) const { 1794 if (!BuildDefs::dump()) 1795 return; 1796 Ostream &Str = Func->getContext()->getStrEmit(); 1797 assert(this->getSrcSize() == 3); 1798 Str << "\t" << this->Opcode 1799 << Traits::TypeAttributes[this->getDest()->getType()].SpSdString << "\t"; 1800 this->getSrc(1)->emit(Func); 1801 Str << ", "; 1802 this->getSrc(0)->emit(Func); 1803 Str << ", "; 1804 this->getDest()->emit(Func); 1805 } 1806 1807 template <typename TraitsType> 1808 void InstImpl<TraitsType>::InstX86Round::emitIAS(const Cfg *Func) const { 1809 assert(this->getSrcSize() == 2); 1810 assert(InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1); 1811 const Variable *Dest = this->getDest(); 1812 Type Ty = Dest->getType(); 1813 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = { 1814 &Assembler::round, &Assembler::round}; 1815 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm, 1816 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0), 1817 this->getSrc(1), Emitter); 1818 } 1819 1820 template <typename TraitsType> 1821 void InstImpl<TraitsType>::InstX86Icmp::emit(const Cfg *Func) const { 1822 if (!BuildDefs::dump()) 1823 return; 1824 Ostream &Str = Func->getContext()->getStrEmit(); 1825 assert(this->getSrcSize() == 2); 1826 Str << "\t" 1827 "cmp" << this->getWidthString(this->getSrc(0)->getType()) << "\t"; 1828 this->getSrc(1)->emit(Func); 1829 Str << ", "; 1830 this->getSrc(0)->emit(Func); 1831 } 1832 1833 template <typename TraitsType> 1834 void InstImpl<TraitsType>::InstX86Icmp::emitIAS(const Cfg *Func) const { 1835 assert(this->getSrcSize() == 2); 1836 const Operand *Src0 = this->getSrc(0); 1837 const Operand *Src1 = this->getSrc(1); 1838 Type Ty = Src0->getType(); 1839 static const GPREmitterRegOp RegEmitter = {&Assembler::cmp, &Assembler::cmp, 1840 &Assembler::cmp}; 1841 static const GPREmitterAddrOp AddrEmitter = {&Assembler::cmp, 1842 &Assembler::cmp}; 1843 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) { 1844 if (SrcVar0->hasReg()) { 1845 constexpr bool NotLea = false; 1846 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter); 1847 return; 1848 } 1849 } 1850 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter); 1851 } 1852 1853 template <typename TraitsType> 1854 void InstImpl<TraitsType>::InstX86Icmp::dump(const Cfg *Func) const { 1855 if (!BuildDefs::dump()) 1856 return; 1857 Ostream &Str = Func->getContext()->getStrDump(); 1858 Str << "cmp." << this->getSrc(0)->getType() << " "; 1859 this->dumpSources(Func); 1860 } 1861 1862 template <typename TraitsType> 1863 void InstImpl<TraitsType>::InstX86Ucomiss::emit(const Cfg *Func) const { 1864 if (!BuildDefs::dump()) 1865 return; 1866 Ostream &Str = Func->getContext()->getStrEmit(); 1867 assert(this->getSrcSize() == 2); 1868 Str << "\t" 1869 "ucomi" 1870 << Traits::TypeAttributes[this->getSrc(0)->getType()].SdSsString << "\t"; 1871 this->getSrc(1)->emit(Func); 1872 Str << ", "; 1873 this->getSrc(0)->emit(Func); 1874 } 1875 1876 template <typename TraitsType> 1877 void InstImpl<TraitsType>::InstX86Ucomiss::emitIAS(const Cfg *Func) const { 1878 assert(this->getSrcSize() == 2); 1879 // Currently src0 is always a variable by convention, to avoid having two 1880 // memory operands. 1881 assert(llvm::isa<Variable>(this->getSrc(0))); 1882 const auto *Src0Var = llvm::cast<Variable>(this->getSrc(0)); 1883 Type Ty = Src0Var->getType(); 1884 static const XmmEmitterRegOp Emitter = {&Assembler::ucomiss, 1885 &Assembler::ucomiss}; 1886 emitIASRegOpTyXMM(Func, Ty, Src0Var, this->getSrc(1), Emitter); 1887 } 1888 1889 template <typename TraitsType> 1890 void InstImpl<TraitsType>::InstX86Ucomiss::dump(const Cfg *Func) const { 1891 if (!BuildDefs::dump()) 1892 return; 1893 Ostream &Str = Func->getContext()->getStrDump(); 1894 Str << "ucomiss." << this->getSrc(0)->getType() << " "; 1895 this->dumpSources(Func); 1896 } 1897 1898 template <typename TraitsType> 1899 void InstImpl<TraitsType>::InstX86UD2::emit(const Cfg *Func) const { 1900 if (!BuildDefs::dump()) 1901 return; 1902 Ostream &Str = Func->getContext()->getStrEmit(); 1903 assert(this->getSrcSize() == 0); 1904 Str << "\t" 1905 "ud2"; 1906 } 1907 1908 template <typename TraitsType> 1909 void InstImpl<TraitsType>::InstX86UD2::emitIAS(const Cfg *Func) const { 1910 Assembler *Asm = Func->getAssembler<Assembler>(); 1911 Asm->ud2(); 1912 } 1913 1914 template <typename TraitsType> 1915 void InstImpl<TraitsType>::InstX86UD2::dump(const Cfg *Func) const { 1916 if (!BuildDefs::dump()) 1917 return; 1918 Ostream &Str = Func->getContext()->getStrDump(); 1919 Str << "ud2"; 1920 } 1921 1922 template <typename TraitsType> 1923 void InstImpl<TraitsType>::InstX86Int3::emit(const Cfg *Func) const { 1924 if (!BuildDefs::dump()) 1925 return; 1926 Ostream &Str = Func->getContext()->getStrEmit(); 1927 assert(this->getSrcSize() == 0); 1928 Str << "\t" 1929 "int 3"; 1930 } 1931 1932 template <typename TraitsType> 1933 void InstImpl<TraitsType>::InstX86Int3::emitIAS(const Cfg *Func) const { 1934 Assembler *Asm = Func->getAssembler<Assembler>(); 1935 Asm->int3(); 1936 } 1937 1938 template <typename TraitsType> 1939 void InstImpl<TraitsType>::InstX86Int3::dump(const Cfg *Func) const { 1940 if (!BuildDefs::dump()) 1941 return; 1942 Ostream &Str = Func->getContext()->getStrDump(); 1943 Str << "int 3"; 1944 } 1945 1946 template <typename TraitsType> 1947 void InstImpl<TraitsType>::InstX86Test::emit(const Cfg *Func) const { 1948 if (!BuildDefs::dump()) 1949 return; 1950 Ostream &Str = Func->getContext()->getStrEmit(); 1951 assert(this->getSrcSize() == 2); 1952 Str << "\t" 1953 "test" << this->getWidthString(this->getSrc(0)->getType()) << "\t"; 1954 this->getSrc(1)->emit(Func); 1955 Str << ", "; 1956 this->getSrc(0)->emit(Func); 1957 } 1958 1959 template <typename TraitsType> 1960 void InstImpl<TraitsType>::InstX86Test::emitIAS(const Cfg *Func) const { 1961 assert(this->getSrcSize() == 2); 1962 const Operand *Src0 = this->getSrc(0); 1963 const Operand *Src1 = this->getSrc(1); 1964 Type Ty = Src0->getType(); 1965 // The Reg/Addr form of test is not encodeable. 1966 static const GPREmitterRegOp RegEmitter = {&Assembler::test, nullptr, 1967 &Assembler::test}; 1968 static const GPREmitterAddrOp AddrEmitter = {&Assembler::test, 1969 &Assembler::test}; 1970 if (const auto *SrcVar0 = llvm::dyn_cast<Variable>(Src0)) { 1971 if (SrcVar0->hasReg()) { 1972 constexpr bool NotLea = false; 1973 emitIASRegOpTyGPR(Func, NotLea, Ty, SrcVar0, Src1, RegEmitter); 1974 return; 1975 } 1976 } 1977 emitIASAsAddrOpTyGPR(Func, Ty, Src0, Src1, AddrEmitter); 1978 } 1979 1980 template <typename TraitsType> 1981 void InstImpl<TraitsType>::InstX86Test::dump(const Cfg *Func) const { 1982 if (!BuildDefs::dump()) 1983 return; 1984 Ostream &Str = Func->getContext()->getStrDump(); 1985 Str << "test." << this->getSrc(0)->getType() << " "; 1986 this->dumpSources(Func); 1987 } 1988 1989 template <typename TraitsType> 1990 void InstImpl<TraitsType>::InstX86Mfence::emit(const Cfg *Func) const { 1991 if (!BuildDefs::dump()) 1992 return; 1993 Ostream &Str = Func->getContext()->getStrEmit(); 1994 assert(this->getSrcSize() == 0); 1995 Str << "\t" 1996 "mfence"; 1997 } 1998 1999 template <typename TraitsType> 2000 void InstImpl<TraitsType>::InstX86Mfence::emitIAS(const Cfg *Func) const { 2001 Assembler *Asm = Func->getAssembler<Assembler>(); 2002 Asm->mfence(); 2003 } 2004 2005 template <typename TraitsType> 2006 void InstImpl<TraitsType>::InstX86Mfence::dump(const Cfg *Func) const { 2007 if (!BuildDefs::dump()) 2008 return; 2009 Ostream &Str = Func->getContext()->getStrDump(); 2010 Str << "mfence"; 2011 } 2012 2013 template <typename TraitsType> 2014 void InstImpl<TraitsType>::InstX86Store::emit(const Cfg *Func) const { 2015 if (!BuildDefs::dump()) 2016 return; 2017 Ostream &Str = Func->getContext()->getStrEmit(); 2018 assert(this->getSrcSize() == 2); 2019 Type Ty = this->getSrc(0)->getType(); 2020 Str << "\t" 2021 "mov" << this->getWidthString(Ty) 2022 << Traits::TypeAttributes[Ty].SdSsString << "\t"; 2023 this->getSrc(0)->emit(Func); 2024 Str << ", "; 2025 this->getSrc(1)->emit(Func); 2026 } 2027 2028 template <typename TraitsType> 2029 void InstImpl<TraitsType>::InstX86Store::emitIAS(const Cfg *Func) const { 2030 assert(this->getSrcSize() == 2); 2031 const Operand *Dest = this->getSrc(1); 2032 const Operand *Src = this->getSrc(0); 2033 Type DestTy = Dest->getType(); 2034 if (isScalarFloatingType(DestTy)) { 2035 // Src must be a register, since Dest is a Mem operand of some kind. 2036 const auto *SrcVar = llvm::cast<Variable>(Src); 2037 assert(SrcVar->hasReg()); 2038 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum()); 2039 Assembler *Asm = Func->getAssembler<Assembler>(); 2040 auto *Target = InstX86Base::getTarget(Func); 2041 if (const auto *DestVar = llvm::dyn_cast<Variable>(Dest)) { 2042 assert(!DestVar->hasReg()); 2043 Address StackAddr(Target->stackVarToAsmOperand(DestVar)); 2044 Asm->movss(DestTy, StackAddr, SrcReg); 2045 } else { 2046 const auto DestMem = llvm::cast<X86OperandMem>(Dest); 2047 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2048 Asm->movss(DestTy, DestMem->toAsmAddress(Asm, Target), SrcReg); 2049 } 2050 return; 2051 } else { 2052 assert(isScalarIntegerType(DestTy)); 2053 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov, 2054 &Assembler::mov}; 2055 emitIASAsAddrOpTyGPR(Func, DestTy, Dest, Src, GPRAddrEmitter); 2056 } 2057 } 2058 2059 template <typename TraitsType> 2060 void InstImpl<TraitsType>::InstX86Store::dump(const Cfg *Func) const { 2061 if (!BuildDefs::dump()) 2062 return; 2063 Ostream &Str = Func->getContext()->getStrDump(); 2064 Str << "mov." << this->getSrc(0)->getType() << " "; 2065 this->getSrc(1)->dump(Func); 2066 Str << ", "; 2067 this->getSrc(0)->dump(Func); 2068 } 2069 2070 template <typename TraitsType> 2071 void InstImpl<TraitsType>::InstX86StoreP::emit(const Cfg *Func) const { 2072 if (!BuildDefs::dump()) 2073 return; 2074 Ostream &Str = Func->getContext()->getStrEmit(); 2075 assert(this->getSrcSize() == 2); 2076 assert(isVectorType(this->getSrc(1)->getType())); 2077 Str << "\t" 2078 "movups\t"; 2079 this->getSrc(0)->emit(Func); 2080 Str << ", "; 2081 this->getSrc(1)->emit(Func); 2082 } 2083 2084 template <typename TraitsType> 2085 void InstImpl<TraitsType>::InstX86StoreP::emitIAS(const Cfg *Func) const { 2086 Assembler *Asm = Func->getAssembler<Assembler>(); 2087 assert(this->getSrcSize() == 2); 2088 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0)); 2089 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1)); 2090 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2091 assert(SrcVar->hasReg()); 2092 auto *Target = InstX86Base::getTarget(Func); 2093 Asm->movups(DestMem->toAsmAddress(Asm, Target), 2094 Traits::getEncodedXmm(SrcVar->getRegNum())); 2095 } 2096 2097 template <typename TraitsType> 2098 void InstImpl<TraitsType>::InstX86StoreP::dump(const Cfg *Func) const { 2099 if (!BuildDefs::dump()) 2100 return; 2101 Ostream &Str = Func->getContext()->getStrDump(); 2102 Str << "storep." << this->getSrc(0)->getType() << " "; 2103 this->getSrc(1)->dump(Func); 2104 Str << ", "; 2105 this->getSrc(0)->dump(Func); 2106 } 2107 2108 template <typename TraitsType> 2109 void InstImpl<TraitsType>::InstX86StoreQ::emit(const Cfg *Func) const { 2110 if (!BuildDefs::dump()) 2111 return; 2112 Ostream &Str = Func->getContext()->getStrEmit(); 2113 assert(this->getSrcSize() == 2); 2114 assert(this->getSrc(1)->getType() == IceType_i64 || 2115 this->getSrc(1)->getType() == IceType_f64 || 2116 isVectorType(this->getSrc(1)->getType())); 2117 Str << "\t" 2118 "movq\t"; 2119 this->getSrc(0)->emit(Func); 2120 Str << ", "; 2121 this->getSrc(1)->emit(Func); 2122 } 2123 2124 template <typename TraitsType> 2125 void InstImpl<TraitsType>::InstX86StoreQ::emitIAS(const Cfg *Func) const { 2126 Assembler *Asm = Func->getAssembler<Assembler>(); 2127 assert(this->getSrcSize() == 2); 2128 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0)); 2129 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1)); 2130 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2131 assert(SrcVar->hasReg()); 2132 auto *Target = InstX86Base::getTarget(Func); 2133 Asm->movq(DestMem->toAsmAddress(Asm, Target), 2134 Traits::getEncodedXmm(SrcVar->getRegNum())); 2135 } 2136 2137 template <typename TraitsType> 2138 void InstImpl<TraitsType>::InstX86StoreQ::dump(const Cfg *Func) const { 2139 if (!BuildDefs::dump()) 2140 return; 2141 Ostream &Str = Func->getContext()->getStrDump(); 2142 Str << "storeq." << this->getSrc(0)->getType() << " "; 2143 this->getSrc(1)->dump(Func); 2144 Str << ", "; 2145 this->getSrc(0)->dump(Func); 2146 } 2147 2148 template <typename TraitsType> 2149 void InstImpl<TraitsType>::InstX86StoreD::emit(const Cfg *Func) const { 2150 if (!BuildDefs::dump()) 2151 return; 2152 Ostream &Str = Func->getContext()->getStrEmit(); 2153 assert(this->getSrcSize() == 2); 2154 assert(this->getSrc(1)->getType() == IceType_i64 || 2155 this->getSrc(1)->getType() == IceType_f64 || 2156 isVectorType(this->getSrc(1)->getType())); 2157 Str << "\t" 2158 "movd\t"; 2159 this->getSrc(0)->emit(Func); 2160 Str << ", "; 2161 this->getSrc(1)->emit(Func); 2162 } 2163 2164 template <typename TraitsType> 2165 void InstImpl<TraitsType>::InstX86StoreD::emitIAS(const Cfg *Func) const { 2166 Assembler *Asm = Func->getAssembler<Assembler>(); 2167 assert(this->getSrcSize() == 2); 2168 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(0)); 2169 const auto DestMem = llvm::cast<X86OperandMem>(this->getSrc(1)); 2170 assert(DestMem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2171 assert(SrcVar->hasReg()); 2172 auto *Target = InstX86Base::getTarget(Func); 2173 Asm->movd(SrcVar->getType(), DestMem->toAsmAddress(Asm, Target), 2174 Traits::getEncodedXmm(SrcVar->getRegNum())); 2175 } 2176 2177 template <typename TraitsType> 2178 void InstImpl<TraitsType>::InstX86StoreD::dump(const Cfg *Func) const { 2179 if (!BuildDefs::dump()) 2180 return; 2181 Ostream &Str = Func->getContext()->getStrDump(); 2182 Str << "stored." << this->getSrc(0)->getType() << " "; 2183 this->getSrc(1)->dump(Func); 2184 Str << ", "; 2185 this->getSrc(0)->dump(Func); 2186 } 2187 2188 template <typename TraitsType> 2189 void InstImpl<TraitsType>::InstX86Lea::emit(const Cfg *Func) const { 2190 if (!BuildDefs::dump()) 2191 return; 2192 if (auto *Add = this->deoptLeaToAddOrNull(Func)) { 2193 Add->emit(Func); 2194 return; 2195 } 2196 2197 Ostream &Str = Func->getContext()->getStrEmit(); 2198 assert(this->getSrcSize() == 1); 2199 assert(this->getDest()->hasReg()); 2200 Str << "\t" 2201 "lea" << this->getWidthString(this->getDest()->getType()) << "\t"; 2202 Operand *Src0 = this->getSrc(0); 2203 if (const auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) { 2204 Type Ty = Src0Var->getType(); 2205 // lea on x86-32 doesn't accept mem128 operands, so cast VSrc0 to an 2206 // acceptable type. 2207 Src0Var->asType(Func, isVectorType(Ty) ? IceType_i32 : Ty, RegNumT()) 2208 ->emit(Func); 2209 } else { 2210 Src0->emit(Func); 2211 } 2212 Str << ", "; 2213 this->getDest()->emit(Func); 2214 } 2215 2216 template <typename TraitsType> 2217 void InstImpl<TraitsType>::InstX86Mov::emit(const Cfg *Func) const { 2218 if (!BuildDefs::dump()) 2219 return; 2220 Ostream &Str = Func->getContext()->getStrEmit(); 2221 assert(this->getSrcSize() == 1); 2222 Operand *Src = this->getSrc(0); 2223 Type SrcTy = Src->getType(); 2224 Type DestTy = this->getDest()->getType(); 2225 if (Traits::Is64Bit && DestTy == IceType_i64 && 2226 llvm::isa<ConstantInteger64>(Src) && 2227 !Utils::IsInt(32, llvm::cast<ConstantInteger64>(Src)->getValue())) { 2228 Str << "\t" 2229 "movabs" 2230 "\t"; 2231 } else { 2232 Str << "\t" 2233 "mov" << (!isScalarFloatingType(DestTy) 2234 ? this->getWidthString(DestTy) 2235 : Traits::TypeAttributes[DestTy].SdSsString) << "\t"; 2236 } 2237 // For an integer truncation operation, src is wider than dest. In this case, 2238 // we use a mov instruction whose data width matches the narrower dest. 2239 // TODO: This assert disallows usages such as copying a floating 2240 // point value between a vector and a scalar (which movss is used for). Clean 2241 // this up. 2242 assert(InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(DestTy) == 2243 InstX86Base::getTarget(Func)->typeWidthInBytesOnStack(SrcTy)); 2244 const Operand *NewSrc = Src; 2245 if (auto *SrcVar = llvm::dyn_cast<Variable>(Src)) { 2246 RegNumT NewRegNum; 2247 if (SrcVar->hasReg()) 2248 NewRegNum = Traits::getGprForType(DestTy, SrcVar->getRegNum()); 2249 if (SrcTy != DestTy) 2250 NewSrc = SrcVar->asType(Func, DestTy, NewRegNum); 2251 } 2252 NewSrc->emit(Func); 2253 Str << ", "; 2254 this->getDest()->emit(Func); 2255 } 2256 2257 template <typename TraitsType> 2258 void InstImpl<TraitsType>::InstX86Mov::emitIAS(const Cfg *Func) const { 2259 assert(this->getSrcSize() == 1); 2260 const Variable *Dest = this->getDest(); 2261 const Operand *Src = this->getSrc(0); 2262 Type DestTy = Dest->getType(); 2263 Type SrcTy = Src->getType(); 2264 // Mov can be used for GPRs or XMM registers. Also, the type does not 2265 // necessarily match (Mov can be used for bitcasts). However, when the type 2266 // does not match, one of the operands must be a register. Thus, the strategy 2267 // is to find out if Src or Dest are a register, then use that register's 2268 // type to decide on which emitter set to use. The emitter set will include 2269 // reg-reg movs, but that case should be unused when the types don't match. 2270 static const XmmEmitterRegOp XmmRegEmitter = {&Assembler::movss, 2271 &Assembler::movss}; 2272 static const GPREmitterRegOp GPRRegEmitter = { 2273 &Assembler::mov, &Assembler::mov, &Assembler::mov}; 2274 static const GPREmitterAddrOp GPRAddrEmitter = {&Assembler::mov, 2275 &Assembler::mov}; 2276 // For an integer truncation operation, src is wider than dest. In this case, 2277 // we use a mov instruction whose data width matches the narrower dest. 2278 // TODO: This assert disallows usages such as copying a floating 2279 // point value between a vector and a scalar (which movss is used for). Clean 2280 // this up. 2281 auto *Target = InstX86Base::getTarget(Func); 2282 assert(Target->typeWidthInBytesOnStack(this->getDest()->getType()) == 2283 Target->typeWidthInBytesOnStack(Src->getType())); 2284 if (Dest->hasReg()) { 2285 if (isScalarFloatingType(DestTy)) { 2286 emitIASRegOpTyXMM(Func, DestTy, Dest, Src, XmmRegEmitter); 2287 return; 2288 } else { 2289 assert(isScalarIntegerType(DestTy)); 2290 // Widen DestTy for truncation (see above note). We should only do this 2291 // when both Src and Dest are integer types. 2292 if (Traits::Is64Bit && DestTy == IceType_i64) { 2293 if (const auto *C64 = llvm::dyn_cast<ConstantInteger64>(Src)) { 2294 Func->getAssembler<Assembler>()->movabs( 2295 Traits::getEncodedGPR(Dest->getRegNum()), C64->getValue()); 2296 return; 2297 } 2298 } 2299 if (isScalarIntegerType(SrcTy)) { 2300 SrcTy = DestTy; 2301 } 2302 constexpr bool NotLea = false; 2303 emitIASRegOpTyGPR(Func, NotLea, DestTy, Dest, Src, GPRRegEmitter); 2304 return; 2305 } 2306 } else { 2307 // Dest must be Stack and Src *could* be a register. Use Src's type to 2308 // decide on the emitters. 2309 Address StackAddr(Target->stackVarToAsmOperand(Dest)); 2310 if (isScalarFloatingType(SrcTy)) { 2311 // Src must be a register. 2312 const auto *SrcVar = llvm::cast<Variable>(Src); 2313 assert(SrcVar->hasReg()); 2314 Assembler *Asm = Func->getAssembler<Assembler>(); 2315 Asm->movss(SrcTy, StackAddr, Traits::getEncodedXmm(SrcVar->getRegNum())); 2316 return; 2317 } else { 2318 // Src can be a register or immediate. 2319 assert(isScalarIntegerType(SrcTy)); 2320 emitIASAddrOpTyGPR(Func, SrcTy, StackAddr, Src, GPRAddrEmitter); 2321 return; 2322 } 2323 return; 2324 } 2325 } 2326 2327 template <typename TraitsType> 2328 void InstImpl<TraitsType>::InstX86Movd::emit(const Cfg *Func) const { 2329 if (!BuildDefs::dump()) 2330 return; 2331 assert(this->getSrcSize() == 1); 2332 Variable *Dest = this->getDest(); 2333 Operand *Src = this->getSrc(0); 2334 2335 if (Dest->getType() == IceType_i64 || Src->getType() == IceType_i64) { 2336 assert(Dest->getType() == IceType_f64 || Src->getType() == IceType_f64); 2337 assert(Dest->getType() != Src->getType()); 2338 Ostream &Str = Func->getContext()->getStrEmit(); 2339 Str << "\t" 2340 "movq" 2341 "\t"; 2342 Src->emit(Func); 2343 Str << ", "; 2344 Dest->emit(Func); 2345 return; 2346 } 2347 2348 InstX86BaseUnaryopXmm<InstX86Base::Movd>::emit(Func); 2349 } 2350 2351 template <typename TraitsType> 2352 void InstImpl<TraitsType>::InstX86Movd::emitIAS(const Cfg *Func) const { 2353 Assembler *Asm = Func->getAssembler<Assembler>(); 2354 assert(this->getSrcSize() == 1); 2355 const Variable *Dest = this->getDest(); 2356 auto *Target = InstX86Base::getTarget(Func); 2357 // For insert/extract element (one of Src/Dest is an Xmm vector and the other 2358 // is an int type). 2359 if (const auto *SrcVar = llvm::dyn_cast<Variable>(this->getSrc(0))) { 2360 if (SrcVar->getType() == IceType_i32 || 2361 (Traits::Is64Bit && SrcVar->getType() == IceType_i64)) { 2362 assert(isVectorType(Dest->getType()) || 2363 (isScalarFloatingType(Dest->getType()) && 2364 typeWidthInBytes(SrcVar->getType()) == 2365 typeWidthInBytes(Dest->getType()))); 2366 assert(Dest->hasReg()); 2367 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum()); 2368 if (SrcVar->hasReg()) { 2369 Asm->movd(SrcVar->getType(), DestReg, 2370 Traits::getEncodedGPR(SrcVar->getRegNum())); 2371 } else { 2372 Address StackAddr(Target->stackVarToAsmOperand(SrcVar)); 2373 Asm->movd(SrcVar->getType(), DestReg, StackAddr); 2374 } 2375 } else { 2376 assert(isVectorType(SrcVar->getType()) || 2377 (isScalarFloatingType(SrcVar->getType()) && 2378 typeWidthInBytes(SrcVar->getType()) == 2379 typeWidthInBytes(Dest->getType()))); 2380 assert(SrcVar->hasReg()); 2381 assert(Dest->getType() == IceType_i32 || 2382 (Traits::Is64Bit && Dest->getType() == IceType_i64)); 2383 XmmRegister SrcReg = Traits::getEncodedXmm(SrcVar->getRegNum()); 2384 if (Dest->hasReg()) { 2385 Asm->movd(Dest->getType(), Traits::getEncodedGPR(Dest->getRegNum()), 2386 SrcReg); 2387 } else { 2388 Address StackAddr(Target->stackVarToAsmOperand(Dest)); 2389 Asm->movd(Dest->getType(), StackAddr, SrcReg); 2390 } 2391 } 2392 } else { 2393 assert(Dest->hasReg()); 2394 XmmRegister DestReg = Traits::getEncodedXmm(Dest->getRegNum()); 2395 auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0)); 2396 Asm->movd(Mem->getType(), DestReg, Mem->toAsmAddress(Asm, Target)); 2397 } 2398 } 2399 2400 template <typename TraitsType> 2401 void InstImpl<TraitsType>::InstX86Movp::emit(const Cfg *Func) const { 2402 if (!BuildDefs::dump()) 2403 return; 2404 // TODO(wala,stichnot): movups works with all vector operands, but there 2405 // exist other instructions (movaps, movdqa, movdqu) that may perform better, 2406 // depending on the data type and alignment of the operands. 2407 Ostream &Str = Func->getContext()->getStrEmit(); 2408 assert(this->getSrcSize() == 1); 2409 Str << "\t" 2410 "movups\t"; 2411 this->getSrc(0)->emit(Func); 2412 Str << ", "; 2413 this->getDest()->emit(Func); 2414 } 2415 2416 template <typename TraitsType> 2417 void InstImpl<TraitsType>::InstX86Movp::emitIAS(const Cfg *Func) const { 2418 assert(this->getSrcSize() == 1); 2419 assert(isVectorType(this->getDest()->getType())); 2420 const Variable *Dest = this->getDest(); 2421 const Operand *Src = this->getSrc(0); 2422 static const XmmEmitterMovOps Emitter = { 2423 &Assembler::movups, &Assembler::movups, &Assembler::movups}; 2424 emitIASMovlikeXMM(Func, Dest, Src, Emitter); 2425 } 2426 2427 template <typename TraitsType> 2428 void InstImpl<TraitsType>::InstX86Movq::emit(const Cfg *Func) const { 2429 if (!BuildDefs::dump()) 2430 return; 2431 Ostream &Str = Func->getContext()->getStrEmit(); 2432 assert(this->getSrcSize() == 1); 2433 assert(this->getDest()->getType() == IceType_i64 || 2434 this->getDest()->getType() == IceType_f64); 2435 Str << "\t" 2436 "movq" 2437 "\t"; 2438 this->getSrc(0)->emit(Func); 2439 Str << ", "; 2440 this->getDest()->emit(Func); 2441 } 2442 2443 template <typename TraitsType> 2444 void InstImpl<TraitsType>::InstX86Movq::emitIAS(const Cfg *Func) const { 2445 assert(this->getSrcSize() == 1); 2446 assert(this->getDest()->getType() == IceType_i64 || 2447 this->getDest()->getType() == IceType_f64 || 2448 isVectorType(this->getDest()->getType())); 2449 const Variable *Dest = this->getDest(); 2450 const Operand *Src = this->getSrc(0); 2451 static const XmmEmitterMovOps Emitter = {&Assembler::movq, &Assembler::movq, 2452 &Assembler::movq}; 2453 emitIASMovlikeXMM(Func, Dest, Src, Emitter); 2454 } 2455 2456 template <typename TraitsType> 2457 void InstImpl<TraitsType>::InstX86MovssRegs::emitIAS(const Cfg *Func) const { 2458 // This is Binop variant is only intended to be used for reg-reg moves where 2459 // part of the Dest register is untouched. 2460 assert(this->getSrcSize() == 2); 2461 const Variable *Dest = this->getDest(); 2462 assert(Dest == this->getSrc(0)); 2463 const auto *SrcVar = llvm::cast<Variable>(this->getSrc(1)); 2464 assert(Dest->hasReg() && SrcVar->hasReg()); 2465 Assembler *Asm = Func->getAssembler<Assembler>(); 2466 Asm->movss(IceType_f32, Traits::getEncodedXmm(Dest->getRegNum()), 2467 Traits::getEncodedXmm(SrcVar->getRegNum())); 2468 } 2469 2470 template <typename TraitsType> 2471 void InstImpl<TraitsType>::InstX86Movsx::emitIAS(const Cfg *Func) const { 2472 assert(this->getSrcSize() == 1); 2473 const Variable *Dest = this->getDest(); 2474 const Operand *Src = this->getSrc(0); 2475 // Dest must be a > 8-bit register, but Src can be 8-bit. In practice we just 2476 // use the full register for Dest to avoid having an OperandSizeOverride 2477 // prefix. It also allows us to only dispatch on SrcTy. 2478 Type SrcTy = Src->getType(); 2479 assert(typeWidthInBytes(Dest->getType()) > 1); 2480 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy)); 2481 constexpr bool NotLea = false; 2482 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter); 2483 } 2484 2485 template <typename TraitsType> 2486 bool InstImpl<TraitsType>::InstX86Movzx::mayBeElided( 2487 const Variable *Dest, const Operand *SrcOpnd) const { 2488 assert(Traits::Is64Bit); 2489 const auto *Src = llvm::dyn_cast<Variable>(SrcOpnd); 2490 2491 // Src is not a Variable, so it does not have a register. Movzx can't be 2492 // elided. 2493 if (Src == nullptr) 2494 return false; 2495 2496 // Movzx to/from memory can't be elided. 2497 if (!Src->hasReg() || !Dest->hasReg()) 2498 return false; 2499 2500 // Reg/reg move with different source and dest can't be elided. 2501 if (Traits::getEncodedGPR(Src->getRegNum()) != 2502 Traits::getEncodedGPR(Dest->getRegNum())) 2503 return false; 2504 2505 // A must-keep movzx 32- to 64-bit is sometimes needed in x86-64 sandboxing. 2506 return !MustKeep; 2507 } 2508 2509 template <typename TraitsType> 2510 void InstImpl<TraitsType>::InstX86Movzx::emit(const Cfg *Func) const { 2511 if (!BuildDefs::dump()) 2512 return; 2513 if (Traits::Is64Bit) { 2514 // There's no movzx %eXX, %rXX. To zero extend 32- to 64-bits, we emit a 2515 // mov %eXX, %eXX. The processor will still do a movzx[bw]q. 2516 assert(this->getSrcSize() == 1); 2517 const Operand *Src = this->getSrc(0); 2518 const Variable *Dest = this->Dest; 2519 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64) { 2520 Ostream &Str = Func->getContext()->getStrEmit(); 2521 if (mayBeElided(Dest, Src)) { 2522 Str << "\t/* elided movzx */"; 2523 } else { 2524 Str << "\t" 2525 "mov" 2526 "\t"; 2527 Src->emit(Func); 2528 Str << ", "; 2529 Dest->asType(Func, IceType_i32, 2530 Traits::getGprForType(IceType_i32, Dest->getRegNum())) 2531 ->emit(Func); 2532 Str << " /* movzx */"; 2533 } 2534 return; 2535 } 2536 } 2537 InstX86BaseUnaryopGPR<InstX86Base::Movzx>::emit(Func); 2538 } 2539 2540 template <typename TraitsType> 2541 void InstImpl<TraitsType>::InstX86Movzx::emitIAS(const Cfg *Func) const { 2542 assert(this->getSrcSize() == 1); 2543 const Variable *Dest = this->getDest(); 2544 const Operand *Src = this->getSrc(0); 2545 Type SrcTy = Src->getType(); 2546 assert(typeWidthInBytes(Dest->getType()) > 1); 2547 assert(typeWidthInBytes(Dest->getType()) > typeWidthInBytes(SrcTy)); 2548 if (Traits::Is64Bit) { 2549 if (Src->getType() == IceType_i32 && Dest->getType() == IceType_i64 && 2550 mayBeElided(Dest, Src)) { 2551 return; 2552 } 2553 } 2554 constexpr bool NotLea = false; 2555 emitIASRegOpTyGPR<false, true>(Func, NotLea, SrcTy, Dest, Src, this->Emitter); 2556 } 2557 2558 template <typename TraitsType> 2559 void InstImpl<TraitsType>::InstX86Nop::emit(const Cfg *Func) const { 2560 if (!BuildDefs::dump()) 2561 return; 2562 Ostream &Str = Func->getContext()->getStrEmit(); 2563 // TODO: Emit the right code for each variant. 2564 Str << "\t" 2565 "nop\t/* variant = " << Variant << " */"; 2566 } 2567 2568 template <typename TraitsType> 2569 void InstImpl<TraitsType>::InstX86Nop::emitIAS(const Cfg *Func) const { 2570 Assembler *Asm = Func->getAssembler<Assembler>(); 2571 // TODO: Emit the right code for the variant. 2572 Asm->nop(); 2573 } 2574 2575 template <typename TraitsType> 2576 void InstImpl<TraitsType>::InstX86Nop::dump(const Cfg *Func) const { 2577 if (!BuildDefs::dump()) 2578 return; 2579 Ostream &Str = Func->getContext()->getStrDump(); 2580 Str << "nop (variant = " << Variant << ")"; 2581 } 2582 2583 template <typename TraitsType> 2584 void InstImpl<TraitsType>::InstX86Fld::emit(const Cfg *Func) const { 2585 if (!BuildDefs::dump()) 2586 return; 2587 Ostream &Str = Func->getContext()->getStrEmit(); 2588 assert(this->getSrcSize() == 1); 2589 Type Ty = this->getSrc(0)->getType(); 2590 const auto *Var = llvm::dyn_cast<Variable>(this->getSrc(0)); 2591 if (Var && Var->hasReg()) { 2592 // This is a physical xmm register, so we need to spill it to a temporary 2593 // stack slot. Function prolog emission guarantees that there is sufficient 2594 // space to do this. 2595 Str << "\t" 2596 "mov" << Traits::TypeAttributes[Ty].SdSsString << "\t"; 2597 Var->emit(Func); 2598 Str << ", (%esp)\n" 2599 "\t" 2600 "fld" << this->getFldString(Ty) << "\t" 2601 "(%esp)"; 2602 return; 2603 } 2604 Str << "\t" 2605 "fld" << this->getFldString(Ty) << "\t"; 2606 this->getSrc(0)->emit(Func); 2607 } 2608 2609 template <typename TraitsType> 2610 void InstImpl<TraitsType>::InstX86Fld::emitIAS(const Cfg *Func) const { 2611 Assembler *Asm = Func->getAssembler<Assembler>(); 2612 assert(this->getSrcSize() == 1); 2613 const Operand *Src = this->getSrc(0); 2614 auto *Target = InstX86Base::getTarget(Func); 2615 Type Ty = Src->getType(); 2616 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) { 2617 if (Var->hasReg()) { 2618 // This is a physical xmm register, so we need to spill it to a temporary 2619 // stack slot. Function prolog emission guarantees that there is 2620 // sufficient space to do this. 2621 Address StackSlot = 2622 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup); 2623 Asm->movss(Ty, StackSlot, Traits::getEncodedXmm(Var->getRegNum())); 2624 Asm->fld(Ty, StackSlot); 2625 } else { 2626 Address StackAddr(Target->stackVarToAsmOperand(Var)); 2627 Asm->fld(Ty, StackAddr); 2628 } 2629 } else if (const auto *Mem = llvm::dyn_cast<X86OperandMem>(Src)) { 2630 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2631 Asm->fld(Ty, Mem->toAsmAddress(Asm, Target)); 2632 } else if (const auto *Imm = llvm::dyn_cast<Constant>(Src)) { 2633 Asm->fld(Ty, Traits::Address::ofConstPool(Asm, Imm)); 2634 } else { 2635 llvm_unreachable("Unexpected operand type"); 2636 } 2637 } 2638 2639 template <typename TraitsType> 2640 void InstImpl<TraitsType>::InstX86Fld::dump(const Cfg *Func) const { 2641 if (!BuildDefs::dump()) 2642 return; 2643 Ostream &Str = Func->getContext()->getStrDump(); 2644 Str << "fld." << this->getSrc(0)->getType() << " "; 2645 this->dumpSources(Func); 2646 } 2647 2648 template <typename TraitsType> 2649 void InstImpl<TraitsType>::InstX86Fstp::emit(const Cfg *Func) const { 2650 if (!BuildDefs::dump()) 2651 return; 2652 Ostream &Str = Func->getContext()->getStrEmit(); 2653 assert(this->getSrcSize() == 0); 2654 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to 2655 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused, 2656 // the fstp should be kept for the SideEffects of popping the stack. 2657 if (!this->getDest()) { 2658 Str << "\t" 2659 "fstp\t" 2660 "st(0)"; 2661 return; 2662 } 2663 Type Ty = this->getDest()->getType(); 2664 if (!this->getDest()->hasReg()) { 2665 Str << "\t" 2666 "fstp" << this->getFldString(Ty) << "\t"; 2667 this->getDest()->emit(Func); 2668 return; 2669 } 2670 // Dest is a physical (xmm) register, so st(0) needs to go through memory. 2671 // Hack this by using caller-reserved memory at the top of stack, spilling 2672 // st(0) there, and loading it into the xmm register. 2673 Str << "\t" 2674 "fstp" << this->getFldString(Ty) << "\t" 2675 "(%esp)\n"; 2676 Str << "\t" 2677 "mov" << Traits::TypeAttributes[Ty].SdSsString << "\t" 2678 "(%esp), "; 2679 this->getDest()->emit(Func); 2680 } 2681 2682 template <typename TraitsType> 2683 void InstImpl<TraitsType>::InstX86Fstp::emitIAS(const Cfg *Func) const { 2684 Assembler *Asm = Func->getAssembler<Assembler>(); 2685 assert(this->getSrcSize() == 0); 2686 const Variable *Dest = this->getDest(); 2687 // TODO(jvoung,stichnot): Utilize this by setting Dest to nullptr to 2688 // "partially" delete the fstp if the Dest is unused. Even if Dest is unused, 2689 // the fstp should be kept for the SideEffects of popping the stack. 2690 if (!Dest) { 2691 Asm->fstp(RegisterSet::getEncodedSTReg(0)); 2692 return; 2693 } 2694 auto *Target = InstX86Base::getTarget(Func); 2695 Type Ty = Dest->getType(); 2696 if (!Dest->hasReg()) { 2697 Address StackAddr(Target->stackVarToAsmOperand(Dest)); 2698 Asm->fstp(Ty, StackAddr); 2699 } else { 2700 // Dest is a physical (xmm) register, so st(0) needs to go through memory. 2701 // Hack this by using caller-reserved memory at the top of stack, spilling 2702 // st(0) there, and loading it into the xmm register. 2703 Address StackSlot = 2704 Address(RegisterSet::Encoded_Reg_esp, 0, AssemblerFixup::NoFixup); 2705 Asm->fstp(Ty, StackSlot); 2706 Asm->movss(Ty, Traits::getEncodedXmm(Dest->getRegNum()), StackSlot); 2707 } 2708 } 2709 2710 template <typename TraitsType> 2711 void InstImpl<TraitsType>::InstX86Fstp::dump(const Cfg *Func) const { 2712 if (!BuildDefs::dump()) 2713 return; 2714 Ostream &Str = Func->getContext()->getStrDump(); 2715 this->dumpDest(Func); 2716 Str << " = fstp." << this->getDest()->getType() << ", st(0)"; 2717 } 2718 2719 template <typename TraitsType> 2720 void InstImpl<TraitsType>::InstX86Pextr::emit(const Cfg *Func) const { 2721 if (!BuildDefs::dump()) 2722 return; 2723 Ostream &Str = Func->getContext()->getStrEmit(); 2724 assert(this->getSrcSize() == 2); 2725 // pextrb and pextrd are SSE4.1 instructions. 2726 Str << "\t" << this->Opcode 2727 << Traits::TypeAttributes[this->getSrc(0)->getType()].IntegralString 2728 << "\t"; 2729 this->getSrc(1)->emit(Func); 2730 Str << ", "; 2731 this->getSrc(0)->emit(Func); 2732 Str << ", "; 2733 Variable *Dest = this->getDest(); 2734 // pextrw must take a register dest. There is an SSE4.1 version that takes a 2735 // memory dest, but we aren't using it. For uniformity, just restrict them 2736 // all to have a register dest for now. 2737 assert(Dest->hasReg()); 2738 Dest->asType(Func, IceType_i32, Dest->getRegNum())->emit(Func); 2739 } 2740 2741 template <typename TraitsType> 2742 void InstImpl<TraitsType>::InstX86Pextr::emitIAS(const Cfg *Func) const { 2743 assert(this->getSrcSize() == 2); 2744 // pextrb and pextrd are SSE4.1 instructions. 2745 const Variable *Dest = this->getDest(); 2746 Type DispatchTy = Traits::getInVectorElementType(this->getSrc(0)->getType()); 2747 // pextrw must take a register dest. There is an SSE4.1 version that takes a 2748 // memory dest, but we aren't using it. For uniformity, just restrict them 2749 // all to have a register dest for now. 2750 assert(Dest->hasReg()); 2751 // pextrw's Src(0) must be a register (both SSE4.1 and SSE2). 2752 assert(llvm::cast<Variable>(this->getSrc(0))->hasReg()); 2753 static const ThreeOpImmEmitter<GPRRegister, XmmRegister> Emitter = { 2754 &Assembler::pextr, nullptr}; 2755 emitIASThreeOpImmOps<GPRRegister, XmmRegister, Traits::getEncodedGPR, 2756 Traits::getEncodedXmm>( 2757 Func, DispatchTy, Dest, this->getSrc(0), this->getSrc(1), Emitter); 2758 } 2759 2760 template <typename TraitsType> 2761 void InstImpl<TraitsType>::InstX86Pinsr::emit(const Cfg *Func) const { 2762 if (!BuildDefs::dump()) 2763 return; 2764 Ostream &Str = Func->getContext()->getStrEmit(); 2765 assert(this->getSrcSize() == 3); 2766 Str << "\t" << this->Opcode 2767 << Traits::TypeAttributes[this->getDest()->getType()].IntegralString 2768 << "\t"; 2769 this->getSrc(2)->emit(Func); 2770 Str << ", "; 2771 Operand *Src1 = this->getSrc(1); 2772 if (const auto *Src1Var = llvm::dyn_cast<Variable>(Src1)) { 2773 // If src1 is a register, it should always be r32. 2774 if (Src1Var->hasReg()) { 2775 const auto NewRegNum = Traits::getBaseReg(Src1Var->getRegNum()); 2776 const Variable *NewSrc = Src1Var->asType(Func, IceType_i32, NewRegNum); 2777 NewSrc->emit(Func); 2778 } else { 2779 Src1Var->emit(Func); 2780 } 2781 } else { 2782 Src1->emit(Func); 2783 } 2784 Str << ", "; 2785 this->getDest()->emit(Func); 2786 } 2787 2788 template <typename TraitsType> 2789 void InstImpl<TraitsType>::InstX86Pinsr::emitIAS(const Cfg *Func) const { 2790 assert(this->getSrcSize() == 3); 2791 assert(this->getDest() == this->getSrc(0)); 2792 // pinsrb and pinsrd are SSE4.1 instructions. 2793 const Operand *Src0 = this->getSrc(1); 2794 Type DispatchTy = Src0->getType(); 2795 // If src1 is a register, it should always be r32 (this should fall out from 2796 // the encodings for ByteRegs overlapping the encodings for r32), but we have 2797 // to make sure the register allocator didn't choose an 8-bit high register 2798 // like "ah". 2799 if (BuildDefs::asserts()) { 2800 if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0)) { 2801 if (Src0Var->hasReg()) { 2802 const auto RegNum = Src0Var->getRegNum(); 2803 const auto BaseRegNum = Traits::getBaseReg(RegNum); 2804 (void)BaseRegNum; 2805 assert(Traits::getEncodedGPR(RegNum) == 2806 Traits::getEncodedGPR(BaseRegNum)); 2807 } 2808 } 2809 } 2810 static const ThreeOpImmEmitter<XmmRegister, GPRRegister> Emitter = { 2811 &Assembler::pinsr, &Assembler::pinsr}; 2812 emitIASThreeOpImmOps<XmmRegister, GPRRegister, Traits::getEncodedXmm, 2813 Traits::getEncodedGPR>(Func, DispatchTy, this->getDest(), 2814 Src0, this->getSrc(2), Emitter); 2815 } 2816 2817 template <typename TraitsType> 2818 void InstImpl<TraitsType>::InstX86Pshufd::emitIAS(const Cfg *Func) const { 2819 assert(this->getSrcSize() == 2); 2820 const Variable *Dest = this->getDest(); 2821 Type Ty = Dest->getType(); 2822 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = { 2823 &Assembler::pshufd, &Assembler::pshufd}; 2824 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm, 2825 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(0), 2826 this->getSrc(1), Emitter); 2827 } 2828 2829 template <typename TraitsType> 2830 void InstImpl<TraitsType>::InstX86Shufps::emitIAS(const Cfg *Func) const { 2831 assert(this->getSrcSize() == 3); 2832 const Variable *Dest = this->getDest(); 2833 assert(Dest == this->getSrc(0)); 2834 Type Ty = Dest->getType(); 2835 static const ThreeOpImmEmitter<XmmRegister, XmmRegister> Emitter = { 2836 &Assembler::shufps, &Assembler::shufps}; 2837 emitIASThreeOpImmOps<XmmRegister, XmmRegister, Traits::getEncodedXmm, 2838 Traits::getEncodedXmm>(Func, Ty, Dest, this->getSrc(1), 2839 this->getSrc(2), Emitter); 2840 } 2841 2842 template <typename TraitsType> 2843 void InstImpl<TraitsType>::InstX86Pop::emit(const Cfg *Func) const { 2844 if (!BuildDefs::dump()) 2845 return; 2846 Ostream &Str = Func->getContext()->getStrEmit(); 2847 assert(this->getSrcSize() == 0); 2848 Str << "\t" 2849 "pop\t"; 2850 this->getDest()->emit(Func); 2851 } 2852 2853 template <typename TraitsType> 2854 void InstImpl<TraitsType>::InstX86Pop::emitIAS(const Cfg *Func) const { 2855 assert(this->getSrcSize() == 0); 2856 Assembler *Asm = Func->getAssembler<Assembler>(); 2857 if (this->getDest()->hasReg()) { 2858 Asm->popl(Traits::getEncodedGPR(this->getDest()->getRegNum())); 2859 } else { 2860 auto *Target = InstX86Base::getTarget(Func); 2861 Asm->popl(Target->stackVarToAsmOperand(this->getDest())); 2862 } 2863 } 2864 2865 template <typename TraitsType> 2866 void InstImpl<TraitsType>::InstX86Pop::dump(const Cfg *Func) const { 2867 if (!BuildDefs::dump()) 2868 return; 2869 Ostream &Str = Func->getContext()->getStrDump(); 2870 this->dumpDest(Func); 2871 Str << " = pop." << this->getDest()->getType() << " "; 2872 } 2873 2874 template <typename TraitsType> 2875 void InstImpl<TraitsType>::InstX86Push::emit(const Cfg *Func) const { 2876 if (!BuildDefs::dump()) 2877 return; 2878 Ostream &Str = Func->getContext()->getStrEmit(); 2879 Str << "\t" 2880 "push" 2881 "\t"; 2882 assert(this->getSrcSize() == 1); 2883 const Operand *Src = this->getSrc(0); 2884 Src->emit(Func); 2885 } 2886 2887 template <typename TraitsType> 2888 void InstImpl<TraitsType>::InstX86Push::emitIAS(const Cfg *Func) const { 2889 Assembler *Asm = Func->getAssembler<Assembler>(); 2890 2891 assert(this->getSrcSize() == 1); 2892 const Operand *Src = this->getSrc(0); 2893 2894 if (const auto *Var = llvm::dyn_cast<Variable>(Src)) { 2895 Asm->pushl(Traits::getEncodedGPR(Var->getRegNum())); 2896 } else if (const auto *Const32 = llvm::dyn_cast<ConstantInteger32>(Src)) { 2897 Asm->pushl(AssemblerImmediate(Const32->getValue())); 2898 } else if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Src)) { 2899 Asm->pushl(CR); 2900 } else { 2901 llvm_unreachable("Unexpected operand type"); 2902 } 2903 } 2904 2905 template <typename TraitsType> 2906 void InstImpl<TraitsType>::InstX86Push::dump(const Cfg *Func) const { 2907 if (!BuildDefs::dump()) 2908 return; 2909 Ostream &Str = Func->getContext()->getStrDump(); 2910 Str << "push." << this->getSrc(0)->getType() << " "; 2911 this->dumpSources(Func); 2912 } 2913 2914 template <typename TraitsType> 2915 void InstImpl<TraitsType>::InstX86Ret::emit(const Cfg *Func) const { 2916 if (!BuildDefs::dump()) 2917 return; 2918 Ostream &Str = Func->getContext()->getStrEmit(); 2919 Str << "\t" 2920 "ret"; 2921 } 2922 2923 template <typename TraitsType> 2924 void InstImpl<TraitsType>::InstX86Ret::emitIAS(const Cfg *Func) const { 2925 Assembler *Asm = Func->getAssembler<Assembler>(); 2926 Asm->ret(); 2927 } 2928 2929 template <typename TraitsType> 2930 void InstImpl<TraitsType>::InstX86Ret::dump(const Cfg *Func) const { 2931 if (!BuildDefs::dump()) 2932 return; 2933 Ostream &Str = Func->getContext()->getStrDump(); 2934 Type Ty = 2935 (this->getSrcSize() == 0 ? IceType_void : this->getSrc(0)->getType()); 2936 Str << "ret." << Ty << " "; 2937 this->dumpSources(Func); 2938 } 2939 2940 template <typename TraitsType> 2941 void InstImpl<TraitsType>::InstX86Setcc::emit(const Cfg *Func) const { 2942 if (!BuildDefs::dump()) 2943 return; 2944 Ostream &Str = Func->getContext()->getStrEmit(); 2945 Str << "\t" 2946 "set" << Traits::InstBrAttributes[Condition].DisplayString << "\t"; 2947 this->Dest->emit(Func); 2948 } 2949 2950 template <typename TraitsType> 2951 void InstImpl<TraitsType>::InstX86Setcc::emitIAS(const Cfg *Func) const { 2952 assert(Condition != Cond::Br_None); 2953 assert(this->getDest()->getType() == IceType_i1); 2954 assert(this->getSrcSize() == 0); 2955 Assembler *Asm = Func->getAssembler<Assembler>(); 2956 auto *Target = InstX86Base::getTarget(Func); 2957 if (this->getDest()->hasReg()) 2958 Asm->setcc(Condition, 2959 Traits::getEncodedByteReg(this->getDest()->getRegNum())); 2960 else 2961 Asm->setcc(Condition, Target->stackVarToAsmOperand(this->getDest())); 2962 return; 2963 } 2964 2965 template <typename TraitsType> 2966 void InstImpl<TraitsType>::InstX86Setcc::dump(const Cfg *Func) const { 2967 if (!BuildDefs::dump()) 2968 return; 2969 Ostream &Str = Func->getContext()->getStrDump(); 2970 Str << "setcc." << Traits::InstBrAttributes[Condition].DisplayString << " "; 2971 this->dumpDest(Func); 2972 } 2973 2974 template <typename TraitsType> 2975 void InstImpl<TraitsType>::InstX86Xadd::emit(const Cfg *Func) const { 2976 if (!BuildDefs::dump()) 2977 return; 2978 Ostream &Str = Func->getContext()->getStrEmit(); 2979 if (this->Locked) { 2980 Str << "\t" 2981 "lock"; 2982 } 2983 Str << "\t" 2984 "xadd" << this->getWidthString(this->getSrc(0)->getType()) << "\t"; 2985 this->getSrc(1)->emit(Func); 2986 Str << ", "; 2987 this->getSrc(0)->emit(Func); 2988 } 2989 2990 template <typename TraitsType> 2991 void InstImpl<TraitsType>::InstX86Xadd::emitIAS(const Cfg *Func) const { 2992 assert(this->getSrcSize() == 2); 2993 Assembler *Asm = Func->getAssembler<Assembler>(); 2994 Type Ty = this->getSrc(0)->getType(); 2995 const auto Mem = llvm::cast<X86OperandMem>(this->getSrc(0)); 2996 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 2997 auto *Target = InstX86Base::getTarget(Func); 2998 const Address Addr = Mem->toAsmAddress(Asm, Target); 2999 const auto *VarReg = llvm::cast<Variable>(this->getSrc(1)); 3000 assert(VarReg->hasReg()); 3001 const GPRRegister Reg = Traits::getEncodedGPR(VarReg->getRegNum()); 3002 Asm->xadd(Ty, Addr, Reg, this->Locked); 3003 } 3004 3005 template <typename TraitsType> 3006 void InstImpl<TraitsType>::InstX86Xadd::dump(const Cfg *Func) const { 3007 if (!BuildDefs::dump()) 3008 return; 3009 Ostream &Str = Func->getContext()->getStrDump(); 3010 if (this->Locked) { 3011 Str << "lock "; 3012 } 3013 Type Ty = this->getSrc(0)->getType(); 3014 Str << "xadd." << Ty << " "; 3015 this->dumpSources(Func); 3016 } 3017 3018 template <typename TraitsType> 3019 void InstImpl<TraitsType>::InstX86Xchg::emit(const Cfg *Func) const { 3020 if (!BuildDefs::dump()) 3021 return; 3022 Ostream &Str = Func->getContext()->getStrEmit(); 3023 Str << "\t" 3024 "xchg" << this->getWidthString(this->getSrc(0)->getType()) << "\t"; 3025 this->getSrc(1)->emit(Func); 3026 Str << ", "; 3027 this->getSrc(0)->emit(Func); 3028 } 3029 3030 template <typename TraitsType> 3031 void InstImpl<TraitsType>::InstX86Xchg::emitIAS(const Cfg *Func) const { 3032 assert(this->getSrcSize() == 2); 3033 Assembler *Asm = Func->getAssembler<Assembler>(); 3034 Type Ty = this->getSrc(0)->getType(); 3035 const auto *VarReg1 = llvm::cast<Variable>(this->getSrc(1)); 3036 assert(VarReg1->hasReg()); 3037 const GPRRegister Reg1 = Traits::getEncodedGPR(VarReg1->getRegNum()); 3038 3039 if (const auto *VarReg0 = llvm::dyn_cast<Variable>(this->getSrc(0))) { 3040 assert(VarReg0->hasReg()); 3041 const GPRRegister Reg0 = Traits::getEncodedGPR(VarReg0->getRegNum()); 3042 Asm->xchg(Ty, Reg0, Reg1); 3043 return; 3044 } 3045 3046 const auto *Mem = llvm::cast<X86OperandMem>(this->getSrc(0)); 3047 assert(Mem->getSegmentRegister() == X86OperandMem::DefaultSegment); 3048 auto *Target = InstX86Base::getTarget(Func); 3049 const Address Addr = Mem->toAsmAddress(Asm, Target); 3050 Asm->xchg(Ty, Addr, Reg1); 3051 } 3052 3053 template <typename TraitsType> 3054 void InstImpl<TraitsType>::InstX86Xchg::dump(const Cfg *Func) const { 3055 if (!BuildDefs::dump()) 3056 return; 3057 Ostream &Str = Func->getContext()->getStrDump(); 3058 Type Ty = this->getSrc(0)->getType(); 3059 Str << "xchg." << Ty << " "; 3060 this->dumpSources(Func); 3061 } 3062 3063 template <typename TraitsType> 3064 void InstImpl<TraitsType>::InstX86IacaStart::emit(const Cfg *Func) const { 3065 if (!BuildDefs::dump()) 3066 return; 3067 Ostream &Str = Func->getContext()->getStrEmit(); 3068 Str << "\t# IACA_START\n" 3069 "\t.byte 0x0F, 0x0B\n" 3070 "\t" 3071 "movl\t$111, %ebx\n" 3072 "\t.byte 0x64, 0x67, 0x90"; 3073 } 3074 3075 template <typename TraitsType> 3076 void InstImpl<TraitsType>::InstX86IacaStart::emitIAS(const Cfg *Func) const { 3077 Assembler *Asm = Func->getAssembler<Assembler>(); 3078 Asm->iaca_start(); 3079 } 3080 3081 template <typename TraitsType> 3082 void InstImpl<TraitsType>::InstX86IacaStart::dump(const Cfg *Func) const { 3083 if (!BuildDefs::dump()) 3084 return; 3085 Ostream &Str = Func->getContext()->getStrDump(); 3086 Str << "IACA_START"; 3087 } 3088 3089 template <typename TraitsType> 3090 void InstImpl<TraitsType>::InstX86IacaEnd::emit(const Cfg *Func) const { 3091 if (!BuildDefs::dump()) 3092 return; 3093 Ostream &Str = Func->getContext()->getStrEmit(); 3094 Str << "\t# IACA_END\n" 3095 "\t" 3096 "movl\t$222, %ebx\n" 3097 "\t.byte 0x64, 0x67, 0x90\n" 3098 "\t.byte 0x0F, 0x0B"; 3099 } 3100 3101 template <typename TraitsType> 3102 void InstImpl<TraitsType>::InstX86IacaEnd::emitIAS(const Cfg *Func) const { 3103 Assembler *Asm = Func->getAssembler<Assembler>(); 3104 Asm->iaca_end(); 3105 } 3106 3107 template <typename TraitsType> 3108 void InstImpl<TraitsType>::InstX86IacaEnd::dump(const Cfg *Func) const { 3109 if (!BuildDefs::dump()) 3110 return; 3111 Ostream &Str = Func->getContext()->getStrDump(); 3112 Str << "IACA_END"; 3113 } 3114 3115 } // end of namespace X86NAMESPACE 3116 3117 } // end of namespace Ice 3118 3119 #endif // SUBZERO_SRC_ICEINSTX86BASEIMPL_H 3120
