1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "assembler_x86_64.h" 18 19 #include "base/casts.h" 20 #include "entrypoints/quick/quick_entrypoints.h" 21 #include "memory_region.h" 22 #include "thread.h" 23 24 namespace art { 25 namespace x86_64 { 26 27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 28 return os << reg.AsRegister(); 29 } 30 31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 32 return os << reg.AsFloatRegister(); 33 } 34 35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 36 return os << "ST" << static_cast<int>(reg); 37 } 38 39 void X86_64Assembler::call(CpuRegister reg) { 40 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 41 EmitOptionalRex32(reg); 42 EmitUint8(0xFF); 43 EmitRegisterOperand(2, reg.LowBits()); 44 } 45 46 47 void X86_64Assembler::call(const Address& address) { 48 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 49 EmitOptionalRex32(address); 50 EmitUint8(0xFF); 51 EmitOperand(2, address); 52 } 53 54 55 void X86_64Assembler::call(Label* label) { 56 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 57 EmitUint8(0xE8); 58 static const int kSize = 5; 59 // Offset by one because we already have emitted the opcode. 60 EmitLabel(label, kSize - 1); 61 } 62 63 void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67 } 68 69 70 void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75 } 76 77 78 void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88 } 89 90 91 void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95 } 96 97 98 void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103 } 104 105 106 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119 } 120 121 122 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128 } 129 130 131 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) { 132 CHECK(imm.is_int32()); 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 EmitRex64(dst); 135 EmitUint8(0xC7); 136 EmitOperand(0, dst); 137 EmitImmediate(imm); 138 } 139 140 141 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 144 EmitRex64(src, dst); 145 EmitUint8(0x89); 146 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 147 } 148 149 150 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 151 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 152 EmitOptionalRex32(dst, src); 153 EmitUint8(0x8B); 154 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 155 } 156 157 158 void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 159 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 160 EmitRex64(dst, src); 161 EmitUint8(0x8B); 162 EmitOperand(dst.LowBits(), src); 163 } 164 165 166 void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 168 EmitOptionalRex32(dst, src); 169 EmitUint8(0x8B); 170 EmitOperand(dst.LowBits(), src); 171 } 172 173 174 void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 175 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 176 EmitRex64(src, dst); 177 EmitUint8(0x89); 178 EmitOperand(src.LowBits(), dst); 179 } 180 181 182 void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 183 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 184 EmitOptionalRex32(src, dst); 185 EmitUint8(0x89); 186 EmitOperand(src.LowBits(), dst); 187 } 188 189 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 190 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 191 EmitOptionalRex32(dst); 192 EmitUint8(0xC7); 193 EmitOperand(0, dst); 194 EmitImmediate(imm); 195 } 196 197 void X86_64Assembler::movntl(const Address& dst, CpuRegister src) { 198 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 199 EmitOptionalRex32(src, dst); 200 EmitUint8(0x0F); 201 EmitUint8(0xC3); 202 EmitOperand(src.LowBits(), dst); 203 } 204 205 void X86_64Assembler::movntq(const Address& dst, CpuRegister src) { 206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 207 EmitRex64(src, dst); 208 EmitUint8(0x0F); 209 EmitUint8(0xC3); 210 EmitOperand(src.LowBits(), dst); 211 } 212 213 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 214 cmov(c, dst, src, true); 215 } 216 217 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 218 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 219 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 220 EmitUint8(0x0F); 221 EmitUint8(0x40 + c); 222 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 223 } 224 225 226 void X86_64Assembler::cmov(Condition c, CpuRegister dst, const Address& src, bool is64bit) { 227 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 228 if (is64bit) { 229 EmitRex64(dst, src); 230 } else { 231 EmitOptionalRex32(dst, src); 232 } 233 EmitUint8(0x0F); 234 EmitUint8(0x40 + c); 235 EmitOperand(dst.LowBits(), src); 236 } 237 238 239 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 240 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 241 EmitOptionalByteRegNormalizingRex32(dst, src); 242 EmitUint8(0x0F); 243 EmitUint8(0xB6); 244 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 245 } 246 247 248 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 249 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 250 // Byte register is only in the source register form, so we don't use 251 // EmitOptionalByteRegNormalizingRex32(dst, src); 252 EmitOptionalRex32(dst, src); 253 EmitUint8(0x0F); 254 EmitUint8(0xB6); 255 EmitOperand(dst.LowBits(), src); 256 } 257 258 259 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 260 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 261 EmitOptionalByteRegNormalizingRex32(dst, src); 262 EmitUint8(0x0F); 263 EmitUint8(0xBE); 264 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 265 } 266 267 268 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 269 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 270 // Byte register is only in the source register form, so we don't use 271 // EmitOptionalByteRegNormalizingRex32(dst, src); 272 EmitOptionalRex32(dst, src); 273 EmitUint8(0x0F); 274 EmitUint8(0xBE); 275 EmitOperand(dst.LowBits(), src); 276 } 277 278 279 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 280 LOG(FATAL) << "Use movzxb or movsxb instead."; 281 } 282 283 284 void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 285 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 286 EmitOptionalByteRegNormalizingRex32(src, dst); 287 EmitUint8(0x88); 288 EmitOperand(src.LowBits(), dst); 289 } 290 291 292 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 293 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 294 EmitOptionalRex32(dst); 295 EmitUint8(0xC6); 296 EmitOperand(Register::RAX, dst); 297 CHECK(imm.is_int8()); 298 EmitUint8(imm.value() & 0xFF); 299 } 300 301 302 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 303 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 304 EmitOptionalRex32(dst, src); 305 EmitUint8(0x0F); 306 EmitUint8(0xB7); 307 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 308 } 309 310 311 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 312 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 313 EmitOptionalRex32(dst, src); 314 EmitUint8(0x0F); 315 EmitUint8(0xB7); 316 EmitOperand(dst.LowBits(), src); 317 } 318 319 320 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 321 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 322 EmitOptionalRex32(dst, src); 323 EmitUint8(0x0F); 324 EmitUint8(0xBF); 325 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 326 } 327 328 329 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 330 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 331 EmitOptionalRex32(dst, src); 332 EmitUint8(0x0F); 333 EmitUint8(0xBF); 334 EmitOperand(dst.LowBits(), src); 335 } 336 337 338 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 339 LOG(FATAL) << "Use movzxw or movsxw instead."; 340 } 341 342 343 void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 345 EmitOperandSizeOverride(); 346 EmitOptionalRex32(src, dst); 347 EmitUint8(0x89); 348 EmitOperand(src.LowBits(), dst); 349 } 350 351 352 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 354 EmitOperandSizeOverride(); 355 EmitOptionalRex32(dst); 356 EmitUint8(0xC7); 357 EmitOperand(Register::RAX, dst); 358 CHECK(imm.is_uint16() || imm.is_int16()); 359 EmitUint8(imm.value() & 0xFF); 360 EmitUint8(imm.value() >> 8); 361 } 362 363 364 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 365 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 366 EmitRex64(dst, src); 367 EmitUint8(0x8D); 368 EmitOperand(dst.LowBits(), src); 369 } 370 371 372 void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 373 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 374 EmitOptionalRex32(dst, src); 375 EmitUint8(0x8D); 376 EmitOperand(dst.LowBits(), src); 377 } 378 379 380 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 381 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 382 EmitOptionalRex32(dst, src); 383 EmitUint8(0x0F); 384 EmitUint8(0x28); 385 EmitXmmRegisterOperand(dst.LowBits(), src); 386 } 387 388 389 void X86_64Assembler::movaps(XmmRegister dst, const Address& src) { 390 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 391 EmitOptionalRex32(dst, src); 392 EmitUint8(0x0F); 393 EmitUint8(0x28); 394 EmitOperand(dst.LowBits(), src); 395 } 396 397 398 void X86_64Assembler::movups(XmmRegister dst, const Address& src) { 399 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 400 EmitOptionalRex32(dst, src); 401 EmitUint8(0x0F); 402 EmitUint8(0x10); 403 EmitOperand(dst.LowBits(), src); 404 } 405 406 407 void X86_64Assembler::movaps(const Address& dst, XmmRegister src) { 408 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 409 EmitOptionalRex32(src, dst); 410 EmitUint8(0x0F); 411 EmitUint8(0x29); 412 EmitOperand(src.LowBits(), dst); 413 } 414 415 416 void X86_64Assembler::movups(const Address& dst, XmmRegister src) { 417 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 418 EmitOptionalRex32(src, dst); 419 EmitUint8(0x0F); 420 EmitUint8(0x11); 421 EmitOperand(src.LowBits(), dst); 422 } 423 424 425 void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 426 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 427 EmitUint8(0xF3); 428 EmitOptionalRex32(dst, src); 429 EmitUint8(0x0F); 430 EmitUint8(0x10); 431 EmitOperand(dst.LowBits(), src); 432 } 433 434 435 void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 436 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 437 EmitUint8(0xF3); 438 EmitOptionalRex32(src, dst); 439 EmitUint8(0x0F); 440 EmitUint8(0x11); 441 EmitOperand(src.LowBits(), dst); 442 } 443 444 445 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 447 EmitUint8(0xF3); 448 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 449 EmitUint8(0x0F); 450 EmitUint8(0x11); 451 EmitXmmRegisterOperand(src.LowBits(), dst); 452 } 453 454 455 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 456 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 457 EmitRex64(dst, src); 458 EmitUint8(0x63); 459 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 460 } 461 462 463 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 464 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 465 EmitRex64(dst, src); 466 EmitUint8(0x63); 467 EmitOperand(dst.LowBits(), src); 468 } 469 470 471 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 472 movd(dst, src, true); 473 } 474 475 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 476 movd(dst, src, true); 477 } 478 479 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 480 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 481 EmitUint8(0x66); 482 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 483 EmitUint8(0x0F); 484 EmitUint8(0x6E); 485 EmitOperand(dst.LowBits(), Operand(src)); 486 } 487 488 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 489 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 490 EmitUint8(0x66); 491 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 492 EmitUint8(0x0F); 493 EmitUint8(0x7E); 494 EmitOperand(src.LowBits(), Operand(dst)); 495 } 496 497 498 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 499 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 500 EmitUint8(0xF3); 501 EmitOptionalRex32(dst, src); 502 EmitUint8(0x0F); 503 EmitUint8(0x58); 504 EmitXmmRegisterOperand(dst.LowBits(), src); 505 } 506 507 508 void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 509 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 510 EmitUint8(0xF3); 511 EmitOptionalRex32(dst, src); 512 EmitUint8(0x0F); 513 EmitUint8(0x58); 514 EmitOperand(dst.LowBits(), src); 515 } 516 517 518 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 519 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 520 EmitUint8(0xF3); 521 EmitOptionalRex32(dst, src); 522 EmitUint8(0x0F); 523 EmitUint8(0x5C); 524 EmitXmmRegisterOperand(dst.LowBits(), src); 525 } 526 527 528 void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 529 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 530 EmitUint8(0xF3); 531 EmitOptionalRex32(dst, src); 532 EmitUint8(0x0F); 533 EmitUint8(0x5C); 534 EmitOperand(dst.LowBits(), src); 535 } 536 537 538 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 539 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 540 EmitUint8(0xF3); 541 EmitOptionalRex32(dst, src); 542 EmitUint8(0x0F); 543 EmitUint8(0x59); 544 EmitXmmRegisterOperand(dst.LowBits(), src); 545 } 546 547 548 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 549 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 550 EmitUint8(0xF3); 551 EmitOptionalRex32(dst, src); 552 EmitUint8(0x0F); 553 EmitUint8(0x59); 554 EmitOperand(dst.LowBits(), src); 555 } 556 557 558 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 559 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 560 EmitUint8(0xF3); 561 EmitOptionalRex32(dst, src); 562 EmitUint8(0x0F); 563 EmitUint8(0x5E); 564 EmitXmmRegisterOperand(dst.LowBits(), src); 565 } 566 567 568 void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 569 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 570 EmitUint8(0xF3); 571 EmitOptionalRex32(dst, src); 572 EmitUint8(0x0F); 573 EmitUint8(0x5E); 574 EmitOperand(dst.LowBits(), src); 575 } 576 577 578 void X86_64Assembler::addps(XmmRegister dst, XmmRegister src) { 579 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 580 EmitOptionalRex32(dst, src); 581 EmitUint8(0x0F); 582 EmitUint8(0x58); 583 EmitXmmRegisterOperand(dst.LowBits(), src); 584 } 585 586 587 void X86_64Assembler::subps(XmmRegister dst, XmmRegister src) { 588 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 589 EmitOptionalRex32(dst, src); 590 EmitUint8(0x0F); 591 EmitUint8(0x5C); 592 EmitXmmRegisterOperand(dst.LowBits(), src); 593 } 594 595 596 void X86_64Assembler::mulps(XmmRegister dst, XmmRegister src) { 597 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 598 EmitOptionalRex32(dst, src); 599 EmitUint8(0x0F); 600 EmitUint8(0x59); 601 EmitXmmRegisterOperand(dst.LowBits(), src); 602 } 603 604 605 void X86_64Assembler::divps(XmmRegister dst, XmmRegister src) { 606 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 607 EmitOptionalRex32(dst, src); 608 EmitUint8(0x0F); 609 EmitUint8(0x5E); 610 EmitXmmRegisterOperand(dst.LowBits(), src); 611 } 612 613 614 void X86_64Assembler::flds(const Address& src) { 615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 616 EmitUint8(0xD9); 617 EmitOperand(0, src); 618 } 619 620 621 void X86_64Assembler::fsts(const Address& dst) { 622 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 623 EmitUint8(0xD9); 624 EmitOperand(2, dst); 625 } 626 627 628 void X86_64Assembler::fstps(const Address& dst) { 629 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 630 EmitUint8(0xD9); 631 EmitOperand(3, dst); 632 } 633 634 635 void X86_64Assembler::movapd(XmmRegister dst, XmmRegister src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0x66); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0x28); 641 EmitXmmRegisterOperand(dst.LowBits(), src); 642 } 643 644 645 void X86_64Assembler::movapd(XmmRegister dst, const Address& src) { 646 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 647 EmitUint8(0x66); 648 EmitOptionalRex32(dst, src); 649 EmitUint8(0x0F); 650 EmitUint8(0x28); 651 EmitOperand(dst.LowBits(), src); 652 } 653 654 655 void X86_64Assembler::movupd(XmmRegister dst, const Address& src) { 656 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 657 EmitUint8(0x66); 658 EmitOptionalRex32(dst, src); 659 EmitUint8(0x0F); 660 EmitUint8(0x10); 661 EmitOperand(dst.LowBits(), src); 662 } 663 664 665 void X86_64Assembler::movapd(const Address& dst, XmmRegister src) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0x66); 668 EmitOptionalRex32(src, dst); 669 EmitUint8(0x0F); 670 EmitUint8(0x29); 671 EmitOperand(src.LowBits(), dst); 672 } 673 674 675 void X86_64Assembler::movupd(const Address& dst, XmmRegister src) { 676 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 677 EmitUint8(0x66); 678 EmitOptionalRex32(src, dst); 679 EmitUint8(0x0F); 680 EmitUint8(0x11); 681 EmitOperand(src.LowBits(), dst); 682 } 683 684 685 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 686 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 687 EmitUint8(0xF2); 688 EmitOptionalRex32(dst, src); 689 EmitUint8(0x0F); 690 EmitUint8(0x10); 691 EmitOperand(dst.LowBits(), src); 692 } 693 694 695 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 696 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 697 EmitUint8(0xF2); 698 EmitOptionalRex32(src, dst); 699 EmitUint8(0x0F); 700 EmitUint8(0x11); 701 EmitOperand(src.LowBits(), dst); 702 } 703 704 705 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 706 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 707 EmitUint8(0xF2); 708 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 709 EmitUint8(0x0F); 710 EmitUint8(0x11); 711 EmitXmmRegisterOperand(src.LowBits(), dst); 712 } 713 714 715 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 716 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 717 EmitUint8(0xF2); 718 EmitOptionalRex32(dst, src); 719 EmitUint8(0x0F); 720 EmitUint8(0x58); 721 EmitXmmRegisterOperand(dst.LowBits(), src); 722 } 723 724 725 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 726 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 727 EmitUint8(0xF2); 728 EmitOptionalRex32(dst, src); 729 EmitUint8(0x0F); 730 EmitUint8(0x58); 731 EmitOperand(dst.LowBits(), src); 732 } 733 734 735 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 736 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 737 EmitUint8(0xF2); 738 EmitOptionalRex32(dst, src); 739 EmitUint8(0x0F); 740 EmitUint8(0x5C); 741 EmitXmmRegisterOperand(dst.LowBits(), src); 742 } 743 744 745 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 746 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 747 EmitUint8(0xF2); 748 EmitOptionalRex32(dst, src); 749 EmitUint8(0x0F); 750 EmitUint8(0x5C); 751 EmitOperand(dst.LowBits(), src); 752 } 753 754 755 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 756 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 757 EmitUint8(0xF2); 758 EmitOptionalRex32(dst, src); 759 EmitUint8(0x0F); 760 EmitUint8(0x59); 761 EmitXmmRegisterOperand(dst.LowBits(), src); 762 } 763 764 765 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 766 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 767 EmitUint8(0xF2); 768 EmitOptionalRex32(dst, src); 769 EmitUint8(0x0F); 770 EmitUint8(0x59); 771 EmitOperand(dst.LowBits(), src); 772 } 773 774 775 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 776 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 777 EmitUint8(0xF2); 778 EmitOptionalRex32(dst, src); 779 EmitUint8(0x0F); 780 EmitUint8(0x5E); 781 EmitXmmRegisterOperand(dst.LowBits(), src); 782 } 783 784 785 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 786 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 787 EmitUint8(0xF2); 788 EmitOptionalRex32(dst, src); 789 EmitUint8(0x0F); 790 EmitUint8(0x5E); 791 EmitOperand(dst.LowBits(), src); 792 } 793 794 795 void X86_64Assembler::addpd(XmmRegister dst, XmmRegister src) { 796 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 797 EmitUint8(0x66); 798 EmitOptionalRex32(dst, src); 799 EmitUint8(0x0F); 800 EmitUint8(0x58); 801 EmitXmmRegisterOperand(dst.LowBits(), src); 802 } 803 804 805 void X86_64Assembler::subpd(XmmRegister dst, XmmRegister src) { 806 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 807 EmitUint8(0x66); 808 EmitOptionalRex32(dst, src); 809 EmitUint8(0x0F); 810 EmitUint8(0x5C); 811 EmitXmmRegisterOperand(dst.LowBits(), src); 812 } 813 814 815 void X86_64Assembler::mulpd(XmmRegister dst, XmmRegister src) { 816 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 817 EmitUint8(0x66); 818 EmitOptionalRex32(dst, src); 819 EmitUint8(0x0F); 820 EmitUint8(0x59); 821 EmitXmmRegisterOperand(dst.LowBits(), src); 822 } 823 824 825 void X86_64Assembler::divpd(XmmRegister dst, XmmRegister src) { 826 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 827 EmitUint8(0x66); 828 EmitOptionalRex32(dst, src); 829 EmitUint8(0x0F); 830 EmitUint8(0x5E); 831 EmitXmmRegisterOperand(dst.LowBits(), src); 832 } 833 834 835 void X86_64Assembler::movdqa(XmmRegister dst, XmmRegister src) { 836 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 837 EmitUint8(0x66); 838 EmitOptionalRex32(dst, src); 839 EmitUint8(0x0F); 840 EmitUint8(0x6F); 841 EmitXmmRegisterOperand(dst.LowBits(), src); 842 } 843 844 845 void X86_64Assembler::movdqa(XmmRegister dst, const Address& src) { 846 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 847 EmitUint8(0x66); 848 EmitOptionalRex32(dst, src); 849 EmitUint8(0x0F); 850 EmitUint8(0x6F); 851 EmitOperand(dst.LowBits(), src); 852 } 853 854 855 void X86_64Assembler::movdqu(XmmRegister dst, const Address& src) { 856 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 857 EmitUint8(0xF3); 858 EmitOptionalRex32(dst, src); 859 EmitUint8(0x0F); 860 EmitUint8(0x6F); 861 EmitOperand(dst.LowBits(), src); 862 } 863 864 865 void X86_64Assembler::movdqa(const Address& dst, XmmRegister src) { 866 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 867 EmitUint8(0x66); 868 EmitOptionalRex32(src, dst); 869 EmitUint8(0x0F); 870 EmitUint8(0x7F); 871 EmitOperand(src.LowBits(), dst); 872 } 873 874 875 void X86_64Assembler::movdqu(const Address& dst, XmmRegister src) { 876 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 877 EmitUint8(0xF3); 878 EmitOptionalRex32(src, dst); 879 EmitUint8(0x0F); 880 EmitUint8(0x7F); 881 EmitOperand(src.LowBits(), dst); 882 } 883 884 885 void X86_64Assembler::paddb(XmmRegister dst, XmmRegister src) { 886 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 887 EmitUint8(0x66); 888 EmitOptionalRex32(dst, src); 889 EmitUint8(0x0F); 890 EmitUint8(0xFC); 891 EmitXmmRegisterOperand(dst.LowBits(), src); 892 } 893 894 895 void X86_64Assembler::psubb(XmmRegister dst, XmmRegister src) { 896 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 897 EmitUint8(0x66); 898 EmitOptionalRex32(dst, src); 899 EmitUint8(0x0F); 900 EmitUint8(0xF8); 901 EmitXmmRegisterOperand(dst.LowBits(), src); 902 } 903 904 905 void X86_64Assembler::paddw(XmmRegister dst, XmmRegister src) { 906 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 907 EmitUint8(0x66); 908 EmitOptionalRex32(dst, src); 909 EmitUint8(0x0F); 910 EmitUint8(0xFD); 911 EmitXmmRegisterOperand(dst.LowBits(), src); 912 } 913 914 915 void X86_64Assembler::psubw(XmmRegister dst, XmmRegister src) { 916 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 917 EmitUint8(0x66); 918 EmitOptionalRex32(dst, src); 919 EmitUint8(0x0F); 920 EmitUint8(0xF9); 921 EmitXmmRegisterOperand(dst.LowBits(), src); 922 } 923 924 925 void X86_64Assembler::pmullw(XmmRegister dst, XmmRegister src) { 926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 927 EmitUint8(0x66); 928 EmitOptionalRex32(dst, src); 929 EmitUint8(0x0F); 930 EmitUint8(0xD5); 931 EmitXmmRegisterOperand(dst.LowBits(), src); 932 } 933 934 935 void X86_64Assembler::paddd(XmmRegister dst, XmmRegister src) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 EmitUint8(0x66); 938 EmitOptionalRex32(dst, src); 939 EmitUint8(0x0F); 940 EmitUint8(0xFE); 941 EmitXmmRegisterOperand(dst.LowBits(), src); 942 } 943 944 945 void X86_64Assembler::psubd(XmmRegister dst, XmmRegister src) { 946 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 947 EmitUint8(0x66); 948 EmitOptionalRex32(dst, src); 949 EmitUint8(0x0F); 950 EmitUint8(0xFA); 951 EmitXmmRegisterOperand(dst.LowBits(), src); 952 } 953 954 955 void X86_64Assembler::pmulld(XmmRegister dst, XmmRegister src) { 956 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 957 EmitUint8(0x66); 958 EmitOptionalRex32(dst, src); 959 EmitUint8(0x0F); 960 EmitUint8(0x38); 961 EmitUint8(0x40); 962 EmitXmmRegisterOperand(dst.LowBits(), src); 963 } 964 965 966 void X86_64Assembler::paddq(XmmRegister dst, XmmRegister src) { 967 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 968 EmitUint8(0x66); 969 EmitOptionalRex32(dst, src); 970 EmitUint8(0x0F); 971 EmitUint8(0xD4); 972 EmitXmmRegisterOperand(dst.LowBits(), src); 973 } 974 975 976 void X86_64Assembler::psubq(XmmRegister dst, XmmRegister src) { 977 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 978 EmitUint8(0x66); 979 EmitOptionalRex32(dst, src); 980 EmitUint8(0x0F); 981 EmitUint8(0xFB); 982 EmitXmmRegisterOperand(dst.LowBits(), src); 983 } 984 985 986 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 987 cvtsi2ss(dst, src, false); 988 } 989 990 991 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 992 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 993 EmitUint8(0xF3); 994 if (is64bit) { 995 // Emit a REX.W prefix if the operand size is 64 bits. 996 EmitRex64(dst, src); 997 } else { 998 EmitOptionalRex32(dst, src); 999 } 1000 EmitUint8(0x0F); 1001 EmitUint8(0x2A); 1002 EmitOperand(dst.LowBits(), Operand(src)); 1003 } 1004 1005 1006 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) { 1007 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1008 EmitUint8(0xF3); 1009 if (is64bit) { 1010 // Emit a REX.W prefix if the operand size is 64 bits. 1011 EmitRex64(dst, src); 1012 } else { 1013 EmitOptionalRex32(dst, src); 1014 } 1015 EmitUint8(0x0F); 1016 EmitUint8(0x2A); 1017 EmitOperand(dst.LowBits(), src); 1018 } 1019 1020 1021 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 1022 cvtsi2sd(dst, src, false); 1023 } 1024 1025 1026 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 1027 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1028 EmitUint8(0xF2); 1029 if (is64bit) { 1030 // Emit a REX.W prefix if the operand size is 64 bits. 1031 EmitRex64(dst, src); 1032 } else { 1033 EmitOptionalRex32(dst, src); 1034 } 1035 EmitUint8(0x0F); 1036 EmitUint8(0x2A); 1037 EmitOperand(dst.LowBits(), Operand(src)); 1038 } 1039 1040 1041 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) { 1042 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1043 EmitUint8(0xF2); 1044 if (is64bit) { 1045 // Emit a REX.W prefix if the operand size is 64 bits. 1046 EmitRex64(dst, src); 1047 } else { 1048 EmitOptionalRex32(dst, src); 1049 } 1050 EmitUint8(0x0F); 1051 EmitUint8(0x2A); 1052 EmitOperand(dst.LowBits(), src); 1053 } 1054 1055 1056 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 1057 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1058 EmitUint8(0xF3); 1059 EmitOptionalRex32(dst, src); 1060 EmitUint8(0x0F); 1061 EmitUint8(0x2D); 1062 EmitXmmRegisterOperand(dst.LowBits(), src); 1063 } 1064 1065 1066 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 1067 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1068 EmitUint8(0xF3); 1069 EmitOptionalRex32(dst, src); 1070 EmitUint8(0x0F); 1071 EmitUint8(0x5A); 1072 EmitXmmRegisterOperand(dst.LowBits(), src); 1073 } 1074 1075 1076 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) { 1077 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1078 EmitUint8(0xF3); 1079 EmitOptionalRex32(dst, src); 1080 EmitUint8(0x0F); 1081 EmitUint8(0x5A); 1082 EmitOperand(dst.LowBits(), src); 1083 } 1084 1085 1086 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 1087 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1088 EmitUint8(0xF2); 1089 EmitOptionalRex32(dst, src); 1090 EmitUint8(0x0F); 1091 EmitUint8(0x2D); 1092 EmitXmmRegisterOperand(dst.LowBits(), src); 1093 } 1094 1095 1096 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 1097 cvttss2si(dst, src, false); 1098 } 1099 1100 1101 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 1102 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1103 EmitUint8(0xF3); 1104 if (is64bit) { 1105 // Emit a REX.W prefix if the operand size is 64 bits. 1106 EmitRex64(dst, src); 1107 } else { 1108 EmitOptionalRex32(dst, src); 1109 } 1110 EmitUint8(0x0F); 1111 EmitUint8(0x2C); 1112 EmitXmmRegisterOperand(dst.LowBits(), src); 1113 } 1114 1115 1116 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 1117 cvttsd2si(dst, src, false); 1118 } 1119 1120 1121 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 1122 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1123 EmitUint8(0xF2); 1124 if (is64bit) { 1125 // Emit a REX.W prefix if the operand size is 64 bits. 1126 EmitRex64(dst, src); 1127 } else { 1128 EmitOptionalRex32(dst, src); 1129 } 1130 EmitUint8(0x0F); 1131 EmitUint8(0x2C); 1132 EmitXmmRegisterOperand(dst.LowBits(), src); 1133 } 1134 1135 1136 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 1137 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1138 EmitUint8(0xF2); 1139 EmitOptionalRex32(dst, src); 1140 EmitUint8(0x0F); 1141 EmitUint8(0x5A); 1142 EmitXmmRegisterOperand(dst.LowBits(), src); 1143 } 1144 1145 1146 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) { 1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1148 EmitUint8(0xF2); 1149 EmitOptionalRex32(dst, src); 1150 EmitUint8(0x0F); 1151 EmitUint8(0x5A); 1152 EmitOperand(dst.LowBits(), src); 1153 } 1154 1155 1156 void X86_64Assembler::cvtdq2ps(XmmRegister dst, XmmRegister src) { 1157 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1158 EmitOptionalRex32(dst, src); 1159 EmitUint8(0x0F); 1160 EmitUint8(0x5B); 1161 EmitXmmRegisterOperand(dst.LowBits(), src); 1162 } 1163 1164 1165 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 1166 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1167 EmitUint8(0xF3); 1168 EmitOptionalRex32(dst, src); 1169 EmitUint8(0x0F); 1170 EmitUint8(0xE6); 1171 EmitXmmRegisterOperand(dst.LowBits(), src); 1172 } 1173 1174 1175 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 1176 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1177 EmitOptionalRex32(a, b); 1178 EmitUint8(0x0F); 1179 EmitUint8(0x2F); 1180 EmitXmmRegisterOperand(a.LowBits(), b); 1181 } 1182 1183 1184 void X86_64Assembler::comiss(XmmRegister a, const Address& b) { 1185 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1186 EmitOptionalRex32(a, b); 1187 EmitUint8(0x0F); 1188 EmitUint8(0x2F); 1189 EmitOperand(a.LowBits(), b); 1190 } 1191 1192 1193 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 1194 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1195 EmitUint8(0x66); 1196 EmitOptionalRex32(a, b); 1197 EmitUint8(0x0F); 1198 EmitUint8(0x2F); 1199 EmitXmmRegisterOperand(a.LowBits(), b); 1200 } 1201 1202 1203 void X86_64Assembler::comisd(XmmRegister a, const Address& b) { 1204 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1205 EmitUint8(0x66); 1206 EmitOptionalRex32(a, b); 1207 EmitUint8(0x0F); 1208 EmitUint8(0x2F); 1209 EmitOperand(a.LowBits(), b); 1210 } 1211 1212 1213 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 1214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1215 EmitOptionalRex32(a, b); 1216 EmitUint8(0x0F); 1217 EmitUint8(0x2E); 1218 EmitXmmRegisterOperand(a.LowBits(), b); 1219 } 1220 1221 1222 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 EmitOptionalRex32(a, b); 1225 EmitUint8(0x0F); 1226 EmitUint8(0x2E); 1227 EmitOperand(a.LowBits(), b); 1228 } 1229 1230 1231 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 1232 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1233 EmitUint8(0x66); 1234 EmitOptionalRex32(a, b); 1235 EmitUint8(0x0F); 1236 EmitUint8(0x2E); 1237 EmitXmmRegisterOperand(a.LowBits(), b); 1238 } 1239 1240 1241 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) { 1242 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1243 EmitUint8(0x66); 1244 EmitOptionalRex32(a, b); 1245 EmitUint8(0x0F); 1246 EmitUint8(0x2E); 1247 EmitOperand(a.LowBits(), b); 1248 } 1249 1250 1251 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 1252 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1253 EmitUint8(0x66); 1254 EmitOptionalRex32(dst, src); 1255 EmitUint8(0x0F); 1256 EmitUint8(0x3A); 1257 EmitUint8(0x0B); 1258 EmitXmmRegisterOperand(dst.LowBits(), src); 1259 EmitUint8(imm.value()); 1260 } 1261 1262 1263 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) { 1264 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1265 EmitUint8(0x66); 1266 EmitOptionalRex32(dst, src); 1267 EmitUint8(0x0F); 1268 EmitUint8(0x3A); 1269 EmitUint8(0x0A); 1270 EmitXmmRegisterOperand(dst.LowBits(), src); 1271 EmitUint8(imm.value()); 1272 } 1273 1274 1275 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 1276 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1277 EmitUint8(0xF2); 1278 EmitOptionalRex32(dst, src); 1279 EmitUint8(0x0F); 1280 EmitUint8(0x51); 1281 EmitXmmRegisterOperand(dst.LowBits(), src); 1282 } 1283 1284 1285 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitUint8(0xF3); 1288 EmitOptionalRex32(dst, src); 1289 EmitUint8(0x0F); 1290 EmitUint8(0x51); 1291 EmitXmmRegisterOperand(dst.LowBits(), src); 1292 } 1293 1294 1295 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 1296 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1297 EmitUint8(0x66); 1298 EmitOptionalRex32(dst, src); 1299 EmitUint8(0x0F); 1300 EmitUint8(0x57); 1301 EmitOperand(dst.LowBits(), src); 1302 } 1303 1304 1305 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 1306 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1307 EmitUint8(0x66); 1308 EmitOptionalRex32(dst, src); 1309 EmitUint8(0x0F); 1310 EmitUint8(0x57); 1311 EmitXmmRegisterOperand(dst.LowBits(), src); 1312 } 1313 1314 1315 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 1316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1317 EmitOptionalRex32(dst, src); 1318 EmitUint8(0x0F); 1319 EmitUint8(0x57); 1320 EmitOperand(dst.LowBits(), src); 1321 } 1322 1323 1324 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 1325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1326 EmitOptionalRex32(dst, src); 1327 EmitUint8(0x0F); 1328 EmitUint8(0x57); 1329 EmitXmmRegisterOperand(dst.LowBits(), src); 1330 } 1331 1332 1333 void X86_64Assembler::pxor(XmmRegister dst, XmmRegister src) { 1334 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1335 EmitUint8(0x66); 1336 EmitOptionalRex32(dst, src); 1337 EmitUint8(0x0F); 1338 EmitUint8(0xEF); 1339 EmitXmmRegisterOperand(dst.LowBits(), src); 1340 } 1341 1342 1343 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1345 EmitUint8(0x66); 1346 EmitOptionalRex32(dst, src); 1347 EmitUint8(0x0F); 1348 EmitUint8(0x54); 1349 EmitOperand(dst.LowBits(), src); 1350 } 1351 1352 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 1353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1354 EmitUint8(0x66); 1355 EmitOptionalRex32(dst, src); 1356 EmitUint8(0x0F); 1357 EmitUint8(0x54); 1358 EmitXmmRegisterOperand(dst.LowBits(), src); 1359 } 1360 1361 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 1362 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1363 EmitOptionalRex32(dst, src); 1364 EmitUint8(0x0F); 1365 EmitUint8(0x54); 1366 EmitXmmRegisterOperand(dst.LowBits(), src); 1367 } 1368 1369 void X86_64Assembler::pand(XmmRegister dst, XmmRegister src) { 1370 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1371 EmitUint8(0x66); 1372 EmitOptionalRex32(dst, src); 1373 EmitUint8(0x0F); 1374 EmitUint8(0xDB); 1375 EmitXmmRegisterOperand(dst.LowBits(), src); 1376 } 1377 1378 void X86_64Assembler::andnpd(XmmRegister dst, XmmRegister src) { 1379 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1380 EmitUint8(0x66); 1381 EmitOptionalRex32(dst, src); 1382 EmitUint8(0x0F); 1383 EmitUint8(0x55); 1384 EmitXmmRegisterOperand(dst.LowBits(), src); 1385 } 1386 1387 void X86_64Assembler::andnps(XmmRegister dst, XmmRegister src) { 1388 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1389 EmitOptionalRex32(dst, src); 1390 EmitUint8(0x0F); 1391 EmitUint8(0x55); 1392 EmitXmmRegisterOperand(dst.LowBits(), src); 1393 } 1394 1395 void X86_64Assembler::pandn(XmmRegister dst, XmmRegister src) { 1396 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1397 EmitUint8(0x66); 1398 EmitOptionalRex32(dst, src); 1399 EmitUint8(0x0F); 1400 EmitUint8(0xDF); 1401 EmitXmmRegisterOperand(dst.LowBits(), src); 1402 } 1403 1404 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 1405 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1406 EmitUint8(0x66); 1407 EmitOptionalRex32(dst, src); 1408 EmitUint8(0x0F); 1409 EmitUint8(0x56); 1410 EmitXmmRegisterOperand(dst.LowBits(), src); 1411 } 1412 1413 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 1414 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1415 EmitOptionalRex32(dst, src); 1416 EmitUint8(0x0F); 1417 EmitUint8(0x56); 1418 EmitXmmRegisterOperand(dst.LowBits(), src); 1419 } 1420 1421 void X86_64Assembler::por(XmmRegister dst, XmmRegister src) { 1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1423 EmitUint8(0x66); 1424 EmitOptionalRex32(dst, src); 1425 EmitUint8(0x0F); 1426 EmitUint8(0xEB); 1427 EmitXmmRegisterOperand(dst.LowBits(), src); 1428 } 1429 1430 void X86_64Assembler::pavgb(XmmRegister dst, XmmRegister src) { 1431 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1432 EmitUint8(0x66); 1433 EmitOptionalRex32(dst, src); 1434 EmitUint8(0x0F); 1435 EmitUint8(0xE0); 1436 EmitXmmRegisterOperand(dst.LowBits(), src); 1437 } 1438 1439 void X86_64Assembler::pavgw(XmmRegister dst, XmmRegister src) { 1440 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1441 EmitUint8(0x66); 1442 EmitOptionalRex32(dst, src); 1443 EmitUint8(0x0F); 1444 EmitUint8(0xE3); 1445 EmitXmmRegisterOperand(dst.LowBits(), src); 1446 } 1447 1448 void X86_64Assembler::pminsb(XmmRegister dst, XmmRegister src) { 1449 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1450 EmitUint8(0x66); 1451 EmitOptionalRex32(dst, src); 1452 EmitUint8(0x0F); 1453 EmitUint8(0x38); 1454 EmitUint8(0x38); 1455 EmitXmmRegisterOperand(dst.LowBits(), src); 1456 } 1457 1458 void X86_64Assembler::pmaxsb(XmmRegister dst, XmmRegister src) { 1459 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1460 EmitUint8(0x66); 1461 EmitOptionalRex32(dst, src); 1462 EmitUint8(0x0F); 1463 EmitUint8(0x38); 1464 EmitUint8(0x3C); 1465 EmitXmmRegisterOperand(dst.LowBits(), src); 1466 } 1467 1468 void X86_64Assembler::pminsw(XmmRegister dst, XmmRegister src) { 1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1470 EmitUint8(0x66); 1471 EmitOptionalRex32(dst, src); 1472 EmitUint8(0x0F); 1473 EmitUint8(0xEA); 1474 EmitXmmRegisterOperand(dst.LowBits(), src); 1475 } 1476 1477 void X86_64Assembler::pmaxsw(XmmRegister dst, XmmRegister src) { 1478 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1479 EmitUint8(0x66); 1480 EmitOptionalRex32(dst, src); 1481 EmitUint8(0x0F); 1482 EmitUint8(0xEE); 1483 EmitXmmRegisterOperand(dst.LowBits(), src); 1484 } 1485 1486 void X86_64Assembler::pminsd(XmmRegister dst, XmmRegister src) { 1487 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1488 EmitUint8(0x66); 1489 EmitOptionalRex32(dst, src); 1490 EmitUint8(0x0F); 1491 EmitUint8(0x38); 1492 EmitUint8(0x39); 1493 EmitXmmRegisterOperand(dst.LowBits(), src); 1494 } 1495 1496 void X86_64Assembler::pmaxsd(XmmRegister dst, XmmRegister src) { 1497 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1498 EmitUint8(0x66); 1499 EmitOptionalRex32(dst, src); 1500 EmitUint8(0x0F); 1501 EmitUint8(0x38); 1502 EmitUint8(0x3D); 1503 EmitXmmRegisterOperand(dst.LowBits(), src); 1504 } 1505 1506 void X86_64Assembler::pminub(XmmRegister dst, XmmRegister src) { 1507 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1508 EmitUint8(0x66); 1509 EmitOptionalRex32(dst, src); 1510 EmitUint8(0x0F); 1511 EmitUint8(0xDA); 1512 EmitXmmRegisterOperand(dst.LowBits(), src); 1513 } 1514 1515 void X86_64Assembler::pmaxub(XmmRegister dst, XmmRegister src) { 1516 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1517 EmitUint8(0x66); 1518 EmitOptionalRex32(dst, src); 1519 EmitUint8(0x0F); 1520 EmitUint8(0xDE); 1521 EmitXmmRegisterOperand(dst.LowBits(), src); 1522 } 1523 1524 void X86_64Assembler::pminuw(XmmRegister dst, XmmRegister src) { 1525 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1526 EmitUint8(0x66); 1527 EmitOptionalRex32(dst, src); 1528 EmitUint8(0x0F); 1529 EmitUint8(0x38); 1530 EmitUint8(0x3A); 1531 EmitXmmRegisterOperand(dst.LowBits(), src); 1532 } 1533 1534 void X86_64Assembler::pmaxuw(XmmRegister dst, XmmRegister src) { 1535 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1536 EmitUint8(0x66); 1537 EmitOptionalRex32(dst, src); 1538 EmitUint8(0x0F); 1539 EmitUint8(0x38); 1540 EmitUint8(0x3E); 1541 EmitXmmRegisterOperand(dst.LowBits(), src); 1542 } 1543 1544 void X86_64Assembler::pminud(XmmRegister dst, XmmRegister src) { 1545 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1546 EmitUint8(0x66); 1547 EmitOptionalRex32(dst, src); 1548 EmitUint8(0x0F); 1549 EmitUint8(0x38); 1550 EmitUint8(0x3B); 1551 EmitXmmRegisterOperand(dst.LowBits(), src); 1552 } 1553 1554 void X86_64Assembler::pmaxud(XmmRegister dst, XmmRegister src) { 1555 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1556 EmitUint8(0x66); 1557 EmitOptionalRex32(dst, src); 1558 EmitUint8(0x0F); 1559 EmitUint8(0x38); 1560 EmitUint8(0x3F); 1561 EmitXmmRegisterOperand(dst.LowBits(), src); 1562 } 1563 1564 void X86_64Assembler::minps(XmmRegister dst, XmmRegister src) { 1565 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1566 EmitOptionalRex32(dst, src); 1567 EmitUint8(0x0F); 1568 EmitUint8(0x5D); 1569 EmitXmmRegisterOperand(dst.LowBits(), src); 1570 } 1571 1572 void X86_64Assembler::maxps(XmmRegister dst, XmmRegister src) { 1573 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1574 EmitOptionalRex32(dst, src); 1575 EmitUint8(0x0F); 1576 EmitUint8(0x5F); 1577 EmitXmmRegisterOperand(dst.LowBits(), src); 1578 } 1579 1580 void X86_64Assembler::minpd(XmmRegister dst, XmmRegister src) { 1581 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1582 EmitUint8(0x66); 1583 EmitOptionalRex32(dst, src); 1584 EmitUint8(0x0F); 1585 EmitUint8(0x5D); 1586 EmitXmmRegisterOperand(dst.LowBits(), src); 1587 } 1588 1589 void X86_64Assembler::maxpd(XmmRegister dst, XmmRegister src) { 1590 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1591 EmitUint8(0x66); 1592 EmitOptionalRex32(dst, src); 1593 EmitUint8(0x0F); 1594 EmitUint8(0x5F); 1595 EmitXmmRegisterOperand(dst.LowBits(), src); 1596 } 1597 1598 void X86_64Assembler::pcmpeqb(XmmRegister dst, XmmRegister src) { 1599 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1600 EmitUint8(0x66); 1601 EmitOptionalRex32(dst, src); 1602 EmitUint8(0x0F); 1603 EmitUint8(0x74); 1604 EmitXmmRegisterOperand(dst.LowBits(), src); 1605 } 1606 1607 void X86_64Assembler::pcmpeqw(XmmRegister dst, XmmRegister src) { 1608 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1609 EmitUint8(0x66); 1610 EmitOptionalRex32(dst, src); 1611 EmitUint8(0x0F); 1612 EmitUint8(0x75); 1613 EmitXmmRegisterOperand(dst.LowBits(), src); 1614 } 1615 1616 void X86_64Assembler::pcmpeqd(XmmRegister dst, XmmRegister src) { 1617 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1618 EmitUint8(0x66); 1619 EmitOptionalRex32(dst, src); 1620 EmitUint8(0x0F); 1621 EmitUint8(0x76); 1622 EmitXmmRegisterOperand(dst.LowBits(), src); 1623 } 1624 1625 void X86_64Assembler::pcmpeqq(XmmRegister dst, XmmRegister src) { 1626 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1627 EmitUint8(0x66); 1628 EmitOptionalRex32(dst, src); 1629 EmitUint8(0x0F); 1630 EmitUint8(0x38); 1631 EmitUint8(0x29); 1632 EmitXmmRegisterOperand(dst.LowBits(), src); 1633 } 1634 1635 void X86_64Assembler::pcmpgtb(XmmRegister dst, XmmRegister src) { 1636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1637 EmitUint8(0x66); 1638 EmitOptionalRex32(dst, src); 1639 EmitUint8(0x0F); 1640 EmitUint8(0x64); 1641 EmitXmmRegisterOperand(dst.LowBits(), src); 1642 } 1643 1644 void X86_64Assembler::pcmpgtw(XmmRegister dst, XmmRegister src) { 1645 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1646 EmitUint8(0x66); 1647 EmitOptionalRex32(dst, src); 1648 EmitUint8(0x0F); 1649 EmitUint8(0x65); 1650 EmitXmmRegisterOperand(dst.LowBits(), src); 1651 } 1652 1653 void X86_64Assembler::pcmpgtd(XmmRegister dst, XmmRegister src) { 1654 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1655 EmitUint8(0x66); 1656 EmitOptionalRex32(dst, src); 1657 EmitUint8(0x0F); 1658 EmitUint8(0x66); 1659 EmitXmmRegisterOperand(dst.LowBits(), src); 1660 } 1661 1662 void X86_64Assembler::pcmpgtq(XmmRegister dst, XmmRegister src) { 1663 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1664 EmitUint8(0x66); 1665 EmitOptionalRex32(dst, src); 1666 EmitUint8(0x0F); 1667 EmitUint8(0x38); 1668 EmitUint8(0x37); 1669 EmitXmmRegisterOperand(dst.LowBits(), src); 1670 } 1671 1672 void X86_64Assembler::shufpd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 1673 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1674 EmitUint8(0x66); 1675 EmitOptionalRex32(dst, src); 1676 EmitUint8(0x0F); 1677 EmitUint8(0xC6); 1678 EmitXmmRegisterOperand(dst.LowBits(), src); 1679 EmitUint8(imm.value()); 1680 } 1681 1682 1683 void X86_64Assembler::shufps(XmmRegister dst, XmmRegister src, const Immediate& imm) { 1684 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1685 EmitOptionalRex32(dst, src); 1686 EmitUint8(0x0F); 1687 EmitUint8(0xC6); 1688 EmitXmmRegisterOperand(dst.LowBits(), src); 1689 EmitUint8(imm.value()); 1690 } 1691 1692 1693 void X86_64Assembler::pshufd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 1694 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1695 EmitUint8(0x66); 1696 EmitOptionalRex32(dst, src); 1697 EmitUint8(0x0F); 1698 EmitUint8(0x70); 1699 EmitXmmRegisterOperand(dst.LowBits(), src); 1700 EmitUint8(imm.value()); 1701 } 1702 1703 1704 void X86_64Assembler::punpcklbw(XmmRegister dst, XmmRegister src) { 1705 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1706 EmitUint8(0x66); 1707 EmitOptionalRex32(dst, src); 1708 EmitUint8(0x0F); 1709 EmitUint8(0x60); 1710 EmitXmmRegisterOperand(dst.LowBits(), src); 1711 } 1712 1713 1714 void X86_64Assembler::punpcklwd(XmmRegister dst, XmmRegister src) { 1715 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1716 EmitUint8(0x66); 1717 EmitOptionalRex32(dst, src); 1718 EmitUint8(0x0F); 1719 EmitUint8(0x61); 1720 EmitXmmRegisterOperand(dst.LowBits(), src); 1721 } 1722 1723 1724 void X86_64Assembler::punpckldq(XmmRegister dst, XmmRegister src) { 1725 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1726 EmitUint8(0x66); 1727 EmitOptionalRex32(dst, src); 1728 EmitUint8(0x0F); 1729 EmitUint8(0x62); 1730 EmitXmmRegisterOperand(dst.LowBits(), src); 1731 } 1732 1733 1734 void X86_64Assembler::punpcklqdq(XmmRegister dst, XmmRegister src) { 1735 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1736 EmitUint8(0x66); 1737 EmitOptionalRex32(dst, src); 1738 EmitUint8(0x0F); 1739 EmitUint8(0x6C); 1740 EmitXmmRegisterOperand(dst.LowBits(), src); 1741 } 1742 1743 1744 void X86_64Assembler::psllw(XmmRegister reg, const Immediate& shift_count) { 1745 DCHECK(shift_count.is_uint8()); 1746 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1747 EmitUint8(0x66); 1748 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1749 EmitUint8(0x0F); 1750 EmitUint8(0x71); 1751 EmitXmmRegisterOperand(6, reg); 1752 EmitUint8(shift_count.value()); 1753 } 1754 1755 1756 void X86_64Assembler::pslld(XmmRegister reg, const Immediate& shift_count) { 1757 DCHECK(shift_count.is_uint8()); 1758 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1759 EmitUint8(0x66); 1760 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1761 EmitUint8(0x0F); 1762 EmitUint8(0x72); 1763 EmitXmmRegisterOperand(6, reg); 1764 EmitUint8(shift_count.value()); 1765 } 1766 1767 1768 void X86_64Assembler::psllq(XmmRegister reg, const Immediate& shift_count) { 1769 DCHECK(shift_count.is_uint8()); 1770 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1771 EmitUint8(0x66); 1772 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1773 EmitUint8(0x0F); 1774 EmitUint8(0x73); 1775 EmitXmmRegisterOperand(6, reg); 1776 EmitUint8(shift_count.value()); 1777 } 1778 1779 1780 void X86_64Assembler::psraw(XmmRegister reg, const Immediate& shift_count) { 1781 DCHECK(shift_count.is_uint8()); 1782 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1783 EmitUint8(0x66); 1784 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1785 EmitUint8(0x0F); 1786 EmitUint8(0x71); 1787 EmitXmmRegisterOperand(4, reg); 1788 EmitUint8(shift_count.value()); 1789 } 1790 1791 1792 void X86_64Assembler::psrad(XmmRegister reg, const Immediate& shift_count) { 1793 DCHECK(shift_count.is_uint8()); 1794 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1795 EmitUint8(0x66); 1796 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1797 EmitUint8(0x0F); 1798 EmitUint8(0x72); 1799 EmitXmmRegisterOperand(4, reg); 1800 EmitUint8(shift_count.value()); 1801 } 1802 1803 1804 void X86_64Assembler::psrlw(XmmRegister reg, const Immediate& shift_count) { 1805 DCHECK(shift_count.is_uint8()); 1806 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1807 EmitUint8(0x66); 1808 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1809 EmitUint8(0x0F); 1810 EmitUint8(0x71); 1811 EmitXmmRegisterOperand(2, reg); 1812 EmitUint8(shift_count.value()); 1813 } 1814 1815 1816 void X86_64Assembler::psrld(XmmRegister reg, const Immediate& shift_count) { 1817 DCHECK(shift_count.is_uint8()); 1818 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1819 EmitUint8(0x66); 1820 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1821 EmitUint8(0x0F); 1822 EmitUint8(0x72); 1823 EmitXmmRegisterOperand(2, reg); 1824 EmitUint8(shift_count.value()); 1825 } 1826 1827 1828 void X86_64Assembler::psrlq(XmmRegister reg, const Immediate& shift_count) { 1829 DCHECK(shift_count.is_uint8()); 1830 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1831 EmitUint8(0x66); 1832 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 1833 EmitUint8(0x0F); 1834 EmitUint8(0x73); 1835 EmitXmmRegisterOperand(2, reg); 1836 EmitUint8(shift_count.value()); 1837 } 1838 1839 1840 void X86_64Assembler::fldl(const Address& src) { 1841 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1842 EmitUint8(0xDD); 1843 EmitOperand(0, src); 1844 } 1845 1846 1847 void X86_64Assembler::fstl(const Address& dst) { 1848 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1849 EmitUint8(0xDD); 1850 EmitOperand(2, dst); 1851 } 1852 1853 1854 void X86_64Assembler::fstpl(const Address& dst) { 1855 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1856 EmitUint8(0xDD); 1857 EmitOperand(3, dst); 1858 } 1859 1860 1861 void X86_64Assembler::fstsw() { 1862 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1863 EmitUint8(0x9B); 1864 EmitUint8(0xDF); 1865 EmitUint8(0xE0); 1866 } 1867 1868 1869 void X86_64Assembler::fnstcw(const Address& dst) { 1870 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1871 EmitUint8(0xD9); 1872 EmitOperand(7, dst); 1873 } 1874 1875 1876 void X86_64Assembler::fldcw(const Address& src) { 1877 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1878 EmitUint8(0xD9); 1879 EmitOperand(5, src); 1880 } 1881 1882 1883 void X86_64Assembler::fistpl(const Address& dst) { 1884 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1885 EmitUint8(0xDF); 1886 EmitOperand(7, dst); 1887 } 1888 1889 1890 void X86_64Assembler::fistps(const Address& dst) { 1891 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1892 EmitUint8(0xDB); 1893 EmitOperand(3, dst); 1894 } 1895 1896 1897 void X86_64Assembler::fildl(const Address& src) { 1898 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1899 EmitUint8(0xDF); 1900 EmitOperand(5, src); 1901 } 1902 1903 1904 void X86_64Assembler::filds(const Address& src) { 1905 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1906 EmitUint8(0xDB); 1907 EmitOperand(0, src); 1908 } 1909 1910 1911 void X86_64Assembler::fincstp() { 1912 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1913 EmitUint8(0xD9); 1914 EmitUint8(0xF7); 1915 } 1916 1917 1918 void X86_64Assembler::ffree(const Immediate& index) { 1919 CHECK_LT(index.value(), 7); 1920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1921 EmitUint8(0xDD); 1922 EmitUint8(0xC0 + index.value()); 1923 } 1924 1925 1926 void X86_64Assembler::fsin() { 1927 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1928 EmitUint8(0xD9); 1929 EmitUint8(0xFE); 1930 } 1931 1932 1933 void X86_64Assembler::fcos() { 1934 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1935 EmitUint8(0xD9); 1936 EmitUint8(0xFF); 1937 } 1938 1939 1940 void X86_64Assembler::fptan() { 1941 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1942 EmitUint8(0xD9); 1943 EmitUint8(0xF2); 1944 } 1945 1946 void X86_64Assembler::fucompp() { 1947 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1948 EmitUint8(0xDA); 1949 EmitUint8(0xE9); 1950 } 1951 1952 1953 void X86_64Assembler::fprem() { 1954 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1955 EmitUint8(0xD9); 1956 EmitUint8(0xF8); 1957 } 1958 1959 1960 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1961 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1962 // There is a short version for rax. 1963 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1964 // work. 1965 const bool src_rax = src.AsRegister() == RAX; 1966 const bool dst_rax = dst.AsRegister() == RAX; 1967 if (src_rax || dst_rax) { 1968 EmitOptionalRex32(src_rax ? dst : src); 1969 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1970 return; 1971 } 1972 1973 // General case. 1974 EmitOptionalRex32(src, dst); 1975 EmitUint8(0x87); 1976 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1977 } 1978 1979 1980 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1981 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1982 // There is a short version for rax. 1983 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1984 // work. 1985 const bool src_rax = src.AsRegister() == RAX; 1986 const bool dst_rax = dst.AsRegister() == RAX; 1987 if (src_rax || dst_rax) { 1988 // If src == target, emit a nop instead. 1989 if (src_rax && dst_rax) { 1990 EmitUint8(0x90); 1991 } else { 1992 EmitRex64(src_rax ? dst : src); 1993 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1994 } 1995 return; 1996 } 1997 1998 // General case. 1999 EmitRex64(src, dst); 2000 EmitUint8(0x87); 2001 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 2002 } 2003 2004 2005 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 2006 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2007 EmitOptionalRex32(reg, address); 2008 EmitUint8(0x87); 2009 EmitOperand(reg.LowBits(), address); 2010 } 2011 2012 2013 void X86_64Assembler::cmpb(const Address& address, const Immediate& imm) { 2014 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2015 CHECK(imm.is_int32()); 2016 EmitOptionalRex32(address); 2017 EmitUint8(0x80); 2018 EmitOperand(7, address); 2019 EmitUint8(imm.value() & 0xFF); 2020 } 2021 2022 2023 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 2024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2025 CHECK(imm.is_int32()); 2026 EmitOperandSizeOverride(); 2027 EmitOptionalRex32(address); 2028 EmitComplex(7, address, imm); 2029 } 2030 2031 2032 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 2033 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2034 CHECK(imm.is_int32()); 2035 EmitOptionalRex32(reg); 2036 EmitComplex(7, Operand(reg), imm); 2037 } 2038 2039 2040 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 2041 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2042 EmitOptionalRex32(reg0, reg1); 2043 EmitUint8(0x3B); 2044 EmitOperand(reg0.LowBits(), Operand(reg1)); 2045 } 2046 2047 2048 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 2049 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2050 EmitOptionalRex32(reg, address); 2051 EmitUint8(0x3B); 2052 EmitOperand(reg.LowBits(), address); 2053 } 2054 2055 2056 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 2057 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2058 EmitOptionalRex32(reg, address); 2059 EmitUint8(0x39); 2060 EmitOperand(reg.LowBits(), address); 2061 } 2062 2063 2064 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 2065 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2066 CHECK(imm.is_int32()); 2067 EmitOptionalRex32(address); 2068 EmitComplex(7, address, imm); 2069 } 2070 2071 2072 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 2073 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2074 EmitRex64(reg0, reg1); 2075 EmitUint8(0x3B); 2076 EmitOperand(reg0.LowBits(), Operand(reg1)); 2077 } 2078 2079 2080 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 2081 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2082 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 2083 EmitRex64(reg); 2084 EmitComplex(7, Operand(reg), imm); 2085 } 2086 2087 2088 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 2089 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2090 EmitRex64(reg, address); 2091 EmitUint8(0x3B); 2092 EmitOperand(reg.LowBits(), address); 2093 } 2094 2095 2096 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 2097 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 2098 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2099 EmitRex64(address); 2100 EmitComplex(7, address, imm); 2101 } 2102 2103 2104 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 2105 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2106 EmitOptionalRex32(dst, src); 2107 EmitUint8(0x03); 2108 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 2109 } 2110 2111 2112 void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 2113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2114 EmitOptionalRex32(reg, address); 2115 EmitUint8(0x03); 2116 EmitOperand(reg.LowBits(), address); 2117 } 2118 2119 2120 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 2121 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2122 EmitOptionalRex32(reg1, reg2); 2123 EmitUint8(0x85); 2124 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 2125 } 2126 2127 2128 void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 2129 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2130 EmitOptionalRex32(reg, address); 2131 EmitUint8(0x85); 2132 EmitOperand(reg.LowBits(), address); 2133 } 2134 2135 2136 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 2137 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2138 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 2139 // we only test the byte CpuRegister to keep the encoding short. 2140 if (immediate.is_uint8() && reg.AsRegister() < 4) { 2141 // Use zero-extended 8-bit immediate. 2142 if (reg.AsRegister() == RAX) { 2143 EmitUint8(0xA8); 2144 } else { 2145 EmitUint8(0xF6); 2146 EmitUint8(0xC0 + reg.AsRegister()); 2147 } 2148 EmitUint8(immediate.value() & 0xFF); 2149 } else if (reg.AsRegister() == RAX) { 2150 // Use short form if the destination is RAX. 2151 EmitUint8(0xA9); 2152 EmitImmediate(immediate); 2153 } else { 2154 EmitOptionalRex32(reg); 2155 EmitUint8(0xF7); 2156 EmitOperand(0, Operand(reg)); 2157 EmitImmediate(immediate); 2158 } 2159 } 2160 2161 2162 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 2163 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2164 EmitRex64(reg1, reg2); 2165 EmitUint8(0x85); 2166 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 2167 } 2168 2169 2170 void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 2171 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2172 EmitRex64(reg, address); 2173 EmitUint8(0x85); 2174 EmitOperand(reg.LowBits(), address); 2175 } 2176 2177 2178 void X86_64Assembler::testb(const Address& dst, const Immediate& imm) { 2179 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2180 EmitOptionalRex32(dst); 2181 EmitUint8(0xF6); 2182 EmitOperand(Register::RAX, dst); 2183 CHECK(imm.is_int8()); 2184 EmitUint8(imm.value() & 0xFF); 2185 } 2186 2187 2188 void X86_64Assembler::testl(const Address& dst, const Immediate& imm) { 2189 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2190 EmitOptionalRex32(dst); 2191 EmitUint8(0xF7); 2192 EmitOperand(0, dst); 2193 EmitImmediate(imm); 2194 } 2195 2196 2197 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 2198 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2199 EmitOptionalRex32(dst, src); 2200 EmitUint8(0x23); 2201 EmitOperand(dst.LowBits(), Operand(src)); 2202 } 2203 2204 2205 void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 2206 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2207 EmitOptionalRex32(reg, address); 2208 EmitUint8(0x23); 2209 EmitOperand(reg.LowBits(), address); 2210 } 2211 2212 2213 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 2214 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2215 EmitOptionalRex32(dst); 2216 EmitComplex(4, Operand(dst), imm); 2217 } 2218 2219 2220 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 2221 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2222 CHECK(imm.is_int32()); // andq only supports 32b immediate. 2223 EmitRex64(reg); 2224 EmitComplex(4, Operand(reg), imm); 2225 } 2226 2227 2228 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 2229 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2230 EmitRex64(dst, src); 2231 EmitUint8(0x23); 2232 EmitOperand(dst.LowBits(), Operand(src)); 2233 } 2234 2235 2236 void X86_64Assembler::andq(CpuRegister dst, const Address& src) { 2237 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2238 EmitRex64(dst, src); 2239 EmitUint8(0x23); 2240 EmitOperand(dst.LowBits(), src); 2241 } 2242 2243 2244 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 2245 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2246 EmitOptionalRex32(dst, src); 2247 EmitUint8(0x0B); 2248 EmitOperand(dst.LowBits(), Operand(src)); 2249 } 2250 2251 2252 void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 2253 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2254 EmitOptionalRex32(reg, address); 2255 EmitUint8(0x0B); 2256 EmitOperand(reg.LowBits(), address); 2257 } 2258 2259 2260 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 2261 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2262 EmitOptionalRex32(dst); 2263 EmitComplex(1, Operand(dst), imm); 2264 } 2265 2266 2267 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 2268 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2269 CHECK(imm.is_int32()); // orq only supports 32b immediate. 2270 EmitRex64(dst); 2271 EmitComplex(1, Operand(dst), imm); 2272 } 2273 2274 2275 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 2276 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2277 EmitRex64(dst, src); 2278 EmitUint8(0x0B); 2279 EmitOperand(dst.LowBits(), Operand(src)); 2280 } 2281 2282 2283 void X86_64Assembler::orq(CpuRegister dst, const Address& src) { 2284 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2285 EmitRex64(dst, src); 2286 EmitUint8(0x0B); 2287 EmitOperand(dst.LowBits(), src); 2288 } 2289 2290 2291 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 2292 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2293 EmitOptionalRex32(dst, src); 2294 EmitUint8(0x33); 2295 EmitOperand(dst.LowBits(), Operand(src)); 2296 } 2297 2298 2299 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 2300 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2301 EmitOptionalRex32(reg, address); 2302 EmitUint8(0x33); 2303 EmitOperand(reg.LowBits(), address); 2304 } 2305 2306 2307 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 2308 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2309 EmitOptionalRex32(dst); 2310 EmitComplex(6, Operand(dst), imm); 2311 } 2312 2313 2314 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 2315 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2316 EmitRex64(dst, src); 2317 EmitUint8(0x33); 2318 EmitOperand(dst.LowBits(), Operand(src)); 2319 } 2320 2321 2322 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 2323 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2324 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 2325 EmitRex64(dst); 2326 EmitComplex(6, Operand(dst), imm); 2327 } 2328 2329 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) { 2330 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2331 EmitRex64(dst, src); 2332 EmitUint8(0x33); 2333 EmitOperand(dst.LowBits(), src); 2334 } 2335 2336 2337 #if 0 2338 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 2339 // REX.WRXB 2340 // W - 64-bit operand 2341 // R - MODRM.reg 2342 // X - SIB.index 2343 // B - MODRM.rm/SIB.base 2344 uint8_t rex = force ? 0x40 : 0; 2345 if (w) { 2346 rex |= 0x48; // REX.W000 2347 } 2348 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 2349 rex |= 0x44; // REX.0R00 2350 *r = static_cast<Register>(*r - 8); 2351 } 2352 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 2353 rex |= 0x42; // REX.00X0 2354 *x = static_cast<Register>(*x - 8); 2355 } 2356 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 2357 rex |= 0x41; // REX.000B 2358 *b = static_cast<Register>(*b - 8); 2359 } 2360 if (rex != 0) { 2361 EmitUint8(rex); 2362 } 2363 } 2364 2365 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 2366 // REX.WRXB 2367 // W - 64-bit operand 2368 // R - MODRM.reg 2369 // X - SIB.index 2370 // B - MODRM.rm/SIB.base 2371 uint8_t rex = mem->rex(); 2372 if (force) { 2373 rex |= 0x40; // REX.0000 2374 } 2375 if (w) { 2376 rex |= 0x48; // REX.W000 2377 } 2378 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 2379 rex |= 0x44; // REX.0R00 2380 *dst = static_cast<Register>(*dst - 8); 2381 } 2382 if (rex != 0) { 2383 EmitUint8(rex); 2384 } 2385 } 2386 2387 void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 2388 #endif 2389 2390 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 2391 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2392 EmitOptionalRex32(reg); 2393 EmitComplex(0, Operand(reg), imm); 2394 } 2395 2396 2397 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 2398 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2399 CHECK(imm.is_int32()); // addq only supports 32b immediate. 2400 EmitRex64(reg); 2401 EmitComplex(0, Operand(reg), imm); 2402 } 2403 2404 2405 void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 2406 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2407 EmitRex64(dst, address); 2408 EmitUint8(0x03); 2409 EmitOperand(dst.LowBits(), address); 2410 } 2411 2412 2413 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 2414 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2415 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 2416 EmitRex64(src, dst); 2417 EmitUint8(0x01); 2418 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 2419 } 2420 2421 2422 void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 2423 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2424 EmitOptionalRex32(reg, address); 2425 EmitUint8(0x01); 2426 EmitOperand(reg.LowBits(), address); 2427 } 2428 2429 2430 void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 2431 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2432 EmitOptionalRex32(address); 2433 EmitComplex(0, address, imm); 2434 } 2435 2436 2437 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 2438 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2439 EmitOptionalRex32(dst, src); 2440 EmitUint8(0x2B); 2441 EmitOperand(dst.LowBits(), Operand(src)); 2442 } 2443 2444 2445 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 2446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2447 EmitOptionalRex32(reg); 2448 EmitComplex(5, Operand(reg), imm); 2449 } 2450 2451 2452 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 2453 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2454 CHECK(imm.is_int32()); // subq only supports 32b immediate. 2455 EmitRex64(reg); 2456 EmitComplex(5, Operand(reg), imm); 2457 } 2458 2459 2460 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 2461 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2462 EmitRex64(dst, src); 2463 EmitUint8(0x2B); 2464 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 2465 } 2466 2467 2468 void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 2469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2470 EmitRex64(reg, address); 2471 EmitUint8(0x2B); 2472 EmitOperand(reg.LowBits() & 7, address); 2473 } 2474 2475 2476 void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 2477 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2478 EmitOptionalRex32(reg, address); 2479 EmitUint8(0x2B); 2480 EmitOperand(reg.LowBits(), address); 2481 } 2482 2483 2484 void X86_64Assembler::cdq() { 2485 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2486 EmitUint8(0x99); 2487 } 2488 2489 2490 void X86_64Assembler::cqo() { 2491 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2492 EmitRex64(); 2493 EmitUint8(0x99); 2494 } 2495 2496 2497 void X86_64Assembler::idivl(CpuRegister reg) { 2498 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2499 EmitOptionalRex32(reg); 2500 EmitUint8(0xF7); 2501 EmitUint8(0xF8 | reg.LowBits()); 2502 } 2503 2504 2505 void X86_64Assembler::idivq(CpuRegister reg) { 2506 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2507 EmitRex64(reg); 2508 EmitUint8(0xF7); 2509 EmitUint8(0xF8 | reg.LowBits()); 2510 } 2511 2512 2513 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 2514 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2515 EmitOptionalRex32(dst, src); 2516 EmitUint8(0x0F); 2517 EmitUint8(0xAF); 2518 EmitOperand(dst.LowBits(), Operand(src)); 2519 } 2520 2521 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src, const Immediate& imm) { 2522 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2523 CHECK(imm.is_int32()); // imull only supports 32b immediate. 2524 2525 EmitOptionalRex32(dst, src); 2526 2527 // See whether imm can be represented as a sign-extended 8bit value. 2528 int32_t v32 = static_cast<int32_t>(imm.value()); 2529 if (IsInt<8>(v32)) { 2530 // Sign-extension works. 2531 EmitUint8(0x6B); 2532 EmitOperand(dst.LowBits(), Operand(src)); 2533 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 2534 } else { 2535 // Not representable, use full immediate. 2536 EmitUint8(0x69); 2537 EmitOperand(dst.LowBits(), Operand(src)); 2538 EmitImmediate(imm); 2539 } 2540 } 2541 2542 2543 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 2544 imull(reg, reg, imm); 2545 } 2546 2547 2548 void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 2549 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2550 EmitOptionalRex32(reg, address); 2551 EmitUint8(0x0F); 2552 EmitUint8(0xAF); 2553 EmitOperand(reg.LowBits(), address); 2554 } 2555 2556 2557 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 2558 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2559 EmitRex64(dst, src); 2560 EmitUint8(0x0F); 2561 EmitUint8(0xAF); 2562 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 2563 } 2564 2565 2566 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 2567 imulq(reg, reg, imm); 2568 } 2569 2570 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 2571 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2572 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 2573 2574 EmitRex64(dst, reg); 2575 2576 // See whether imm can be represented as a sign-extended 8bit value. 2577 int64_t v64 = imm.value(); 2578 if (IsInt<8>(v64)) { 2579 // Sign-extension works. 2580 EmitUint8(0x6B); 2581 EmitOperand(dst.LowBits(), Operand(reg)); 2582 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 2583 } else { 2584 // Not representable, use full immediate. 2585 EmitUint8(0x69); 2586 EmitOperand(dst.LowBits(), Operand(reg)); 2587 EmitImmediate(imm); 2588 } 2589 } 2590 2591 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 2592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2593 EmitRex64(reg, address); 2594 EmitUint8(0x0F); 2595 EmitUint8(0xAF); 2596 EmitOperand(reg.LowBits(), address); 2597 } 2598 2599 2600 void X86_64Assembler::imull(CpuRegister reg) { 2601 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2602 EmitOptionalRex32(reg); 2603 EmitUint8(0xF7); 2604 EmitOperand(5, Operand(reg)); 2605 } 2606 2607 2608 void X86_64Assembler::imulq(CpuRegister reg) { 2609 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2610 EmitRex64(reg); 2611 EmitUint8(0xF7); 2612 EmitOperand(5, Operand(reg)); 2613 } 2614 2615 2616 void X86_64Assembler::imull(const Address& address) { 2617 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2618 EmitOptionalRex32(address); 2619 EmitUint8(0xF7); 2620 EmitOperand(5, address); 2621 } 2622 2623 2624 void X86_64Assembler::mull(CpuRegister reg) { 2625 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2626 EmitOptionalRex32(reg); 2627 EmitUint8(0xF7); 2628 EmitOperand(4, Operand(reg)); 2629 } 2630 2631 2632 void X86_64Assembler::mull(const Address& address) { 2633 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2634 EmitOptionalRex32(address); 2635 EmitUint8(0xF7); 2636 EmitOperand(4, address); 2637 } 2638 2639 2640 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 2641 EmitGenericShift(false, 4, reg, imm); 2642 } 2643 2644 2645 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 2646 EmitGenericShift(true, 4, reg, imm); 2647 } 2648 2649 2650 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 2651 EmitGenericShift(false, 4, operand, shifter); 2652 } 2653 2654 2655 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 2656 EmitGenericShift(true, 4, operand, shifter); 2657 } 2658 2659 2660 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 2661 EmitGenericShift(false, 5, reg, imm); 2662 } 2663 2664 2665 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 2666 EmitGenericShift(true, 5, reg, imm); 2667 } 2668 2669 2670 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 2671 EmitGenericShift(false, 5, operand, shifter); 2672 } 2673 2674 2675 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 2676 EmitGenericShift(true, 5, operand, shifter); 2677 } 2678 2679 2680 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 2681 EmitGenericShift(false, 7, reg, imm); 2682 } 2683 2684 2685 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 2686 EmitGenericShift(false, 7, operand, shifter); 2687 } 2688 2689 2690 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 2691 EmitGenericShift(true, 7, reg, imm); 2692 } 2693 2694 2695 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 2696 EmitGenericShift(true, 7, operand, shifter); 2697 } 2698 2699 2700 void X86_64Assembler::roll(CpuRegister reg, const Immediate& imm) { 2701 EmitGenericShift(false, 0, reg, imm); 2702 } 2703 2704 2705 void X86_64Assembler::roll(CpuRegister operand, CpuRegister shifter) { 2706 EmitGenericShift(false, 0, operand, shifter); 2707 } 2708 2709 2710 void X86_64Assembler::rorl(CpuRegister reg, const Immediate& imm) { 2711 EmitGenericShift(false, 1, reg, imm); 2712 } 2713 2714 2715 void X86_64Assembler::rorl(CpuRegister operand, CpuRegister shifter) { 2716 EmitGenericShift(false, 1, operand, shifter); 2717 } 2718 2719 2720 void X86_64Assembler::rolq(CpuRegister reg, const Immediate& imm) { 2721 EmitGenericShift(true, 0, reg, imm); 2722 } 2723 2724 2725 void X86_64Assembler::rolq(CpuRegister operand, CpuRegister shifter) { 2726 EmitGenericShift(true, 0, operand, shifter); 2727 } 2728 2729 2730 void X86_64Assembler::rorq(CpuRegister reg, const Immediate& imm) { 2731 EmitGenericShift(true, 1, reg, imm); 2732 } 2733 2734 2735 void X86_64Assembler::rorq(CpuRegister operand, CpuRegister shifter) { 2736 EmitGenericShift(true, 1, operand, shifter); 2737 } 2738 2739 2740 void X86_64Assembler::negl(CpuRegister reg) { 2741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2742 EmitOptionalRex32(reg); 2743 EmitUint8(0xF7); 2744 EmitOperand(3, Operand(reg)); 2745 } 2746 2747 2748 void X86_64Assembler::negq(CpuRegister reg) { 2749 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2750 EmitRex64(reg); 2751 EmitUint8(0xF7); 2752 EmitOperand(3, Operand(reg)); 2753 } 2754 2755 2756 void X86_64Assembler::notl(CpuRegister reg) { 2757 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2758 EmitOptionalRex32(reg); 2759 EmitUint8(0xF7); 2760 EmitUint8(0xD0 | reg.LowBits()); 2761 } 2762 2763 2764 void X86_64Assembler::notq(CpuRegister reg) { 2765 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2766 EmitRex64(reg); 2767 EmitUint8(0xF7); 2768 EmitOperand(2, Operand(reg)); 2769 } 2770 2771 2772 void X86_64Assembler::enter(const Immediate& imm) { 2773 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2774 EmitUint8(0xC8); 2775 CHECK(imm.is_uint16()) << imm.value(); 2776 EmitUint8(imm.value() & 0xFF); 2777 EmitUint8((imm.value() >> 8) & 0xFF); 2778 EmitUint8(0x00); 2779 } 2780 2781 2782 void X86_64Assembler::leave() { 2783 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2784 EmitUint8(0xC9); 2785 } 2786 2787 2788 void X86_64Assembler::ret() { 2789 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2790 EmitUint8(0xC3); 2791 } 2792 2793 2794 void X86_64Assembler::ret(const Immediate& imm) { 2795 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2796 EmitUint8(0xC2); 2797 CHECK(imm.is_uint16()); 2798 EmitUint8(imm.value() & 0xFF); 2799 EmitUint8((imm.value() >> 8) & 0xFF); 2800 } 2801 2802 2803 2804 void X86_64Assembler::nop() { 2805 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2806 EmitUint8(0x90); 2807 } 2808 2809 2810 void X86_64Assembler::int3() { 2811 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2812 EmitUint8(0xCC); 2813 } 2814 2815 2816 void X86_64Assembler::hlt() { 2817 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2818 EmitUint8(0xF4); 2819 } 2820 2821 2822 void X86_64Assembler::j(Condition condition, Label* label) { 2823 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2824 if (label->IsBound()) { 2825 static const int kShortSize = 2; 2826 static const int kLongSize = 6; 2827 int offset = label->Position() - buffer_.Size(); 2828 CHECK_LE(offset, 0); 2829 if (IsInt<8>(offset - kShortSize)) { 2830 EmitUint8(0x70 + condition); 2831 EmitUint8((offset - kShortSize) & 0xFF); 2832 } else { 2833 EmitUint8(0x0F); 2834 EmitUint8(0x80 + condition); 2835 EmitInt32(offset - kLongSize); 2836 } 2837 } else { 2838 EmitUint8(0x0F); 2839 EmitUint8(0x80 + condition); 2840 EmitLabelLink(label); 2841 } 2842 } 2843 2844 2845 void X86_64Assembler::j(Condition condition, NearLabel* label) { 2846 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2847 if (label->IsBound()) { 2848 static const int kShortSize = 2; 2849 int offset = label->Position() - buffer_.Size(); 2850 CHECK_LE(offset, 0); 2851 CHECK(IsInt<8>(offset - kShortSize)); 2852 EmitUint8(0x70 + condition); 2853 EmitUint8((offset - kShortSize) & 0xFF); 2854 } else { 2855 EmitUint8(0x70 + condition); 2856 EmitLabelLink(label); 2857 } 2858 } 2859 2860 2861 void X86_64Assembler::jrcxz(NearLabel* label) { 2862 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2863 if (label->IsBound()) { 2864 static const int kShortSize = 2; 2865 int offset = label->Position() - buffer_.Size(); 2866 CHECK_LE(offset, 0); 2867 CHECK(IsInt<8>(offset - kShortSize)); 2868 EmitUint8(0xE3); 2869 EmitUint8((offset - kShortSize) & 0xFF); 2870 } else { 2871 EmitUint8(0xE3); 2872 EmitLabelLink(label); 2873 } 2874 } 2875 2876 2877 void X86_64Assembler::jmp(CpuRegister reg) { 2878 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2879 EmitOptionalRex32(reg); 2880 EmitUint8(0xFF); 2881 EmitRegisterOperand(4, reg.LowBits()); 2882 } 2883 2884 void X86_64Assembler::jmp(const Address& address) { 2885 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2886 EmitOptionalRex32(address); 2887 EmitUint8(0xFF); 2888 EmitOperand(4, address); 2889 } 2890 2891 void X86_64Assembler::jmp(Label* label) { 2892 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2893 if (label->IsBound()) { 2894 static const int kShortSize = 2; 2895 static const int kLongSize = 5; 2896 int offset = label->Position() - buffer_.Size(); 2897 CHECK_LE(offset, 0); 2898 if (IsInt<8>(offset - kShortSize)) { 2899 EmitUint8(0xEB); 2900 EmitUint8((offset - kShortSize) & 0xFF); 2901 } else { 2902 EmitUint8(0xE9); 2903 EmitInt32(offset - kLongSize); 2904 } 2905 } else { 2906 EmitUint8(0xE9); 2907 EmitLabelLink(label); 2908 } 2909 } 2910 2911 2912 void X86_64Assembler::jmp(NearLabel* label) { 2913 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2914 if (label->IsBound()) { 2915 static const int kShortSize = 2; 2916 int offset = label->Position() - buffer_.Size(); 2917 CHECK_LE(offset, 0); 2918 CHECK(IsInt<8>(offset - kShortSize)); 2919 EmitUint8(0xEB); 2920 EmitUint8((offset - kShortSize) & 0xFF); 2921 } else { 2922 EmitUint8(0xEB); 2923 EmitLabelLink(label); 2924 } 2925 } 2926 2927 2928 void X86_64Assembler::rep_movsw() { 2929 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2930 EmitUint8(0x66); 2931 EmitUint8(0xF3); 2932 EmitUint8(0xA5); 2933 } 2934 2935 2936 X86_64Assembler* X86_64Assembler::lock() { 2937 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2938 EmitUint8(0xF0); 2939 return this; 2940 } 2941 2942 2943 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 2944 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2945 EmitOptionalRex32(reg, address); 2946 EmitUint8(0x0F); 2947 EmitUint8(0xB1); 2948 EmitOperand(reg.LowBits(), address); 2949 } 2950 2951 2952 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) { 2953 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2954 EmitRex64(reg, address); 2955 EmitUint8(0x0F); 2956 EmitUint8(0xB1); 2957 EmitOperand(reg.LowBits(), address); 2958 } 2959 2960 2961 void X86_64Assembler::mfence() { 2962 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2963 EmitUint8(0x0F); 2964 EmitUint8(0xAE); 2965 EmitUint8(0xF0); 2966 } 2967 2968 2969 X86_64Assembler* X86_64Assembler::gs() { 2970 // TODO: gs is a prefix and not an instruction 2971 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2972 EmitUint8(0x65); 2973 return this; 2974 } 2975 2976 2977 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 2978 int value = imm.value(); 2979 if (value != 0) { 2980 if (value > 0) { 2981 addl(reg, imm); 2982 } else { 2983 subl(reg, Immediate(value)); 2984 } 2985 } 2986 } 2987 2988 2989 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 2990 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2991 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 2992 if (dst.NeedsRex() || dst.AsRegister() > 3) { 2993 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 2994 } 2995 EmitUint8(0x0F); 2996 EmitUint8(0x90 + condition); 2997 EmitUint8(0xC0 + dst.LowBits()); 2998 } 2999 3000 void X86_64Assembler::bswapl(CpuRegister dst) { 3001 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3002 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 3003 EmitUint8(0x0F); 3004 EmitUint8(0xC8 + dst.LowBits()); 3005 } 3006 3007 void X86_64Assembler::bswapq(CpuRegister dst) { 3008 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3009 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 3010 EmitUint8(0x0F); 3011 EmitUint8(0xC8 + dst.LowBits()); 3012 } 3013 3014 void X86_64Assembler::bsfl(CpuRegister dst, CpuRegister src) { 3015 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3016 EmitOptionalRex32(dst, src); 3017 EmitUint8(0x0F); 3018 EmitUint8(0xBC); 3019 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3020 } 3021 3022 void X86_64Assembler::bsfl(CpuRegister dst, const Address& src) { 3023 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3024 EmitOptionalRex32(dst, src); 3025 EmitUint8(0x0F); 3026 EmitUint8(0xBC); 3027 EmitOperand(dst.LowBits(), src); 3028 } 3029 3030 void X86_64Assembler::bsfq(CpuRegister dst, CpuRegister src) { 3031 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3032 EmitRex64(dst, src); 3033 EmitUint8(0x0F); 3034 EmitUint8(0xBC); 3035 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3036 } 3037 3038 void X86_64Assembler::bsfq(CpuRegister dst, const Address& src) { 3039 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3040 EmitRex64(dst, src); 3041 EmitUint8(0x0F); 3042 EmitUint8(0xBC); 3043 EmitOperand(dst.LowBits(), src); 3044 } 3045 3046 void X86_64Assembler::bsrl(CpuRegister dst, CpuRegister src) { 3047 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3048 EmitOptionalRex32(dst, src); 3049 EmitUint8(0x0F); 3050 EmitUint8(0xBD); 3051 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3052 } 3053 3054 void X86_64Assembler::bsrl(CpuRegister dst, const Address& src) { 3055 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3056 EmitOptionalRex32(dst, src); 3057 EmitUint8(0x0F); 3058 EmitUint8(0xBD); 3059 EmitOperand(dst.LowBits(), src); 3060 } 3061 3062 void X86_64Assembler::bsrq(CpuRegister dst, CpuRegister src) { 3063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3064 EmitRex64(dst, src); 3065 EmitUint8(0x0F); 3066 EmitUint8(0xBD); 3067 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3068 } 3069 3070 void X86_64Assembler::bsrq(CpuRegister dst, const Address& src) { 3071 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3072 EmitRex64(dst, src); 3073 EmitUint8(0x0F); 3074 EmitUint8(0xBD); 3075 EmitOperand(dst.LowBits(), src); 3076 } 3077 3078 void X86_64Assembler::popcntl(CpuRegister dst, CpuRegister src) { 3079 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3080 EmitUint8(0xF3); 3081 EmitOptionalRex32(dst, src); 3082 EmitUint8(0x0F); 3083 EmitUint8(0xB8); 3084 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3085 } 3086 3087 void X86_64Assembler::popcntl(CpuRegister dst, const Address& src) { 3088 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3089 EmitUint8(0xF3); 3090 EmitOptionalRex32(dst, src); 3091 EmitUint8(0x0F); 3092 EmitUint8(0xB8); 3093 EmitOperand(dst.LowBits(), src); 3094 } 3095 3096 void X86_64Assembler::popcntq(CpuRegister dst, CpuRegister src) { 3097 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3098 EmitUint8(0xF3); 3099 EmitRex64(dst, src); 3100 EmitUint8(0x0F); 3101 EmitUint8(0xB8); 3102 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 3103 } 3104 3105 void X86_64Assembler::popcntq(CpuRegister dst, const Address& src) { 3106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3107 EmitUint8(0xF3); 3108 EmitRex64(dst, src); 3109 EmitUint8(0x0F); 3110 EmitUint8(0xB8); 3111 EmitOperand(dst.LowBits(), src); 3112 } 3113 3114 void X86_64Assembler::repne_scasb() { 3115 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3116 EmitUint8(0xF2); 3117 EmitUint8(0xAE); 3118 } 3119 3120 void X86_64Assembler::repne_scasw() { 3121 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3122 EmitUint8(0x66); 3123 EmitUint8(0xF2); 3124 EmitUint8(0xAF); 3125 } 3126 3127 void X86_64Assembler::repe_cmpsw() { 3128 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3129 EmitUint8(0x66); 3130 EmitUint8(0xF3); 3131 EmitUint8(0xA7); 3132 } 3133 3134 3135 void X86_64Assembler::repe_cmpsl() { 3136 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3137 EmitUint8(0xF3); 3138 EmitUint8(0xA7); 3139 } 3140 3141 3142 void X86_64Assembler::repe_cmpsq() { 3143 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3144 EmitUint8(0xF3); 3145 EmitRex64(); 3146 EmitUint8(0xA7); 3147 } 3148 3149 3150 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 3151 // TODO: Need to have a code constants table. 3152 int64_t constant = bit_cast<int64_t, double>(value); 3153 pushq(Immediate(High32Bits(constant))); 3154 pushq(Immediate(Low32Bits(constant))); 3155 movsd(dst, Address(CpuRegister(RSP), 0)); 3156 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 3157 } 3158 3159 3160 void X86_64Assembler::Align(int alignment, int offset) { 3161 CHECK(IsPowerOfTwo(alignment)); 3162 // Emit nop instruction until the real position is aligned. 3163 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 3164 nop(); 3165 } 3166 } 3167 3168 3169 void X86_64Assembler::Bind(Label* label) { 3170 int bound = buffer_.Size(); 3171 CHECK(!label->IsBound()); // Labels can only be bound once. 3172 while (label->IsLinked()) { 3173 int position = label->LinkPosition(); 3174 int next = buffer_.Load<int32_t>(position); 3175 buffer_.Store<int32_t>(position, bound - (position + 4)); 3176 label->position_ = next; 3177 } 3178 label->BindTo(bound); 3179 } 3180 3181 3182 void X86_64Assembler::Bind(NearLabel* label) { 3183 int bound = buffer_.Size(); 3184 CHECK(!label->IsBound()); // Labels can only be bound once. 3185 while (label->IsLinked()) { 3186 int position = label->LinkPosition(); 3187 uint8_t delta = buffer_.Load<uint8_t>(position); 3188 int offset = bound - (position + 1); 3189 CHECK(IsInt<8>(offset)); 3190 buffer_.Store<int8_t>(position, offset); 3191 label->position_ = delta != 0u ? label->position_ - delta : 0; 3192 } 3193 label->BindTo(bound); 3194 } 3195 3196 3197 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 3198 CHECK_GE(reg_or_opcode, 0); 3199 CHECK_LT(reg_or_opcode, 8); 3200 const int length = operand.length_; 3201 CHECK_GT(length, 0); 3202 // Emit the ModRM byte updated with the given reg value. 3203 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 3204 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 3205 // Emit the rest of the encoded operand. 3206 for (int i = 1; i < length; i++) { 3207 EmitUint8(operand.encoding_[i]); 3208 } 3209 AssemblerFixup* fixup = operand.GetFixup(); 3210 if (fixup != nullptr) { 3211 EmitFixup(fixup); 3212 } 3213 } 3214 3215 3216 void X86_64Assembler::EmitImmediate(const Immediate& imm) { 3217 if (imm.is_int32()) { 3218 EmitInt32(static_cast<int32_t>(imm.value())); 3219 } else { 3220 EmitInt64(imm.value()); 3221 } 3222 } 3223 3224 3225 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 3226 const Operand& operand, 3227 const Immediate& immediate) { 3228 CHECK_GE(reg_or_opcode, 0); 3229 CHECK_LT(reg_or_opcode, 8); 3230 if (immediate.is_int8()) { 3231 // Use sign-extended 8-bit immediate. 3232 EmitUint8(0x83); 3233 EmitOperand(reg_or_opcode, operand); 3234 EmitUint8(immediate.value() & 0xFF); 3235 } else if (operand.IsRegister(CpuRegister(RAX))) { 3236 // Use short form if the destination is eax. 3237 EmitUint8(0x05 + (reg_or_opcode << 3)); 3238 EmitImmediate(immediate); 3239 } else { 3240 EmitUint8(0x81); 3241 EmitOperand(reg_or_opcode, operand); 3242 EmitImmediate(immediate); 3243 } 3244 } 3245 3246 3247 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 3248 if (label->IsBound()) { 3249 int offset = label->Position() - buffer_.Size(); 3250 CHECK_LE(offset, 0); 3251 EmitInt32(offset - instruction_size); 3252 } else { 3253 EmitLabelLink(label); 3254 } 3255 } 3256 3257 3258 void X86_64Assembler::EmitLabelLink(Label* label) { 3259 CHECK(!label->IsBound()); 3260 int position = buffer_.Size(); 3261 EmitInt32(label->position_); 3262 label->LinkTo(position); 3263 } 3264 3265 3266 void X86_64Assembler::EmitLabelLink(NearLabel* label) { 3267 CHECK(!label->IsBound()); 3268 int position = buffer_.Size(); 3269 if (label->IsLinked()) { 3270 // Save the delta in the byte that we have to play with. 3271 uint32_t delta = position - label->LinkPosition(); 3272 CHECK(IsUint<8>(delta)); 3273 EmitUint8(delta & 0xFF); 3274 } else { 3275 EmitUint8(0); 3276 } 3277 label->LinkTo(position); 3278 } 3279 3280 3281 void X86_64Assembler::EmitGenericShift(bool wide, 3282 int reg_or_opcode, 3283 CpuRegister reg, 3284 const Immediate& imm) { 3285 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3286 CHECK(imm.is_int8()); 3287 if (wide) { 3288 EmitRex64(reg); 3289 } else { 3290 EmitOptionalRex32(reg); 3291 } 3292 if (imm.value() == 1) { 3293 EmitUint8(0xD1); 3294 EmitOperand(reg_or_opcode, Operand(reg)); 3295 } else { 3296 EmitUint8(0xC1); 3297 EmitOperand(reg_or_opcode, Operand(reg)); 3298 EmitUint8(imm.value() & 0xFF); 3299 } 3300 } 3301 3302 3303 void X86_64Assembler::EmitGenericShift(bool wide, 3304 int reg_or_opcode, 3305 CpuRegister operand, 3306 CpuRegister shifter) { 3307 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3308 CHECK_EQ(shifter.AsRegister(), RCX); 3309 if (wide) { 3310 EmitRex64(operand); 3311 } else { 3312 EmitOptionalRex32(operand); 3313 } 3314 EmitUint8(0xD3); 3315 EmitOperand(reg_or_opcode, Operand(operand)); 3316 } 3317 3318 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 3319 // REX.WRXB 3320 // W - 64-bit operand 3321 // R - MODRM.reg 3322 // X - SIB.index 3323 // B - MODRM.rm/SIB.base 3324 uint8_t rex = force ? 0x40 : 0; 3325 if (w) { 3326 rex |= 0x48; // REX.W000 3327 } 3328 if (r) { 3329 rex |= 0x44; // REX.0R00 3330 } 3331 if (x) { 3332 rex |= 0x42; // REX.00X0 3333 } 3334 if (b) { 3335 rex |= 0x41; // REX.000B 3336 } 3337 if (rex != 0) { 3338 EmitUint8(rex); 3339 } 3340 } 3341 3342 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 3343 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 3344 } 3345 3346 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 3347 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 3348 } 3349 3350 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 3351 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 3352 } 3353 3354 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 3355 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 3356 } 3357 3358 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 3359 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 3360 } 3361 3362 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 3363 uint8_t rex = operand.rex(); 3364 if (rex != 0) { 3365 EmitUint8(rex); 3366 } 3367 } 3368 3369 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 3370 uint8_t rex = operand.rex(); 3371 if (dst.NeedsRex()) { 3372 rex |= 0x44; // REX.0R00 3373 } 3374 if (rex != 0) { 3375 EmitUint8(rex); 3376 } 3377 } 3378 3379 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 3380 uint8_t rex = operand.rex(); 3381 if (dst.NeedsRex()) { 3382 rex |= 0x44; // REX.0R00 3383 } 3384 if (rex != 0) { 3385 EmitUint8(rex); 3386 } 3387 } 3388 3389 void X86_64Assembler::EmitRex64() { 3390 EmitOptionalRex(false, true, false, false, false); 3391 } 3392 3393 void X86_64Assembler::EmitRex64(CpuRegister reg) { 3394 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 3395 } 3396 3397 void X86_64Assembler::EmitRex64(const Operand& operand) { 3398 uint8_t rex = operand.rex(); 3399 rex |= 0x48; // REX.W000 3400 EmitUint8(rex); 3401 } 3402 3403 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 3404 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 3405 } 3406 3407 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 3408 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 3409 } 3410 3411 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 3412 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 3413 } 3414 3415 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 3416 uint8_t rex = 0x48 | operand.rex(); // REX.W000 3417 if (dst.NeedsRex()) { 3418 rex |= 0x44; // REX.0R00 3419 } 3420 EmitUint8(rex); 3421 } 3422 3423 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) { 3424 uint8_t rex = 0x48 | operand.rex(); // REX.W000 3425 if (dst.NeedsRex()) { 3426 rex |= 0x44; // REX.0R00 3427 } 3428 EmitUint8(rex); 3429 } 3430 3431 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 3432 // For src, SPL, BPL, SIL, DIL need the rex prefix. 3433 bool force = src.AsRegister() > 3; 3434 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex()); 3435 } 3436 3437 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 3438 uint8_t rex = operand.rex(); 3439 // For dst, SPL, BPL, SIL, DIL need the rex prefix. 3440 bool force = dst.AsRegister() > 3; 3441 if (force) { 3442 rex |= 0x40; // REX.0000 3443 } 3444 if (dst.NeedsRex()) { 3445 rex |= 0x44; // REX.0R00 3446 } 3447 if (rex != 0) { 3448 EmitUint8(rex); 3449 } 3450 } 3451 3452 void X86_64Assembler::AddConstantArea() { 3453 ArrayRef<const int32_t> area = constant_area_.GetBuffer(); 3454 for (size_t i = 0, e = area.size(); i < e; i++) { 3455 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 3456 EmitInt32(area[i]); 3457 } 3458 } 3459 3460 size_t ConstantArea::AppendInt32(int32_t v) { 3461 size_t result = buffer_.size() * elem_size_; 3462 buffer_.push_back(v); 3463 return result; 3464 } 3465 3466 size_t ConstantArea::AddInt32(int32_t v) { 3467 // Look for an existing match. 3468 for (size_t i = 0, e = buffer_.size(); i < e; i++) { 3469 if (v == buffer_[i]) { 3470 return i * elem_size_; 3471 } 3472 } 3473 3474 // Didn't match anything. 3475 return AppendInt32(v); 3476 } 3477 3478 size_t ConstantArea::AddInt64(int64_t v) { 3479 int32_t v_low = v; 3480 int32_t v_high = v >> 32; 3481 if (buffer_.size() > 1) { 3482 // Ensure we don't pass the end of the buffer. 3483 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) { 3484 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) { 3485 return i * elem_size_; 3486 } 3487 } 3488 } 3489 3490 // Didn't match anything. 3491 size_t result = buffer_.size() * elem_size_; 3492 buffer_.push_back(v_low); 3493 buffer_.push_back(v_high); 3494 return result; 3495 } 3496 3497 size_t ConstantArea::AddDouble(double v) { 3498 // Treat the value as a 64-bit integer value. 3499 return AddInt64(bit_cast<int64_t, double>(v)); 3500 } 3501 3502 size_t ConstantArea::AddFloat(float v) { 3503 // Treat the value as a 32-bit integer value. 3504 return AddInt32(bit_cast<int32_t, float>(v)); 3505 } 3506 3507 } // namespace x86_64 3508 } // namespace art 3509