1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "assembler_x86_64.h" 18 19 #include "base/casts.h" 20 #include "entrypoints/quick/quick_entrypoints.h" 21 #include "memory_region.h" 22 #include "thread.h" 23 24 namespace art { 25 namespace x86_64 { 26 27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) { 28 return os << reg.AsRegister(); 29 } 30 31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) { 32 return os << reg.AsFloatRegister(); 33 } 34 35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) { 36 return os << "ST" << static_cast<int>(reg); 37 } 38 39 void X86_64Assembler::call(CpuRegister reg) { 40 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 41 EmitOptionalRex32(reg); 42 EmitUint8(0xFF); 43 EmitRegisterOperand(2, reg.LowBits()); 44 } 45 46 47 void X86_64Assembler::call(const Address& address) { 48 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 49 EmitOptionalRex32(address); 50 EmitUint8(0xFF); 51 EmitOperand(2, address); 52 } 53 54 55 void X86_64Assembler::call(Label* label) { 56 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 57 EmitUint8(0xE8); 58 static const int kSize = 5; 59 // Offset by one because we already have emitted the opcode. 60 EmitLabel(label, kSize - 1); 61 } 62 63 void X86_64Assembler::pushq(CpuRegister reg) { 64 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 65 EmitOptionalRex32(reg); 66 EmitUint8(0x50 + reg.LowBits()); 67 } 68 69 70 void X86_64Assembler::pushq(const Address& address) { 71 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 72 EmitOptionalRex32(address); 73 EmitUint8(0xFF); 74 EmitOperand(6, address); 75 } 76 77 78 void X86_64Assembler::pushq(const Immediate& imm) { 79 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 80 CHECK(imm.is_int32()); // pushq only supports 32b immediate. 81 if (imm.is_int8()) { 82 EmitUint8(0x6A); 83 EmitUint8(imm.value() & 0xFF); 84 } else { 85 EmitUint8(0x68); 86 EmitImmediate(imm); 87 } 88 } 89 90 91 void X86_64Assembler::popq(CpuRegister reg) { 92 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 93 EmitOptionalRex32(reg); 94 EmitUint8(0x58 + reg.LowBits()); 95 } 96 97 98 void X86_64Assembler::popq(const Address& address) { 99 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 100 EmitOptionalRex32(address); 101 EmitUint8(0x8F); 102 EmitOperand(0, address); 103 } 104 105 106 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) { 107 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 108 if (imm.is_int32()) { 109 // 32 bit. Note: sign-extends. 110 EmitRex64(dst); 111 EmitUint8(0xC7); 112 EmitRegisterOperand(0, dst.LowBits()); 113 EmitInt32(static_cast<int32_t>(imm.value())); 114 } else { 115 EmitRex64(dst); 116 EmitUint8(0xB8 + dst.LowBits()); 117 EmitInt64(imm.value()); 118 } 119 } 120 121 122 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) { 123 CHECK(imm.is_int32()); 124 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 125 EmitOptionalRex32(dst); 126 EmitUint8(0xB8 + dst.LowBits()); 127 EmitImmediate(imm); 128 } 129 130 131 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) { 132 CHECK(imm.is_int32()); 133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 134 EmitRex64(dst); 135 EmitUint8(0xC7); 136 EmitOperand(0, dst); 137 EmitImmediate(imm); 138 } 139 140 141 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) { 142 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 144 EmitRex64(src, dst); 145 EmitUint8(0x89); 146 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 147 } 148 149 150 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) { 151 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 152 EmitOptionalRex32(dst, src); 153 EmitUint8(0x8B); 154 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 155 } 156 157 158 void X86_64Assembler::movq(CpuRegister dst, const Address& src) { 159 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 160 EmitRex64(dst, src); 161 EmitUint8(0x8B); 162 EmitOperand(dst.LowBits(), src); 163 } 164 165 166 void X86_64Assembler::movl(CpuRegister dst, const Address& src) { 167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 168 EmitOptionalRex32(dst, src); 169 EmitUint8(0x8B); 170 EmitOperand(dst.LowBits(), src); 171 } 172 173 174 void X86_64Assembler::movq(const Address& dst, CpuRegister src) { 175 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 176 EmitRex64(src, dst); 177 EmitUint8(0x89); 178 EmitOperand(src.LowBits(), dst); 179 } 180 181 182 void X86_64Assembler::movl(const Address& dst, CpuRegister src) { 183 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 184 EmitOptionalRex32(src, dst); 185 EmitUint8(0x89); 186 EmitOperand(src.LowBits(), dst); 187 } 188 189 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) { 190 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 191 EmitOptionalRex32(dst); 192 EmitUint8(0xC7); 193 EmitOperand(0, dst); 194 EmitImmediate(imm); 195 } 196 197 198 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) { 199 cmov(c, dst, src, true); 200 } 201 202 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) { 203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 204 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 205 EmitUint8(0x0F); 206 EmitUint8(0x40 + c); 207 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 208 } 209 210 211 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) { 212 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 213 EmitOptionalByteRegNormalizingRex32(dst, src); 214 EmitUint8(0x0F); 215 EmitUint8(0xB6); 216 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 217 } 218 219 220 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) { 221 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 222 // Byte register is only in the source register form, so we don't use 223 // EmitOptionalByteRegNormalizingRex32(dst, src); 224 EmitOptionalRex32(dst, src); 225 EmitUint8(0x0F); 226 EmitUint8(0xB6); 227 EmitOperand(dst.LowBits(), src); 228 } 229 230 231 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) { 232 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 233 EmitOptionalByteRegNormalizingRex32(dst, src); 234 EmitUint8(0x0F); 235 EmitUint8(0xBE); 236 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 237 } 238 239 240 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) { 241 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 242 // Byte register is only in the source register form, so we don't use 243 // EmitOptionalByteRegNormalizingRex32(dst, src); 244 EmitOptionalRex32(dst, src); 245 EmitUint8(0x0F); 246 EmitUint8(0xBE); 247 EmitOperand(dst.LowBits(), src); 248 } 249 250 251 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) { 252 LOG(FATAL) << "Use movzxb or movsxb instead."; 253 } 254 255 256 void X86_64Assembler::movb(const Address& dst, CpuRegister src) { 257 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 258 EmitOptionalByteRegNormalizingRex32(src, dst); 259 EmitUint8(0x88); 260 EmitOperand(src.LowBits(), dst); 261 } 262 263 264 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) { 265 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 266 EmitOptionalRex32(dst); 267 EmitUint8(0xC6); 268 EmitOperand(Register::RAX, dst); 269 CHECK(imm.is_int8()); 270 EmitUint8(imm.value() & 0xFF); 271 } 272 273 274 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) { 275 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 276 EmitOptionalRex32(dst, src); 277 EmitUint8(0x0F); 278 EmitUint8(0xB7); 279 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 280 } 281 282 283 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) { 284 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 285 EmitOptionalRex32(dst, src); 286 EmitUint8(0x0F); 287 EmitUint8(0xB7); 288 EmitOperand(dst.LowBits(), src); 289 } 290 291 292 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) { 293 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 294 EmitOptionalRex32(dst, src); 295 EmitUint8(0x0F); 296 EmitUint8(0xBF); 297 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 298 } 299 300 301 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) { 302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 303 EmitOptionalRex32(dst, src); 304 EmitUint8(0x0F); 305 EmitUint8(0xBF); 306 EmitOperand(dst.LowBits(), src); 307 } 308 309 310 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) { 311 LOG(FATAL) << "Use movzxw or movsxw instead."; 312 } 313 314 315 void X86_64Assembler::movw(const Address& dst, CpuRegister src) { 316 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 317 EmitOperandSizeOverride(); 318 EmitOptionalRex32(src, dst); 319 EmitUint8(0x89); 320 EmitOperand(src.LowBits(), dst); 321 } 322 323 324 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) { 325 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 326 EmitOperandSizeOverride(); 327 EmitOptionalRex32(dst); 328 EmitUint8(0xC7); 329 EmitOperand(Register::RAX, dst); 330 CHECK(imm.is_uint16() || imm.is_int16()); 331 EmitUint8(imm.value() & 0xFF); 332 EmitUint8(imm.value() >> 8); 333 } 334 335 336 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) { 337 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 338 EmitRex64(dst, src); 339 EmitUint8(0x8D); 340 EmitOperand(dst.LowBits(), src); 341 } 342 343 344 void X86_64Assembler::leal(CpuRegister dst, const Address& src) { 345 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 346 EmitOptionalRex32(dst, src); 347 EmitUint8(0x8D); 348 EmitOperand(dst.LowBits(), src); 349 } 350 351 352 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) { 353 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 354 EmitOptionalRex32(dst, src); 355 EmitUint8(0x0F); 356 EmitUint8(0x28); 357 EmitXmmRegisterOperand(dst.LowBits(), src); 358 } 359 360 361 void X86_64Assembler::movss(XmmRegister dst, const Address& src) { 362 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 363 EmitUint8(0xF3); 364 EmitOptionalRex32(dst, src); 365 EmitUint8(0x0F); 366 EmitUint8(0x10); 367 EmitOperand(dst.LowBits(), src); 368 } 369 370 371 void X86_64Assembler::movss(const Address& dst, XmmRegister src) { 372 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 373 EmitUint8(0xF3); 374 EmitOptionalRex32(src, dst); 375 EmitUint8(0x0F); 376 EmitUint8(0x11); 377 EmitOperand(src.LowBits(), dst); 378 } 379 380 381 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) { 382 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 383 EmitUint8(0xF3); 384 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM. 385 EmitUint8(0x0F); 386 EmitUint8(0x11); 387 EmitXmmRegisterOperand(src.LowBits(), dst); 388 } 389 390 391 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) { 392 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 393 EmitRex64(dst, src); 394 EmitUint8(0x63); 395 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 396 } 397 398 399 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) { 400 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 401 EmitRex64(dst, src); 402 EmitUint8(0x63); 403 EmitOperand(dst.LowBits(), src); 404 } 405 406 407 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) { 408 movd(dst, src, true); 409 } 410 411 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) { 412 movd(dst, src, true); 413 } 414 415 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) { 416 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 417 EmitUint8(0x66); 418 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex()); 419 EmitUint8(0x0F); 420 EmitUint8(0x6E); 421 EmitOperand(dst.LowBits(), Operand(src)); 422 } 423 424 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) { 425 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 426 EmitUint8(0x66); 427 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex()); 428 EmitUint8(0x0F); 429 EmitUint8(0x7E); 430 EmitOperand(src.LowBits(), Operand(dst)); 431 } 432 433 434 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) { 435 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 436 EmitUint8(0xF3); 437 EmitOptionalRex32(dst, src); 438 EmitUint8(0x0F); 439 EmitUint8(0x58); 440 EmitXmmRegisterOperand(dst.LowBits(), src); 441 } 442 443 444 void X86_64Assembler::addss(XmmRegister dst, const Address& src) { 445 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 446 EmitUint8(0xF3); 447 EmitOptionalRex32(dst, src); 448 EmitUint8(0x0F); 449 EmitUint8(0x58); 450 EmitOperand(dst.LowBits(), src); 451 } 452 453 454 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) { 455 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 456 EmitUint8(0xF3); 457 EmitOptionalRex32(dst, src); 458 EmitUint8(0x0F); 459 EmitUint8(0x5C); 460 EmitXmmRegisterOperand(dst.LowBits(), src); 461 } 462 463 464 void X86_64Assembler::subss(XmmRegister dst, const Address& src) { 465 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 466 EmitUint8(0xF3); 467 EmitOptionalRex32(dst, src); 468 EmitUint8(0x0F); 469 EmitUint8(0x5C); 470 EmitOperand(dst.LowBits(), src); 471 } 472 473 474 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) { 475 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 476 EmitUint8(0xF3); 477 EmitOptionalRex32(dst, src); 478 EmitUint8(0x0F); 479 EmitUint8(0x59); 480 EmitXmmRegisterOperand(dst.LowBits(), src); 481 } 482 483 484 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) { 485 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 486 EmitUint8(0xF3); 487 EmitOptionalRex32(dst, src); 488 EmitUint8(0x0F); 489 EmitUint8(0x59); 490 EmitOperand(dst.LowBits(), src); 491 } 492 493 494 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) { 495 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 496 EmitUint8(0xF3); 497 EmitOptionalRex32(dst, src); 498 EmitUint8(0x0F); 499 EmitUint8(0x5E); 500 EmitXmmRegisterOperand(dst.LowBits(), src); 501 } 502 503 504 void X86_64Assembler::divss(XmmRegister dst, const Address& src) { 505 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 506 EmitUint8(0xF3); 507 EmitOptionalRex32(dst, src); 508 EmitUint8(0x0F); 509 EmitUint8(0x5E); 510 EmitOperand(dst.LowBits(), src); 511 } 512 513 514 void X86_64Assembler::flds(const Address& src) { 515 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 516 EmitUint8(0xD9); 517 EmitOperand(0, src); 518 } 519 520 521 void X86_64Assembler::fsts(const Address& dst) { 522 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 523 EmitUint8(0xD9); 524 EmitOperand(2, dst); 525 } 526 527 528 void X86_64Assembler::fstps(const Address& dst) { 529 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 530 EmitUint8(0xD9); 531 EmitOperand(3, dst); 532 } 533 534 535 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) { 536 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 537 EmitUint8(0xF2); 538 EmitOptionalRex32(dst, src); 539 EmitUint8(0x0F); 540 EmitUint8(0x10); 541 EmitOperand(dst.LowBits(), src); 542 } 543 544 545 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) { 546 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 547 EmitUint8(0xF2); 548 EmitOptionalRex32(src, dst); 549 EmitUint8(0x0F); 550 EmitUint8(0x11); 551 EmitOperand(src.LowBits(), dst); 552 } 553 554 555 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) { 556 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 557 EmitUint8(0xF2); 558 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM. 559 EmitUint8(0x0F); 560 EmitUint8(0x11); 561 EmitXmmRegisterOperand(src.LowBits(), dst); 562 } 563 564 565 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) { 566 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 567 EmitUint8(0xF2); 568 EmitOptionalRex32(dst, src); 569 EmitUint8(0x0F); 570 EmitUint8(0x58); 571 EmitXmmRegisterOperand(dst.LowBits(), src); 572 } 573 574 575 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) { 576 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 577 EmitUint8(0xF2); 578 EmitOptionalRex32(dst, src); 579 EmitUint8(0x0F); 580 EmitUint8(0x58); 581 EmitOperand(dst.LowBits(), src); 582 } 583 584 585 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) { 586 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 587 EmitUint8(0xF2); 588 EmitOptionalRex32(dst, src); 589 EmitUint8(0x0F); 590 EmitUint8(0x5C); 591 EmitXmmRegisterOperand(dst.LowBits(), src); 592 } 593 594 595 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) { 596 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 597 EmitUint8(0xF2); 598 EmitOptionalRex32(dst, src); 599 EmitUint8(0x0F); 600 EmitUint8(0x5C); 601 EmitOperand(dst.LowBits(), src); 602 } 603 604 605 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) { 606 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 607 EmitUint8(0xF2); 608 EmitOptionalRex32(dst, src); 609 EmitUint8(0x0F); 610 EmitUint8(0x59); 611 EmitXmmRegisterOperand(dst.LowBits(), src); 612 } 613 614 615 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) { 616 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 617 EmitUint8(0xF2); 618 EmitOptionalRex32(dst, src); 619 EmitUint8(0x0F); 620 EmitUint8(0x59); 621 EmitOperand(dst.LowBits(), src); 622 } 623 624 625 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) { 626 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 627 EmitUint8(0xF2); 628 EmitOptionalRex32(dst, src); 629 EmitUint8(0x0F); 630 EmitUint8(0x5E); 631 EmitXmmRegisterOperand(dst.LowBits(), src); 632 } 633 634 635 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) { 636 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 637 EmitUint8(0xF2); 638 EmitOptionalRex32(dst, src); 639 EmitUint8(0x0F); 640 EmitUint8(0x5E); 641 EmitOperand(dst.LowBits(), src); 642 } 643 644 645 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { 646 cvtsi2ss(dst, src, false); 647 } 648 649 650 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) { 651 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 652 EmitUint8(0xF3); 653 if (is64bit) { 654 // Emit a REX.W prefix if the operand size is 64 bits. 655 EmitRex64(dst, src); 656 } else { 657 EmitOptionalRex32(dst, src); 658 } 659 EmitUint8(0x0F); 660 EmitUint8(0x2A); 661 EmitOperand(dst.LowBits(), Operand(src)); 662 } 663 664 665 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) { 666 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 667 EmitUint8(0xF3); 668 if (is64bit) { 669 // Emit a REX.W prefix if the operand size is 64 bits. 670 EmitRex64(dst, src); 671 } else { 672 EmitOptionalRex32(dst, src); 673 } 674 EmitUint8(0x0F); 675 EmitUint8(0x2A); 676 EmitOperand(dst.LowBits(), src); 677 } 678 679 680 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) { 681 cvtsi2sd(dst, src, false); 682 } 683 684 685 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) { 686 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 687 EmitUint8(0xF2); 688 if (is64bit) { 689 // Emit a REX.W prefix if the operand size is 64 bits. 690 EmitRex64(dst, src); 691 } else { 692 EmitOptionalRex32(dst, src); 693 } 694 EmitUint8(0x0F); 695 EmitUint8(0x2A); 696 EmitOperand(dst.LowBits(), Operand(src)); 697 } 698 699 700 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) { 701 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 702 EmitUint8(0xF2); 703 if (is64bit) { 704 // Emit a REX.W prefix if the operand size is 64 bits. 705 EmitRex64(dst, src); 706 } else { 707 EmitOptionalRex32(dst, src); 708 } 709 EmitUint8(0x0F); 710 EmitUint8(0x2A); 711 EmitOperand(dst.LowBits(), src); 712 } 713 714 715 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) { 716 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 717 EmitUint8(0xF3); 718 EmitOptionalRex32(dst, src); 719 EmitUint8(0x0F); 720 EmitUint8(0x2D); 721 EmitXmmRegisterOperand(dst.LowBits(), src); 722 } 723 724 725 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) { 726 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 727 EmitUint8(0xF3); 728 EmitOptionalRex32(dst, src); 729 EmitUint8(0x0F); 730 EmitUint8(0x5A); 731 EmitXmmRegisterOperand(dst.LowBits(), src); 732 } 733 734 735 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) { 736 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 737 EmitUint8(0xF3); 738 EmitOptionalRex32(dst, src); 739 EmitUint8(0x0F); 740 EmitUint8(0x5A); 741 EmitOperand(dst.LowBits(), src); 742 } 743 744 745 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) { 746 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 747 EmitUint8(0xF2); 748 EmitOptionalRex32(dst, src); 749 EmitUint8(0x0F); 750 EmitUint8(0x2D); 751 EmitXmmRegisterOperand(dst.LowBits(), src); 752 } 753 754 755 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) { 756 cvttss2si(dst, src, false); 757 } 758 759 760 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) { 761 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 762 EmitUint8(0xF3); 763 if (is64bit) { 764 // Emit a REX.W prefix if the operand size is 64 bits. 765 EmitRex64(dst, src); 766 } else { 767 EmitOptionalRex32(dst, src); 768 } 769 EmitUint8(0x0F); 770 EmitUint8(0x2C); 771 EmitXmmRegisterOperand(dst.LowBits(), src); 772 } 773 774 775 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) { 776 cvttsd2si(dst, src, false); 777 } 778 779 780 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) { 781 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 782 EmitUint8(0xF2); 783 if (is64bit) { 784 // Emit a REX.W prefix if the operand size is 64 bits. 785 EmitRex64(dst, src); 786 } else { 787 EmitOptionalRex32(dst, src); 788 } 789 EmitUint8(0x0F); 790 EmitUint8(0x2C); 791 EmitXmmRegisterOperand(dst.LowBits(), src); 792 } 793 794 795 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) { 796 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 797 EmitUint8(0xF2); 798 EmitOptionalRex32(dst, src); 799 EmitUint8(0x0F); 800 EmitUint8(0x5A); 801 EmitXmmRegisterOperand(dst.LowBits(), src); 802 } 803 804 805 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) { 806 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 807 EmitUint8(0xF2); 808 EmitOptionalRex32(dst, src); 809 EmitUint8(0x0F); 810 EmitUint8(0x5A); 811 EmitOperand(dst.LowBits(), src); 812 } 813 814 815 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) { 816 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 817 EmitUint8(0xF3); 818 EmitOptionalRex32(dst, src); 819 EmitUint8(0x0F); 820 EmitUint8(0xE6); 821 EmitXmmRegisterOperand(dst.LowBits(), src); 822 } 823 824 825 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) { 826 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 827 EmitOptionalRex32(a, b); 828 EmitUint8(0x0F); 829 EmitUint8(0x2F); 830 EmitXmmRegisterOperand(a.LowBits(), b); 831 } 832 833 834 void X86_64Assembler::comiss(XmmRegister a, const Address& b) { 835 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 836 EmitOptionalRex32(a, b); 837 EmitUint8(0x0F); 838 EmitUint8(0x2F); 839 EmitOperand(a.LowBits(), b); 840 } 841 842 843 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) { 844 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 845 EmitUint8(0x66); 846 EmitOptionalRex32(a, b); 847 EmitUint8(0x0F); 848 EmitUint8(0x2F); 849 EmitXmmRegisterOperand(a.LowBits(), b); 850 } 851 852 853 void X86_64Assembler::comisd(XmmRegister a, const Address& b) { 854 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 855 EmitUint8(0x66); 856 EmitOptionalRex32(a, b); 857 EmitUint8(0x0F); 858 EmitUint8(0x2F); 859 EmitOperand(a.LowBits(), b); 860 } 861 862 863 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) { 864 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 865 EmitOptionalRex32(a, b); 866 EmitUint8(0x0F); 867 EmitUint8(0x2E); 868 EmitXmmRegisterOperand(a.LowBits(), b); 869 } 870 871 872 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) { 873 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 874 EmitOptionalRex32(a, b); 875 EmitUint8(0x0F); 876 EmitUint8(0x2E); 877 EmitOperand(a.LowBits(), b); 878 } 879 880 881 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) { 882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 883 EmitUint8(0x66); 884 EmitOptionalRex32(a, b); 885 EmitUint8(0x0F); 886 EmitUint8(0x2E); 887 EmitXmmRegisterOperand(a.LowBits(), b); 888 } 889 890 891 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) { 892 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 893 EmitUint8(0x66); 894 EmitOptionalRex32(a, b); 895 EmitUint8(0x0F); 896 EmitUint8(0x2E); 897 EmitOperand(a.LowBits(), b); 898 } 899 900 901 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) { 902 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 903 EmitUint8(0x66); 904 EmitOptionalRex32(dst, src); 905 EmitUint8(0x0F); 906 EmitUint8(0x3A); 907 EmitUint8(0x0B); 908 EmitXmmRegisterOperand(dst.LowBits(), src); 909 EmitUint8(imm.value()); 910 } 911 912 913 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) { 914 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 915 EmitUint8(0x66); 916 EmitOptionalRex32(dst, src); 917 EmitUint8(0x0F); 918 EmitUint8(0x3A); 919 EmitUint8(0x0A); 920 EmitXmmRegisterOperand(dst.LowBits(), src); 921 EmitUint8(imm.value()); 922 } 923 924 925 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) { 926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 927 EmitUint8(0xF2); 928 EmitOptionalRex32(dst, src); 929 EmitUint8(0x0F); 930 EmitUint8(0x51); 931 EmitXmmRegisterOperand(dst.LowBits(), src); 932 } 933 934 935 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) { 936 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 937 EmitUint8(0xF3); 938 EmitOptionalRex32(dst, src); 939 EmitUint8(0x0F); 940 EmitUint8(0x51); 941 EmitXmmRegisterOperand(dst.LowBits(), src); 942 } 943 944 945 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) { 946 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 947 EmitUint8(0x66); 948 EmitOptionalRex32(dst, src); 949 EmitUint8(0x0F); 950 EmitUint8(0x57); 951 EmitOperand(dst.LowBits(), src); 952 } 953 954 955 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) { 956 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 957 EmitUint8(0x66); 958 EmitOptionalRex32(dst, src); 959 EmitUint8(0x0F); 960 EmitUint8(0x57); 961 EmitXmmRegisterOperand(dst.LowBits(), src); 962 } 963 964 965 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) { 966 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 967 EmitOptionalRex32(dst, src); 968 EmitUint8(0x0F); 969 EmitUint8(0x57); 970 EmitOperand(dst.LowBits(), src); 971 } 972 973 974 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) { 975 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 976 EmitOptionalRex32(dst, src); 977 EmitUint8(0x0F); 978 EmitUint8(0x57); 979 EmitXmmRegisterOperand(dst.LowBits(), src); 980 } 981 982 983 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { 984 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 985 EmitUint8(0x66); 986 EmitOptionalRex32(dst, src); 987 EmitUint8(0x0F); 988 EmitUint8(0x54); 989 EmitOperand(dst.LowBits(), src); 990 } 991 992 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) { 993 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 994 EmitUint8(0x66); 995 EmitOptionalRex32(dst, src); 996 EmitUint8(0x0F); 997 EmitUint8(0x54); 998 EmitXmmRegisterOperand(dst.LowBits(), src); 999 } 1000 1001 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) { 1002 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1003 EmitOptionalRex32(dst, src); 1004 EmitUint8(0x0F); 1005 EmitUint8(0x54); 1006 EmitXmmRegisterOperand(dst.LowBits(), src); 1007 } 1008 1009 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { 1010 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1011 EmitUint8(0x66); 1012 EmitOptionalRex32(dst, src); 1013 EmitUint8(0x0F); 1014 EmitUint8(0x56); 1015 EmitXmmRegisterOperand(dst.LowBits(), src); 1016 } 1017 1018 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) { 1019 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1020 EmitOptionalRex32(dst, src); 1021 EmitUint8(0x0F); 1022 EmitUint8(0x56); 1023 EmitXmmRegisterOperand(dst.LowBits(), src); 1024 } 1025 1026 void X86_64Assembler::fldl(const Address& src) { 1027 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1028 EmitUint8(0xDD); 1029 EmitOperand(0, src); 1030 } 1031 1032 1033 void X86_64Assembler::fstl(const Address& dst) { 1034 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1035 EmitUint8(0xDD); 1036 EmitOperand(2, dst); 1037 } 1038 1039 1040 void X86_64Assembler::fstpl(const Address& dst) { 1041 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1042 EmitUint8(0xDD); 1043 EmitOperand(3, dst); 1044 } 1045 1046 1047 void X86_64Assembler::fstsw() { 1048 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1049 EmitUint8(0x9B); 1050 EmitUint8(0xDF); 1051 EmitUint8(0xE0); 1052 } 1053 1054 1055 void X86_64Assembler::fnstcw(const Address& dst) { 1056 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1057 EmitUint8(0xD9); 1058 EmitOperand(7, dst); 1059 } 1060 1061 1062 void X86_64Assembler::fldcw(const Address& src) { 1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1064 EmitUint8(0xD9); 1065 EmitOperand(5, src); 1066 } 1067 1068 1069 void X86_64Assembler::fistpl(const Address& dst) { 1070 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1071 EmitUint8(0xDF); 1072 EmitOperand(7, dst); 1073 } 1074 1075 1076 void X86_64Assembler::fistps(const Address& dst) { 1077 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1078 EmitUint8(0xDB); 1079 EmitOperand(3, dst); 1080 } 1081 1082 1083 void X86_64Assembler::fildl(const Address& src) { 1084 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1085 EmitUint8(0xDF); 1086 EmitOperand(5, src); 1087 } 1088 1089 1090 void X86_64Assembler::filds(const Address& src) { 1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1092 EmitUint8(0xDB); 1093 EmitOperand(0, src); 1094 } 1095 1096 1097 void X86_64Assembler::fincstp() { 1098 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1099 EmitUint8(0xD9); 1100 EmitUint8(0xF7); 1101 } 1102 1103 1104 void X86_64Assembler::ffree(const Immediate& index) { 1105 CHECK_LT(index.value(), 7); 1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1107 EmitUint8(0xDD); 1108 EmitUint8(0xC0 + index.value()); 1109 } 1110 1111 1112 void X86_64Assembler::fsin() { 1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1114 EmitUint8(0xD9); 1115 EmitUint8(0xFE); 1116 } 1117 1118 1119 void X86_64Assembler::fcos() { 1120 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1121 EmitUint8(0xD9); 1122 EmitUint8(0xFF); 1123 } 1124 1125 1126 void X86_64Assembler::fptan() { 1127 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1128 EmitUint8(0xD9); 1129 EmitUint8(0xF2); 1130 } 1131 1132 void X86_64Assembler::fucompp() { 1133 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1134 EmitUint8(0xDA); 1135 EmitUint8(0xE9); 1136 } 1137 1138 1139 void X86_64Assembler::fprem() { 1140 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1141 EmitUint8(0xD9); 1142 EmitUint8(0xF8); 1143 } 1144 1145 1146 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) { 1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1148 // There is a short version for rax. 1149 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1150 // work. 1151 const bool src_rax = src.AsRegister() == RAX; 1152 const bool dst_rax = dst.AsRegister() == RAX; 1153 if (src_rax || dst_rax) { 1154 EmitOptionalRex32(src_rax ? dst : src); 1155 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1156 return; 1157 } 1158 1159 // General case. 1160 EmitOptionalRex32(src, dst); 1161 EmitUint8(0x87); 1162 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1163 } 1164 1165 1166 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) { 1167 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1168 // There is a short version for rax. 1169 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't 1170 // work. 1171 const bool src_rax = src.AsRegister() == RAX; 1172 const bool dst_rax = dst.AsRegister() == RAX; 1173 if (src_rax || dst_rax) { 1174 // If src == target, emit a nop instead. 1175 if (src_rax && dst_rax) { 1176 EmitUint8(0x90); 1177 } else { 1178 EmitRex64(src_rax ? dst : src); 1179 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits())); 1180 } 1181 return; 1182 } 1183 1184 // General case. 1185 EmitRex64(src, dst); 1186 EmitUint8(0x87); 1187 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1188 } 1189 1190 1191 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) { 1192 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1193 EmitOptionalRex32(reg, address); 1194 EmitUint8(0x87); 1195 EmitOperand(reg.LowBits(), address); 1196 } 1197 1198 1199 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) { 1200 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1201 EmitOptionalRex32(address); 1202 EmitUint8(0x66); 1203 EmitComplex(7, address, imm); 1204 } 1205 1206 1207 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) { 1208 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1209 EmitOptionalRex32(reg); 1210 EmitComplex(7, Operand(reg), imm); 1211 } 1212 1213 1214 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) { 1215 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1216 EmitOptionalRex32(reg0, reg1); 1217 EmitUint8(0x3B); 1218 EmitOperand(reg0.LowBits(), Operand(reg1)); 1219 } 1220 1221 1222 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) { 1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1224 EmitOptionalRex32(reg, address); 1225 EmitUint8(0x3B); 1226 EmitOperand(reg.LowBits(), address); 1227 } 1228 1229 1230 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) { 1231 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1232 EmitOptionalRex32(reg, address); 1233 EmitUint8(0x39); 1234 EmitOperand(reg.LowBits(), address); 1235 } 1236 1237 1238 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) { 1239 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1240 EmitOptionalRex32(address); 1241 EmitComplex(7, address, imm); 1242 } 1243 1244 1245 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) { 1246 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1247 EmitRex64(reg0, reg1); 1248 EmitUint8(0x3B); 1249 EmitOperand(reg0.LowBits(), Operand(reg1)); 1250 } 1251 1252 1253 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) { 1254 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1255 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1256 EmitRex64(reg); 1257 EmitComplex(7, Operand(reg), imm); 1258 } 1259 1260 1261 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) { 1262 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1263 EmitRex64(reg, address); 1264 EmitUint8(0x3B); 1265 EmitOperand(reg.LowBits(), address); 1266 } 1267 1268 1269 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) { 1270 CHECK(imm.is_int32()); // cmpq only supports 32b immediate. 1271 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1272 EmitRex64(address); 1273 EmitComplex(7, address, imm); 1274 } 1275 1276 1277 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) { 1278 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1279 EmitOptionalRex32(dst, src); 1280 EmitUint8(0x03); 1281 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1282 } 1283 1284 1285 void X86_64Assembler::addl(CpuRegister reg, const Address& address) { 1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1287 EmitOptionalRex32(reg, address); 1288 EmitUint8(0x03); 1289 EmitOperand(reg.LowBits(), address); 1290 } 1291 1292 1293 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) { 1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1295 EmitOptionalRex32(reg1, reg2); 1296 EmitUint8(0x85); 1297 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1298 } 1299 1300 1301 void X86_64Assembler::testl(CpuRegister reg, const Address& address) { 1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1303 EmitOptionalRex32(reg, address); 1304 EmitUint8(0x85); 1305 EmitOperand(reg.LowBits(), address); 1306 } 1307 1308 1309 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) { 1310 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1311 // For registers that have a byte variant (RAX, RBX, RCX, and RDX) 1312 // we only test the byte CpuRegister to keep the encoding short. 1313 if (immediate.is_uint8() && reg.AsRegister() < 4) { 1314 // Use zero-extended 8-bit immediate. 1315 if (reg.AsRegister() == RAX) { 1316 EmitUint8(0xA8); 1317 } else { 1318 EmitUint8(0xF6); 1319 EmitUint8(0xC0 + reg.AsRegister()); 1320 } 1321 EmitUint8(immediate.value() & 0xFF); 1322 } else if (reg.AsRegister() == RAX) { 1323 // Use short form if the destination is RAX. 1324 EmitUint8(0xA9); 1325 EmitImmediate(immediate); 1326 } else { 1327 EmitOptionalRex32(reg); 1328 EmitUint8(0xF7); 1329 EmitOperand(0, Operand(reg)); 1330 EmitImmediate(immediate); 1331 } 1332 } 1333 1334 1335 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) { 1336 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1337 EmitRex64(reg1, reg2); 1338 EmitUint8(0x85); 1339 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits()); 1340 } 1341 1342 1343 void X86_64Assembler::testq(CpuRegister reg, const Address& address) { 1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1345 EmitRex64(reg, address); 1346 EmitUint8(0x85); 1347 EmitOperand(reg.LowBits(), address); 1348 } 1349 1350 1351 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) { 1352 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1353 EmitOptionalRex32(dst, src); 1354 EmitUint8(0x23); 1355 EmitOperand(dst.LowBits(), Operand(src)); 1356 } 1357 1358 1359 void X86_64Assembler::andl(CpuRegister reg, const Address& address) { 1360 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1361 EmitOptionalRex32(reg, address); 1362 EmitUint8(0x23); 1363 EmitOperand(reg.LowBits(), address); 1364 } 1365 1366 1367 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) { 1368 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1369 EmitOptionalRex32(dst); 1370 EmitComplex(4, Operand(dst), imm); 1371 } 1372 1373 1374 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) { 1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1376 CHECK(imm.is_int32()); // andq only supports 32b immediate. 1377 EmitRex64(reg); 1378 EmitComplex(4, Operand(reg), imm); 1379 } 1380 1381 1382 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) { 1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1384 EmitRex64(dst, src); 1385 EmitUint8(0x23); 1386 EmitOperand(dst.LowBits(), Operand(src)); 1387 } 1388 1389 1390 void X86_64Assembler::andq(CpuRegister dst, const Address& src) { 1391 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1392 EmitRex64(dst, src); 1393 EmitUint8(0x23); 1394 EmitOperand(dst.LowBits(), src); 1395 } 1396 1397 1398 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) { 1399 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1400 EmitOptionalRex32(dst, src); 1401 EmitUint8(0x0B); 1402 EmitOperand(dst.LowBits(), Operand(src)); 1403 } 1404 1405 1406 void X86_64Assembler::orl(CpuRegister reg, const Address& address) { 1407 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1408 EmitOptionalRex32(reg, address); 1409 EmitUint8(0x0B); 1410 EmitOperand(reg.LowBits(), address); 1411 } 1412 1413 1414 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) { 1415 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1416 EmitOptionalRex32(dst); 1417 EmitComplex(1, Operand(dst), imm); 1418 } 1419 1420 1421 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) { 1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1423 CHECK(imm.is_int32()); // orq only supports 32b immediate. 1424 EmitRex64(dst); 1425 EmitComplex(1, Operand(dst), imm); 1426 } 1427 1428 1429 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) { 1430 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1431 EmitRex64(dst, src); 1432 EmitUint8(0x0B); 1433 EmitOperand(dst.LowBits(), Operand(src)); 1434 } 1435 1436 1437 void X86_64Assembler::orq(CpuRegister dst, const Address& src) { 1438 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1439 EmitRex64(dst, src); 1440 EmitUint8(0x0B); 1441 EmitOperand(dst.LowBits(), src); 1442 } 1443 1444 1445 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) { 1446 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1447 EmitOptionalRex32(dst, src); 1448 EmitUint8(0x33); 1449 EmitOperand(dst.LowBits(), Operand(src)); 1450 } 1451 1452 1453 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) { 1454 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1455 EmitOptionalRex32(reg, address); 1456 EmitUint8(0x33); 1457 EmitOperand(reg.LowBits(), address); 1458 } 1459 1460 1461 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) { 1462 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1463 EmitOptionalRex32(dst); 1464 EmitComplex(6, Operand(dst), imm); 1465 } 1466 1467 1468 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) { 1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1470 EmitRex64(dst, src); 1471 EmitUint8(0x33); 1472 EmitOperand(dst.LowBits(), Operand(src)); 1473 } 1474 1475 1476 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) { 1477 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1478 CHECK(imm.is_int32()); // xorq only supports 32b immediate. 1479 EmitRex64(dst); 1480 EmitComplex(6, Operand(dst), imm); 1481 } 1482 1483 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) { 1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1485 EmitRex64(dst, src); 1486 EmitUint8(0x33); 1487 EmitOperand(dst.LowBits(), src); 1488 } 1489 1490 1491 #if 0 1492 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) { 1493 // REX.WRXB 1494 // W - 64-bit operand 1495 // R - MODRM.reg 1496 // X - SIB.index 1497 // B - MODRM.rm/SIB.base 1498 uint8_t rex = force ? 0x40 : 0; 1499 if (w) { 1500 rex |= 0x48; // REX.W000 1501 } 1502 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) { 1503 rex |= 0x44; // REX.0R00 1504 *r = static_cast<Register>(*r - 8); 1505 } 1506 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) { 1507 rex |= 0x42; // REX.00X0 1508 *x = static_cast<Register>(*x - 8); 1509 } 1510 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) { 1511 rex |= 0x41; // REX.000B 1512 *b = static_cast<Register>(*b - 8); 1513 } 1514 if (rex != 0) { 1515 EmitUint8(rex); 1516 } 1517 } 1518 1519 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) { 1520 // REX.WRXB 1521 // W - 64-bit operand 1522 // R - MODRM.reg 1523 // X - SIB.index 1524 // B - MODRM.rm/SIB.base 1525 uint8_t rex = mem->rex(); 1526 if (force) { 1527 rex |= 0x40; // REX.0000 1528 } 1529 if (w) { 1530 rex |= 0x48; // REX.W000 1531 } 1532 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) { 1533 rex |= 0x44; // REX.0R00 1534 *dst = static_cast<Register>(*dst - 8); 1535 } 1536 if (rex != 0) { 1537 EmitUint8(rex); 1538 } 1539 } 1540 1541 void rex_mem_reg(bool force, bool w, Address* mem, Register* src); 1542 #endif 1543 1544 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) { 1545 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1546 EmitOptionalRex32(reg); 1547 EmitComplex(0, Operand(reg), imm); 1548 } 1549 1550 1551 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) { 1552 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1553 CHECK(imm.is_int32()); // addq only supports 32b immediate. 1554 EmitRex64(reg); 1555 EmitComplex(0, Operand(reg), imm); 1556 } 1557 1558 1559 void X86_64Assembler::addq(CpuRegister dst, const Address& address) { 1560 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1561 EmitRex64(dst, address); 1562 EmitUint8(0x03); 1563 EmitOperand(dst.LowBits(), address); 1564 } 1565 1566 1567 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) { 1568 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1569 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64 1570 EmitRex64(src, dst); 1571 EmitUint8(0x01); 1572 EmitRegisterOperand(src.LowBits(), dst.LowBits()); 1573 } 1574 1575 1576 void X86_64Assembler::addl(const Address& address, CpuRegister reg) { 1577 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1578 EmitOptionalRex32(reg, address); 1579 EmitUint8(0x01); 1580 EmitOperand(reg.LowBits(), address); 1581 } 1582 1583 1584 void X86_64Assembler::addl(const Address& address, const Immediate& imm) { 1585 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1586 EmitOptionalRex32(address); 1587 EmitComplex(0, address, imm); 1588 } 1589 1590 1591 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) { 1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1593 EmitOptionalRex32(dst, src); 1594 EmitUint8(0x2B); 1595 EmitOperand(dst.LowBits(), Operand(src)); 1596 } 1597 1598 1599 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) { 1600 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1601 EmitOptionalRex32(reg); 1602 EmitComplex(5, Operand(reg), imm); 1603 } 1604 1605 1606 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) { 1607 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1608 CHECK(imm.is_int32()); // subq only supports 32b immediate. 1609 EmitRex64(reg); 1610 EmitComplex(5, Operand(reg), imm); 1611 } 1612 1613 1614 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) { 1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1616 EmitRex64(dst, src); 1617 EmitUint8(0x2B); 1618 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1619 } 1620 1621 1622 void X86_64Assembler::subq(CpuRegister reg, const Address& address) { 1623 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1624 EmitRex64(reg, address); 1625 EmitUint8(0x2B); 1626 EmitOperand(reg.LowBits() & 7, address); 1627 } 1628 1629 1630 void X86_64Assembler::subl(CpuRegister reg, const Address& address) { 1631 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1632 EmitOptionalRex32(reg, address); 1633 EmitUint8(0x2B); 1634 EmitOperand(reg.LowBits(), address); 1635 } 1636 1637 1638 void X86_64Assembler::cdq() { 1639 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1640 EmitUint8(0x99); 1641 } 1642 1643 1644 void X86_64Assembler::cqo() { 1645 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1646 EmitRex64(); 1647 EmitUint8(0x99); 1648 } 1649 1650 1651 void X86_64Assembler::idivl(CpuRegister reg) { 1652 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1653 EmitOptionalRex32(reg); 1654 EmitUint8(0xF7); 1655 EmitUint8(0xF8 | reg.LowBits()); 1656 } 1657 1658 1659 void X86_64Assembler::idivq(CpuRegister reg) { 1660 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1661 EmitRex64(reg); 1662 EmitUint8(0xF7); 1663 EmitUint8(0xF8 | reg.LowBits()); 1664 } 1665 1666 1667 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) { 1668 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1669 EmitOptionalRex32(dst, src); 1670 EmitUint8(0x0F); 1671 EmitUint8(0xAF); 1672 EmitOperand(dst.LowBits(), Operand(src)); 1673 } 1674 1675 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) { 1676 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1677 CHECK(imm.is_int32()); // imull only supports 32b immediate. 1678 1679 EmitOptionalRex32(reg, reg); 1680 1681 // See whether imm can be represented as a sign-extended 8bit value. 1682 int32_t v32 = static_cast<int32_t>(imm.value()); 1683 if (IsInt<8>(v32)) { 1684 // Sign-extension works. 1685 EmitUint8(0x6B); 1686 EmitOperand(reg.LowBits(), Operand(reg)); 1687 EmitUint8(static_cast<uint8_t>(v32 & 0xFF)); 1688 } else { 1689 // Not representable, use full immediate. 1690 EmitUint8(0x69); 1691 EmitOperand(reg.LowBits(), Operand(reg)); 1692 EmitImmediate(imm); 1693 } 1694 } 1695 1696 1697 void X86_64Assembler::imull(CpuRegister reg, const Address& address) { 1698 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1699 EmitOptionalRex32(reg, address); 1700 EmitUint8(0x0F); 1701 EmitUint8(0xAF); 1702 EmitOperand(reg.LowBits(), address); 1703 } 1704 1705 1706 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) { 1707 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1708 EmitRex64(dst, src); 1709 EmitUint8(0x0F); 1710 EmitUint8(0xAF); 1711 EmitRegisterOperand(dst.LowBits(), src.LowBits()); 1712 } 1713 1714 1715 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) { 1716 imulq(reg, reg, imm); 1717 } 1718 1719 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) { 1720 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1721 CHECK(imm.is_int32()); // imulq only supports 32b immediate. 1722 1723 EmitRex64(dst, reg); 1724 1725 // See whether imm can be represented as a sign-extended 8bit value. 1726 int64_t v64 = imm.value(); 1727 if (IsInt<8>(v64)) { 1728 // Sign-extension works. 1729 EmitUint8(0x6B); 1730 EmitOperand(dst.LowBits(), Operand(reg)); 1731 EmitUint8(static_cast<uint8_t>(v64 & 0xFF)); 1732 } else { 1733 // Not representable, use full immediate. 1734 EmitUint8(0x69); 1735 EmitOperand(dst.LowBits(), Operand(reg)); 1736 EmitImmediate(imm); 1737 } 1738 } 1739 1740 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) { 1741 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1742 EmitRex64(reg, address); 1743 EmitUint8(0x0F); 1744 EmitUint8(0xAF); 1745 EmitOperand(reg.LowBits(), address); 1746 } 1747 1748 1749 void X86_64Assembler::imull(CpuRegister reg) { 1750 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1751 EmitOptionalRex32(reg); 1752 EmitUint8(0xF7); 1753 EmitOperand(5, Operand(reg)); 1754 } 1755 1756 1757 void X86_64Assembler::imulq(CpuRegister reg) { 1758 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1759 EmitRex64(reg); 1760 EmitUint8(0xF7); 1761 EmitOperand(5, Operand(reg)); 1762 } 1763 1764 1765 void X86_64Assembler::imull(const Address& address) { 1766 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1767 EmitOptionalRex32(address); 1768 EmitUint8(0xF7); 1769 EmitOperand(5, address); 1770 } 1771 1772 1773 void X86_64Assembler::mull(CpuRegister reg) { 1774 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1775 EmitOptionalRex32(reg); 1776 EmitUint8(0xF7); 1777 EmitOperand(4, Operand(reg)); 1778 } 1779 1780 1781 void X86_64Assembler::mull(const Address& address) { 1782 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1783 EmitOptionalRex32(address); 1784 EmitUint8(0xF7); 1785 EmitOperand(4, address); 1786 } 1787 1788 1789 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) { 1790 EmitGenericShift(false, 4, reg, imm); 1791 } 1792 1793 1794 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) { 1795 EmitGenericShift(true, 4, reg, imm); 1796 } 1797 1798 1799 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) { 1800 EmitGenericShift(false, 4, operand, shifter); 1801 } 1802 1803 1804 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) { 1805 EmitGenericShift(true, 4, operand, shifter); 1806 } 1807 1808 1809 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) { 1810 EmitGenericShift(false, 5, reg, imm); 1811 } 1812 1813 1814 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) { 1815 EmitGenericShift(true, 5, reg, imm); 1816 } 1817 1818 1819 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) { 1820 EmitGenericShift(false, 5, operand, shifter); 1821 } 1822 1823 1824 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) { 1825 EmitGenericShift(true, 5, operand, shifter); 1826 } 1827 1828 1829 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) { 1830 EmitGenericShift(false, 7, reg, imm); 1831 } 1832 1833 1834 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) { 1835 EmitGenericShift(false, 7, operand, shifter); 1836 } 1837 1838 1839 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) { 1840 EmitGenericShift(true, 7, reg, imm); 1841 } 1842 1843 1844 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) { 1845 EmitGenericShift(true, 7, operand, shifter); 1846 } 1847 1848 1849 void X86_64Assembler::negl(CpuRegister reg) { 1850 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1851 EmitOptionalRex32(reg); 1852 EmitUint8(0xF7); 1853 EmitOperand(3, Operand(reg)); 1854 } 1855 1856 1857 void X86_64Assembler::negq(CpuRegister reg) { 1858 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1859 EmitRex64(reg); 1860 EmitUint8(0xF7); 1861 EmitOperand(3, Operand(reg)); 1862 } 1863 1864 1865 void X86_64Assembler::notl(CpuRegister reg) { 1866 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1867 EmitOptionalRex32(reg); 1868 EmitUint8(0xF7); 1869 EmitUint8(0xD0 | reg.LowBits()); 1870 } 1871 1872 1873 void X86_64Assembler::notq(CpuRegister reg) { 1874 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1875 EmitRex64(reg); 1876 EmitUint8(0xF7); 1877 EmitOperand(2, Operand(reg)); 1878 } 1879 1880 1881 void X86_64Assembler::enter(const Immediate& imm) { 1882 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1883 EmitUint8(0xC8); 1884 CHECK(imm.is_uint16()) << imm.value(); 1885 EmitUint8(imm.value() & 0xFF); 1886 EmitUint8((imm.value() >> 8) & 0xFF); 1887 EmitUint8(0x00); 1888 } 1889 1890 1891 void X86_64Assembler::leave() { 1892 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1893 EmitUint8(0xC9); 1894 } 1895 1896 1897 void X86_64Assembler::ret() { 1898 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1899 EmitUint8(0xC3); 1900 } 1901 1902 1903 void X86_64Assembler::ret(const Immediate& imm) { 1904 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1905 EmitUint8(0xC2); 1906 CHECK(imm.is_uint16()); 1907 EmitUint8(imm.value() & 0xFF); 1908 EmitUint8((imm.value() >> 8) & 0xFF); 1909 } 1910 1911 1912 1913 void X86_64Assembler::nop() { 1914 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1915 EmitUint8(0x90); 1916 } 1917 1918 1919 void X86_64Assembler::int3() { 1920 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1921 EmitUint8(0xCC); 1922 } 1923 1924 1925 void X86_64Assembler::hlt() { 1926 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1927 EmitUint8(0xF4); 1928 } 1929 1930 1931 void X86_64Assembler::j(Condition condition, Label* label) { 1932 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1933 if (label->IsBound()) { 1934 static const int kShortSize = 2; 1935 static const int kLongSize = 6; 1936 int offset = label->Position() - buffer_.Size(); 1937 CHECK_LE(offset, 0); 1938 if (IsInt<8>(offset - kShortSize)) { 1939 EmitUint8(0x70 + condition); 1940 EmitUint8((offset - kShortSize) & 0xFF); 1941 } else { 1942 EmitUint8(0x0F); 1943 EmitUint8(0x80 + condition); 1944 EmitInt32(offset - kLongSize); 1945 } 1946 } else { 1947 EmitUint8(0x0F); 1948 EmitUint8(0x80 + condition); 1949 EmitLabelLink(label); 1950 } 1951 } 1952 1953 1954 void X86_64Assembler::jmp(CpuRegister reg) { 1955 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1956 EmitOptionalRex32(reg); 1957 EmitUint8(0xFF); 1958 EmitRegisterOperand(4, reg.LowBits()); 1959 } 1960 1961 void X86_64Assembler::jmp(const Address& address) { 1962 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1963 EmitOptionalRex32(address); 1964 EmitUint8(0xFF); 1965 EmitOperand(4, address); 1966 } 1967 1968 void X86_64Assembler::jmp(Label* label) { 1969 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1970 if (label->IsBound()) { 1971 static const int kShortSize = 2; 1972 static const int kLongSize = 5; 1973 int offset = label->Position() - buffer_.Size(); 1974 CHECK_LE(offset, 0); 1975 if (IsInt<8>(offset - kShortSize)) { 1976 EmitUint8(0xEB); 1977 EmitUint8((offset - kShortSize) & 0xFF); 1978 } else { 1979 EmitUint8(0xE9); 1980 EmitInt32(offset - kLongSize); 1981 } 1982 } else { 1983 EmitUint8(0xE9); 1984 EmitLabelLink(label); 1985 } 1986 } 1987 1988 1989 X86_64Assembler* X86_64Assembler::lock() { 1990 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1991 EmitUint8(0xF0); 1992 return this; 1993 } 1994 1995 1996 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) { 1997 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 1998 EmitOptionalRex32(reg, address); 1999 EmitUint8(0x0F); 2000 EmitUint8(0xB1); 2001 EmitOperand(reg.LowBits(), address); 2002 } 2003 2004 2005 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) { 2006 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2007 EmitRex64(reg, address); 2008 EmitUint8(0x0F); 2009 EmitUint8(0xB1); 2010 EmitOperand(reg.LowBits(), address); 2011 } 2012 2013 2014 void X86_64Assembler::mfence() { 2015 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2016 EmitUint8(0x0F); 2017 EmitUint8(0xAE); 2018 EmitUint8(0xF0); 2019 } 2020 2021 2022 X86_64Assembler* X86_64Assembler::gs() { 2023 // TODO: gs is a prefix and not an instruction 2024 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2025 EmitUint8(0x65); 2026 return this; 2027 } 2028 2029 2030 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) { 2031 int value = imm.value(); 2032 if (value != 0) { 2033 if (value > 0) { 2034 addl(reg, imm); 2035 } else { 2036 subl(reg, Immediate(value)); 2037 } 2038 } 2039 } 2040 2041 2042 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) { 2043 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2044 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh). 2045 if (dst.NeedsRex() || dst.AsRegister() > 3) { 2046 EmitOptionalRex(true, false, false, false, dst.NeedsRex()); 2047 } 2048 EmitUint8(0x0F); 2049 EmitUint8(0x90 + condition); 2050 EmitUint8(0xC0 + dst.LowBits()); 2051 } 2052 2053 void X86_64Assembler::bswapl(CpuRegister dst) { 2054 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2055 EmitOptionalRex(false, false, false, false, dst.NeedsRex()); 2056 EmitUint8(0x0F); 2057 EmitUint8(0xC8 + dst.LowBits()); 2058 } 2059 2060 void X86_64Assembler::bswapq(CpuRegister dst) { 2061 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2062 EmitOptionalRex(false, true, false, false, dst.NeedsRex()); 2063 EmitUint8(0x0F); 2064 EmitUint8(0xC8 + dst.LowBits()); 2065 } 2066 2067 2068 void X86_64Assembler::repne_scasw() { 2069 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2070 EmitUint8(0x66); 2071 EmitUint8(0xF2); 2072 EmitUint8(0xAF); 2073 } 2074 2075 2076 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) { 2077 // TODO: Need to have a code constants table. 2078 int64_t constant = bit_cast<int64_t, double>(value); 2079 pushq(Immediate(High32Bits(constant))); 2080 pushq(Immediate(Low32Bits(constant))); 2081 movsd(dst, Address(CpuRegister(RSP), 0)); 2082 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t))); 2083 } 2084 2085 2086 void X86_64Assembler::Align(int alignment, int offset) { 2087 CHECK(IsPowerOfTwo(alignment)); 2088 // Emit nop instruction until the real position is aligned. 2089 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) { 2090 nop(); 2091 } 2092 } 2093 2094 2095 void X86_64Assembler::Bind(Label* label) { 2096 int bound = buffer_.Size(); 2097 CHECK(!label->IsBound()); // Labels can only be bound once. 2098 while (label->IsLinked()) { 2099 int position = label->LinkPosition(); 2100 int next = buffer_.Load<int32_t>(position); 2101 buffer_.Store<int32_t>(position, bound - (position + 4)); 2102 label->position_ = next; 2103 } 2104 label->BindTo(bound); 2105 } 2106 2107 2108 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) { 2109 CHECK_GE(reg_or_opcode, 0); 2110 CHECK_LT(reg_or_opcode, 8); 2111 const int length = operand.length_; 2112 CHECK_GT(length, 0); 2113 // Emit the ModRM byte updated with the given reg value. 2114 CHECK_EQ(operand.encoding_[0] & 0x38, 0); 2115 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3)); 2116 // Emit the rest of the encoded operand. 2117 for (int i = 1; i < length; i++) { 2118 EmitUint8(operand.encoding_[i]); 2119 } 2120 AssemblerFixup* fixup = operand.GetFixup(); 2121 if (fixup != nullptr) { 2122 EmitFixup(fixup); 2123 } 2124 } 2125 2126 2127 void X86_64Assembler::EmitImmediate(const Immediate& imm) { 2128 if (imm.is_int32()) { 2129 EmitInt32(static_cast<int32_t>(imm.value())); 2130 } else { 2131 EmitInt64(imm.value()); 2132 } 2133 } 2134 2135 2136 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode, 2137 const Operand& operand, 2138 const Immediate& immediate) { 2139 CHECK_GE(reg_or_opcode, 0); 2140 CHECK_LT(reg_or_opcode, 8); 2141 if (immediate.is_int8()) { 2142 // Use sign-extended 8-bit immediate. 2143 EmitUint8(0x83); 2144 EmitOperand(reg_or_opcode, operand); 2145 EmitUint8(immediate.value() & 0xFF); 2146 } else if (operand.IsRegister(CpuRegister(RAX))) { 2147 // Use short form if the destination is eax. 2148 EmitUint8(0x05 + (reg_or_opcode << 3)); 2149 EmitImmediate(immediate); 2150 } else { 2151 EmitUint8(0x81); 2152 EmitOperand(reg_or_opcode, operand); 2153 EmitImmediate(immediate); 2154 } 2155 } 2156 2157 2158 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) { 2159 if (label->IsBound()) { 2160 int offset = label->Position() - buffer_.Size(); 2161 CHECK_LE(offset, 0); 2162 EmitInt32(offset - instruction_size); 2163 } else { 2164 EmitLabelLink(label); 2165 } 2166 } 2167 2168 2169 void X86_64Assembler::EmitLabelLink(Label* label) { 2170 CHECK(!label->IsBound()); 2171 int position = buffer_.Size(); 2172 EmitInt32(label->position_); 2173 label->LinkTo(position); 2174 } 2175 2176 2177 void X86_64Assembler::EmitGenericShift(bool wide, 2178 int reg_or_opcode, 2179 CpuRegister reg, 2180 const Immediate& imm) { 2181 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2182 CHECK(imm.is_int8()); 2183 if (wide) { 2184 EmitRex64(reg); 2185 } else { 2186 EmitOptionalRex32(reg); 2187 } 2188 if (imm.value() == 1) { 2189 EmitUint8(0xD1); 2190 EmitOperand(reg_or_opcode, Operand(reg)); 2191 } else { 2192 EmitUint8(0xC1); 2193 EmitOperand(reg_or_opcode, Operand(reg)); 2194 EmitUint8(imm.value() & 0xFF); 2195 } 2196 } 2197 2198 2199 void X86_64Assembler::EmitGenericShift(bool wide, 2200 int reg_or_opcode, 2201 CpuRegister operand, 2202 CpuRegister shifter) { 2203 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2204 CHECK_EQ(shifter.AsRegister(), RCX); 2205 if (wide) { 2206 EmitRex64(operand); 2207 } else { 2208 EmitOptionalRex32(operand); 2209 } 2210 EmitUint8(0xD3); 2211 EmitOperand(reg_or_opcode, Operand(operand)); 2212 } 2213 2214 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) { 2215 // REX.WRXB 2216 // W - 64-bit operand 2217 // R - MODRM.reg 2218 // X - SIB.index 2219 // B - MODRM.rm/SIB.base 2220 uint8_t rex = force ? 0x40 : 0; 2221 if (w) { 2222 rex |= 0x48; // REX.W000 2223 } 2224 if (r) { 2225 rex |= 0x44; // REX.0R00 2226 } 2227 if (x) { 2228 rex |= 0x42; // REX.00X0 2229 } 2230 if (b) { 2231 rex |= 0x41; // REX.000B 2232 } 2233 if (rex != 0) { 2234 EmitUint8(rex); 2235 } 2236 } 2237 2238 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) { 2239 EmitOptionalRex(false, false, false, false, reg.NeedsRex()); 2240 } 2241 2242 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) { 2243 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2244 } 2245 2246 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) { 2247 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2248 } 2249 2250 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) { 2251 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2252 } 2253 2254 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) { 2255 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex()); 2256 } 2257 2258 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) { 2259 uint8_t rex = operand.rex(); 2260 if (rex != 0) { 2261 EmitUint8(rex); 2262 } 2263 } 2264 2265 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) { 2266 uint8_t rex = operand.rex(); 2267 if (dst.NeedsRex()) { 2268 rex |= 0x44; // REX.0R00 2269 } 2270 if (rex != 0) { 2271 EmitUint8(rex); 2272 } 2273 } 2274 2275 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) { 2276 uint8_t rex = operand.rex(); 2277 if (dst.NeedsRex()) { 2278 rex |= 0x44; // REX.0R00 2279 } 2280 if (rex != 0) { 2281 EmitUint8(rex); 2282 } 2283 } 2284 2285 void X86_64Assembler::EmitRex64() { 2286 EmitOptionalRex(false, true, false, false, false); 2287 } 2288 2289 void X86_64Assembler::EmitRex64(CpuRegister reg) { 2290 EmitOptionalRex(false, true, false, false, reg.NeedsRex()); 2291 } 2292 2293 void X86_64Assembler::EmitRex64(const Operand& operand) { 2294 uint8_t rex = operand.rex(); 2295 rex |= 0x48; // REX.W000 2296 EmitUint8(rex); 2297 } 2298 2299 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) { 2300 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2301 } 2302 2303 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) { 2304 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2305 } 2306 2307 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) { 2308 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex()); 2309 } 2310 2311 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) { 2312 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2313 if (dst.NeedsRex()) { 2314 rex |= 0x44; // REX.0R00 2315 } 2316 EmitUint8(rex); 2317 } 2318 2319 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) { 2320 uint8_t rex = 0x48 | operand.rex(); // REX.W000 2321 if (dst.NeedsRex()) { 2322 rex |= 0x44; // REX.0R00 2323 } 2324 EmitUint8(rex); 2325 } 2326 2327 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) { 2328 // For src, SPL, BPL, SIL, DIL need the rex prefix. 2329 bool force = src.AsRegister() > 3; 2330 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex()); 2331 } 2332 2333 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) { 2334 uint8_t rex = operand.rex(); 2335 // For dst, SPL, BPL, SIL, DIL need the rex prefix. 2336 bool force = dst.AsRegister() > 3; 2337 if (force) { 2338 rex |= 0x40; // REX.0000 2339 } 2340 if (dst.NeedsRex()) { 2341 rex |= 0x44; // REX.0R00 2342 } 2343 if (rex != 0) { 2344 EmitUint8(rex); 2345 } 2346 } 2347 2348 static dwarf::Reg DWARFReg(Register reg) { 2349 return dwarf::Reg::X86_64Core(static_cast<int>(reg)); 2350 } 2351 static dwarf::Reg DWARFReg(FloatRegister reg) { 2352 return dwarf::Reg::X86_64Fp(static_cast<int>(reg)); 2353 } 2354 2355 constexpr size_t kFramePointerSize = 8; 2356 2357 void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg, 2358 const std::vector<ManagedRegister>& spill_regs, 2359 const ManagedRegisterEntrySpills& entry_spills) { 2360 DCHECK_EQ(buffer_.Size(), 0U); // Nothing emitted yet. 2361 cfi_.SetCurrentCFAOffset(8); // Return address on stack. 2362 CHECK_ALIGNED(frame_size, kStackAlignment); 2363 int gpr_count = 0; 2364 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2365 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2366 if (spill.IsCpuRegister()) { 2367 pushq(spill.AsCpuRegister()); 2368 gpr_count++; 2369 cfi_.AdjustCFAOffset(kFramePointerSize); 2370 cfi_.RelOffset(DWARFReg(spill.AsCpuRegister().AsRegister()), 0); 2371 } 2372 } 2373 // return address then method on stack. 2374 int64_t rest_of_frame = static_cast<int64_t>(frame_size) 2375 - (gpr_count * kFramePointerSize) 2376 - kFramePointerSize /*return address*/; 2377 subq(CpuRegister(RSP), Immediate(rest_of_frame)); 2378 cfi_.AdjustCFAOffset(rest_of_frame); 2379 2380 // spill xmms 2381 int64_t offset = rest_of_frame; 2382 for (int i = spill_regs.size() - 1; i >= 0; --i) { 2383 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2384 if (spill.IsXmmRegister()) { 2385 offset -= sizeof(double); 2386 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister()); 2387 cfi_.RelOffset(DWARFReg(spill.AsXmmRegister().AsFloatRegister()), offset); 2388 } 2389 } 2390 2391 DCHECK_EQ(kX86_64PointerSize, kFramePointerSize); 2392 2393 movq(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister()); 2394 2395 for (size_t i = 0; i < entry_spills.size(); ++i) { 2396 ManagedRegisterSpill spill = entry_spills.at(i); 2397 if (spill.AsX86_64().IsCpuRegister()) { 2398 if (spill.getSize() == 8) { 2399 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), 2400 spill.AsX86_64().AsCpuRegister()); 2401 } else { 2402 CHECK_EQ(spill.getSize(), 4); 2403 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister()); 2404 } 2405 } else { 2406 if (spill.getSize() == 8) { 2407 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2408 } else { 2409 CHECK_EQ(spill.getSize(), 4); 2410 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister()); 2411 } 2412 } 2413 } 2414 } 2415 2416 void X86_64Assembler::RemoveFrame(size_t frame_size, 2417 const std::vector<ManagedRegister>& spill_regs) { 2418 CHECK_ALIGNED(frame_size, kStackAlignment); 2419 cfi_.RememberState(); 2420 int gpr_count = 0; 2421 // unspill xmms 2422 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize; 2423 for (size_t i = 0; i < spill_regs.size(); ++i) { 2424 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2425 if (spill.IsXmmRegister()) { 2426 offset += sizeof(double); 2427 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset)); 2428 cfi_.Restore(DWARFReg(spill.AsXmmRegister().AsFloatRegister())); 2429 } else { 2430 gpr_count++; 2431 } 2432 } 2433 int adjust = static_cast<int>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize; 2434 addq(CpuRegister(RSP), Immediate(adjust)); 2435 cfi_.AdjustCFAOffset(-adjust); 2436 for (size_t i = 0; i < spill_regs.size(); ++i) { 2437 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64(); 2438 if (spill.IsCpuRegister()) { 2439 popq(spill.AsCpuRegister()); 2440 cfi_.AdjustCFAOffset(-static_cast<int>(kFramePointerSize)); 2441 cfi_.Restore(DWARFReg(spill.AsCpuRegister().AsRegister())); 2442 } 2443 } 2444 ret(); 2445 // The CFI should be restored for any code that follows the exit block. 2446 cfi_.RestoreState(); 2447 cfi_.DefCFAOffset(frame_size); 2448 } 2449 2450 void X86_64Assembler::IncreaseFrameSize(size_t adjust) { 2451 CHECK_ALIGNED(adjust, kStackAlignment); 2452 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust))); 2453 cfi_.AdjustCFAOffset(adjust); 2454 } 2455 2456 void X86_64Assembler::DecreaseFrameSize(size_t adjust) { 2457 CHECK_ALIGNED(adjust, kStackAlignment); 2458 addq(CpuRegister(RSP), Immediate(adjust)); 2459 cfi_.AdjustCFAOffset(-adjust); 2460 } 2461 2462 void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) { 2463 X86_64ManagedRegister src = msrc.AsX86_64(); 2464 if (src.IsNoRegister()) { 2465 CHECK_EQ(0u, size); 2466 } else if (src.IsCpuRegister()) { 2467 if (size == 4) { 2468 CHECK_EQ(4u, size); 2469 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2470 } else { 2471 CHECK_EQ(8u, size); 2472 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister()); 2473 } 2474 } else if (src.IsRegisterPair()) { 2475 CHECK_EQ(0u, size); 2476 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow()); 2477 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)), 2478 src.AsRegisterPairHigh()); 2479 } else if (src.IsX87Register()) { 2480 if (size == 4) { 2481 fstps(Address(CpuRegister(RSP), offs)); 2482 } else { 2483 fstpl(Address(CpuRegister(RSP), offs)); 2484 } 2485 } else { 2486 CHECK(src.IsXmmRegister()); 2487 if (size == 4) { 2488 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2489 } else { 2490 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister()); 2491 } 2492 } 2493 } 2494 2495 void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { 2496 X86_64ManagedRegister src = msrc.AsX86_64(); 2497 CHECK(src.IsCpuRegister()); 2498 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2499 } 2500 2501 void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { 2502 X86_64ManagedRegister src = msrc.AsX86_64(); 2503 CHECK(src.IsCpuRegister()); 2504 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister()); 2505 } 2506 2507 void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm, 2508 ManagedRegister) { 2509 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq? 2510 } 2511 2512 void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm, 2513 ManagedRegister) { 2514 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq? 2515 } 2516 2517 void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs, 2518 FrameOffset fr_offs, 2519 ManagedRegister mscratch) { 2520 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2521 CHECK(scratch.IsCpuRegister()); 2522 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs)); 2523 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2524 } 2525 2526 void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) { 2527 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP)); 2528 } 2529 2530 void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/, 2531 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) { 2532 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM 2533 } 2534 2535 void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) { 2536 X86_64ManagedRegister dest = mdest.AsX86_64(); 2537 if (dest.IsNoRegister()) { 2538 CHECK_EQ(0u, size); 2539 } else if (dest.IsCpuRegister()) { 2540 if (size == 4) { 2541 CHECK_EQ(4u, size); 2542 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2543 } else { 2544 CHECK_EQ(8u, size); 2545 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2546 } 2547 } else if (dest.IsRegisterPair()) { 2548 CHECK_EQ(0u, size); 2549 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src)); 2550 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4))); 2551 } else if (dest.IsX87Register()) { 2552 if (size == 4) { 2553 flds(Address(CpuRegister(RSP), src)); 2554 } else { 2555 fldl(Address(CpuRegister(RSP), src)); 2556 } 2557 } else { 2558 CHECK(dest.IsXmmRegister()); 2559 if (size == 4) { 2560 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2561 } else { 2562 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src)); 2563 } 2564 } 2565 } 2566 2567 void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) { 2568 X86_64ManagedRegister dest = mdest.AsX86_64(); 2569 if (dest.IsNoRegister()) { 2570 CHECK_EQ(0u, size); 2571 } else if (dest.IsCpuRegister()) { 2572 CHECK_EQ(4u, size); 2573 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true)); 2574 } else if (dest.IsRegisterPair()) { 2575 CHECK_EQ(8u, size); 2576 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true)); 2577 } else if (dest.IsX87Register()) { 2578 if (size == 4) { 2579 gs()->flds(Address::Absolute(src, true)); 2580 } else { 2581 gs()->fldl(Address::Absolute(src, true)); 2582 } 2583 } else { 2584 CHECK(dest.IsXmmRegister()); 2585 if (size == 4) { 2586 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true)); 2587 } else { 2588 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true)); 2589 } 2590 } 2591 } 2592 2593 void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) { 2594 X86_64ManagedRegister dest = mdest.AsX86_64(); 2595 CHECK(dest.IsCpuRegister()); 2596 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2597 } 2598 2599 void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, MemberOffset offs, 2600 bool poison_reference) { 2601 X86_64ManagedRegister dest = mdest.AsX86_64(); 2602 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2603 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2604 if (kPoisonHeapReferences && poison_reference) { 2605 negl(dest.AsCpuRegister()); 2606 } 2607 } 2608 2609 void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base, 2610 Offset offs) { 2611 X86_64ManagedRegister dest = mdest.AsX86_64(); 2612 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister()); 2613 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs)); 2614 } 2615 2616 void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) { 2617 X86_64ManagedRegister dest = mdest.AsX86_64(); 2618 CHECK(dest.IsCpuRegister()); 2619 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true)); 2620 } 2621 2622 void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) { 2623 X86_64ManagedRegister reg = mreg.AsX86_64(); 2624 CHECK(size == 1 || size == 2) << size; 2625 CHECK(reg.IsCpuRegister()) << reg; 2626 if (size == 1) { 2627 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2628 } else { 2629 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2630 } 2631 } 2632 2633 void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) { 2634 X86_64ManagedRegister reg = mreg.AsX86_64(); 2635 CHECK(size == 1 || size == 2) << size; 2636 CHECK(reg.IsCpuRegister()) << reg; 2637 if (size == 1) { 2638 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister()); 2639 } else { 2640 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister()); 2641 } 2642 } 2643 2644 void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) { 2645 X86_64ManagedRegister dest = mdest.AsX86_64(); 2646 X86_64ManagedRegister src = msrc.AsX86_64(); 2647 if (!dest.Equals(src)) { 2648 if (dest.IsCpuRegister() && src.IsCpuRegister()) { 2649 movq(dest.AsCpuRegister(), src.AsCpuRegister()); 2650 } else if (src.IsX87Register() && dest.IsXmmRegister()) { 2651 // Pass via stack and pop X87 register 2652 subl(CpuRegister(RSP), Immediate(16)); 2653 if (size == 4) { 2654 CHECK_EQ(src.AsX87Register(), ST0); 2655 fstps(Address(CpuRegister(RSP), 0)); 2656 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2657 } else { 2658 CHECK_EQ(src.AsX87Register(), ST0); 2659 fstpl(Address(CpuRegister(RSP), 0)); 2660 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0)); 2661 } 2662 addq(CpuRegister(RSP), Immediate(16)); 2663 } else { 2664 // TODO: x87, SSE 2665 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src; 2666 } 2667 } 2668 } 2669 2670 void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) { 2671 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2672 CHECK(scratch.IsCpuRegister()); 2673 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src)); 2674 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister()); 2675 } 2676 2677 void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs, 2678 ThreadOffset<8> thr_offs, 2679 ManagedRegister mscratch) { 2680 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2681 CHECK(scratch.IsCpuRegister()); 2682 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true)); 2683 Store(fr_offs, scratch, 8); 2684 } 2685 2686 void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs, 2687 FrameOffset fr_offs, 2688 ManagedRegister mscratch) { 2689 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2690 CHECK(scratch.IsCpuRegister()); 2691 Load(scratch, fr_offs, 8); 2692 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister()); 2693 } 2694 2695 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, ManagedRegister mscratch, 2696 size_t size) { 2697 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2698 if (scratch.IsCpuRegister() && size == 8) { 2699 Load(scratch, src, 4); 2700 Store(dest, scratch, 4); 2701 Load(scratch, FrameOffset(src.Int32Value() + 4), 4); 2702 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4); 2703 } else { 2704 Load(scratch, src, size); 2705 Store(dest, scratch, size); 2706 } 2707 } 2708 2709 void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/, 2710 ManagedRegister /*scratch*/, size_t /*size*/) { 2711 UNIMPLEMENTED(FATAL); 2712 } 2713 2714 void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, 2715 ManagedRegister scratch, size_t size) { 2716 CHECK(scratch.IsNoRegister()); 2717 CHECK_EQ(size, 4u); 2718 pushq(Address(CpuRegister(RSP), src)); 2719 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset)); 2720 } 2721 2722 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, 2723 ManagedRegister mscratch, size_t size) { 2724 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2725 CHECK_EQ(size, 4u); 2726 movq(scratch, Address(CpuRegister(RSP), src_base)); 2727 movq(scratch, Address(scratch, src_offset)); 2728 movq(Address(CpuRegister(RSP), dest), scratch); 2729 } 2730 2731 void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset, 2732 ManagedRegister src, Offset src_offset, 2733 ManagedRegister scratch, size_t size) { 2734 CHECK_EQ(size, 4u); 2735 CHECK(scratch.IsNoRegister()); 2736 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset)); 2737 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset)); 2738 } 2739 2740 void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, 2741 ManagedRegister mscratch, size_t size) { 2742 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2743 CHECK_EQ(size, 4u); 2744 CHECK_EQ(dest.Int32Value(), src.Int32Value()); 2745 movq(scratch, Address(CpuRegister(RSP), src)); 2746 pushq(Address(scratch, src_offset)); 2747 popq(Address(scratch, dest_offset)); 2748 } 2749 2750 void X86_64Assembler::MemoryBarrier(ManagedRegister) { 2751 mfence(); 2752 } 2753 2754 void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg, 2755 FrameOffset handle_scope_offset, 2756 ManagedRegister min_reg, bool null_allowed) { 2757 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2758 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2759 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed 2760 // Use out_reg as indicator of null. 2761 in_reg = out_reg; 2762 // TODO: movzwl 2763 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2764 } 2765 CHECK(in_reg.IsCpuRegister()); 2766 CHECK(out_reg.IsCpuRegister()); 2767 VerifyObject(in_reg, null_allowed); 2768 if (null_allowed) { 2769 Label null_arg; 2770 if (!out_reg.Equals(in_reg)) { 2771 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2772 } 2773 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2774 j(kZero, &null_arg); 2775 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2776 Bind(&null_arg); 2777 } else { 2778 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2779 } 2780 } 2781 2782 void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off, 2783 FrameOffset handle_scope_offset, 2784 ManagedRegister mscratch, 2785 bool null_allowed) { 2786 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2787 CHECK(scratch.IsCpuRegister()); 2788 if (null_allowed) { 2789 Label null_arg; 2790 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2791 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister()); 2792 j(kZero, &null_arg); 2793 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2794 Bind(&null_arg); 2795 } else { 2796 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset)); 2797 } 2798 Store(out_off, scratch, 8); 2799 } 2800 2801 // Given a handle scope entry, load the associated reference. 2802 void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, 2803 ManagedRegister min_reg) { 2804 X86_64ManagedRegister out_reg = mout_reg.AsX86_64(); 2805 X86_64ManagedRegister in_reg = min_reg.AsX86_64(); 2806 CHECK(out_reg.IsCpuRegister()); 2807 CHECK(in_reg.IsCpuRegister()); 2808 Label null_arg; 2809 if (!out_reg.Equals(in_reg)) { 2810 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister()); 2811 } 2812 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister()); 2813 j(kZero, &null_arg); 2814 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0)); 2815 Bind(&null_arg); 2816 } 2817 2818 void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { 2819 // TODO: not validating references 2820 } 2821 2822 void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { 2823 // TODO: not validating references 2824 } 2825 2826 void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) { 2827 X86_64ManagedRegister base = mbase.AsX86_64(); 2828 CHECK(base.IsCpuRegister()); 2829 call(Address(base.AsCpuRegister(), offset.Int32Value())); 2830 // TODO: place reference map on call 2831 } 2832 2833 void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { 2834 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister(); 2835 movq(scratch, Address(CpuRegister(RSP), base)); 2836 call(Address(scratch, offset)); 2837 } 2838 2839 void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) { 2840 gs()->call(Address::Absolute(offset, true)); 2841 } 2842 2843 void X86_64Assembler::GetCurrentThread(ManagedRegister tr) { 2844 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2845 } 2846 2847 void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) { 2848 X86_64ManagedRegister scratch = mscratch.AsX86_64(); 2849 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true)); 2850 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister()); 2851 } 2852 2853 // Slowpath entered when Thread::Current()->_exception is non-null 2854 class X86_64ExceptionSlowPath FINAL : public SlowPath { 2855 public: 2856 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {} 2857 virtual void Emit(Assembler *sp_asm) OVERRIDE; 2858 private: 2859 const size_t stack_adjust_; 2860 }; 2861 2862 void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) { 2863 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust); 2864 buffer_.EnqueueSlowPath(slow); 2865 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0)); 2866 j(kNotEqual, slow->Entry()); 2867 } 2868 2869 void X86_64ExceptionSlowPath::Emit(Assembler *sasm) { 2870 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm); 2871 #define __ sp_asm-> 2872 __ Bind(&entry_); 2873 // Note: the return value is dead 2874 if (stack_adjust_ != 0) { // Fix up the frame. 2875 __ DecreaseFrameSize(stack_adjust_); 2876 } 2877 // Pass exception as argument in RDI 2878 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true)); 2879 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true)); 2880 // this call should never return 2881 __ int3(); 2882 #undef __ 2883 } 2884 2885 void X86_64Assembler::AddConstantArea() { 2886 const std::vector<int32_t>& area = constant_area_.GetBuffer(); 2887 for (size_t i = 0, e = area.size(); i < e; i++) { 2888 AssemblerBuffer::EnsureCapacity ensured(&buffer_); 2889 EmitInt32(area[i]); 2890 } 2891 } 2892 2893 int ConstantArea::AddInt32(int32_t v) { 2894 for (size_t i = 0, e = buffer_.size(); i < e; i++) { 2895 if (v == buffer_[i]) { 2896 return i * elem_size_; 2897 } 2898 } 2899 2900 // Didn't match anything. 2901 int result = buffer_.size() * elem_size_; 2902 buffer_.push_back(v); 2903 return result; 2904 } 2905 2906 int ConstantArea::AddInt64(int64_t v) { 2907 int32_t v_low = v; 2908 int32_t v_high = v >> 32; 2909 if (buffer_.size() > 1) { 2910 // Ensure we don't pass the end of the buffer. 2911 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) { 2912 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) { 2913 return i * elem_size_; 2914 } 2915 } 2916 } 2917 2918 // Didn't match anything. 2919 int result = buffer_.size() * elem_size_; 2920 buffer_.push_back(v_low); 2921 buffer_.push_back(v_high); 2922 return result; 2923 } 2924 2925 int ConstantArea::AddDouble(double v) { 2926 // Treat the value as a 64-bit integer value. 2927 return AddInt64(bit_cast<int64_t, double>(v)); 2928 } 2929 2930 int ConstantArea::AddFloat(float v) { 2931 // Treat the value as a 32-bit integer value. 2932 return AddInt32(bit_cast<int32_t, float>(v)); 2933 } 2934 2935 } // namespace x86_64 2936 } // namespace art 2937
