1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/x64/assembler-x64.h" 6 7 #include <cstring> 8 9 #if V8_TARGET_ARCH_X64 10 11 #if V8_LIBC_MSVCRT 12 #include <intrin.h> // _xgetbv() 13 #endif 14 #if V8_OS_MACOSX 15 #include <sys/sysctl.h> 16 #endif 17 18 #include "src/base/bits.h" 19 #include "src/macro-assembler.h" 20 #include "src/v8.h" 21 22 namespace v8 { 23 namespace internal { 24 25 // ----------------------------------------------------------------------------- 26 // Implementation of CpuFeatures 27 28 namespace { 29 30 #if !V8_LIBC_MSVCRT 31 32 V8_INLINE uint64_t _xgetbv(unsigned int xcr) { 33 unsigned eax, edx; 34 // Check xgetbv; this uses a .byte sequence instead of the instruction 35 // directly because older assemblers do not include support for xgetbv and 36 // there is no easy way to conditionally compile based on the assembler 37 // used. 38 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 39 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32); 40 } 41 42 #define _XCR_XFEATURE_ENABLED_MASK 0 43 44 #endif // !V8_LIBC_MSVCRT 45 46 47 bool OSHasAVXSupport() { 48 #if V8_OS_MACOSX 49 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being 50 // caused by ISRs, so we detect that here and disable AVX in that case. 51 char buffer[128]; 52 size_t buffer_size = arraysize(buffer); 53 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE}; 54 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) { 55 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version"); 56 } 57 // The buffer now contains a string of the form XX.YY.ZZ, where 58 // XX is the major kernel version component. 59 char* period_pos = strchr(buffer, '.'); 60 DCHECK_NOT_NULL(period_pos); 61 *period_pos = '\0'; 62 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT 63 if (kernel_version_major <= 13) return false; 64 #endif // V8_OS_MACOSX 65 // Check whether OS claims to support AVX. 66 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); 67 return (feature_mask & 0x6) == 0x6; 68 } 69 70 } // namespace 71 72 73 void CpuFeatures::ProbeImpl(bool cross_compile) { 74 base::CPU cpu; 75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory. 76 CHECK(cpu.has_cmov()); // CMOV support is mandatory. 77 78 // Only use statically determined features for cross compile (snapshot). 79 if (cross_compile) return; 80 81 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1; 82 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; 83 // SAHF is not generally available in long mode. 84 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; 85 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && 86 OSHasAVXSupport()) { 87 supported_ |= 1u << AVX; 88 } 89 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() && 90 OSHasAVXSupport()) { 91 supported_ |= 1u << FMA3; 92 } 93 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; 94 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; 95 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; 96 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT; 97 if (strcmp(FLAG_mcpu, "auto") == 0) { 98 if (cpu.is_atom()) supported_ |= 1u << ATOM; 99 } else if (strcmp(FLAG_mcpu, "atom") == 0) { 100 supported_ |= 1u << ATOM; 101 } 102 } 103 104 105 void CpuFeatures::PrintTarget() { } 106 void CpuFeatures::PrintFeatures() { 107 printf( 108 "SSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d " 109 "POPCNT=%d ATOM=%d\n", 110 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSE4_1), 111 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX), 112 CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1), 113 CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT), 114 CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM)); 115 } 116 117 118 // ----------------------------------------------------------------------------- 119 // Implementation of Operand 120 121 Operand::Operand(Register base, int32_t disp) : rex_(0) { 122 len_ = 1; 123 if (base.is(rsp) || base.is(r12)) { 124 // SIB byte is needed to encode (rsp + offset) or (r12 + offset). 125 set_sib(times_1, rsp, base); 126 } 127 128 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 129 set_modrm(0, base); 130 } else if (is_int8(disp)) { 131 set_modrm(1, base); 132 set_disp8(disp); 133 } else { 134 set_modrm(2, base); 135 set_disp32(disp); 136 } 137 } 138 139 140 Operand::Operand(Register base, 141 Register index, 142 ScaleFactor scale, 143 int32_t disp) : rex_(0) { 144 DCHECK(!index.is(rsp)); 145 len_ = 1; 146 set_sib(scale, index, base); 147 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 148 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits 149 // possibly set by set_sib. 150 set_modrm(0, rsp); 151 } else if (is_int8(disp)) { 152 set_modrm(1, rsp); 153 set_disp8(disp); 154 } else { 155 set_modrm(2, rsp); 156 set_disp32(disp); 157 } 158 } 159 160 161 Operand::Operand(Register index, 162 ScaleFactor scale, 163 int32_t disp) : rex_(0) { 164 DCHECK(!index.is(rsp)); 165 len_ = 1; 166 set_modrm(0, rsp); 167 set_sib(scale, index, rbp); 168 set_disp32(disp); 169 } 170 171 172 Operand::Operand(Label* label) : rex_(0), len_(1) { 173 DCHECK_NOT_NULL(label); 174 set_modrm(0, rbp); 175 set_disp64(reinterpret_cast<intptr_t>(label)); 176 } 177 178 179 Operand::Operand(const Operand& operand, int32_t offset) { 180 DCHECK(operand.len_ >= 1); 181 // Operand encodes REX ModR/M [SIB] [Disp]. 182 byte modrm = operand.buf_[0]; 183 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target). 184 bool has_sib = ((modrm & 0x07) == 0x04); 185 byte mode = modrm & 0xC0; 186 int disp_offset = has_sib ? 2 : 1; 187 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07; 188 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit 189 // displacement. 190 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base. 191 int32_t disp_value = 0; 192 if (mode == 0x80 || is_baseless) { 193 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement. 194 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]); 195 } else if (mode == 0x40) { 196 // Mode 1: Byte displacement. 197 disp_value = static_cast<signed char>(operand.buf_[disp_offset]); 198 } 199 200 // Write new operand with same registers, but with modified displacement. 201 DCHECK(offset >= 0 ? disp_value + offset > disp_value 202 : disp_value + offset < disp_value); // No overflow. 203 disp_value += offset; 204 rex_ = operand.rex_; 205 if (!is_int8(disp_value) || is_baseless) { 206 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13. 207 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80); 208 len_ = disp_offset + 4; 209 Memory::int32_at(&buf_[disp_offset]) = disp_value; 210 } else if (disp_value != 0 || (base_reg == 0x05)) { 211 // Need 8 bits of displacement. 212 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1. 213 len_ = disp_offset + 1; 214 buf_[disp_offset] = static_cast<byte>(disp_value); 215 } else { 216 // Need no displacement. 217 buf_[0] = (modrm & 0x3f); // Mode 0. 218 len_ = disp_offset; 219 } 220 if (has_sib) { 221 buf_[1] = operand.buf_[1]; 222 } 223 } 224 225 226 bool Operand::AddressUsesRegister(Register reg) const { 227 int code = reg.code(); 228 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand. 229 // Start with only low three bits of base register. Initial decoding doesn't 230 // distinguish on the REX.B bit. 231 int base_code = buf_[0] & 0x07; 232 if (base_code == rsp.code()) { 233 // SIB byte present in buf_[1]. 234 // Check the index register from the SIB byte + REX.X prefix. 235 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2); 236 // Index code (including REX.X) of 0x04 (rsp) means no index register. 237 if (index_code != rsp.code() && index_code == code) return true; 238 // Add REX.B to get the full base register code. 239 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3); 240 // A base register of 0x05 (rbp) with mod = 0 means no base register. 241 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 242 return code == base_code; 243 } else { 244 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means 245 // no base register. 246 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 247 base_code |= ((rex_ & 0x01) << 3); 248 return code == base_code; 249 } 250 } 251 252 253 // ----------------------------------------------------------------------------- 254 // Implementation of Assembler. 255 256 #ifdef GENERATED_CODE_COVERAGE 257 static void InitCoverageLog(); 258 #endif 259 260 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 261 : AssemblerBase(isolate, buffer, buffer_size), 262 code_targets_(100), 263 positions_recorder_(this) { 264 // Clear the buffer in debug mode unless it was provided by the 265 // caller in which case we can't be sure it's okay to overwrite 266 // existing code in it. 267 #ifdef DEBUG 268 if (own_buffer_) { 269 memset(buffer_, 0xCC, buffer_size_); // int3 270 } 271 #endif 272 273 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_); 274 275 276 #ifdef GENERATED_CODE_COVERAGE 277 InitCoverageLog(); 278 #endif 279 } 280 281 282 void Assembler::GetCode(CodeDesc* desc) { 283 // Finalize code (at this point overflow() may be true, but the gap ensures 284 // that we are still not overlapping instructions and relocation info). 285 reloc_info_writer.Finish(); 286 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap. 287 // Set up code descriptor. 288 desc->buffer = buffer_; 289 desc->buffer_size = buffer_size_; 290 desc->instr_size = pc_offset(); 291 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system. 292 desc->reloc_size = 293 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos()); 294 desc->origin = this; 295 desc->constant_pool_size = 0; 296 } 297 298 299 void Assembler::Align(int m) { 300 DCHECK(base::bits::IsPowerOfTwo32(m)); 301 int delta = (m - (pc_offset() & (m - 1))) & (m - 1); 302 Nop(delta); 303 } 304 305 306 void Assembler::CodeTargetAlign() { 307 Align(16); // Preferred alignment of jump targets on x64. 308 } 309 310 311 bool Assembler::IsNop(Address addr) { 312 Address a = addr; 313 while (*a == 0x66) a++; 314 if (*a == 0x90) return true; 315 if (a[0] == 0xf && a[1] == 0x1f) return true; 316 return false; 317 } 318 319 320 void Assembler::bind_to(Label* L, int pos) { 321 DCHECK(!L->is_bound()); // Label may only be bound once. 322 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid. 323 if (L->is_linked()) { 324 int current = L->pos(); 325 int next = long_at(current); 326 while (next != current) { 327 if (current >= 4 && long_at(current - 4) == 0) { 328 // Absolute address. 329 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 330 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 331 internal_reference_positions_.push_back(current - 4); 332 } else { 333 // Relative address, relative to point after address. 334 int imm32 = pos - (current + sizeof(int32_t)); 335 long_at_put(current, imm32); 336 } 337 current = next; 338 next = long_at(next); 339 } 340 // Fix up last fixup on linked list. 341 if (current >= 4 && long_at(current - 4) == 0) { 342 // Absolute address. 343 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 344 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 345 internal_reference_positions_.push_back(current - 4); 346 } else { 347 // Relative address, relative to point after address. 348 int imm32 = pos - (current + sizeof(int32_t)); 349 long_at_put(current, imm32); 350 } 351 } 352 while (L->is_near_linked()) { 353 int fixup_pos = L->near_link_pos(); 354 int offset_to_next = 355 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos))); 356 DCHECK(offset_to_next <= 0); 357 int disp = pos - (fixup_pos + sizeof(int8_t)); 358 CHECK(is_int8(disp)); 359 set_byte_at(fixup_pos, disp); 360 if (offset_to_next < 0) { 361 L->link_to(fixup_pos + offset_to_next, Label::kNear); 362 } else { 363 L->UnuseNear(); 364 } 365 } 366 L->bind_to(pos); 367 } 368 369 370 void Assembler::bind(Label* L) { 371 bind_to(L, pc_offset()); 372 } 373 374 375 void Assembler::GrowBuffer() { 376 DCHECK(buffer_overflow()); 377 if (!own_buffer_) FATAL("external code buffer is too small"); 378 379 // Compute new buffer size. 380 CodeDesc desc; // the new buffer 381 desc.buffer_size = 2 * buffer_size_; 382 383 // Some internal data structures overflow for very large buffers, 384 // they must ensure that kMaximalBufferSize is not too large. 385 if ((desc.buffer_size > kMaximalBufferSize) || 386 (desc.buffer_size > isolate()->heap()->MaxOldGenerationSize())) { 387 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer"); 388 } 389 390 // Set up new buffer. 391 desc.buffer = NewArray<byte>(desc.buffer_size); 392 desc.origin = this; 393 desc.instr_size = pc_offset(); 394 desc.reloc_size = 395 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos())); 396 397 // Clear the buffer in debug mode. Use 'int3' instructions to make 398 // sure to get into problems if we ever run uninitialized code. 399 #ifdef DEBUG 400 memset(desc.buffer, 0xCC, desc.buffer_size); 401 #endif 402 403 // Copy the data. 404 intptr_t pc_delta = desc.buffer - buffer_; 405 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 406 (buffer_ + buffer_size_); 407 MemMove(desc.buffer, buffer_, desc.instr_size); 408 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(), 409 desc.reloc_size); 410 411 // Switch buffers. 412 DeleteArray(buffer_); 413 buffer_ = desc.buffer; 414 buffer_size_ = desc.buffer_size; 415 pc_ += pc_delta; 416 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 417 reloc_info_writer.last_pc() + pc_delta); 418 419 // Relocate internal references. 420 for (auto pos : internal_reference_positions_) { 421 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 422 *p += pc_delta; 423 } 424 425 DCHECK(!buffer_overflow()); 426 } 427 428 429 void Assembler::emit_operand(int code, const Operand& adr) { 430 DCHECK(is_uint3(code)); 431 const unsigned length = adr.len_; 432 DCHECK(length > 0); 433 434 // Emit updated ModR/M byte containing the given register. 435 DCHECK((adr.buf_[0] & 0x38) == 0); 436 *pc_++ = adr.buf_[0] | code << 3; 437 438 // Recognize RIP relative addressing. 439 if (adr.buf_[0] == 5) { 440 DCHECK_EQ(9u, length); 441 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]); 442 if (label->is_bound()) { 443 int offset = label->pos() - pc_offset() - sizeof(int32_t); 444 DCHECK_GE(0, offset); 445 emitl(offset); 446 } else if (label->is_linked()) { 447 emitl(label->pos()); 448 label->link_to(pc_offset() - sizeof(int32_t)); 449 } else { 450 DCHECK(label->is_unused()); 451 int32_t current = pc_offset(); 452 emitl(current); 453 label->link_to(current); 454 } 455 } else { 456 // Emit the rest of the encoded operand. 457 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i]; 458 } 459 } 460 461 462 // Assembler Instruction implementations. 463 464 void Assembler::arithmetic_op(byte opcode, 465 Register reg, 466 const Operand& op, 467 int size) { 468 EnsureSpace ensure_space(this); 469 emit_rex(reg, op, size); 470 emit(opcode); 471 emit_operand(reg, op); 472 } 473 474 475 void Assembler::arithmetic_op(byte opcode, 476 Register reg, 477 Register rm_reg, 478 int size) { 479 EnsureSpace ensure_space(this); 480 DCHECK((opcode & 0xC6) == 2); 481 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 482 // Swap reg and rm_reg and change opcode operand order. 483 emit_rex(rm_reg, reg, size); 484 emit(opcode ^ 0x02); 485 emit_modrm(rm_reg, reg); 486 } else { 487 emit_rex(reg, rm_reg, size); 488 emit(opcode); 489 emit_modrm(reg, rm_reg); 490 } 491 } 492 493 494 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) { 495 EnsureSpace ensure_space(this); 496 DCHECK((opcode & 0xC6) == 2); 497 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 498 // Swap reg and rm_reg and change opcode operand order. 499 emit(0x66); 500 emit_optional_rex_32(rm_reg, reg); 501 emit(opcode ^ 0x02); 502 emit_modrm(rm_reg, reg); 503 } else { 504 emit(0x66); 505 emit_optional_rex_32(reg, rm_reg); 506 emit(opcode); 507 emit_modrm(reg, rm_reg); 508 } 509 } 510 511 512 void Assembler::arithmetic_op_16(byte opcode, 513 Register reg, 514 const Operand& rm_reg) { 515 EnsureSpace ensure_space(this); 516 emit(0x66); 517 emit_optional_rex_32(reg, rm_reg); 518 emit(opcode); 519 emit_operand(reg, rm_reg); 520 } 521 522 523 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) { 524 EnsureSpace ensure_space(this); 525 if (!reg.is_byte_register()) { 526 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 527 emit_rex_32(reg); 528 } 529 emit(opcode); 530 emit_operand(reg, op); 531 } 532 533 534 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) { 535 EnsureSpace ensure_space(this); 536 DCHECK((opcode & 0xC6) == 2); 537 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 538 // Swap reg and rm_reg and change opcode operand order. 539 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) { 540 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 541 emit_rex_32(rm_reg, reg); 542 } 543 emit(opcode ^ 0x02); 544 emit_modrm(rm_reg, reg); 545 } else { 546 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) { 547 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 548 emit_rex_32(reg, rm_reg); 549 } 550 emit(opcode); 551 emit_modrm(reg, rm_reg); 552 } 553 } 554 555 556 void Assembler::immediate_arithmetic_op(byte subcode, 557 Register dst, 558 Immediate src, 559 int size) { 560 EnsureSpace ensure_space(this); 561 emit_rex(dst, size); 562 if (is_int8(src.value_)) { 563 emit(0x83); 564 emit_modrm(subcode, dst); 565 emit(src.value_); 566 } else if (dst.is(rax)) { 567 emit(0x05 | (subcode << 3)); 568 emitl(src.value_); 569 } else { 570 emit(0x81); 571 emit_modrm(subcode, dst); 572 emitl(src.value_); 573 } 574 } 575 576 void Assembler::immediate_arithmetic_op(byte subcode, 577 const Operand& dst, 578 Immediate src, 579 int size) { 580 EnsureSpace ensure_space(this); 581 emit_rex(dst, size); 582 if (is_int8(src.value_)) { 583 emit(0x83); 584 emit_operand(subcode, dst); 585 emit(src.value_); 586 } else { 587 emit(0x81); 588 emit_operand(subcode, dst); 589 emitl(src.value_); 590 } 591 } 592 593 594 void Assembler::immediate_arithmetic_op_16(byte subcode, 595 Register dst, 596 Immediate src) { 597 EnsureSpace ensure_space(this); 598 emit(0x66); // Operand size override prefix. 599 emit_optional_rex_32(dst); 600 if (is_int8(src.value_)) { 601 emit(0x83); 602 emit_modrm(subcode, dst); 603 emit(src.value_); 604 } else if (dst.is(rax)) { 605 emit(0x05 | (subcode << 3)); 606 emitw(src.value_); 607 } else { 608 emit(0x81); 609 emit_modrm(subcode, dst); 610 emitw(src.value_); 611 } 612 } 613 614 615 void Assembler::immediate_arithmetic_op_16(byte subcode, 616 const Operand& dst, 617 Immediate src) { 618 EnsureSpace ensure_space(this); 619 emit(0x66); // Operand size override prefix. 620 emit_optional_rex_32(dst); 621 if (is_int8(src.value_)) { 622 emit(0x83); 623 emit_operand(subcode, dst); 624 emit(src.value_); 625 } else { 626 emit(0x81); 627 emit_operand(subcode, dst); 628 emitw(src.value_); 629 } 630 } 631 632 633 void Assembler::immediate_arithmetic_op_8(byte subcode, 634 const Operand& dst, 635 Immediate src) { 636 EnsureSpace ensure_space(this); 637 emit_optional_rex_32(dst); 638 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 639 emit(0x80); 640 emit_operand(subcode, dst); 641 emit(src.value_); 642 } 643 644 645 void Assembler::immediate_arithmetic_op_8(byte subcode, 646 Register dst, 647 Immediate src) { 648 EnsureSpace ensure_space(this); 649 if (!dst.is_byte_register()) { 650 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 651 emit_rex_32(dst); 652 } 653 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 654 emit(0x80); 655 emit_modrm(subcode, dst); 656 emit(src.value_); 657 } 658 659 660 void Assembler::shift(Register dst, 661 Immediate shift_amount, 662 int subcode, 663 int size) { 664 EnsureSpace ensure_space(this); 665 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 666 : is_uint5(shift_amount.value_)); 667 if (shift_amount.value_ == 1) { 668 emit_rex(dst, size); 669 emit(0xD1); 670 emit_modrm(subcode, dst); 671 } else { 672 emit_rex(dst, size); 673 emit(0xC1); 674 emit_modrm(subcode, dst); 675 emit(shift_amount.value_); 676 } 677 } 678 679 680 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode, 681 int size) { 682 EnsureSpace ensure_space(this); 683 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 684 : is_uint5(shift_amount.value_)); 685 if (shift_amount.value_ == 1) { 686 emit_rex(dst, size); 687 emit(0xD1); 688 emit_operand(subcode, dst); 689 } else { 690 emit_rex(dst, size); 691 emit(0xC1); 692 emit_operand(subcode, dst); 693 emit(shift_amount.value_); 694 } 695 } 696 697 698 void Assembler::shift(Register dst, int subcode, int size) { 699 EnsureSpace ensure_space(this); 700 emit_rex(dst, size); 701 emit(0xD3); 702 emit_modrm(subcode, dst); 703 } 704 705 706 void Assembler::shift(Operand dst, int subcode, int size) { 707 EnsureSpace ensure_space(this); 708 emit_rex(dst, size); 709 emit(0xD3); 710 emit_operand(subcode, dst); 711 } 712 713 714 void Assembler::bt(const Operand& dst, Register src) { 715 EnsureSpace ensure_space(this); 716 emit_rex_64(src, dst); 717 emit(0x0F); 718 emit(0xA3); 719 emit_operand(src, dst); 720 } 721 722 723 void Assembler::bts(const Operand& dst, Register src) { 724 EnsureSpace ensure_space(this); 725 emit_rex_64(src, dst); 726 emit(0x0F); 727 emit(0xAB); 728 emit_operand(src, dst); 729 } 730 731 732 void Assembler::bsrl(Register dst, Register src) { 733 EnsureSpace ensure_space(this); 734 emit_optional_rex_32(dst, src); 735 emit(0x0F); 736 emit(0xBD); 737 emit_modrm(dst, src); 738 } 739 740 741 void Assembler::bsrl(Register dst, const Operand& src) { 742 EnsureSpace ensure_space(this); 743 emit_optional_rex_32(dst, src); 744 emit(0x0F); 745 emit(0xBD); 746 emit_operand(dst, src); 747 } 748 749 750 void Assembler::bsrq(Register dst, Register src) { 751 EnsureSpace ensure_space(this); 752 emit_rex_64(dst, src); 753 emit(0x0F); 754 emit(0xBD); 755 emit_modrm(dst, src); 756 } 757 758 759 void Assembler::bsrq(Register dst, const Operand& src) { 760 EnsureSpace ensure_space(this); 761 emit_rex_64(dst, src); 762 emit(0x0F); 763 emit(0xBD); 764 emit_operand(dst, src); 765 } 766 767 768 void Assembler::bsfl(Register dst, Register src) { 769 EnsureSpace ensure_space(this); 770 emit_optional_rex_32(dst, src); 771 emit(0x0F); 772 emit(0xBC); 773 emit_modrm(dst, src); 774 } 775 776 777 void Assembler::bsfl(Register dst, const Operand& src) { 778 EnsureSpace ensure_space(this); 779 emit_optional_rex_32(dst, src); 780 emit(0x0F); 781 emit(0xBC); 782 emit_operand(dst, src); 783 } 784 785 786 void Assembler::bsfq(Register dst, Register src) { 787 EnsureSpace ensure_space(this); 788 emit_rex_64(dst, src); 789 emit(0x0F); 790 emit(0xBC); 791 emit_modrm(dst, src); 792 } 793 794 795 void Assembler::bsfq(Register dst, const Operand& src) { 796 EnsureSpace ensure_space(this); 797 emit_rex_64(dst, src); 798 emit(0x0F); 799 emit(0xBC); 800 emit_operand(dst, src); 801 } 802 803 804 void Assembler::call(Label* L) { 805 positions_recorder()->WriteRecordedPositions(); 806 EnsureSpace ensure_space(this); 807 // 1110 1000 #32-bit disp. 808 emit(0xE8); 809 if (L->is_bound()) { 810 int offset = L->pos() - pc_offset() - sizeof(int32_t); 811 DCHECK(offset <= 0); 812 emitl(offset); 813 } else if (L->is_linked()) { 814 emitl(L->pos()); 815 L->link_to(pc_offset() - sizeof(int32_t)); 816 } else { 817 DCHECK(L->is_unused()); 818 int32_t current = pc_offset(); 819 emitl(current); 820 L->link_to(current); 821 } 822 } 823 824 825 void Assembler::call(Address entry, RelocInfo::Mode rmode) { 826 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 827 positions_recorder()->WriteRecordedPositions(); 828 EnsureSpace ensure_space(this); 829 // 1110 1000 #32-bit disp. 830 emit(0xE8); 831 emit_runtime_entry(entry, rmode); 832 } 833 834 835 void Assembler::call(Handle<Code> target, 836 RelocInfo::Mode rmode, 837 TypeFeedbackId ast_id) { 838 positions_recorder()->WriteRecordedPositions(); 839 EnsureSpace ensure_space(this); 840 // 1110 1000 #32-bit disp. 841 emit(0xE8); 842 emit_code_target(target, rmode, ast_id); 843 } 844 845 846 void Assembler::call(Register adr) { 847 positions_recorder()->WriteRecordedPositions(); 848 EnsureSpace ensure_space(this); 849 // Opcode: FF /2 r64. 850 emit_optional_rex_32(adr); 851 emit(0xFF); 852 emit_modrm(0x2, adr); 853 } 854 855 856 void Assembler::call(const Operand& op) { 857 positions_recorder()->WriteRecordedPositions(); 858 EnsureSpace ensure_space(this); 859 // Opcode: FF /2 m64. 860 emit_optional_rex_32(op); 861 emit(0xFF); 862 emit_operand(0x2, op); 863 } 864 865 866 // Calls directly to the given address using a relative offset. 867 // Should only ever be used in Code objects for calls within the 868 // same Code object. Should not be used when generating new code (use labels), 869 // but only when patching existing code. 870 void Assembler::call(Address target) { 871 positions_recorder()->WriteRecordedPositions(); 872 EnsureSpace ensure_space(this); 873 // 1110 1000 #32-bit disp. 874 emit(0xE8); 875 Address source = pc_ + 4; 876 intptr_t displacement = target - source; 877 DCHECK(is_int32(displacement)); 878 emitl(static_cast<int32_t>(displacement)); 879 } 880 881 882 void Assembler::clc() { 883 EnsureSpace ensure_space(this); 884 emit(0xF8); 885 } 886 887 888 void Assembler::cld() { 889 EnsureSpace ensure_space(this); 890 emit(0xFC); 891 } 892 893 894 void Assembler::cdq() { 895 EnsureSpace ensure_space(this); 896 emit(0x99); 897 } 898 899 900 void Assembler::cmovq(Condition cc, Register dst, Register src) { 901 if (cc == always) { 902 movq(dst, src); 903 } else if (cc == never) { 904 return; 905 } 906 // No need to check CpuInfo for CMOV support, it's a required part of the 907 // 64-bit architecture. 908 DCHECK(cc >= 0); // Use mov for unconditional moves. 909 EnsureSpace ensure_space(this); 910 // Opcode: REX.W 0f 40 + cc /r. 911 emit_rex_64(dst, src); 912 emit(0x0f); 913 emit(0x40 + cc); 914 emit_modrm(dst, src); 915 } 916 917 918 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) { 919 if (cc == always) { 920 movq(dst, src); 921 } else if (cc == never) { 922 return; 923 } 924 DCHECK(cc >= 0); 925 EnsureSpace ensure_space(this); 926 // Opcode: REX.W 0f 40 + cc /r. 927 emit_rex_64(dst, src); 928 emit(0x0f); 929 emit(0x40 + cc); 930 emit_operand(dst, src); 931 } 932 933 934 void Assembler::cmovl(Condition cc, Register dst, Register src) { 935 if (cc == always) { 936 movl(dst, src); 937 } else if (cc == never) { 938 return; 939 } 940 DCHECK(cc >= 0); 941 EnsureSpace ensure_space(this); 942 // Opcode: 0f 40 + cc /r. 943 emit_optional_rex_32(dst, src); 944 emit(0x0f); 945 emit(0x40 + cc); 946 emit_modrm(dst, src); 947 } 948 949 950 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) { 951 if (cc == always) { 952 movl(dst, src); 953 } else if (cc == never) { 954 return; 955 } 956 DCHECK(cc >= 0); 957 EnsureSpace ensure_space(this); 958 // Opcode: 0f 40 + cc /r. 959 emit_optional_rex_32(dst, src); 960 emit(0x0f); 961 emit(0x40 + cc); 962 emit_operand(dst, src); 963 } 964 965 966 void Assembler::cmpb_al(Immediate imm8) { 967 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_)); 968 EnsureSpace ensure_space(this); 969 emit(0x3c); 970 emit(imm8.value_); 971 } 972 973 974 void Assembler::cpuid() { 975 EnsureSpace ensure_space(this); 976 emit(0x0F); 977 emit(0xA2); 978 } 979 980 981 void Assembler::cqo() { 982 EnsureSpace ensure_space(this); 983 emit_rex_64(); 984 emit(0x99); 985 } 986 987 988 void Assembler::emit_dec(Register dst, int size) { 989 EnsureSpace ensure_space(this); 990 emit_rex(dst, size); 991 emit(0xFF); 992 emit_modrm(0x1, dst); 993 } 994 995 996 void Assembler::emit_dec(const Operand& dst, int size) { 997 EnsureSpace ensure_space(this); 998 emit_rex(dst, size); 999 emit(0xFF); 1000 emit_operand(1, dst); 1001 } 1002 1003 1004 void Assembler::decb(Register dst) { 1005 EnsureSpace ensure_space(this); 1006 if (!dst.is_byte_register()) { 1007 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1008 emit_rex_32(dst); 1009 } 1010 emit(0xFE); 1011 emit_modrm(0x1, dst); 1012 } 1013 1014 1015 void Assembler::decb(const Operand& dst) { 1016 EnsureSpace ensure_space(this); 1017 emit_optional_rex_32(dst); 1018 emit(0xFE); 1019 emit_operand(1, dst); 1020 } 1021 1022 1023 void Assembler::enter(Immediate size) { 1024 EnsureSpace ensure_space(this); 1025 emit(0xC8); 1026 emitw(size.value_); // 16 bit operand, always. 1027 emit(0); 1028 } 1029 1030 1031 void Assembler::hlt() { 1032 EnsureSpace ensure_space(this); 1033 emit(0xF4); 1034 } 1035 1036 1037 void Assembler::emit_idiv(Register src, int size) { 1038 EnsureSpace ensure_space(this); 1039 emit_rex(src, size); 1040 emit(0xF7); 1041 emit_modrm(0x7, src); 1042 } 1043 1044 1045 void Assembler::emit_div(Register src, int size) { 1046 EnsureSpace ensure_space(this); 1047 emit_rex(src, size); 1048 emit(0xF7); 1049 emit_modrm(0x6, src); 1050 } 1051 1052 1053 void Assembler::emit_imul(Register src, int size) { 1054 EnsureSpace ensure_space(this); 1055 emit_rex(src, size); 1056 emit(0xF7); 1057 emit_modrm(0x5, src); 1058 } 1059 1060 1061 void Assembler::emit_imul(const Operand& src, int size) { 1062 EnsureSpace ensure_space(this); 1063 emit_rex(src, size); 1064 emit(0xF7); 1065 emit_operand(0x5, src); 1066 } 1067 1068 1069 void Assembler::emit_imul(Register dst, Register src, int size) { 1070 EnsureSpace ensure_space(this); 1071 emit_rex(dst, src, size); 1072 emit(0x0F); 1073 emit(0xAF); 1074 emit_modrm(dst, src); 1075 } 1076 1077 1078 void Assembler::emit_imul(Register dst, const Operand& src, int size) { 1079 EnsureSpace ensure_space(this); 1080 emit_rex(dst, src, size); 1081 emit(0x0F); 1082 emit(0xAF); 1083 emit_operand(dst, src); 1084 } 1085 1086 1087 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) { 1088 EnsureSpace ensure_space(this); 1089 emit_rex(dst, src, size); 1090 if (is_int8(imm.value_)) { 1091 emit(0x6B); 1092 emit_modrm(dst, src); 1093 emit(imm.value_); 1094 } else { 1095 emit(0x69); 1096 emit_modrm(dst, src); 1097 emitl(imm.value_); 1098 } 1099 } 1100 1101 1102 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm, 1103 int size) { 1104 EnsureSpace ensure_space(this); 1105 emit_rex(dst, src, size); 1106 if (is_int8(imm.value_)) { 1107 emit(0x6B); 1108 emit_operand(dst, src); 1109 emit(imm.value_); 1110 } else { 1111 emit(0x69); 1112 emit_operand(dst, src); 1113 emitl(imm.value_); 1114 } 1115 } 1116 1117 1118 void Assembler::emit_inc(Register dst, int size) { 1119 EnsureSpace ensure_space(this); 1120 emit_rex(dst, size); 1121 emit(0xFF); 1122 emit_modrm(0x0, dst); 1123 } 1124 1125 1126 void Assembler::emit_inc(const Operand& dst, int size) { 1127 EnsureSpace ensure_space(this); 1128 emit_rex(dst, size); 1129 emit(0xFF); 1130 emit_operand(0, dst); 1131 } 1132 1133 1134 void Assembler::int3() { 1135 EnsureSpace ensure_space(this); 1136 emit(0xCC); 1137 } 1138 1139 1140 void Assembler::j(Condition cc, Label* L, Label::Distance distance) { 1141 if (cc == always) { 1142 jmp(L); 1143 return; 1144 } else if (cc == never) { 1145 return; 1146 } 1147 EnsureSpace ensure_space(this); 1148 DCHECK(is_uint4(cc)); 1149 if (L->is_bound()) { 1150 const int short_size = 2; 1151 const int long_size = 6; 1152 int offs = L->pos() - pc_offset(); 1153 DCHECK(offs <= 0); 1154 // Determine whether we can use 1-byte offsets for backwards branches, 1155 // which have a max range of 128 bytes. 1156 1157 // We also need to check predictable_code_size() flag here, because on x64, 1158 // when the full code generator recompiles code for debugging, some places 1159 // need to be padded out to a certain size. The debugger is keeping track of 1160 // how often it did this so that it can adjust return addresses on the 1161 // stack, but if the size of jump instructions can also change, that's not 1162 // enough and the calculated offsets would be incorrect. 1163 if (is_int8(offs - short_size) && !predictable_code_size()) { 1164 // 0111 tttn #8-bit disp. 1165 emit(0x70 | cc); 1166 emit((offs - short_size) & 0xFF); 1167 } else { 1168 // 0000 1111 1000 tttn #32-bit disp. 1169 emit(0x0F); 1170 emit(0x80 | cc); 1171 emitl(offs - long_size); 1172 } 1173 } else if (distance == Label::kNear) { 1174 // 0111 tttn #8-bit disp 1175 emit(0x70 | cc); 1176 byte disp = 0x00; 1177 if (L->is_near_linked()) { 1178 int offset = L->near_link_pos() - pc_offset(); 1179 DCHECK(is_int8(offset)); 1180 disp = static_cast<byte>(offset & 0xFF); 1181 } 1182 L->link_to(pc_offset(), Label::kNear); 1183 emit(disp); 1184 } else if (L->is_linked()) { 1185 // 0000 1111 1000 tttn #32-bit disp. 1186 emit(0x0F); 1187 emit(0x80 | cc); 1188 emitl(L->pos()); 1189 L->link_to(pc_offset() - sizeof(int32_t)); 1190 } else { 1191 DCHECK(L->is_unused()); 1192 emit(0x0F); 1193 emit(0x80 | cc); 1194 int32_t current = pc_offset(); 1195 emitl(current); 1196 L->link_to(current); 1197 } 1198 } 1199 1200 1201 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) { 1202 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1203 EnsureSpace ensure_space(this); 1204 DCHECK(is_uint4(cc)); 1205 emit(0x0F); 1206 emit(0x80 | cc); 1207 emit_runtime_entry(entry, rmode); 1208 } 1209 1210 1211 void Assembler::j(Condition cc, 1212 Handle<Code> target, 1213 RelocInfo::Mode rmode) { 1214 EnsureSpace ensure_space(this); 1215 DCHECK(is_uint4(cc)); 1216 // 0000 1111 1000 tttn #32-bit disp. 1217 emit(0x0F); 1218 emit(0x80 | cc); 1219 emit_code_target(target, rmode); 1220 } 1221 1222 1223 void Assembler::jmp(Label* L, Label::Distance distance) { 1224 EnsureSpace ensure_space(this); 1225 const int short_size = sizeof(int8_t); 1226 const int long_size = sizeof(int32_t); 1227 if (L->is_bound()) { 1228 int offs = L->pos() - pc_offset() - 1; 1229 DCHECK(offs <= 0); 1230 if (is_int8(offs - short_size) && !predictable_code_size()) { 1231 // 1110 1011 #8-bit disp. 1232 emit(0xEB); 1233 emit((offs - short_size) & 0xFF); 1234 } else { 1235 // 1110 1001 #32-bit disp. 1236 emit(0xE9); 1237 emitl(offs - long_size); 1238 } 1239 } else if (distance == Label::kNear) { 1240 emit(0xEB); 1241 byte disp = 0x00; 1242 if (L->is_near_linked()) { 1243 int offset = L->near_link_pos() - pc_offset(); 1244 DCHECK(is_int8(offset)); 1245 disp = static_cast<byte>(offset & 0xFF); 1246 } 1247 L->link_to(pc_offset(), Label::kNear); 1248 emit(disp); 1249 } else if (L->is_linked()) { 1250 // 1110 1001 #32-bit disp. 1251 emit(0xE9); 1252 emitl(L->pos()); 1253 L->link_to(pc_offset() - long_size); 1254 } else { 1255 // 1110 1001 #32-bit disp. 1256 DCHECK(L->is_unused()); 1257 emit(0xE9); 1258 int32_t current = pc_offset(); 1259 emitl(current); 1260 L->link_to(current); 1261 } 1262 } 1263 1264 1265 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) { 1266 EnsureSpace ensure_space(this); 1267 // 1110 1001 #32-bit disp. 1268 emit(0xE9); 1269 emit_code_target(target, rmode); 1270 } 1271 1272 1273 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { 1274 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1275 EnsureSpace ensure_space(this); 1276 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1277 emit(0xE9); 1278 emit_runtime_entry(entry, rmode); 1279 } 1280 1281 1282 void Assembler::jmp(Register target) { 1283 EnsureSpace ensure_space(this); 1284 // Opcode FF/4 r64. 1285 emit_optional_rex_32(target); 1286 emit(0xFF); 1287 emit_modrm(0x4, target); 1288 } 1289 1290 1291 void Assembler::jmp(const Operand& src) { 1292 EnsureSpace ensure_space(this); 1293 // Opcode FF/4 m64. 1294 emit_optional_rex_32(src); 1295 emit(0xFF); 1296 emit_operand(0x4, src); 1297 } 1298 1299 1300 void Assembler::emit_lea(Register dst, const Operand& src, int size) { 1301 EnsureSpace ensure_space(this); 1302 emit_rex(dst, src, size); 1303 emit(0x8D); 1304 emit_operand(dst, src); 1305 } 1306 1307 1308 void Assembler::load_rax(void* value, RelocInfo::Mode mode) { 1309 EnsureSpace ensure_space(this); 1310 if (kPointerSize == kInt64Size) { 1311 emit(0x48); // REX.W 1312 emit(0xA1); 1313 emitp(value, mode); 1314 } else { 1315 DCHECK(kPointerSize == kInt32Size); 1316 emit(0xA1); 1317 emitp(value, mode); 1318 // In 64-bit mode, need to zero extend the operand to 8 bytes. 1319 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 1320 // Developer's Manual Volume 2. 1321 emitl(0); 1322 } 1323 } 1324 1325 1326 void Assembler::load_rax(ExternalReference ref) { 1327 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 1328 } 1329 1330 1331 void Assembler::leave() { 1332 EnsureSpace ensure_space(this); 1333 emit(0xC9); 1334 } 1335 1336 1337 void Assembler::movb(Register dst, const Operand& src) { 1338 EnsureSpace ensure_space(this); 1339 if (!dst.is_byte_register()) { 1340 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1341 emit_rex_32(dst, src); 1342 } else { 1343 emit_optional_rex_32(dst, src); 1344 } 1345 emit(0x8A); 1346 emit_operand(dst, src); 1347 } 1348 1349 1350 void Assembler::movb(Register dst, Immediate imm) { 1351 EnsureSpace ensure_space(this); 1352 if (!dst.is_byte_register()) { 1353 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1354 emit_rex_32(dst); 1355 } 1356 emit(0xB0 + dst.low_bits()); 1357 emit(imm.value_); 1358 } 1359 1360 1361 void Assembler::movb(const Operand& dst, Register src) { 1362 EnsureSpace ensure_space(this); 1363 if (!src.is_byte_register()) { 1364 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1365 emit_rex_32(src, dst); 1366 } else { 1367 emit_optional_rex_32(src, dst); 1368 } 1369 emit(0x88); 1370 emit_operand(src, dst); 1371 } 1372 1373 1374 void Assembler::movb(const Operand& dst, Immediate imm) { 1375 EnsureSpace ensure_space(this); 1376 emit_optional_rex_32(dst); 1377 emit(0xC6); 1378 emit_operand(0x0, dst); 1379 emit(static_cast<byte>(imm.value_)); 1380 } 1381 1382 1383 void Assembler::movw(Register dst, const Operand& src) { 1384 EnsureSpace ensure_space(this); 1385 emit(0x66); 1386 emit_optional_rex_32(dst, src); 1387 emit(0x8B); 1388 emit_operand(dst, src); 1389 } 1390 1391 1392 void Assembler::movw(const Operand& dst, Register src) { 1393 EnsureSpace ensure_space(this); 1394 emit(0x66); 1395 emit_optional_rex_32(src, dst); 1396 emit(0x89); 1397 emit_operand(src, dst); 1398 } 1399 1400 1401 void Assembler::movw(const Operand& dst, Immediate imm) { 1402 EnsureSpace ensure_space(this); 1403 emit(0x66); 1404 emit_optional_rex_32(dst); 1405 emit(0xC7); 1406 emit_operand(0x0, dst); 1407 emit(static_cast<byte>(imm.value_ & 0xff)); 1408 emit(static_cast<byte>(imm.value_ >> 8)); 1409 } 1410 1411 1412 void Assembler::emit_mov(Register dst, const Operand& src, int size) { 1413 EnsureSpace ensure_space(this); 1414 emit_rex(dst, src, size); 1415 emit(0x8B); 1416 emit_operand(dst, src); 1417 } 1418 1419 1420 void Assembler::emit_mov(Register dst, Register src, int size) { 1421 EnsureSpace ensure_space(this); 1422 if (src.low_bits() == 4) { 1423 emit_rex(src, dst, size); 1424 emit(0x89); 1425 emit_modrm(src, dst); 1426 } else { 1427 emit_rex(dst, src, size); 1428 emit(0x8B); 1429 emit_modrm(dst, src); 1430 } 1431 } 1432 1433 1434 void Assembler::emit_mov(const Operand& dst, Register src, int size) { 1435 EnsureSpace ensure_space(this); 1436 emit_rex(src, dst, size); 1437 emit(0x89); 1438 emit_operand(src, dst); 1439 } 1440 1441 1442 void Assembler::emit_mov(Register dst, Immediate value, int size) { 1443 EnsureSpace ensure_space(this); 1444 emit_rex(dst, size); 1445 if (size == kInt64Size) { 1446 emit(0xC7); 1447 emit_modrm(0x0, dst); 1448 } else { 1449 DCHECK(size == kInt32Size); 1450 emit(0xB8 + dst.low_bits()); 1451 } 1452 emit(value); 1453 } 1454 1455 1456 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) { 1457 EnsureSpace ensure_space(this); 1458 emit_rex(dst, size); 1459 emit(0xC7); 1460 emit_operand(0x0, dst); 1461 emit(value); 1462 } 1463 1464 1465 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) { 1466 EnsureSpace ensure_space(this); 1467 emit_rex(dst, kPointerSize); 1468 emit(0xB8 | dst.low_bits()); 1469 emitp(value, rmode); 1470 } 1471 1472 1473 void Assembler::movq(Register dst, int64_t value) { 1474 EnsureSpace ensure_space(this); 1475 emit_rex_64(dst); 1476 emit(0xB8 | dst.low_bits()); 1477 emitq(value); 1478 } 1479 1480 1481 void Assembler::movq(Register dst, uint64_t value) { 1482 movq(dst, static_cast<int64_t>(value)); 1483 } 1484 1485 1486 // Loads the ip-relative location of the src label into the target location 1487 // (as a 32-bit offset sign extended to 64-bit). 1488 void Assembler::movl(const Operand& dst, Label* src) { 1489 EnsureSpace ensure_space(this); 1490 emit_optional_rex_32(dst); 1491 emit(0xC7); 1492 emit_operand(0, dst); 1493 if (src->is_bound()) { 1494 int offset = src->pos() - pc_offset() - sizeof(int32_t); 1495 DCHECK(offset <= 0); 1496 emitl(offset); 1497 } else if (src->is_linked()) { 1498 emitl(src->pos()); 1499 src->link_to(pc_offset() - sizeof(int32_t)); 1500 } else { 1501 DCHECK(src->is_unused()); 1502 int32_t current = pc_offset(); 1503 emitl(current); 1504 src->link_to(current); 1505 } 1506 } 1507 1508 1509 void Assembler::movsxbl(Register dst, Register src) { 1510 EnsureSpace ensure_space(this); 1511 if (!src.is_byte_register()) { 1512 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1513 emit_rex_32(dst, src); 1514 } else { 1515 emit_optional_rex_32(dst, src); 1516 } 1517 emit(0x0F); 1518 emit(0xBE); 1519 emit_modrm(dst, src); 1520 } 1521 1522 1523 void Assembler::movsxbl(Register dst, const Operand& src) { 1524 EnsureSpace ensure_space(this); 1525 emit_optional_rex_32(dst, src); 1526 emit(0x0F); 1527 emit(0xBE); 1528 emit_operand(dst, src); 1529 } 1530 1531 1532 void Assembler::movsxbq(Register dst, const Operand& src) { 1533 EnsureSpace ensure_space(this); 1534 emit_rex_64(dst, src); 1535 emit(0x0F); 1536 emit(0xBE); 1537 emit_operand(dst, src); 1538 } 1539 1540 1541 void Assembler::movsxwl(Register dst, Register src) { 1542 EnsureSpace ensure_space(this); 1543 emit_optional_rex_32(dst, src); 1544 emit(0x0F); 1545 emit(0xBF); 1546 emit_modrm(dst, src); 1547 } 1548 1549 1550 void Assembler::movsxwl(Register dst, const Operand& src) { 1551 EnsureSpace ensure_space(this); 1552 emit_optional_rex_32(dst, src); 1553 emit(0x0F); 1554 emit(0xBF); 1555 emit_operand(dst, src); 1556 } 1557 1558 1559 void Assembler::movsxwq(Register dst, const Operand& src) { 1560 EnsureSpace ensure_space(this); 1561 emit_rex_64(dst, src); 1562 emit(0x0F); 1563 emit(0xBF); 1564 emit_operand(dst, src); 1565 } 1566 1567 1568 void Assembler::movsxlq(Register dst, Register src) { 1569 EnsureSpace ensure_space(this); 1570 emit_rex_64(dst, src); 1571 emit(0x63); 1572 emit_modrm(dst, src); 1573 } 1574 1575 1576 void Assembler::movsxlq(Register dst, const Operand& src) { 1577 EnsureSpace ensure_space(this); 1578 emit_rex_64(dst, src); 1579 emit(0x63); 1580 emit_operand(dst, src); 1581 } 1582 1583 1584 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) { 1585 EnsureSpace ensure_space(this); 1586 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1587 // there is no need to make this a 64 bit operation. 1588 emit_optional_rex_32(dst, src); 1589 emit(0x0F); 1590 emit(0xB6); 1591 emit_operand(dst, src); 1592 } 1593 1594 1595 void Assembler::emit_movzxb(Register dst, Register src, int size) { 1596 EnsureSpace ensure_space(this); 1597 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1598 // there is no need to make this a 64 bit operation. 1599 if (!src.is_byte_register()) { 1600 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1601 emit_rex_32(dst, src); 1602 } else { 1603 emit_optional_rex_32(dst, src); 1604 } 1605 emit(0x0F); 1606 emit(0xB6); 1607 emit_modrm(dst, src); 1608 } 1609 1610 1611 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) { 1612 EnsureSpace ensure_space(this); 1613 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1614 // there is no need to make this a 64 bit operation. 1615 emit_optional_rex_32(dst, src); 1616 emit(0x0F); 1617 emit(0xB7); 1618 emit_operand(dst, src); 1619 } 1620 1621 1622 void Assembler::emit_movzxw(Register dst, Register src, int size) { 1623 EnsureSpace ensure_space(this); 1624 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1625 // there is no need to make this a 64 bit operation. 1626 emit_optional_rex_32(dst, src); 1627 emit(0x0F); 1628 emit(0xB7); 1629 emit_modrm(dst, src); 1630 } 1631 1632 1633 void Assembler::repmovsb() { 1634 EnsureSpace ensure_space(this); 1635 emit(0xF3); 1636 emit(0xA4); 1637 } 1638 1639 1640 void Assembler::repmovsw() { 1641 EnsureSpace ensure_space(this); 1642 emit(0x66); // Operand size override. 1643 emit(0xF3); 1644 emit(0xA4); 1645 } 1646 1647 1648 void Assembler::emit_repmovs(int size) { 1649 EnsureSpace ensure_space(this); 1650 emit(0xF3); 1651 emit_rex(size); 1652 emit(0xA5); 1653 } 1654 1655 1656 void Assembler::mull(Register src) { 1657 EnsureSpace ensure_space(this); 1658 emit_optional_rex_32(src); 1659 emit(0xF7); 1660 emit_modrm(0x4, src); 1661 } 1662 1663 1664 void Assembler::mull(const Operand& src) { 1665 EnsureSpace ensure_space(this); 1666 emit_optional_rex_32(src); 1667 emit(0xF7); 1668 emit_operand(0x4, src); 1669 } 1670 1671 1672 void Assembler::mulq(Register src) { 1673 EnsureSpace ensure_space(this); 1674 emit_rex_64(src); 1675 emit(0xF7); 1676 emit_modrm(0x4, src); 1677 } 1678 1679 1680 void Assembler::emit_neg(Register dst, int size) { 1681 EnsureSpace ensure_space(this); 1682 emit_rex(dst, size); 1683 emit(0xF7); 1684 emit_modrm(0x3, dst); 1685 } 1686 1687 1688 void Assembler::emit_neg(const Operand& dst, int size) { 1689 EnsureSpace ensure_space(this); 1690 emit_rex_64(dst); 1691 emit(0xF7); 1692 emit_operand(3, dst); 1693 } 1694 1695 1696 void Assembler::nop() { 1697 EnsureSpace ensure_space(this); 1698 emit(0x90); 1699 } 1700 1701 1702 void Assembler::emit_not(Register dst, int size) { 1703 EnsureSpace ensure_space(this); 1704 emit_rex(dst, size); 1705 emit(0xF7); 1706 emit_modrm(0x2, dst); 1707 } 1708 1709 1710 void Assembler::emit_not(const Operand& dst, int size) { 1711 EnsureSpace ensure_space(this); 1712 emit_rex(dst, size); 1713 emit(0xF7); 1714 emit_operand(2, dst); 1715 } 1716 1717 1718 void Assembler::Nop(int n) { 1719 // The recommended muti-byte sequences of NOP instructions from the Intel 64 1720 // and IA-32 Architectures Software Developer's Manual. 1721 // 1722 // Length Assembly Byte Sequence 1723 // 2 bytes 66 NOP 66 90H 1724 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H 1725 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H 1726 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H 1727 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H 1728 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H 1729 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H 1730 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00 1731 // 00000000H] 00H 1732 1733 EnsureSpace ensure_space(this); 1734 while (n > 0) { 1735 switch (n) { 1736 case 2: 1737 emit(0x66); 1738 case 1: 1739 emit(0x90); 1740 return; 1741 case 3: 1742 emit(0x0f); 1743 emit(0x1f); 1744 emit(0x00); 1745 return; 1746 case 4: 1747 emit(0x0f); 1748 emit(0x1f); 1749 emit(0x40); 1750 emit(0x00); 1751 return; 1752 case 6: 1753 emit(0x66); 1754 case 5: 1755 emit(0x0f); 1756 emit(0x1f); 1757 emit(0x44); 1758 emit(0x00); 1759 emit(0x00); 1760 return; 1761 case 7: 1762 emit(0x0f); 1763 emit(0x1f); 1764 emit(0x80); 1765 emit(0x00); 1766 emit(0x00); 1767 emit(0x00); 1768 emit(0x00); 1769 return; 1770 default: 1771 case 11: 1772 emit(0x66); 1773 n--; 1774 case 10: 1775 emit(0x66); 1776 n--; 1777 case 9: 1778 emit(0x66); 1779 n--; 1780 case 8: 1781 emit(0x0f); 1782 emit(0x1f); 1783 emit(0x84); 1784 emit(0x00); 1785 emit(0x00); 1786 emit(0x00); 1787 emit(0x00); 1788 emit(0x00); 1789 n -= 8; 1790 } 1791 } 1792 } 1793 1794 1795 void Assembler::popq(Register dst) { 1796 EnsureSpace ensure_space(this); 1797 emit_optional_rex_32(dst); 1798 emit(0x58 | dst.low_bits()); 1799 } 1800 1801 1802 void Assembler::popq(const Operand& dst) { 1803 EnsureSpace ensure_space(this); 1804 emit_optional_rex_32(dst); 1805 emit(0x8F); 1806 emit_operand(0, dst); 1807 } 1808 1809 1810 void Assembler::popfq() { 1811 EnsureSpace ensure_space(this); 1812 emit(0x9D); 1813 } 1814 1815 1816 void Assembler::pushq(Register src) { 1817 EnsureSpace ensure_space(this); 1818 emit_optional_rex_32(src); 1819 emit(0x50 | src.low_bits()); 1820 } 1821 1822 1823 void Assembler::pushq(const Operand& src) { 1824 EnsureSpace ensure_space(this); 1825 emit_optional_rex_32(src); 1826 emit(0xFF); 1827 emit_operand(6, src); 1828 } 1829 1830 1831 void Assembler::pushq(Immediate value) { 1832 EnsureSpace ensure_space(this); 1833 if (is_int8(value.value_)) { 1834 emit(0x6A); 1835 emit(value.value_); // Emit low byte of value. 1836 } else { 1837 emit(0x68); 1838 emitl(value.value_); 1839 } 1840 } 1841 1842 1843 void Assembler::pushq_imm32(int32_t imm32) { 1844 EnsureSpace ensure_space(this); 1845 emit(0x68); 1846 emitl(imm32); 1847 } 1848 1849 1850 void Assembler::pushfq() { 1851 EnsureSpace ensure_space(this); 1852 emit(0x9C); 1853 } 1854 1855 1856 void Assembler::ret(int imm16) { 1857 EnsureSpace ensure_space(this); 1858 DCHECK(is_uint16(imm16)); 1859 if (imm16 == 0) { 1860 emit(0xC3); 1861 } else { 1862 emit(0xC2); 1863 emit(imm16 & 0xFF); 1864 emit((imm16 >> 8) & 0xFF); 1865 } 1866 } 1867 1868 1869 void Assembler::ud2() { 1870 EnsureSpace ensure_space(this); 1871 emit(0x0F); 1872 emit(0x0B); 1873 } 1874 1875 1876 void Assembler::setcc(Condition cc, Register reg) { 1877 if (cc > last_condition) { 1878 movb(reg, Immediate(cc == always ? 1 : 0)); 1879 return; 1880 } 1881 EnsureSpace ensure_space(this); 1882 DCHECK(is_uint4(cc)); 1883 if (!reg.is_byte_register()) { 1884 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1885 emit_rex_32(reg); 1886 } 1887 emit(0x0F); 1888 emit(0x90 | cc); 1889 emit_modrm(0x0, reg); 1890 } 1891 1892 1893 void Assembler::shld(Register dst, Register src) { 1894 EnsureSpace ensure_space(this); 1895 emit_rex_64(src, dst); 1896 emit(0x0F); 1897 emit(0xA5); 1898 emit_modrm(src, dst); 1899 } 1900 1901 1902 void Assembler::shrd(Register dst, Register src) { 1903 EnsureSpace ensure_space(this); 1904 emit_rex_64(src, dst); 1905 emit(0x0F); 1906 emit(0xAD); 1907 emit_modrm(src, dst); 1908 } 1909 1910 1911 void Assembler::emit_xchg(Register dst, Register src, int size) { 1912 EnsureSpace ensure_space(this); 1913 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding 1914 Register other = src.is(rax) ? dst : src; 1915 emit_rex(other, size); 1916 emit(0x90 | other.low_bits()); 1917 } else if (dst.low_bits() == 4) { 1918 emit_rex(dst, src, size); 1919 emit(0x87); 1920 emit_modrm(dst, src); 1921 } else { 1922 emit_rex(src, dst, size); 1923 emit(0x87); 1924 emit_modrm(src, dst); 1925 } 1926 } 1927 1928 1929 void Assembler::emit_xchg(Register dst, const Operand& src, int size) { 1930 EnsureSpace ensure_space(this); 1931 emit_rex(dst, src, size); 1932 emit(0x87); 1933 emit_operand(dst, src); 1934 } 1935 1936 1937 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) { 1938 EnsureSpace ensure_space(this); 1939 if (kPointerSize == kInt64Size) { 1940 emit(0x48); // REX.W 1941 emit(0xA3); 1942 emitp(dst, mode); 1943 } else { 1944 DCHECK(kPointerSize == kInt32Size); 1945 emit(0xA3); 1946 emitp(dst, mode); 1947 // In 64-bit mode, need to zero extend the operand to 8 bytes. 1948 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 1949 // Developer's Manual Volume 2. 1950 emitl(0); 1951 } 1952 } 1953 1954 1955 void Assembler::store_rax(ExternalReference ref) { 1956 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 1957 } 1958 1959 1960 void Assembler::testb(Register dst, Register src) { 1961 EnsureSpace ensure_space(this); 1962 if (src.low_bits() == 4) { 1963 emit_rex_32(src, dst); 1964 emit(0x84); 1965 emit_modrm(src, dst); 1966 } else { 1967 if (!dst.is_byte_register() || !src.is_byte_register()) { 1968 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1969 emit_rex_32(dst, src); 1970 } 1971 emit(0x84); 1972 emit_modrm(dst, src); 1973 } 1974 } 1975 1976 1977 void Assembler::testb(Register reg, Immediate mask) { 1978 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 1979 EnsureSpace ensure_space(this); 1980 if (reg.is(rax)) { 1981 emit(0xA8); 1982 emit(mask.value_); // Low byte emitted. 1983 } else { 1984 if (!reg.is_byte_register()) { 1985 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1986 emit_rex_32(reg); 1987 } 1988 emit(0xF6); 1989 emit_modrm(0x0, reg); 1990 emit(mask.value_); // Low byte emitted. 1991 } 1992 } 1993 1994 1995 void Assembler::testb(const Operand& op, Immediate mask) { 1996 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 1997 EnsureSpace ensure_space(this); 1998 emit_optional_rex_32(rax, op); 1999 emit(0xF6); 2000 emit_operand(rax, op); // Operation code 0 2001 emit(mask.value_); // Low byte emitted. 2002 } 2003 2004 2005 void Assembler::testb(const Operand& op, Register reg) { 2006 EnsureSpace ensure_space(this); 2007 if (!reg.is_byte_register()) { 2008 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2009 emit_rex_32(reg, op); 2010 } else { 2011 emit_optional_rex_32(reg, op); 2012 } 2013 emit(0x84); 2014 emit_operand(reg, op); 2015 } 2016 2017 2018 void Assembler::emit_test(Register dst, Register src, int size) { 2019 EnsureSpace ensure_space(this); 2020 if (src.low_bits() == 4) { 2021 emit_rex(src, dst, size); 2022 emit(0x85); 2023 emit_modrm(src, dst); 2024 } else { 2025 emit_rex(dst, src, size); 2026 emit(0x85); 2027 emit_modrm(dst, src); 2028 } 2029 } 2030 2031 2032 void Assembler::emit_test(Register reg, Immediate mask, int size) { 2033 // testl with a mask that fits in the low byte is exactly testb. 2034 if (is_uint8(mask.value_)) { 2035 testb(reg, mask); 2036 return; 2037 } 2038 EnsureSpace ensure_space(this); 2039 if (reg.is(rax)) { 2040 emit_rex(rax, size); 2041 emit(0xA9); 2042 emit(mask); 2043 } else { 2044 emit_rex(reg, size); 2045 emit(0xF7); 2046 emit_modrm(0x0, reg); 2047 emit(mask); 2048 } 2049 } 2050 2051 2052 void Assembler::emit_test(const Operand& op, Immediate mask, int size) { 2053 // testl with a mask that fits in the low byte is exactly testb. 2054 if (is_uint8(mask.value_)) { 2055 testb(op, mask); 2056 return; 2057 } 2058 EnsureSpace ensure_space(this); 2059 emit_rex(rax, op, size); 2060 emit(0xF7); 2061 emit_operand(rax, op); // Operation code 0 2062 emit(mask); 2063 } 2064 2065 2066 void Assembler::emit_test(const Operand& op, Register reg, int size) { 2067 EnsureSpace ensure_space(this); 2068 emit_rex(reg, op, size); 2069 emit(0x85); 2070 emit_operand(reg, op); 2071 } 2072 2073 2074 // FPU instructions. 2075 2076 2077 void Assembler::fld(int i) { 2078 EnsureSpace ensure_space(this); 2079 emit_farith(0xD9, 0xC0, i); 2080 } 2081 2082 2083 void Assembler::fld1() { 2084 EnsureSpace ensure_space(this); 2085 emit(0xD9); 2086 emit(0xE8); 2087 } 2088 2089 2090 void Assembler::fldz() { 2091 EnsureSpace ensure_space(this); 2092 emit(0xD9); 2093 emit(0xEE); 2094 } 2095 2096 2097 void Assembler::fldpi() { 2098 EnsureSpace ensure_space(this); 2099 emit(0xD9); 2100 emit(0xEB); 2101 } 2102 2103 2104 void Assembler::fldln2() { 2105 EnsureSpace ensure_space(this); 2106 emit(0xD9); 2107 emit(0xED); 2108 } 2109 2110 2111 void Assembler::fld_s(const Operand& adr) { 2112 EnsureSpace ensure_space(this); 2113 emit_optional_rex_32(adr); 2114 emit(0xD9); 2115 emit_operand(0, adr); 2116 } 2117 2118 2119 void Assembler::fld_d(const Operand& adr) { 2120 EnsureSpace ensure_space(this); 2121 emit_optional_rex_32(adr); 2122 emit(0xDD); 2123 emit_operand(0, adr); 2124 } 2125 2126 2127 void Assembler::fstp_s(const Operand& adr) { 2128 EnsureSpace ensure_space(this); 2129 emit_optional_rex_32(adr); 2130 emit(0xD9); 2131 emit_operand(3, adr); 2132 } 2133 2134 2135 void Assembler::fstp_d(const Operand& adr) { 2136 EnsureSpace ensure_space(this); 2137 emit_optional_rex_32(adr); 2138 emit(0xDD); 2139 emit_operand(3, adr); 2140 } 2141 2142 2143 void Assembler::fstp(int index) { 2144 DCHECK(is_uint3(index)); 2145 EnsureSpace ensure_space(this); 2146 emit_farith(0xDD, 0xD8, index); 2147 } 2148 2149 2150 void Assembler::fild_s(const Operand& adr) { 2151 EnsureSpace ensure_space(this); 2152 emit_optional_rex_32(adr); 2153 emit(0xDB); 2154 emit_operand(0, adr); 2155 } 2156 2157 2158 void Assembler::fild_d(const Operand& adr) { 2159 EnsureSpace ensure_space(this); 2160 emit_optional_rex_32(adr); 2161 emit(0xDF); 2162 emit_operand(5, adr); 2163 } 2164 2165 2166 void Assembler::fistp_s(const Operand& adr) { 2167 EnsureSpace ensure_space(this); 2168 emit_optional_rex_32(adr); 2169 emit(0xDB); 2170 emit_operand(3, adr); 2171 } 2172 2173 2174 void Assembler::fisttp_s(const Operand& adr) { 2175 DCHECK(IsEnabled(SSE3)); 2176 EnsureSpace ensure_space(this); 2177 emit_optional_rex_32(adr); 2178 emit(0xDB); 2179 emit_operand(1, adr); 2180 } 2181 2182 2183 void Assembler::fisttp_d(const Operand& adr) { 2184 DCHECK(IsEnabled(SSE3)); 2185 EnsureSpace ensure_space(this); 2186 emit_optional_rex_32(adr); 2187 emit(0xDD); 2188 emit_operand(1, adr); 2189 } 2190 2191 2192 void Assembler::fist_s(const Operand& adr) { 2193 EnsureSpace ensure_space(this); 2194 emit_optional_rex_32(adr); 2195 emit(0xDB); 2196 emit_operand(2, adr); 2197 } 2198 2199 2200 void Assembler::fistp_d(const Operand& adr) { 2201 EnsureSpace ensure_space(this); 2202 emit_optional_rex_32(adr); 2203 emit(0xDF); 2204 emit_operand(7, adr); 2205 } 2206 2207 2208 void Assembler::fabs() { 2209 EnsureSpace ensure_space(this); 2210 emit(0xD9); 2211 emit(0xE1); 2212 } 2213 2214 2215 void Assembler::fchs() { 2216 EnsureSpace ensure_space(this); 2217 emit(0xD9); 2218 emit(0xE0); 2219 } 2220 2221 2222 void Assembler::fcos() { 2223 EnsureSpace ensure_space(this); 2224 emit(0xD9); 2225 emit(0xFF); 2226 } 2227 2228 2229 void Assembler::fsin() { 2230 EnsureSpace ensure_space(this); 2231 emit(0xD9); 2232 emit(0xFE); 2233 } 2234 2235 2236 void Assembler::fptan() { 2237 EnsureSpace ensure_space(this); 2238 emit(0xD9); 2239 emit(0xF2); 2240 } 2241 2242 2243 void Assembler::fyl2x() { 2244 EnsureSpace ensure_space(this); 2245 emit(0xD9); 2246 emit(0xF1); 2247 } 2248 2249 2250 void Assembler::f2xm1() { 2251 EnsureSpace ensure_space(this); 2252 emit(0xD9); 2253 emit(0xF0); 2254 } 2255 2256 2257 void Assembler::fscale() { 2258 EnsureSpace ensure_space(this); 2259 emit(0xD9); 2260 emit(0xFD); 2261 } 2262 2263 2264 void Assembler::fninit() { 2265 EnsureSpace ensure_space(this); 2266 emit(0xDB); 2267 emit(0xE3); 2268 } 2269 2270 2271 void Assembler::fadd(int i) { 2272 EnsureSpace ensure_space(this); 2273 emit_farith(0xDC, 0xC0, i); 2274 } 2275 2276 2277 void Assembler::fsub(int i) { 2278 EnsureSpace ensure_space(this); 2279 emit_farith(0xDC, 0xE8, i); 2280 } 2281 2282 2283 void Assembler::fisub_s(const Operand& adr) { 2284 EnsureSpace ensure_space(this); 2285 emit_optional_rex_32(adr); 2286 emit(0xDA); 2287 emit_operand(4, adr); 2288 } 2289 2290 2291 void Assembler::fmul(int i) { 2292 EnsureSpace ensure_space(this); 2293 emit_farith(0xDC, 0xC8, i); 2294 } 2295 2296 2297 void Assembler::fdiv(int i) { 2298 EnsureSpace ensure_space(this); 2299 emit_farith(0xDC, 0xF8, i); 2300 } 2301 2302 2303 void Assembler::faddp(int i) { 2304 EnsureSpace ensure_space(this); 2305 emit_farith(0xDE, 0xC0, i); 2306 } 2307 2308 2309 void Assembler::fsubp(int i) { 2310 EnsureSpace ensure_space(this); 2311 emit_farith(0xDE, 0xE8, i); 2312 } 2313 2314 2315 void Assembler::fsubrp(int i) { 2316 EnsureSpace ensure_space(this); 2317 emit_farith(0xDE, 0xE0, i); 2318 } 2319 2320 2321 void Assembler::fmulp(int i) { 2322 EnsureSpace ensure_space(this); 2323 emit_farith(0xDE, 0xC8, i); 2324 } 2325 2326 2327 void Assembler::fdivp(int i) { 2328 EnsureSpace ensure_space(this); 2329 emit_farith(0xDE, 0xF8, i); 2330 } 2331 2332 2333 void Assembler::fprem() { 2334 EnsureSpace ensure_space(this); 2335 emit(0xD9); 2336 emit(0xF8); 2337 } 2338 2339 2340 void Assembler::fprem1() { 2341 EnsureSpace ensure_space(this); 2342 emit(0xD9); 2343 emit(0xF5); 2344 } 2345 2346 2347 void Assembler::fxch(int i) { 2348 EnsureSpace ensure_space(this); 2349 emit_farith(0xD9, 0xC8, i); 2350 } 2351 2352 2353 void Assembler::fincstp() { 2354 EnsureSpace ensure_space(this); 2355 emit(0xD9); 2356 emit(0xF7); 2357 } 2358 2359 2360 void Assembler::ffree(int i) { 2361 EnsureSpace ensure_space(this); 2362 emit_farith(0xDD, 0xC0, i); 2363 } 2364 2365 2366 void Assembler::ftst() { 2367 EnsureSpace ensure_space(this); 2368 emit(0xD9); 2369 emit(0xE4); 2370 } 2371 2372 2373 void Assembler::fucomp(int i) { 2374 EnsureSpace ensure_space(this); 2375 emit_farith(0xDD, 0xE8, i); 2376 } 2377 2378 2379 void Assembler::fucompp() { 2380 EnsureSpace ensure_space(this); 2381 emit(0xDA); 2382 emit(0xE9); 2383 } 2384 2385 2386 void Assembler::fucomi(int i) { 2387 EnsureSpace ensure_space(this); 2388 emit(0xDB); 2389 emit(0xE8 + i); 2390 } 2391 2392 2393 void Assembler::fucomip() { 2394 EnsureSpace ensure_space(this); 2395 emit(0xDF); 2396 emit(0xE9); 2397 } 2398 2399 2400 void Assembler::fcompp() { 2401 EnsureSpace ensure_space(this); 2402 emit(0xDE); 2403 emit(0xD9); 2404 } 2405 2406 2407 void Assembler::fnstsw_ax() { 2408 EnsureSpace ensure_space(this); 2409 emit(0xDF); 2410 emit(0xE0); 2411 } 2412 2413 2414 void Assembler::fwait() { 2415 EnsureSpace ensure_space(this); 2416 emit(0x9B); 2417 } 2418 2419 2420 void Assembler::frndint() { 2421 EnsureSpace ensure_space(this); 2422 emit(0xD9); 2423 emit(0xFC); 2424 } 2425 2426 2427 void Assembler::fnclex() { 2428 EnsureSpace ensure_space(this); 2429 emit(0xDB); 2430 emit(0xE2); 2431 } 2432 2433 2434 void Assembler::sahf() { 2435 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf 2436 // in 64-bit mode. Test CpuID. 2437 DCHECK(IsEnabled(SAHF)); 2438 EnsureSpace ensure_space(this); 2439 emit(0x9E); 2440 } 2441 2442 2443 void Assembler::emit_farith(int b1, int b2, int i) { 2444 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode 2445 DCHECK(is_uint3(i)); // illegal stack offset 2446 emit(b1); 2447 emit(b2 + i); 2448 } 2449 2450 2451 // SSE operations. 2452 2453 void Assembler::andps(XMMRegister dst, XMMRegister src) { 2454 EnsureSpace ensure_space(this); 2455 emit_optional_rex_32(dst, src); 2456 emit(0x0F); 2457 emit(0x54); 2458 emit_sse_operand(dst, src); 2459 } 2460 2461 2462 void Assembler::andps(XMMRegister dst, const Operand& src) { 2463 EnsureSpace ensure_space(this); 2464 emit_optional_rex_32(dst, src); 2465 emit(0x0F); 2466 emit(0x54); 2467 emit_sse_operand(dst, src); 2468 } 2469 2470 2471 void Assembler::orps(XMMRegister dst, XMMRegister src) { 2472 EnsureSpace ensure_space(this); 2473 emit_optional_rex_32(dst, src); 2474 emit(0x0F); 2475 emit(0x56); 2476 emit_sse_operand(dst, src); 2477 } 2478 2479 2480 void Assembler::orps(XMMRegister dst, const Operand& src) { 2481 EnsureSpace ensure_space(this); 2482 emit_optional_rex_32(dst, src); 2483 emit(0x0F); 2484 emit(0x56); 2485 emit_sse_operand(dst, src); 2486 } 2487 2488 2489 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 2490 DCHECK(!IsEnabled(AVX)); 2491 EnsureSpace ensure_space(this); 2492 emit_optional_rex_32(dst, src); 2493 emit(0x0F); 2494 emit(0x57); 2495 emit_sse_operand(dst, src); 2496 } 2497 2498 2499 void Assembler::xorps(XMMRegister dst, const Operand& src) { 2500 DCHECK(!IsEnabled(AVX)); 2501 EnsureSpace ensure_space(this); 2502 emit_optional_rex_32(dst, src); 2503 emit(0x0F); 2504 emit(0x57); 2505 emit_sse_operand(dst, src); 2506 } 2507 2508 2509 void Assembler::addps(XMMRegister dst, XMMRegister src) { 2510 EnsureSpace ensure_space(this); 2511 emit_optional_rex_32(dst, src); 2512 emit(0x0F); 2513 emit(0x58); 2514 emit_sse_operand(dst, src); 2515 } 2516 2517 2518 void Assembler::addps(XMMRegister dst, const Operand& src) { 2519 EnsureSpace ensure_space(this); 2520 emit_optional_rex_32(dst, src); 2521 emit(0x0F); 2522 emit(0x58); 2523 emit_sse_operand(dst, src); 2524 } 2525 2526 2527 void Assembler::subps(XMMRegister dst, XMMRegister src) { 2528 EnsureSpace ensure_space(this); 2529 emit_optional_rex_32(dst, src); 2530 emit(0x0F); 2531 emit(0x5C); 2532 emit_sse_operand(dst, src); 2533 } 2534 2535 2536 void Assembler::subps(XMMRegister dst, const Operand& src) { 2537 EnsureSpace ensure_space(this); 2538 emit_optional_rex_32(dst, src); 2539 emit(0x0F); 2540 emit(0x5C); 2541 emit_sse_operand(dst, src); 2542 } 2543 2544 2545 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 2546 EnsureSpace ensure_space(this); 2547 emit_optional_rex_32(dst, src); 2548 emit(0x0F); 2549 emit(0x59); 2550 emit_sse_operand(dst, src); 2551 } 2552 2553 2554 void Assembler::mulps(XMMRegister dst, const Operand& src) { 2555 EnsureSpace ensure_space(this); 2556 emit_optional_rex_32(dst, src); 2557 emit(0x0F); 2558 emit(0x59); 2559 emit_sse_operand(dst, src); 2560 } 2561 2562 2563 void Assembler::divps(XMMRegister dst, XMMRegister src) { 2564 EnsureSpace ensure_space(this); 2565 emit_optional_rex_32(dst, src); 2566 emit(0x0F); 2567 emit(0x5E); 2568 emit_sse_operand(dst, src); 2569 } 2570 2571 2572 void Assembler::divps(XMMRegister dst, const Operand& src) { 2573 EnsureSpace ensure_space(this); 2574 emit_optional_rex_32(dst, src); 2575 emit(0x0F); 2576 emit(0x5E); 2577 emit_sse_operand(dst, src); 2578 } 2579 2580 2581 // SSE 2 operations. 2582 2583 void Assembler::movd(XMMRegister dst, Register src) { 2584 DCHECK(!IsEnabled(AVX)); 2585 EnsureSpace ensure_space(this); 2586 emit(0x66); 2587 emit_optional_rex_32(dst, src); 2588 emit(0x0F); 2589 emit(0x6E); 2590 emit_sse_operand(dst, src); 2591 } 2592 2593 2594 void Assembler::movd(XMMRegister dst, const Operand& src) { 2595 DCHECK(!IsEnabled(AVX)); 2596 EnsureSpace ensure_space(this); 2597 emit(0x66); 2598 emit_optional_rex_32(dst, src); 2599 emit(0x0F); 2600 emit(0x6E); 2601 emit_sse_operand(dst, src); 2602 } 2603 2604 2605 void Assembler::movd(Register dst, XMMRegister src) { 2606 DCHECK(!IsEnabled(AVX)); 2607 EnsureSpace ensure_space(this); 2608 emit(0x66); 2609 emit_optional_rex_32(src, dst); 2610 emit(0x0F); 2611 emit(0x7E); 2612 emit_sse_operand(src, dst); 2613 } 2614 2615 2616 void Assembler::movq(XMMRegister dst, Register src) { 2617 DCHECK(!IsEnabled(AVX)); 2618 EnsureSpace ensure_space(this); 2619 emit(0x66); 2620 emit_rex_64(dst, src); 2621 emit(0x0F); 2622 emit(0x6E); 2623 emit_sse_operand(dst, src); 2624 } 2625 2626 2627 void Assembler::movq(Register dst, XMMRegister src) { 2628 DCHECK(!IsEnabled(AVX)); 2629 EnsureSpace ensure_space(this); 2630 emit(0x66); 2631 emit_rex_64(src, dst); 2632 emit(0x0F); 2633 emit(0x7E); 2634 emit_sse_operand(src, dst); 2635 } 2636 2637 2638 void Assembler::movq(XMMRegister dst, XMMRegister src) { 2639 DCHECK(!IsEnabled(AVX)); 2640 EnsureSpace ensure_space(this); 2641 if (dst.low_bits() == 4) { 2642 // Avoid unnecessary SIB byte. 2643 emit(0xf3); 2644 emit_optional_rex_32(dst, src); 2645 emit(0x0F); 2646 emit(0x7e); 2647 emit_sse_operand(dst, src); 2648 } else { 2649 emit(0x66); 2650 emit_optional_rex_32(src, dst); 2651 emit(0x0F); 2652 emit(0xD6); 2653 emit_sse_operand(src, dst); 2654 } 2655 } 2656 2657 2658 void Assembler::movdqa(const Operand& dst, XMMRegister src) { 2659 EnsureSpace ensure_space(this); 2660 emit(0x66); 2661 emit_rex_64(src, dst); 2662 emit(0x0F); 2663 emit(0x7F); 2664 emit_sse_operand(src, dst); 2665 } 2666 2667 2668 void Assembler::movdqa(XMMRegister dst, const Operand& src) { 2669 EnsureSpace ensure_space(this); 2670 emit(0x66); 2671 emit_rex_64(dst, src); 2672 emit(0x0F); 2673 emit(0x6F); 2674 emit_sse_operand(dst, src); 2675 } 2676 2677 2678 void Assembler::movdqu(const Operand& dst, XMMRegister src) { 2679 EnsureSpace ensure_space(this); 2680 emit(0xF3); 2681 emit_rex_64(src, dst); 2682 emit(0x0F); 2683 emit(0x7F); 2684 emit_sse_operand(src, dst); 2685 } 2686 2687 2688 void Assembler::movdqu(XMMRegister dst, const Operand& src) { 2689 EnsureSpace ensure_space(this); 2690 emit(0xF3); 2691 emit_rex_64(dst, src); 2692 emit(0x0F); 2693 emit(0x6F); 2694 emit_sse_operand(dst, src); 2695 } 2696 2697 2698 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) { 2699 DCHECK(IsEnabled(SSE4_1)); 2700 DCHECK(is_uint8(imm8)); 2701 EnsureSpace ensure_space(this); 2702 emit(0x66); 2703 emit_optional_rex_32(src, dst); 2704 emit(0x0F); 2705 emit(0x3A); 2706 emit(0x17); 2707 emit_sse_operand(src, dst); 2708 emit(imm8); 2709 } 2710 2711 2712 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) { 2713 DCHECK(IsEnabled(SSE4_1)); 2714 EnsureSpace ensure_space(this); 2715 emit(0x66); 2716 emit_optional_rex_32(src, dst); 2717 emit(0x0F); 2718 emit(0x3A); 2719 emit(0x16); 2720 emit_sse_operand(src, dst); 2721 emit(imm8); 2722 } 2723 2724 2725 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) { 2726 DCHECK(IsEnabled(SSE4_1)); 2727 EnsureSpace ensure_space(this); 2728 emit(0x66); 2729 emit_optional_rex_32(dst, src); 2730 emit(0x0F); 2731 emit(0x3A); 2732 emit(0x22); 2733 emit_sse_operand(dst, src); 2734 emit(imm8); 2735 } 2736 2737 2738 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) { 2739 DCHECK(IsEnabled(SSE4_1)); 2740 EnsureSpace ensure_space(this); 2741 emit(0x66); 2742 emit_optional_rex_32(dst, src); 2743 emit(0x0F); 2744 emit(0x3A); 2745 emit(0x22); 2746 emit_sse_operand(dst, src); 2747 emit(imm8); 2748 } 2749 2750 2751 void Assembler::movsd(const Operand& dst, XMMRegister src) { 2752 DCHECK(!IsEnabled(AVX)); 2753 EnsureSpace ensure_space(this); 2754 emit(0xF2); // double 2755 emit_optional_rex_32(src, dst); 2756 emit(0x0F); 2757 emit(0x11); // store 2758 emit_sse_operand(src, dst); 2759 } 2760 2761 2762 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 2763 DCHECK(!IsEnabled(AVX)); 2764 EnsureSpace ensure_space(this); 2765 emit(0xF2); // double 2766 emit_optional_rex_32(dst, src); 2767 emit(0x0F); 2768 emit(0x10); // load 2769 emit_sse_operand(dst, src); 2770 } 2771 2772 2773 void Assembler::movsd(XMMRegister dst, const Operand& src) { 2774 DCHECK(!IsEnabled(AVX)); 2775 EnsureSpace ensure_space(this); 2776 emit(0xF2); // double 2777 emit_optional_rex_32(dst, src); 2778 emit(0x0F); 2779 emit(0x10); // load 2780 emit_sse_operand(dst, src); 2781 } 2782 2783 2784 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 2785 DCHECK(!IsEnabled(AVX)); 2786 EnsureSpace ensure_space(this); 2787 if (src.low_bits() == 4) { 2788 // Try to avoid an unnecessary SIB byte. 2789 emit_optional_rex_32(src, dst); 2790 emit(0x0F); 2791 emit(0x29); 2792 emit_sse_operand(src, dst); 2793 } else { 2794 emit_optional_rex_32(dst, src); 2795 emit(0x0F); 2796 emit(0x28); 2797 emit_sse_operand(dst, src); 2798 } 2799 } 2800 2801 2802 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) { 2803 DCHECK(is_uint8(imm8)); 2804 EnsureSpace ensure_space(this); 2805 emit_optional_rex_32(src, dst); 2806 emit(0x0F); 2807 emit(0xC6); 2808 emit_sse_operand(dst, src); 2809 emit(imm8); 2810 } 2811 2812 2813 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 2814 DCHECK(!IsEnabled(AVX)); 2815 EnsureSpace ensure_space(this); 2816 if (src.low_bits() == 4) { 2817 // Try to avoid an unnecessary SIB byte. 2818 emit(0x66); 2819 emit_optional_rex_32(src, dst); 2820 emit(0x0F); 2821 emit(0x29); 2822 emit_sse_operand(src, dst); 2823 } else { 2824 emit(0x66); 2825 emit_optional_rex_32(dst, src); 2826 emit(0x0F); 2827 emit(0x28); 2828 emit_sse_operand(dst, src); 2829 } 2830 } 2831 2832 2833 void Assembler::addss(XMMRegister dst, XMMRegister src) { 2834 EnsureSpace ensure_space(this); 2835 emit(0xF3); 2836 emit_optional_rex_32(dst, src); 2837 emit(0x0F); 2838 emit(0x58); 2839 emit_sse_operand(dst, src); 2840 } 2841 2842 2843 void Assembler::addss(XMMRegister dst, const Operand& src) { 2844 EnsureSpace ensure_space(this); 2845 emit(0xF3); 2846 emit_optional_rex_32(dst, src); 2847 emit(0x0F); 2848 emit(0x58); 2849 emit_sse_operand(dst, src); 2850 } 2851 2852 2853 void Assembler::subss(XMMRegister dst, XMMRegister src) { 2854 EnsureSpace ensure_space(this); 2855 emit(0xF3); 2856 emit_optional_rex_32(dst, src); 2857 emit(0x0F); 2858 emit(0x5C); 2859 emit_sse_operand(dst, src); 2860 } 2861 2862 2863 void Assembler::subss(XMMRegister dst, const Operand& src) { 2864 EnsureSpace ensure_space(this); 2865 emit(0xF3); 2866 emit_optional_rex_32(dst, src); 2867 emit(0x0F); 2868 emit(0x5C); 2869 emit_sse_operand(dst, src); 2870 } 2871 2872 2873 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 2874 EnsureSpace ensure_space(this); 2875 emit(0xF3); 2876 emit_optional_rex_32(dst, src); 2877 emit(0x0F); 2878 emit(0x59); 2879 emit_sse_operand(dst, src); 2880 } 2881 2882 2883 void Assembler::mulss(XMMRegister dst, const Operand& src) { 2884 EnsureSpace ensure_space(this); 2885 emit(0xF3); 2886 emit_optional_rex_32(dst, src); 2887 emit(0x0F); 2888 emit(0x59); 2889 emit_sse_operand(dst, src); 2890 } 2891 2892 2893 void Assembler::divss(XMMRegister dst, XMMRegister src) { 2894 EnsureSpace ensure_space(this); 2895 emit(0xF3); 2896 emit_optional_rex_32(dst, src); 2897 emit(0x0F); 2898 emit(0x5E); 2899 emit_sse_operand(dst, src); 2900 } 2901 2902 2903 void Assembler::divss(XMMRegister dst, const Operand& src) { 2904 EnsureSpace ensure_space(this); 2905 emit(0xF3); 2906 emit_optional_rex_32(dst, src); 2907 emit(0x0F); 2908 emit(0x5E); 2909 emit_sse_operand(dst, src); 2910 } 2911 2912 2913 void Assembler::maxss(XMMRegister dst, XMMRegister src) { 2914 EnsureSpace ensure_space(this); 2915 emit(0xF3); 2916 emit_optional_rex_32(dst, src); 2917 emit(0x0F); 2918 emit(0x5F); 2919 emit_sse_operand(dst, src); 2920 } 2921 2922 2923 void Assembler::maxss(XMMRegister dst, const Operand& src) { 2924 EnsureSpace ensure_space(this); 2925 emit(0xF3); 2926 emit_optional_rex_32(dst, src); 2927 emit(0x0F); 2928 emit(0x5F); 2929 emit_sse_operand(dst, src); 2930 } 2931 2932 2933 void Assembler::minss(XMMRegister dst, XMMRegister src) { 2934 EnsureSpace ensure_space(this); 2935 emit(0xF3); 2936 emit_optional_rex_32(dst, src); 2937 emit(0x0F); 2938 emit(0x5D); 2939 emit_sse_operand(dst, src); 2940 } 2941 2942 2943 void Assembler::minss(XMMRegister dst, const Operand& src) { 2944 EnsureSpace ensure_space(this); 2945 emit(0xF3); 2946 emit_optional_rex_32(dst, src); 2947 emit(0x0F); 2948 emit(0x5D); 2949 emit_sse_operand(dst, src); 2950 } 2951 2952 2953 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 2954 EnsureSpace ensure_space(this); 2955 emit(0xF3); 2956 emit_optional_rex_32(dst, src); 2957 emit(0x0F); 2958 emit(0x51); 2959 emit_sse_operand(dst, src); 2960 } 2961 2962 2963 void Assembler::sqrtss(XMMRegister dst, const Operand& src) { 2964 EnsureSpace ensure_space(this); 2965 emit(0xF3); 2966 emit_optional_rex_32(dst, src); 2967 emit(0x0F); 2968 emit(0x51); 2969 emit_sse_operand(dst, src); 2970 } 2971 2972 2973 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 2974 DCHECK(!IsEnabled(AVX)); 2975 EnsureSpace ensure_space(this); 2976 emit_optional_rex_32(dst, src); 2977 emit(0x0f); 2978 emit(0x2e); 2979 emit_sse_operand(dst, src); 2980 } 2981 2982 2983 void Assembler::ucomiss(XMMRegister dst, const Operand& src) { 2984 DCHECK(!IsEnabled(AVX)); 2985 EnsureSpace ensure_space(this); 2986 emit_optional_rex_32(dst, src); 2987 emit(0x0f); 2988 emit(0x2e); 2989 emit_sse_operand(dst, src); 2990 } 2991 2992 2993 void Assembler::movss(XMMRegister dst, XMMRegister src) { 2994 DCHECK(!IsEnabled(AVX)); 2995 EnsureSpace ensure_space(this); 2996 emit(0xF3); // single 2997 emit_optional_rex_32(dst, src); 2998 emit(0x0F); 2999 emit(0x10); // load 3000 emit_sse_operand(dst, src); 3001 } 3002 3003 3004 void Assembler::movss(XMMRegister dst, const Operand& src) { 3005 DCHECK(!IsEnabled(AVX)); 3006 EnsureSpace ensure_space(this); 3007 emit(0xF3); // single 3008 emit_optional_rex_32(dst, src); 3009 emit(0x0F); 3010 emit(0x10); // load 3011 emit_sse_operand(dst, src); 3012 } 3013 3014 3015 void Assembler::movss(const Operand& src, XMMRegister dst) { 3016 DCHECK(!IsEnabled(AVX)); 3017 EnsureSpace ensure_space(this); 3018 emit(0xF3); // single 3019 emit_optional_rex_32(dst, src); 3020 emit(0x0F); 3021 emit(0x11); // store 3022 emit_sse_operand(dst, src); 3023 } 3024 3025 3026 void Assembler::psllq(XMMRegister reg, byte imm8) { 3027 DCHECK(!IsEnabled(AVX)); 3028 EnsureSpace ensure_space(this); 3029 emit(0x66); 3030 emit_optional_rex_32(reg); 3031 emit(0x0F); 3032 emit(0x73); 3033 emit_sse_operand(rsi, reg); // rsi == 6 3034 emit(imm8); 3035 } 3036 3037 3038 void Assembler::psrlq(XMMRegister reg, byte imm8) { 3039 DCHECK(!IsEnabled(AVX)); 3040 EnsureSpace ensure_space(this); 3041 emit(0x66); 3042 emit_optional_rex_32(reg); 3043 emit(0x0F); 3044 emit(0x73); 3045 emit_sse_operand(rdx, reg); // rdx == 2 3046 emit(imm8); 3047 } 3048 3049 3050 void Assembler::pslld(XMMRegister reg, byte imm8) { 3051 EnsureSpace ensure_space(this); 3052 emit(0x66); 3053 emit_optional_rex_32(reg); 3054 emit(0x0F); 3055 emit(0x72); 3056 emit_sse_operand(rsi, reg); // rsi == 6 3057 emit(imm8); 3058 } 3059 3060 3061 void Assembler::psrld(XMMRegister reg, byte imm8) { 3062 EnsureSpace ensure_space(this); 3063 emit(0x66); 3064 emit_optional_rex_32(reg); 3065 emit(0x0F); 3066 emit(0x72); 3067 emit_sse_operand(rdx, reg); // rdx == 2 3068 emit(imm8); 3069 } 3070 3071 3072 void Assembler::cvttss2si(Register dst, const Operand& src) { 3073 DCHECK(!IsEnabled(AVX)); 3074 EnsureSpace ensure_space(this); 3075 emit(0xF3); 3076 emit_optional_rex_32(dst, src); 3077 emit(0x0F); 3078 emit(0x2C); 3079 emit_operand(dst, src); 3080 } 3081 3082 3083 void Assembler::cvttss2si(Register dst, XMMRegister src) { 3084 DCHECK(!IsEnabled(AVX)); 3085 EnsureSpace ensure_space(this); 3086 emit(0xF3); 3087 emit_optional_rex_32(dst, src); 3088 emit(0x0F); 3089 emit(0x2C); 3090 emit_sse_operand(dst, src); 3091 } 3092 3093 3094 void Assembler::cvttsd2si(Register dst, const Operand& src) { 3095 DCHECK(!IsEnabled(AVX)); 3096 EnsureSpace ensure_space(this); 3097 emit(0xF2); 3098 emit_optional_rex_32(dst, src); 3099 emit(0x0F); 3100 emit(0x2C); 3101 emit_operand(dst, src); 3102 } 3103 3104 3105 void Assembler::cvttsd2si(Register dst, XMMRegister src) { 3106 DCHECK(!IsEnabled(AVX)); 3107 EnsureSpace ensure_space(this); 3108 emit(0xF2); 3109 emit_optional_rex_32(dst, src); 3110 emit(0x0F); 3111 emit(0x2C); 3112 emit_sse_operand(dst, src); 3113 } 3114 3115 3116 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 3117 DCHECK(!IsEnabled(AVX)); 3118 EnsureSpace ensure_space(this); 3119 emit(0xF3); 3120 emit_rex_64(dst, src); 3121 emit(0x0F); 3122 emit(0x2C); 3123 emit_sse_operand(dst, src); 3124 } 3125 3126 3127 void Assembler::cvttss2siq(Register dst, const Operand& src) { 3128 DCHECK(!IsEnabled(AVX)); 3129 EnsureSpace ensure_space(this); 3130 emit(0xF3); 3131 emit_rex_64(dst, src); 3132 emit(0x0F); 3133 emit(0x2C); 3134 emit_sse_operand(dst, src); 3135 } 3136 3137 3138 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 3139 DCHECK(!IsEnabled(AVX)); 3140 EnsureSpace ensure_space(this); 3141 emit(0xF2); 3142 emit_rex_64(dst, src); 3143 emit(0x0F); 3144 emit(0x2C); 3145 emit_sse_operand(dst, src); 3146 } 3147 3148 3149 void Assembler::cvttsd2siq(Register dst, const Operand& src) { 3150 DCHECK(!IsEnabled(AVX)); 3151 EnsureSpace ensure_space(this); 3152 emit(0xF2); 3153 emit_rex_64(dst, src); 3154 emit(0x0F); 3155 emit(0x2C); 3156 emit_sse_operand(dst, src); 3157 } 3158 3159 3160 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) { 3161 DCHECK(!IsEnabled(AVX)); 3162 EnsureSpace ensure_space(this); 3163 emit(0xF2); 3164 emit_optional_rex_32(dst, src); 3165 emit(0x0F); 3166 emit(0x2A); 3167 emit_sse_operand(dst, src); 3168 } 3169 3170 3171 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) { 3172 DCHECK(!IsEnabled(AVX)); 3173 EnsureSpace ensure_space(this); 3174 emit(0xF2); 3175 emit_optional_rex_32(dst, src); 3176 emit(0x0F); 3177 emit(0x2A); 3178 emit_sse_operand(dst, src); 3179 } 3180 3181 3182 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) { 3183 EnsureSpace ensure_space(this); 3184 emit(0xF3); 3185 emit_optional_rex_32(dst, src); 3186 emit(0x0F); 3187 emit(0x2A); 3188 emit_sse_operand(dst, src); 3189 } 3190 3191 3192 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) { 3193 DCHECK(!IsEnabled(AVX)); 3194 EnsureSpace ensure_space(this); 3195 emit(0xF3); 3196 emit_rex_64(dst, src); 3197 emit(0x0F); 3198 emit(0x2A); 3199 emit_sse_operand(dst, src); 3200 } 3201 3202 3203 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) { 3204 DCHECK(!IsEnabled(AVX)); 3205 EnsureSpace ensure_space(this); 3206 emit(0xF3); 3207 emit_rex_64(dst, src); 3208 emit(0x0F); 3209 emit(0x2A); 3210 emit_sse_operand(dst, src); 3211 } 3212 3213 3214 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) { 3215 DCHECK(!IsEnabled(AVX)); 3216 EnsureSpace ensure_space(this); 3217 emit(0xF2); 3218 emit_rex_64(dst, src); 3219 emit(0x0F); 3220 emit(0x2A); 3221 emit_sse_operand(dst, src); 3222 } 3223 3224 3225 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) { 3226 DCHECK(!IsEnabled(AVX)); 3227 EnsureSpace ensure_space(this); 3228 emit(0xF2); 3229 emit_rex_64(dst, src); 3230 emit(0x0F); 3231 emit(0x2A); 3232 emit_sse_operand(dst, src); 3233 } 3234 3235 3236 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 3237 DCHECK(!IsEnabled(AVX)); 3238 EnsureSpace ensure_space(this); 3239 emit(0xF3); 3240 emit_optional_rex_32(dst, src); 3241 emit(0x0F); 3242 emit(0x5A); 3243 emit_sse_operand(dst, src); 3244 } 3245 3246 3247 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) { 3248 DCHECK(!IsEnabled(AVX)); 3249 EnsureSpace ensure_space(this); 3250 emit(0xF3); 3251 emit_optional_rex_32(dst, src); 3252 emit(0x0F); 3253 emit(0x5A); 3254 emit_sse_operand(dst, src); 3255 } 3256 3257 3258 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 3259 DCHECK(!IsEnabled(AVX)); 3260 EnsureSpace ensure_space(this); 3261 emit(0xF2); 3262 emit_optional_rex_32(dst, src); 3263 emit(0x0F); 3264 emit(0x5A); 3265 emit_sse_operand(dst, src); 3266 } 3267 3268 3269 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) { 3270 DCHECK(!IsEnabled(AVX)); 3271 EnsureSpace ensure_space(this); 3272 emit(0xF2); 3273 emit_optional_rex_32(dst, src); 3274 emit(0x0F); 3275 emit(0x5A); 3276 emit_sse_operand(dst, src); 3277 } 3278 3279 3280 void Assembler::cvtsd2si(Register dst, XMMRegister src) { 3281 DCHECK(!IsEnabled(AVX)); 3282 EnsureSpace ensure_space(this); 3283 emit(0xF2); 3284 emit_optional_rex_32(dst, src); 3285 emit(0x0F); 3286 emit(0x2D); 3287 emit_sse_operand(dst, src); 3288 } 3289 3290 3291 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 3292 DCHECK(!IsEnabled(AVX)); 3293 EnsureSpace ensure_space(this); 3294 emit(0xF2); 3295 emit_rex_64(dst, src); 3296 emit(0x0F); 3297 emit(0x2D); 3298 emit_sse_operand(dst, src); 3299 } 3300 3301 3302 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 3303 EnsureSpace ensure_space(this); 3304 emit(0xF2); 3305 emit_optional_rex_32(dst, src); 3306 emit(0x0F); 3307 emit(0x58); 3308 emit_sse_operand(dst, src); 3309 } 3310 3311 3312 void Assembler::addsd(XMMRegister dst, const Operand& src) { 3313 EnsureSpace ensure_space(this); 3314 emit(0xF2); 3315 emit_optional_rex_32(dst, src); 3316 emit(0x0F); 3317 emit(0x58); 3318 emit_sse_operand(dst, src); 3319 } 3320 3321 3322 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 3323 EnsureSpace ensure_space(this); 3324 emit(0xF2); 3325 emit_optional_rex_32(dst, src); 3326 emit(0x0F); 3327 emit(0x59); 3328 emit_sse_operand(dst, src); 3329 } 3330 3331 3332 void Assembler::mulsd(XMMRegister dst, const Operand& src) { 3333 EnsureSpace ensure_space(this); 3334 emit(0xF2); 3335 emit_optional_rex_32(dst, src); 3336 emit(0x0F); 3337 emit(0x59); 3338 emit_sse_operand(dst, src); 3339 } 3340 3341 3342 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 3343 EnsureSpace ensure_space(this); 3344 emit(0xF2); 3345 emit_optional_rex_32(dst, src); 3346 emit(0x0F); 3347 emit(0x5C); 3348 emit_sse_operand(dst, src); 3349 } 3350 3351 3352 void Assembler::subsd(XMMRegister dst, const Operand& src) { 3353 EnsureSpace ensure_space(this); 3354 emit(0xF2); 3355 emit_optional_rex_32(dst, src); 3356 emit(0x0F); 3357 emit(0x5C); 3358 emit_sse_operand(dst, src); 3359 } 3360 3361 3362 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 3363 EnsureSpace ensure_space(this); 3364 emit(0xF2); 3365 emit_optional_rex_32(dst, src); 3366 emit(0x0F); 3367 emit(0x5E); 3368 emit_sse_operand(dst, src); 3369 } 3370 3371 3372 void Assembler::divsd(XMMRegister dst, const Operand& src) { 3373 EnsureSpace ensure_space(this); 3374 emit(0xF2); 3375 emit_optional_rex_32(dst, src); 3376 emit(0x0F); 3377 emit(0x5E); 3378 emit_sse_operand(dst, src); 3379 } 3380 3381 3382 void Assembler::maxsd(XMMRegister dst, XMMRegister src) { 3383 EnsureSpace ensure_space(this); 3384 emit(0xF2); 3385 emit_optional_rex_32(dst, src); 3386 emit(0x0F); 3387 emit(0x5F); 3388 emit_sse_operand(dst, src); 3389 } 3390 3391 3392 void Assembler::maxsd(XMMRegister dst, const Operand& src) { 3393 EnsureSpace ensure_space(this); 3394 emit(0xF2); 3395 emit_optional_rex_32(dst, src); 3396 emit(0x0F); 3397 emit(0x5F); 3398 emit_sse_operand(dst, src); 3399 } 3400 3401 3402 void Assembler::minsd(XMMRegister dst, XMMRegister src) { 3403 EnsureSpace ensure_space(this); 3404 emit(0xF2); 3405 emit_optional_rex_32(dst, src); 3406 emit(0x0F); 3407 emit(0x5D); 3408 emit_sse_operand(dst, src); 3409 } 3410 3411 3412 void Assembler::minsd(XMMRegister dst, const Operand& src) { 3413 EnsureSpace ensure_space(this); 3414 emit(0xF2); 3415 emit_optional_rex_32(dst, src); 3416 emit(0x0F); 3417 emit(0x5D); 3418 emit_sse_operand(dst, src); 3419 } 3420 3421 3422 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 3423 EnsureSpace ensure_space(this); 3424 emit(0x66); 3425 emit_optional_rex_32(dst, src); 3426 emit(0x0F); 3427 emit(0x54); 3428 emit_sse_operand(dst, src); 3429 } 3430 3431 3432 void Assembler::orpd(XMMRegister dst, XMMRegister src) { 3433 EnsureSpace ensure_space(this); 3434 emit(0x66); 3435 emit_optional_rex_32(dst, src); 3436 emit(0x0F); 3437 emit(0x56); 3438 emit_sse_operand(dst, src); 3439 } 3440 3441 3442 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 3443 DCHECK(!IsEnabled(AVX)); 3444 EnsureSpace ensure_space(this); 3445 emit(0x66); 3446 emit_optional_rex_32(dst, src); 3447 emit(0x0F); 3448 emit(0x57); 3449 emit_sse_operand(dst, src); 3450 } 3451 3452 3453 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 3454 DCHECK(!IsEnabled(AVX)); 3455 EnsureSpace ensure_space(this); 3456 emit(0xF2); 3457 emit_optional_rex_32(dst, src); 3458 emit(0x0F); 3459 emit(0x51); 3460 emit_sse_operand(dst, src); 3461 } 3462 3463 3464 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) { 3465 DCHECK(!IsEnabled(AVX)); 3466 EnsureSpace ensure_space(this); 3467 emit(0xF2); 3468 emit_optional_rex_32(dst, src); 3469 emit(0x0F); 3470 emit(0x51); 3471 emit_sse_operand(dst, src); 3472 } 3473 3474 3475 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 3476 DCHECK(!IsEnabled(AVX)); 3477 EnsureSpace ensure_space(this); 3478 emit(0x66); 3479 emit_optional_rex_32(dst, src); 3480 emit(0x0f); 3481 emit(0x2e); 3482 emit_sse_operand(dst, src); 3483 } 3484 3485 3486 void Assembler::ucomisd(XMMRegister dst, const Operand& src) { 3487 DCHECK(!IsEnabled(AVX)); 3488 EnsureSpace ensure_space(this); 3489 emit(0x66); 3490 emit_optional_rex_32(dst, src); 3491 emit(0x0f); 3492 emit(0x2e); 3493 emit_sse_operand(dst, src); 3494 } 3495 3496 3497 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) { 3498 EnsureSpace ensure_space(this); 3499 emit(0xF2); 3500 emit_optional_rex_32(dst, src); 3501 emit(0x0F); 3502 emit(0xC2); 3503 emit_sse_operand(dst, src); 3504 emit(0x01); // LT == 1 3505 } 3506 3507 3508 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3509 DCHECK(!IsEnabled(AVX)); 3510 DCHECK(IsEnabled(SSE4_1)); 3511 EnsureSpace ensure_space(this); 3512 emit(0x66); 3513 emit_optional_rex_32(dst, src); 3514 emit(0x0f); 3515 emit(0x3a); 3516 emit(0x0a); 3517 emit_sse_operand(dst, src); 3518 // Mask precision exception. 3519 emit(static_cast<byte>(mode) | 0x8); 3520 } 3521 3522 3523 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3524 DCHECK(!IsEnabled(AVX)); 3525 DCHECK(IsEnabled(SSE4_1)); 3526 EnsureSpace ensure_space(this); 3527 emit(0x66); 3528 emit_optional_rex_32(dst, src); 3529 emit(0x0f); 3530 emit(0x3a); 3531 emit(0x0b); 3532 emit_sse_operand(dst, src); 3533 // Mask precision exception. 3534 emit(static_cast<byte>(mode) | 0x8); 3535 } 3536 3537 3538 void Assembler::movmskpd(Register dst, XMMRegister src) { 3539 EnsureSpace ensure_space(this); 3540 emit(0x66); 3541 emit_optional_rex_32(dst, src); 3542 emit(0x0f); 3543 emit(0x50); 3544 emit_sse_operand(dst, src); 3545 } 3546 3547 3548 void Assembler::movmskps(Register dst, XMMRegister src) { 3549 EnsureSpace ensure_space(this); 3550 emit_optional_rex_32(dst, src); 3551 emit(0x0f); 3552 emit(0x50); 3553 emit_sse_operand(dst, src); 3554 } 3555 3556 3557 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) { 3558 DCHECK(!IsEnabled(AVX)); 3559 EnsureSpace ensure_space(this); 3560 emit(0x66); 3561 emit_optional_rex_32(dst, src); 3562 emit(0x0F); 3563 emit(0x76); 3564 emit_sse_operand(dst, src); 3565 } 3566 3567 3568 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 3569 EnsureSpace ensure_space(this); 3570 emit(0x66); 3571 emit_optional_rex_32(dst, src); 3572 emit(0x0F); 3573 emit(0x62); 3574 emit_sse_operand(dst, src); 3575 } 3576 3577 3578 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) { 3579 EnsureSpace ensure_space(this); 3580 emit(0x66); 3581 emit_optional_rex_32(dst, src); 3582 emit(0x0F); 3583 emit(0x6A); 3584 emit_sse_operand(dst, src); 3585 } 3586 3587 3588 // AVX instructions 3589 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3590 XMMRegister src2) { 3591 DCHECK(IsEnabled(FMA3)); 3592 EnsureSpace ensure_space(this); 3593 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3594 emit(op); 3595 emit_sse_operand(dst, src2); 3596 } 3597 3598 3599 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3600 const Operand& src2) { 3601 DCHECK(IsEnabled(FMA3)); 3602 EnsureSpace ensure_space(this); 3603 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3604 emit(op); 3605 emit_sse_operand(dst, src2); 3606 } 3607 3608 3609 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3610 XMMRegister src2) { 3611 DCHECK(IsEnabled(FMA3)); 3612 EnsureSpace ensure_space(this); 3613 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3614 emit(op); 3615 emit_sse_operand(dst, src2); 3616 } 3617 3618 3619 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3620 const Operand& src2) { 3621 DCHECK(IsEnabled(FMA3)); 3622 EnsureSpace ensure_space(this); 3623 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3624 emit(op); 3625 emit_sse_operand(dst, src2); 3626 } 3627 3628 3629 void Assembler::vmovd(XMMRegister dst, Register src) { 3630 DCHECK(IsEnabled(AVX)); 3631 EnsureSpace ensure_space(this); 3632 XMMRegister isrc = {src.code()}; 3633 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0); 3634 emit(0x6e); 3635 emit_sse_operand(dst, src); 3636 } 3637 3638 3639 void Assembler::vmovd(XMMRegister dst, const Operand& src) { 3640 DCHECK(IsEnabled(AVX)); 3641 EnsureSpace ensure_space(this); 3642 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0); 3643 emit(0x6e); 3644 emit_sse_operand(dst, src); 3645 } 3646 3647 3648 void Assembler::vmovd(Register dst, XMMRegister src) { 3649 DCHECK(IsEnabled(AVX)); 3650 EnsureSpace ensure_space(this); 3651 XMMRegister idst = {dst.code()}; 3652 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0); 3653 emit(0x7e); 3654 emit_sse_operand(src, dst); 3655 } 3656 3657 3658 void Assembler::vmovq(XMMRegister dst, Register src) { 3659 DCHECK(IsEnabled(AVX)); 3660 EnsureSpace ensure_space(this); 3661 XMMRegister isrc = {src.code()}; 3662 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1); 3663 emit(0x6e); 3664 emit_sse_operand(dst, src); 3665 } 3666 3667 3668 void Assembler::vmovq(XMMRegister dst, const Operand& src) { 3669 DCHECK(IsEnabled(AVX)); 3670 EnsureSpace ensure_space(this); 3671 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1); 3672 emit(0x6e); 3673 emit_sse_operand(dst, src); 3674 } 3675 3676 3677 void Assembler::vmovq(Register dst, XMMRegister src) { 3678 DCHECK(IsEnabled(AVX)); 3679 EnsureSpace ensure_space(this); 3680 XMMRegister idst = {dst.code()}; 3681 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1); 3682 emit(0x7e); 3683 emit_sse_operand(src, dst); 3684 } 3685 3686 3687 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, 3688 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, VexW w) { 3689 DCHECK(IsEnabled(AVX)); 3690 EnsureSpace ensure_space(this); 3691 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 3692 emit(op); 3693 emit_sse_operand(dst, src2); 3694 } 3695 3696 3697 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, 3698 const Operand& src2, SIMDPrefix pp, LeadingOpcode m, 3699 VexW w) { 3700 DCHECK(IsEnabled(AVX)); 3701 EnsureSpace ensure_space(this); 3702 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 3703 emit(op); 3704 emit_sse_operand(dst, src2); 3705 } 3706 3707 3708 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 3709 XMMRegister src2) { 3710 DCHECK(IsEnabled(AVX)); 3711 EnsureSpace ensure_space(this); 3712 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 3713 emit(op); 3714 emit_sse_operand(dst, src2); 3715 } 3716 3717 3718 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 3719 const Operand& src2) { 3720 DCHECK(IsEnabled(AVX)); 3721 EnsureSpace ensure_space(this); 3722 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 3723 emit(op); 3724 emit_sse_operand(dst, src2); 3725 } 3726 3727 3728 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 3729 XMMRegister src2) { 3730 DCHECK(IsEnabled(AVX)); 3731 EnsureSpace ensure_space(this); 3732 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 3733 emit(op); 3734 emit_sse_operand(dst, src2); 3735 } 3736 3737 3738 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 3739 const Operand& src2) { 3740 DCHECK(IsEnabled(AVX)); 3741 EnsureSpace ensure_space(this); 3742 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 3743 emit(op); 3744 emit_sse_operand(dst, src2); 3745 } 3746 3747 3748 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) { 3749 DCHECK(IsEnabled(AVX)); 3750 EnsureSpace ensure_space(this); 3751 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 3752 emit(0x2e); 3753 emit_sse_operand(dst, src); 3754 } 3755 3756 3757 void Assembler::vucomiss(XMMRegister dst, const Operand& src) { 3758 DCHECK(IsEnabled(AVX)); 3759 EnsureSpace ensure_space(this); 3760 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 3761 emit(0x2e); 3762 emit_sse_operand(dst, src); 3763 } 3764 3765 3766 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 3767 XMMRegister src2) { 3768 DCHECK(IsEnabled(AVX)); 3769 EnsureSpace ensure_space(this); 3770 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 3771 emit(op); 3772 emit_sse_operand(dst, src2); 3773 } 3774 3775 3776 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 3777 const Operand& src2) { 3778 DCHECK(IsEnabled(AVX)); 3779 EnsureSpace ensure_space(this); 3780 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 3781 emit(op); 3782 emit_sse_operand(dst, src2); 3783 } 3784 3785 3786 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) { 3787 DCHECK(IsEnabled(BMI1)); 3788 EnsureSpace ensure_space(this); 3789 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 3790 emit(op); 3791 emit_modrm(reg, rm); 3792 } 3793 3794 3795 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) { 3796 DCHECK(IsEnabled(BMI1)); 3797 EnsureSpace ensure_space(this); 3798 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 3799 emit(op); 3800 emit_operand(reg, rm); 3801 } 3802 3803 3804 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) { 3805 DCHECK(IsEnabled(BMI1)); 3806 EnsureSpace ensure_space(this); 3807 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 3808 emit(op); 3809 emit_modrm(reg, rm); 3810 } 3811 3812 3813 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) { 3814 DCHECK(IsEnabled(BMI1)); 3815 EnsureSpace ensure_space(this); 3816 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 3817 emit(op); 3818 emit_operand(reg, rm); 3819 } 3820 3821 3822 void Assembler::tzcntq(Register dst, Register src) { 3823 DCHECK(IsEnabled(BMI1)); 3824 EnsureSpace ensure_space(this); 3825 emit(0xF3); 3826 emit_rex_64(dst, src); 3827 emit(0x0F); 3828 emit(0xBC); 3829 emit_modrm(dst, src); 3830 } 3831 3832 3833 void Assembler::tzcntq(Register dst, const Operand& src) { 3834 DCHECK(IsEnabled(BMI1)); 3835 EnsureSpace ensure_space(this); 3836 emit(0xF3); 3837 emit_rex_64(dst, src); 3838 emit(0x0F); 3839 emit(0xBC); 3840 emit_operand(dst, src); 3841 } 3842 3843 3844 void Assembler::tzcntl(Register dst, Register src) { 3845 DCHECK(IsEnabled(BMI1)); 3846 EnsureSpace ensure_space(this); 3847 emit(0xF3); 3848 emit_optional_rex_32(dst, src); 3849 emit(0x0F); 3850 emit(0xBC); 3851 emit_modrm(dst, src); 3852 } 3853 3854 3855 void Assembler::tzcntl(Register dst, const Operand& src) { 3856 DCHECK(IsEnabled(BMI1)); 3857 EnsureSpace ensure_space(this); 3858 emit(0xF3); 3859 emit_optional_rex_32(dst, src); 3860 emit(0x0F); 3861 emit(0xBC); 3862 emit_operand(dst, src); 3863 } 3864 3865 3866 void Assembler::lzcntq(Register dst, Register src) { 3867 DCHECK(IsEnabled(LZCNT)); 3868 EnsureSpace ensure_space(this); 3869 emit(0xF3); 3870 emit_rex_64(dst, src); 3871 emit(0x0F); 3872 emit(0xBD); 3873 emit_modrm(dst, src); 3874 } 3875 3876 3877 void Assembler::lzcntq(Register dst, const Operand& src) { 3878 DCHECK(IsEnabled(LZCNT)); 3879 EnsureSpace ensure_space(this); 3880 emit(0xF3); 3881 emit_rex_64(dst, src); 3882 emit(0x0F); 3883 emit(0xBD); 3884 emit_operand(dst, src); 3885 } 3886 3887 3888 void Assembler::lzcntl(Register dst, Register src) { 3889 DCHECK(IsEnabled(LZCNT)); 3890 EnsureSpace ensure_space(this); 3891 emit(0xF3); 3892 emit_optional_rex_32(dst, src); 3893 emit(0x0F); 3894 emit(0xBD); 3895 emit_modrm(dst, src); 3896 } 3897 3898 3899 void Assembler::lzcntl(Register dst, const Operand& src) { 3900 DCHECK(IsEnabled(LZCNT)); 3901 EnsureSpace ensure_space(this); 3902 emit(0xF3); 3903 emit_optional_rex_32(dst, src); 3904 emit(0x0F); 3905 emit(0xBD); 3906 emit_operand(dst, src); 3907 } 3908 3909 3910 void Assembler::popcntq(Register dst, Register src) { 3911 DCHECK(IsEnabled(POPCNT)); 3912 EnsureSpace ensure_space(this); 3913 emit(0xF3); 3914 emit_rex_64(dst, src); 3915 emit(0x0F); 3916 emit(0xB8); 3917 emit_modrm(dst, src); 3918 } 3919 3920 3921 void Assembler::popcntq(Register dst, const Operand& src) { 3922 DCHECK(IsEnabled(POPCNT)); 3923 EnsureSpace ensure_space(this); 3924 emit(0xF3); 3925 emit_rex_64(dst, src); 3926 emit(0x0F); 3927 emit(0xB8); 3928 emit_operand(dst, src); 3929 } 3930 3931 3932 void Assembler::popcntl(Register dst, Register src) { 3933 DCHECK(IsEnabled(POPCNT)); 3934 EnsureSpace ensure_space(this); 3935 emit(0xF3); 3936 emit_optional_rex_32(dst, src); 3937 emit(0x0F); 3938 emit(0xB8); 3939 emit_modrm(dst, src); 3940 } 3941 3942 3943 void Assembler::popcntl(Register dst, const Operand& src) { 3944 DCHECK(IsEnabled(POPCNT)); 3945 EnsureSpace ensure_space(this); 3946 emit(0xF3); 3947 emit_optional_rex_32(dst, src); 3948 emit(0x0F); 3949 emit(0xB8); 3950 emit_operand(dst, src); 3951 } 3952 3953 3954 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 3955 Register rm) { 3956 DCHECK(IsEnabled(BMI2)); 3957 EnsureSpace ensure_space(this); 3958 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 3959 emit(op); 3960 emit_modrm(reg, rm); 3961 } 3962 3963 3964 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 3965 const Operand& rm) { 3966 DCHECK(IsEnabled(BMI2)); 3967 EnsureSpace ensure_space(this); 3968 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 3969 emit(op); 3970 emit_operand(reg, rm); 3971 } 3972 3973 3974 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 3975 Register rm) { 3976 DCHECK(IsEnabled(BMI2)); 3977 EnsureSpace ensure_space(this); 3978 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 3979 emit(op); 3980 emit_modrm(reg, rm); 3981 } 3982 3983 3984 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 3985 const Operand& rm) { 3986 DCHECK(IsEnabled(BMI2)); 3987 EnsureSpace ensure_space(this); 3988 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 3989 emit(op); 3990 emit_operand(reg, rm); 3991 } 3992 3993 3994 void Assembler::rorxq(Register dst, Register src, byte imm8) { 3995 DCHECK(IsEnabled(BMI2)); 3996 DCHECK(is_uint8(imm8)); 3997 Register vreg = {0}; // VEX.vvvv unused 3998 EnsureSpace ensure_space(this); 3999 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4000 emit(0xF0); 4001 emit_modrm(dst, src); 4002 emit(imm8); 4003 } 4004 4005 4006 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) { 4007 DCHECK(IsEnabled(BMI2)); 4008 DCHECK(is_uint8(imm8)); 4009 Register vreg = {0}; // VEX.vvvv unused 4010 EnsureSpace ensure_space(this); 4011 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4012 emit(0xF0); 4013 emit_operand(dst, src); 4014 emit(imm8); 4015 } 4016 4017 4018 void Assembler::rorxl(Register dst, Register src, byte imm8) { 4019 DCHECK(IsEnabled(BMI2)); 4020 DCHECK(is_uint8(imm8)); 4021 Register vreg = {0}; // VEX.vvvv unused 4022 EnsureSpace ensure_space(this); 4023 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4024 emit(0xF0); 4025 emit_modrm(dst, src); 4026 emit(imm8); 4027 } 4028 4029 4030 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) { 4031 DCHECK(IsEnabled(BMI2)); 4032 DCHECK(is_uint8(imm8)); 4033 Register vreg = {0}; // VEX.vvvv unused 4034 EnsureSpace ensure_space(this); 4035 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4036 emit(0xF0); 4037 emit_operand(dst, src); 4038 emit(imm8); 4039 } 4040 4041 4042 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) { 4043 Register ireg = { reg.code() }; 4044 emit_operand(ireg, adr); 4045 } 4046 4047 4048 void Assembler::emit_sse_operand(Register reg, const Operand& adr) { 4049 Register ireg = {reg.code()}; 4050 emit_operand(ireg, adr); 4051 } 4052 4053 4054 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) { 4055 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4056 } 4057 4058 4059 void Assembler::emit_sse_operand(XMMRegister dst, Register src) { 4060 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4061 } 4062 4063 4064 void Assembler::emit_sse_operand(Register dst, XMMRegister src) { 4065 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4066 } 4067 4068 4069 void Assembler::db(uint8_t data) { 4070 EnsureSpace ensure_space(this); 4071 emit(data); 4072 } 4073 4074 4075 void Assembler::dd(uint32_t data) { 4076 EnsureSpace ensure_space(this); 4077 emitl(data); 4078 } 4079 4080 4081 void Assembler::dq(uint64_t data) { 4082 EnsureSpace ensure_space(this); 4083 emitq(data); 4084 } 4085 4086 4087 void Assembler::dq(Label* label) { 4088 EnsureSpace ensure_space(this); 4089 if (label->is_bound()) { 4090 internal_reference_positions_.push_back(pc_offset()); 4091 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE); 4092 } else { 4093 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 4094 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address. 4095 if (label->is_linked()) { 4096 emitl(label->pos()); 4097 label->link_to(pc_offset() - sizeof(int32_t)); 4098 } else { 4099 DCHECK(label->is_unused()); 4100 int32_t current = pc_offset(); 4101 emitl(current); 4102 label->link_to(current); 4103 } 4104 } 4105 } 4106 4107 4108 // Relocation information implementations. 4109 4110 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 4111 DCHECK(!RelocInfo::IsNone(rmode)); 4112 // Don't record external references unless the heap will be serialized. 4113 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 4114 !serializer_enabled() && !emit_debug_code()) { 4115 return; 4116 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) { 4117 // Don't record psuedo relocation info for code age sequence mode. 4118 return; 4119 } 4120 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL); 4121 reloc_info_writer.Write(&rinfo); 4122 } 4123 4124 4125 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask | 4126 1 << RelocInfo::RUNTIME_ENTRY | 4127 1 << RelocInfo::INTERNAL_REFERENCE | 4128 1 << RelocInfo::CODE_AGE_SEQUENCE; 4129 4130 4131 bool RelocInfo::IsCodedSpecially() { 4132 // The deserializer needs to know whether a pointer is specially coded. Being 4133 // specially coded on x64 means that it is a relative 32 bit address, as used 4134 // by branch instructions. 4135 return (1 << rmode_) & kApplyMask; 4136 } 4137 4138 4139 bool RelocInfo::IsInConstantPool() { 4140 return false; 4141 } 4142 4143 4144 } // namespace internal 4145 } // namespace v8 4146 4147 #endif // V8_TARGET_ARCH_X64 4148
