1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/x64/assembler-x64.h" 6 7 #include <cstring> 8 9 #if V8_TARGET_ARCH_X64 10 11 #if V8_LIBC_MSVCRT 12 #include <intrin.h> // _xgetbv() 13 #endif 14 #if V8_OS_MACOSX 15 #include <sys/sysctl.h> 16 #endif 17 18 #include "src/base/bits.h" 19 #include "src/macro-assembler.h" 20 #include "src/v8.h" 21 22 namespace v8 { 23 namespace internal { 24 25 // ----------------------------------------------------------------------------- 26 // Implementation of CpuFeatures 27 28 namespace { 29 30 #if !V8_LIBC_MSVCRT 31 32 V8_INLINE uint64_t _xgetbv(unsigned int xcr) { 33 unsigned eax, edx; 34 // Check xgetbv; this uses a .byte sequence instead of the instruction 35 // directly because older assemblers do not include support for xgetbv and 36 // there is no easy way to conditionally compile based on the assembler 37 // used. 38 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 39 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32); 40 } 41 42 #define _XCR_XFEATURE_ENABLED_MASK 0 43 44 #endif // !V8_LIBC_MSVCRT 45 46 47 bool OSHasAVXSupport() { 48 #if V8_OS_MACOSX 49 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being 50 // caused by ISRs, so we detect that here and disable AVX in that case. 51 char buffer[128]; 52 size_t buffer_size = arraysize(buffer); 53 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE}; 54 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) { 55 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version"); 56 } 57 // The buffer now contains a string of the form XX.YY.ZZ, where 58 // XX is the major kernel version component. 59 char* period_pos = strchr(buffer, '.'); 60 DCHECK_NOT_NULL(period_pos); 61 *period_pos = '\0'; 62 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT 63 if (kernel_version_major <= 13) return false; 64 #endif // V8_OS_MACOSX 65 // Check whether OS claims to support AVX. 66 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); 67 return (feature_mask & 0x6) == 0x6; 68 } 69 70 } // namespace 71 72 73 void CpuFeatures::ProbeImpl(bool cross_compile) { 74 base::CPU cpu; 75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory. 76 CHECK(cpu.has_cmov()); // CMOV support is mandatory. 77 78 // Only use statically determined features for cross compile (snapshot). 79 if (cross_compile) return; 80 81 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1; 82 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3; 83 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; 84 // SAHF is not generally available in long mode. 85 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; 86 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && 87 OSHasAVXSupport()) { 88 supported_ |= 1u << AVX; 89 } 90 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() && 91 OSHasAVXSupport()) { 92 supported_ |= 1u << FMA3; 93 } 94 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; 95 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; 96 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; 97 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT; 98 if (strcmp(FLAG_mcpu, "auto") == 0) { 99 if (cpu.is_atom()) supported_ |= 1u << ATOM; 100 } else if (strcmp(FLAG_mcpu, "atom") == 0) { 101 supported_ |= 1u << ATOM; 102 } 103 } 104 105 106 void CpuFeatures::PrintTarget() { } 107 void CpuFeatures::PrintFeatures() { 108 printf( 109 "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d " 110 "LZCNT=%d " 111 "POPCNT=%d ATOM=%d\n", 112 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3), 113 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF), 114 CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3), 115 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2), 116 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT), 117 CpuFeatures::IsSupported(ATOM)); 118 } 119 120 // ----------------------------------------------------------------------------- 121 // Implementation of RelocInfo 122 123 Address RelocInfo::wasm_memory_reference() { 124 DCHECK(IsWasmMemoryReference(rmode_)); 125 return Memory::Address_at(pc_); 126 } 127 128 Address RelocInfo::wasm_global_reference() { 129 DCHECK(IsWasmGlobalReference(rmode_)); 130 return Memory::Address_at(pc_); 131 } 132 133 uint32_t RelocInfo::wasm_memory_size_reference() { 134 DCHECK(IsWasmMemorySizeReference(rmode_)); 135 return Memory::uint32_at(pc_); 136 } 137 138 void RelocInfo::unchecked_update_wasm_memory_reference( 139 Address address, ICacheFlushMode flush_mode) { 140 Memory::Address_at(pc_) = address; 141 } 142 143 void RelocInfo::unchecked_update_wasm_memory_size(uint32_t size, 144 ICacheFlushMode flush_mode) { 145 Memory::uint32_at(pc_) = size; 146 } 147 148 // ----------------------------------------------------------------------------- 149 // Implementation of Operand 150 151 Operand::Operand(Register base, int32_t disp) : rex_(0) { 152 len_ = 1; 153 if (base.is(rsp) || base.is(r12)) { 154 // SIB byte is needed to encode (rsp + offset) or (r12 + offset). 155 set_sib(times_1, rsp, base); 156 } 157 158 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 159 set_modrm(0, base); 160 } else if (is_int8(disp)) { 161 set_modrm(1, base); 162 set_disp8(disp); 163 } else { 164 set_modrm(2, base); 165 set_disp32(disp); 166 } 167 } 168 169 170 Operand::Operand(Register base, 171 Register index, 172 ScaleFactor scale, 173 int32_t disp) : rex_(0) { 174 DCHECK(!index.is(rsp)); 175 len_ = 1; 176 set_sib(scale, index, base); 177 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 178 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits 179 // possibly set by set_sib. 180 set_modrm(0, rsp); 181 } else if (is_int8(disp)) { 182 set_modrm(1, rsp); 183 set_disp8(disp); 184 } else { 185 set_modrm(2, rsp); 186 set_disp32(disp); 187 } 188 } 189 190 191 Operand::Operand(Register index, 192 ScaleFactor scale, 193 int32_t disp) : rex_(0) { 194 DCHECK(!index.is(rsp)); 195 len_ = 1; 196 set_modrm(0, rsp); 197 set_sib(scale, index, rbp); 198 set_disp32(disp); 199 } 200 201 202 Operand::Operand(Label* label) : rex_(0), len_(1) { 203 DCHECK_NOT_NULL(label); 204 set_modrm(0, rbp); 205 set_disp64(reinterpret_cast<intptr_t>(label)); 206 } 207 208 209 Operand::Operand(const Operand& operand, int32_t offset) { 210 DCHECK(operand.len_ >= 1); 211 // Operand encodes REX ModR/M [SIB] [Disp]. 212 byte modrm = operand.buf_[0]; 213 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target). 214 bool has_sib = ((modrm & 0x07) == 0x04); 215 byte mode = modrm & 0xC0; 216 int disp_offset = has_sib ? 2 : 1; 217 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07; 218 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit 219 // displacement. 220 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base. 221 int32_t disp_value = 0; 222 if (mode == 0x80 || is_baseless) { 223 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement. 224 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]); 225 } else if (mode == 0x40) { 226 // Mode 1: Byte displacement. 227 disp_value = static_cast<signed char>(operand.buf_[disp_offset]); 228 } 229 230 // Write new operand with same registers, but with modified displacement. 231 DCHECK(offset >= 0 ? disp_value + offset > disp_value 232 : disp_value + offset < disp_value); // No overflow. 233 disp_value += offset; 234 rex_ = operand.rex_; 235 if (!is_int8(disp_value) || is_baseless) { 236 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13. 237 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80); 238 len_ = disp_offset + 4; 239 Memory::int32_at(&buf_[disp_offset]) = disp_value; 240 } else if (disp_value != 0 || (base_reg == 0x05)) { 241 // Need 8 bits of displacement. 242 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1. 243 len_ = disp_offset + 1; 244 buf_[disp_offset] = static_cast<byte>(disp_value); 245 } else { 246 // Need no displacement. 247 buf_[0] = (modrm & 0x3f); // Mode 0. 248 len_ = disp_offset; 249 } 250 if (has_sib) { 251 buf_[1] = operand.buf_[1]; 252 } 253 } 254 255 256 bool Operand::AddressUsesRegister(Register reg) const { 257 int code = reg.code(); 258 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand. 259 // Start with only low three bits of base register. Initial decoding doesn't 260 // distinguish on the REX.B bit. 261 int base_code = buf_[0] & 0x07; 262 if (base_code == rsp.code()) { 263 // SIB byte present in buf_[1]. 264 // Check the index register from the SIB byte + REX.X prefix. 265 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2); 266 // Index code (including REX.X) of 0x04 (rsp) means no index register. 267 if (index_code != rsp.code() && index_code == code) return true; 268 // Add REX.B to get the full base register code. 269 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3); 270 // A base register of 0x05 (rbp) with mod = 0 means no base register. 271 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 272 return code == base_code; 273 } else { 274 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means 275 // no base register. 276 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 277 base_code |= ((rex_ & 0x01) << 3); 278 return code == base_code; 279 } 280 } 281 282 283 // ----------------------------------------------------------------------------- 284 // Implementation of Assembler. 285 286 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 287 : AssemblerBase(isolate, buffer, buffer_size), code_targets_(100) { 288 // Clear the buffer in debug mode unless it was provided by the 289 // caller in which case we can't be sure it's okay to overwrite 290 // existing code in it. 291 #ifdef DEBUG 292 if (own_buffer_) { 293 memset(buffer_, 0xCC, buffer_size_); // int3 294 } 295 #endif 296 297 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_); 298 } 299 300 301 void Assembler::GetCode(CodeDesc* desc) { 302 // Finalize code (at this point overflow() may be true, but the gap ensures 303 // that we are still not overlapping instructions and relocation info). 304 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap. 305 // Set up code descriptor. 306 desc->buffer = buffer_; 307 desc->buffer_size = buffer_size_; 308 desc->instr_size = pc_offset(); 309 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system. 310 desc->reloc_size = 311 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos()); 312 desc->origin = this; 313 desc->constant_pool_size = 0; 314 desc->unwinding_info_size = 0; 315 desc->unwinding_info = nullptr; 316 } 317 318 319 void Assembler::Align(int m) { 320 DCHECK(base::bits::IsPowerOfTwo32(m)); 321 int delta = (m - (pc_offset() & (m - 1))) & (m - 1); 322 Nop(delta); 323 } 324 325 326 void Assembler::CodeTargetAlign() { 327 Align(16); // Preferred alignment of jump targets on x64. 328 } 329 330 331 bool Assembler::IsNop(Address addr) { 332 Address a = addr; 333 while (*a == 0x66) a++; 334 if (*a == 0x90) return true; 335 if (a[0] == 0xf && a[1] == 0x1f) return true; 336 return false; 337 } 338 339 340 void Assembler::bind_to(Label* L, int pos) { 341 DCHECK(!L->is_bound()); // Label may only be bound once. 342 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid. 343 if (L->is_linked()) { 344 int current = L->pos(); 345 int next = long_at(current); 346 while (next != current) { 347 if (current >= 4 && long_at(current - 4) == 0) { 348 // Absolute address. 349 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 350 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 351 internal_reference_positions_.push_back(current - 4); 352 } else { 353 // Relative address, relative to point after address. 354 int imm32 = pos - (current + sizeof(int32_t)); 355 long_at_put(current, imm32); 356 } 357 current = next; 358 next = long_at(next); 359 } 360 // Fix up last fixup on linked list. 361 if (current >= 4 && long_at(current - 4) == 0) { 362 // Absolute address. 363 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 364 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 365 internal_reference_positions_.push_back(current - 4); 366 } else { 367 // Relative address, relative to point after address. 368 int imm32 = pos - (current + sizeof(int32_t)); 369 long_at_put(current, imm32); 370 } 371 } 372 while (L->is_near_linked()) { 373 int fixup_pos = L->near_link_pos(); 374 int offset_to_next = 375 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos))); 376 DCHECK(offset_to_next <= 0); 377 int disp = pos - (fixup_pos + sizeof(int8_t)); 378 CHECK(is_int8(disp)); 379 set_byte_at(fixup_pos, disp); 380 if (offset_to_next < 0) { 381 L->link_to(fixup_pos + offset_to_next, Label::kNear); 382 } else { 383 L->UnuseNear(); 384 } 385 } 386 L->bind_to(pos); 387 } 388 389 390 void Assembler::bind(Label* L) { 391 bind_to(L, pc_offset()); 392 } 393 394 395 void Assembler::GrowBuffer() { 396 DCHECK(buffer_overflow()); 397 if (!own_buffer_) FATAL("external code buffer is too small"); 398 399 // Compute new buffer size. 400 CodeDesc desc; // the new buffer 401 desc.buffer_size = 2 * buffer_size_; 402 403 // Some internal data structures overflow for very large buffers, 404 // they must ensure that kMaximalBufferSize is not too large. 405 if (desc.buffer_size > kMaximalBufferSize || 406 static_cast<size_t>(desc.buffer_size) > 407 isolate()->heap()->MaxOldGenerationSize()) { 408 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer"); 409 } 410 411 // Set up new buffer. 412 desc.buffer = NewArray<byte>(desc.buffer_size); 413 desc.origin = this; 414 desc.instr_size = pc_offset(); 415 desc.reloc_size = 416 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos())); 417 418 // Clear the buffer in debug mode. Use 'int3' instructions to make 419 // sure to get into problems if we ever run uninitialized code. 420 #ifdef DEBUG 421 memset(desc.buffer, 0xCC, desc.buffer_size); 422 #endif 423 424 // Copy the data. 425 intptr_t pc_delta = desc.buffer - buffer_; 426 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 427 (buffer_ + buffer_size_); 428 MemMove(desc.buffer, buffer_, desc.instr_size); 429 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(), 430 desc.reloc_size); 431 432 // Switch buffers. 433 DeleteArray(buffer_); 434 buffer_ = desc.buffer; 435 buffer_size_ = desc.buffer_size; 436 pc_ += pc_delta; 437 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 438 reloc_info_writer.last_pc() + pc_delta); 439 440 // Relocate internal references. 441 for (auto pos : internal_reference_positions_) { 442 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 443 *p += pc_delta; 444 } 445 446 DCHECK(!buffer_overflow()); 447 } 448 449 450 void Assembler::emit_operand(int code, const Operand& adr) { 451 DCHECK(is_uint3(code)); 452 const unsigned length = adr.len_; 453 DCHECK(length > 0); 454 455 // Emit updated ModR/M byte containing the given register. 456 DCHECK((adr.buf_[0] & 0x38) == 0); 457 *pc_++ = adr.buf_[0] | code << 3; 458 459 // Recognize RIP relative addressing. 460 if (adr.buf_[0] == 5) { 461 DCHECK_EQ(9u, length); 462 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]); 463 if (label->is_bound()) { 464 int offset = label->pos() - pc_offset() - sizeof(int32_t); 465 DCHECK_GE(0, offset); 466 emitl(offset); 467 } else if (label->is_linked()) { 468 emitl(label->pos()); 469 label->link_to(pc_offset() - sizeof(int32_t)); 470 } else { 471 DCHECK(label->is_unused()); 472 int32_t current = pc_offset(); 473 emitl(current); 474 label->link_to(current); 475 } 476 } else { 477 // Emit the rest of the encoded operand. 478 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i]; 479 } 480 } 481 482 483 // Assembler Instruction implementations. 484 485 void Assembler::arithmetic_op(byte opcode, 486 Register reg, 487 const Operand& op, 488 int size) { 489 EnsureSpace ensure_space(this); 490 emit_rex(reg, op, size); 491 emit(opcode); 492 emit_operand(reg, op); 493 } 494 495 496 void Assembler::arithmetic_op(byte opcode, 497 Register reg, 498 Register rm_reg, 499 int size) { 500 EnsureSpace ensure_space(this); 501 DCHECK((opcode & 0xC6) == 2); 502 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 503 // Swap reg and rm_reg and change opcode operand order. 504 emit_rex(rm_reg, reg, size); 505 emit(opcode ^ 0x02); 506 emit_modrm(rm_reg, reg); 507 } else { 508 emit_rex(reg, rm_reg, size); 509 emit(opcode); 510 emit_modrm(reg, rm_reg); 511 } 512 } 513 514 515 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) { 516 EnsureSpace ensure_space(this); 517 DCHECK((opcode & 0xC6) == 2); 518 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 519 // Swap reg and rm_reg and change opcode operand order. 520 emit(0x66); 521 emit_optional_rex_32(rm_reg, reg); 522 emit(opcode ^ 0x02); 523 emit_modrm(rm_reg, reg); 524 } else { 525 emit(0x66); 526 emit_optional_rex_32(reg, rm_reg); 527 emit(opcode); 528 emit_modrm(reg, rm_reg); 529 } 530 } 531 532 533 void Assembler::arithmetic_op_16(byte opcode, 534 Register reg, 535 const Operand& rm_reg) { 536 EnsureSpace ensure_space(this); 537 emit(0x66); 538 emit_optional_rex_32(reg, rm_reg); 539 emit(opcode); 540 emit_operand(reg, rm_reg); 541 } 542 543 544 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) { 545 EnsureSpace ensure_space(this); 546 if (!reg.is_byte_register()) { 547 emit_rex_32(reg, op); 548 } else { 549 emit_optional_rex_32(reg, op); 550 } 551 emit(opcode); 552 emit_operand(reg, op); 553 } 554 555 556 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) { 557 EnsureSpace ensure_space(this); 558 DCHECK((opcode & 0xC6) == 2); 559 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 560 // Swap reg and rm_reg and change opcode operand order. 561 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) { 562 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 563 emit_rex_32(rm_reg, reg); 564 } 565 emit(opcode ^ 0x02); 566 emit_modrm(rm_reg, reg); 567 } else { 568 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) { 569 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 570 emit_rex_32(reg, rm_reg); 571 } 572 emit(opcode); 573 emit_modrm(reg, rm_reg); 574 } 575 } 576 577 578 void Assembler::immediate_arithmetic_op(byte subcode, 579 Register dst, 580 Immediate src, 581 int size) { 582 EnsureSpace ensure_space(this); 583 emit_rex(dst, size); 584 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) { 585 emit(0x83); 586 emit_modrm(subcode, dst); 587 emit(src.value_); 588 } else if (dst.is(rax)) { 589 emit(0x05 | (subcode << 3)); 590 emit(src); 591 } else { 592 emit(0x81); 593 emit_modrm(subcode, dst); 594 emit(src); 595 } 596 } 597 598 void Assembler::immediate_arithmetic_op(byte subcode, 599 const Operand& dst, 600 Immediate src, 601 int size) { 602 EnsureSpace ensure_space(this); 603 emit_rex(dst, size); 604 if (is_int8(src.value_)) { 605 emit(0x83); 606 emit_operand(subcode, dst); 607 if (!RelocInfo::IsNone(src.rmode_)) { 608 RecordRelocInfo(src.rmode_); 609 } 610 emit(src.value_); 611 } else { 612 emit(0x81); 613 emit_operand(subcode, dst); 614 emit(src); 615 } 616 } 617 618 619 void Assembler::immediate_arithmetic_op_16(byte subcode, 620 Register dst, 621 Immediate src) { 622 EnsureSpace ensure_space(this); 623 emit(0x66); // Operand size override prefix. 624 emit_optional_rex_32(dst); 625 if (is_int8(src.value_)) { 626 emit(0x83); 627 emit_modrm(subcode, dst); 628 emit(src.value_); 629 } else if (dst.is(rax)) { 630 emit(0x05 | (subcode << 3)); 631 emitw(src.value_); 632 } else { 633 emit(0x81); 634 emit_modrm(subcode, dst); 635 emitw(src.value_); 636 } 637 } 638 639 640 void Assembler::immediate_arithmetic_op_16(byte subcode, 641 const Operand& dst, 642 Immediate src) { 643 EnsureSpace ensure_space(this); 644 emit(0x66); // Operand size override prefix. 645 emit_optional_rex_32(dst); 646 if (is_int8(src.value_)) { 647 emit(0x83); 648 emit_operand(subcode, dst); 649 emit(src.value_); 650 } else { 651 emit(0x81); 652 emit_operand(subcode, dst); 653 emitw(src.value_); 654 } 655 } 656 657 658 void Assembler::immediate_arithmetic_op_8(byte subcode, 659 const Operand& dst, 660 Immediate src) { 661 EnsureSpace ensure_space(this); 662 emit_optional_rex_32(dst); 663 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 664 emit(0x80); 665 emit_operand(subcode, dst); 666 emit(src.value_); 667 } 668 669 670 void Assembler::immediate_arithmetic_op_8(byte subcode, 671 Register dst, 672 Immediate src) { 673 EnsureSpace ensure_space(this); 674 if (!dst.is_byte_register()) { 675 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 676 emit_rex_32(dst); 677 } 678 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 679 emit(0x80); 680 emit_modrm(subcode, dst); 681 emit(src.value_); 682 } 683 684 685 void Assembler::shift(Register dst, 686 Immediate shift_amount, 687 int subcode, 688 int size) { 689 EnsureSpace ensure_space(this); 690 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 691 : is_uint5(shift_amount.value_)); 692 if (shift_amount.value_ == 1) { 693 emit_rex(dst, size); 694 emit(0xD1); 695 emit_modrm(subcode, dst); 696 } else { 697 emit_rex(dst, size); 698 emit(0xC1); 699 emit_modrm(subcode, dst); 700 emit(shift_amount.value_); 701 } 702 } 703 704 705 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode, 706 int size) { 707 EnsureSpace ensure_space(this); 708 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 709 : is_uint5(shift_amount.value_)); 710 if (shift_amount.value_ == 1) { 711 emit_rex(dst, size); 712 emit(0xD1); 713 emit_operand(subcode, dst); 714 } else { 715 emit_rex(dst, size); 716 emit(0xC1); 717 emit_operand(subcode, dst); 718 emit(shift_amount.value_); 719 } 720 } 721 722 723 void Assembler::shift(Register dst, int subcode, int size) { 724 EnsureSpace ensure_space(this); 725 emit_rex(dst, size); 726 emit(0xD3); 727 emit_modrm(subcode, dst); 728 } 729 730 731 void Assembler::shift(Operand dst, int subcode, int size) { 732 EnsureSpace ensure_space(this); 733 emit_rex(dst, size); 734 emit(0xD3); 735 emit_operand(subcode, dst); 736 } 737 738 739 void Assembler::bt(const Operand& dst, Register src) { 740 EnsureSpace ensure_space(this); 741 emit_rex_64(src, dst); 742 emit(0x0F); 743 emit(0xA3); 744 emit_operand(src, dst); 745 } 746 747 748 void Assembler::bts(const Operand& dst, Register src) { 749 EnsureSpace ensure_space(this); 750 emit_rex_64(src, dst); 751 emit(0x0F); 752 emit(0xAB); 753 emit_operand(src, dst); 754 } 755 756 757 void Assembler::bsrl(Register dst, Register src) { 758 EnsureSpace ensure_space(this); 759 emit_optional_rex_32(dst, src); 760 emit(0x0F); 761 emit(0xBD); 762 emit_modrm(dst, src); 763 } 764 765 766 void Assembler::bsrl(Register dst, const Operand& src) { 767 EnsureSpace ensure_space(this); 768 emit_optional_rex_32(dst, src); 769 emit(0x0F); 770 emit(0xBD); 771 emit_operand(dst, src); 772 } 773 774 775 void Assembler::bsrq(Register dst, Register src) { 776 EnsureSpace ensure_space(this); 777 emit_rex_64(dst, src); 778 emit(0x0F); 779 emit(0xBD); 780 emit_modrm(dst, src); 781 } 782 783 784 void Assembler::bsrq(Register dst, const Operand& src) { 785 EnsureSpace ensure_space(this); 786 emit_rex_64(dst, src); 787 emit(0x0F); 788 emit(0xBD); 789 emit_operand(dst, src); 790 } 791 792 793 void Assembler::bsfl(Register dst, Register src) { 794 EnsureSpace ensure_space(this); 795 emit_optional_rex_32(dst, src); 796 emit(0x0F); 797 emit(0xBC); 798 emit_modrm(dst, src); 799 } 800 801 802 void Assembler::bsfl(Register dst, const Operand& src) { 803 EnsureSpace ensure_space(this); 804 emit_optional_rex_32(dst, src); 805 emit(0x0F); 806 emit(0xBC); 807 emit_operand(dst, src); 808 } 809 810 811 void Assembler::bsfq(Register dst, Register src) { 812 EnsureSpace ensure_space(this); 813 emit_rex_64(dst, src); 814 emit(0x0F); 815 emit(0xBC); 816 emit_modrm(dst, src); 817 } 818 819 820 void Assembler::bsfq(Register dst, const Operand& src) { 821 EnsureSpace ensure_space(this); 822 emit_rex_64(dst, src); 823 emit(0x0F); 824 emit(0xBC); 825 emit_operand(dst, src); 826 } 827 828 829 void Assembler::call(Label* L) { 830 EnsureSpace ensure_space(this); 831 // 1110 1000 #32-bit disp. 832 emit(0xE8); 833 if (L->is_bound()) { 834 int offset = L->pos() - pc_offset() - sizeof(int32_t); 835 DCHECK(offset <= 0); 836 emitl(offset); 837 } else if (L->is_linked()) { 838 emitl(L->pos()); 839 L->link_to(pc_offset() - sizeof(int32_t)); 840 } else { 841 DCHECK(L->is_unused()); 842 int32_t current = pc_offset(); 843 emitl(current); 844 L->link_to(current); 845 } 846 } 847 848 849 void Assembler::call(Address entry, RelocInfo::Mode rmode) { 850 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 851 EnsureSpace ensure_space(this); 852 // 1110 1000 #32-bit disp. 853 emit(0xE8); 854 emit_runtime_entry(entry, rmode); 855 } 856 857 858 void Assembler::call(Handle<Code> target, 859 RelocInfo::Mode rmode, 860 TypeFeedbackId ast_id) { 861 EnsureSpace ensure_space(this); 862 // 1110 1000 #32-bit disp. 863 emit(0xE8); 864 emit_code_target(target, rmode, ast_id); 865 } 866 867 868 void Assembler::call(Register adr) { 869 EnsureSpace ensure_space(this); 870 // Opcode: FF /2 r64. 871 emit_optional_rex_32(adr); 872 emit(0xFF); 873 emit_modrm(0x2, adr); 874 } 875 876 877 void Assembler::call(const Operand& op) { 878 EnsureSpace ensure_space(this); 879 // Opcode: FF /2 m64. 880 emit_optional_rex_32(op); 881 emit(0xFF); 882 emit_operand(0x2, op); 883 } 884 885 886 // Calls directly to the given address using a relative offset. 887 // Should only ever be used in Code objects for calls within the 888 // same Code object. Should not be used when generating new code (use labels), 889 // but only when patching existing code. 890 void Assembler::call(Address target) { 891 EnsureSpace ensure_space(this); 892 // 1110 1000 #32-bit disp. 893 emit(0xE8); 894 Address source = pc_ + 4; 895 intptr_t displacement = target - source; 896 DCHECK(is_int32(displacement)); 897 emitl(static_cast<int32_t>(displacement)); 898 } 899 900 901 void Assembler::clc() { 902 EnsureSpace ensure_space(this); 903 emit(0xF8); 904 } 905 906 907 void Assembler::cld() { 908 EnsureSpace ensure_space(this); 909 emit(0xFC); 910 } 911 912 913 void Assembler::cdq() { 914 EnsureSpace ensure_space(this); 915 emit(0x99); 916 } 917 918 919 void Assembler::cmovq(Condition cc, Register dst, Register src) { 920 if (cc == always) { 921 movq(dst, src); 922 } else if (cc == never) { 923 return; 924 } 925 // No need to check CpuInfo for CMOV support, it's a required part of the 926 // 64-bit architecture. 927 DCHECK(cc >= 0); // Use mov for unconditional moves. 928 EnsureSpace ensure_space(this); 929 // Opcode: REX.W 0f 40 + cc /r. 930 emit_rex_64(dst, src); 931 emit(0x0f); 932 emit(0x40 + cc); 933 emit_modrm(dst, src); 934 } 935 936 937 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) { 938 if (cc == always) { 939 movq(dst, src); 940 } else if (cc == never) { 941 return; 942 } 943 DCHECK(cc >= 0); 944 EnsureSpace ensure_space(this); 945 // Opcode: REX.W 0f 40 + cc /r. 946 emit_rex_64(dst, src); 947 emit(0x0f); 948 emit(0x40 + cc); 949 emit_operand(dst, src); 950 } 951 952 953 void Assembler::cmovl(Condition cc, Register dst, Register src) { 954 if (cc == always) { 955 movl(dst, src); 956 } else if (cc == never) { 957 return; 958 } 959 DCHECK(cc >= 0); 960 EnsureSpace ensure_space(this); 961 // Opcode: 0f 40 + cc /r. 962 emit_optional_rex_32(dst, src); 963 emit(0x0f); 964 emit(0x40 + cc); 965 emit_modrm(dst, src); 966 } 967 968 969 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) { 970 if (cc == always) { 971 movl(dst, src); 972 } else if (cc == never) { 973 return; 974 } 975 DCHECK(cc >= 0); 976 EnsureSpace ensure_space(this); 977 // Opcode: 0f 40 + cc /r. 978 emit_optional_rex_32(dst, src); 979 emit(0x0f); 980 emit(0x40 + cc); 981 emit_operand(dst, src); 982 } 983 984 985 void Assembler::cmpb_al(Immediate imm8) { 986 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_)); 987 EnsureSpace ensure_space(this); 988 emit(0x3c); 989 emit(imm8.value_); 990 } 991 992 void Assembler::lock() { 993 EnsureSpace ensure_space(this); 994 emit(0xf0); 995 } 996 997 void Assembler::cmpxchgb(const Operand& dst, Register src) { 998 EnsureSpace ensure_space(this); 999 if (!src.is_byte_register()) { 1000 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1001 emit_rex_32(src, dst); 1002 } else { 1003 emit_optional_rex_32(src, dst); 1004 } 1005 emit(0x0f); 1006 emit(0xb0); 1007 emit_operand(src, dst); 1008 } 1009 1010 void Assembler::cmpxchgw(const Operand& dst, Register src) { 1011 EnsureSpace ensure_space(this); 1012 emit(0x66); 1013 emit_optional_rex_32(src, dst); 1014 emit(0x0f); 1015 emit(0xb1); 1016 emit_operand(src, dst); 1017 } 1018 1019 void Assembler::emit_cmpxchg(const Operand& dst, Register src, int size) { 1020 EnsureSpace ensure_space(this); 1021 emit_rex(src, dst, size); 1022 emit(0x0f); 1023 emit(0xb1); 1024 emit_operand(src, dst); 1025 } 1026 1027 void Assembler::cpuid() { 1028 EnsureSpace ensure_space(this); 1029 emit(0x0F); 1030 emit(0xA2); 1031 } 1032 1033 1034 void Assembler::cqo() { 1035 EnsureSpace ensure_space(this); 1036 emit_rex_64(); 1037 emit(0x99); 1038 } 1039 1040 1041 void Assembler::emit_dec(Register dst, int size) { 1042 EnsureSpace ensure_space(this); 1043 emit_rex(dst, size); 1044 emit(0xFF); 1045 emit_modrm(0x1, dst); 1046 } 1047 1048 1049 void Assembler::emit_dec(const Operand& dst, int size) { 1050 EnsureSpace ensure_space(this); 1051 emit_rex(dst, size); 1052 emit(0xFF); 1053 emit_operand(1, dst); 1054 } 1055 1056 1057 void Assembler::decb(Register dst) { 1058 EnsureSpace ensure_space(this); 1059 if (!dst.is_byte_register()) { 1060 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1061 emit_rex_32(dst); 1062 } 1063 emit(0xFE); 1064 emit_modrm(0x1, dst); 1065 } 1066 1067 1068 void Assembler::decb(const Operand& dst) { 1069 EnsureSpace ensure_space(this); 1070 emit_optional_rex_32(dst); 1071 emit(0xFE); 1072 emit_operand(1, dst); 1073 } 1074 1075 1076 void Assembler::enter(Immediate size) { 1077 EnsureSpace ensure_space(this); 1078 emit(0xC8); 1079 emitw(size.value_); // 16 bit operand, always. 1080 emit(0); 1081 } 1082 1083 1084 void Assembler::hlt() { 1085 EnsureSpace ensure_space(this); 1086 emit(0xF4); 1087 } 1088 1089 1090 void Assembler::emit_idiv(Register src, int size) { 1091 EnsureSpace ensure_space(this); 1092 emit_rex(src, size); 1093 emit(0xF7); 1094 emit_modrm(0x7, src); 1095 } 1096 1097 1098 void Assembler::emit_div(Register src, int size) { 1099 EnsureSpace ensure_space(this); 1100 emit_rex(src, size); 1101 emit(0xF7); 1102 emit_modrm(0x6, src); 1103 } 1104 1105 1106 void Assembler::emit_imul(Register src, int size) { 1107 EnsureSpace ensure_space(this); 1108 emit_rex(src, size); 1109 emit(0xF7); 1110 emit_modrm(0x5, src); 1111 } 1112 1113 1114 void Assembler::emit_imul(const Operand& src, int size) { 1115 EnsureSpace ensure_space(this); 1116 emit_rex(src, size); 1117 emit(0xF7); 1118 emit_operand(0x5, src); 1119 } 1120 1121 1122 void Assembler::emit_imul(Register dst, Register src, int size) { 1123 EnsureSpace ensure_space(this); 1124 emit_rex(dst, src, size); 1125 emit(0x0F); 1126 emit(0xAF); 1127 emit_modrm(dst, src); 1128 } 1129 1130 1131 void Assembler::emit_imul(Register dst, const Operand& src, int size) { 1132 EnsureSpace ensure_space(this); 1133 emit_rex(dst, src, size); 1134 emit(0x0F); 1135 emit(0xAF); 1136 emit_operand(dst, src); 1137 } 1138 1139 1140 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) { 1141 EnsureSpace ensure_space(this); 1142 emit_rex(dst, src, size); 1143 if (is_int8(imm.value_)) { 1144 emit(0x6B); 1145 emit_modrm(dst, src); 1146 emit(imm.value_); 1147 } else { 1148 emit(0x69); 1149 emit_modrm(dst, src); 1150 emitl(imm.value_); 1151 } 1152 } 1153 1154 1155 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm, 1156 int size) { 1157 EnsureSpace ensure_space(this); 1158 emit_rex(dst, src, size); 1159 if (is_int8(imm.value_)) { 1160 emit(0x6B); 1161 emit_operand(dst, src); 1162 emit(imm.value_); 1163 } else { 1164 emit(0x69); 1165 emit_operand(dst, src); 1166 emitl(imm.value_); 1167 } 1168 } 1169 1170 1171 void Assembler::emit_inc(Register dst, int size) { 1172 EnsureSpace ensure_space(this); 1173 emit_rex(dst, size); 1174 emit(0xFF); 1175 emit_modrm(0x0, dst); 1176 } 1177 1178 1179 void Assembler::emit_inc(const Operand& dst, int size) { 1180 EnsureSpace ensure_space(this); 1181 emit_rex(dst, size); 1182 emit(0xFF); 1183 emit_operand(0, dst); 1184 } 1185 1186 1187 void Assembler::int3() { 1188 EnsureSpace ensure_space(this); 1189 emit(0xCC); 1190 } 1191 1192 1193 void Assembler::j(Condition cc, Label* L, Label::Distance distance) { 1194 if (cc == always) { 1195 jmp(L); 1196 return; 1197 } else if (cc == never) { 1198 return; 1199 } 1200 EnsureSpace ensure_space(this); 1201 DCHECK(is_uint4(cc)); 1202 if (L->is_bound()) { 1203 const int short_size = 2; 1204 const int long_size = 6; 1205 int offs = L->pos() - pc_offset(); 1206 DCHECK(offs <= 0); 1207 // Determine whether we can use 1-byte offsets for backwards branches, 1208 // which have a max range of 128 bytes. 1209 1210 // We also need to check predictable_code_size() flag here, because on x64, 1211 // when the full code generator recompiles code for debugging, some places 1212 // need to be padded out to a certain size. The debugger is keeping track of 1213 // how often it did this so that it can adjust return addresses on the 1214 // stack, but if the size of jump instructions can also change, that's not 1215 // enough and the calculated offsets would be incorrect. 1216 if (is_int8(offs - short_size) && !predictable_code_size()) { 1217 // 0111 tttn #8-bit disp. 1218 emit(0x70 | cc); 1219 emit((offs - short_size) & 0xFF); 1220 } else { 1221 // 0000 1111 1000 tttn #32-bit disp. 1222 emit(0x0F); 1223 emit(0x80 | cc); 1224 emitl(offs - long_size); 1225 } 1226 } else if (distance == Label::kNear) { 1227 // 0111 tttn #8-bit disp 1228 emit(0x70 | cc); 1229 byte disp = 0x00; 1230 if (L->is_near_linked()) { 1231 int offset = L->near_link_pos() - pc_offset(); 1232 DCHECK(is_int8(offset)); 1233 disp = static_cast<byte>(offset & 0xFF); 1234 } 1235 L->link_to(pc_offset(), Label::kNear); 1236 emit(disp); 1237 } else if (L->is_linked()) { 1238 // 0000 1111 1000 tttn #32-bit disp. 1239 emit(0x0F); 1240 emit(0x80 | cc); 1241 emitl(L->pos()); 1242 L->link_to(pc_offset() - sizeof(int32_t)); 1243 } else { 1244 DCHECK(L->is_unused()); 1245 emit(0x0F); 1246 emit(0x80 | cc); 1247 int32_t current = pc_offset(); 1248 emitl(current); 1249 L->link_to(current); 1250 } 1251 } 1252 1253 1254 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) { 1255 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1256 EnsureSpace ensure_space(this); 1257 DCHECK(is_uint4(cc)); 1258 emit(0x0F); 1259 emit(0x80 | cc); 1260 emit_runtime_entry(entry, rmode); 1261 } 1262 1263 1264 void Assembler::j(Condition cc, 1265 Handle<Code> target, 1266 RelocInfo::Mode rmode) { 1267 EnsureSpace ensure_space(this); 1268 DCHECK(is_uint4(cc)); 1269 // 0000 1111 1000 tttn #32-bit disp. 1270 emit(0x0F); 1271 emit(0x80 | cc); 1272 emit_code_target(target, rmode); 1273 } 1274 1275 1276 void Assembler::jmp(Label* L, Label::Distance distance) { 1277 EnsureSpace ensure_space(this); 1278 const int short_size = sizeof(int8_t); 1279 const int long_size = sizeof(int32_t); 1280 if (L->is_bound()) { 1281 int offs = L->pos() - pc_offset() - 1; 1282 DCHECK(offs <= 0); 1283 if (is_int8(offs - short_size) && !predictable_code_size()) { 1284 // 1110 1011 #8-bit disp. 1285 emit(0xEB); 1286 emit((offs - short_size) & 0xFF); 1287 } else { 1288 // 1110 1001 #32-bit disp. 1289 emit(0xE9); 1290 emitl(offs - long_size); 1291 } 1292 } else if (distance == Label::kNear) { 1293 emit(0xEB); 1294 byte disp = 0x00; 1295 if (L->is_near_linked()) { 1296 int offset = L->near_link_pos() - pc_offset(); 1297 DCHECK(is_int8(offset)); 1298 disp = static_cast<byte>(offset & 0xFF); 1299 } 1300 L->link_to(pc_offset(), Label::kNear); 1301 emit(disp); 1302 } else if (L->is_linked()) { 1303 // 1110 1001 #32-bit disp. 1304 emit(0xE9); 1305 emitl(L->pos()); 1306 L->link_to(pc_offset() - long_size); 1307 } else { 1308 // 1110 1001 #32-bit disp. 1309 DCHECK(L->is_unused()); 1310 emit(0xE9); 1311 int32_t current = pc_offset(); 1312 emitl(current); 1313 L->link_to(current); 1314 } 1315 } 1316 1317 1318 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) { 1319 EnsureSpace ensure_space(this); 1320 // 1110 1001 #32-bit disp. 1321 emit(0xE9); 1322 emit_code_target(target, rmode); 1323 } 1324 1325 1326 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { 1327 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1328 EnsureSpace ensure_space(this); 1329 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1330 emit(0xE9); 1331 emit_runtime_entry(entry, rmode); 1332 } 1333 1334 1335 void Assembler::jmp(Register target) { 1336 EnsureSpace ensure_space(this); 1337 // Opcode FF/4 r64. 1338 emit_optional_rex_32(target); 1339 emit(0xFF); 1340 emit_modrm(0x4, target); 1341 } 1342 1343 1344 void Assembler::jmp(const Operand& src) { 1345 EnsureSpace ensure_space(this); 1346 // Opcode FF/4 m64. 1347 emit_optional_rex_32(src); 1348 emit(0xFF); 1349 emit_operand(0x4, src); 1350 } 1351 1352 1353 void Assembler::emit_lea(Register dst, const Operand& src, int size) { 1354 EnsureSpace ensure_space(this); 1355 emit_rex(dst, src, size); 1356 emit(0x8D); 1357 emit_operand(dst, src); 1358 } 1359 1360 1361 void Assembler::load_rax(void* value, RelocInfo::Mode mode) { 1362 EnsureSpace ensure_space(this); 1363 if (kPointerSize == kInt64Size) { 1364 emit(0x48); // REX.W 1365 emit(0xA1); 1366 emitp(value, mode); 1367 } else { 1368 DCHECK(kPointerSize == kInt32Size); 1369 emit(0xA1); 1370 emitp(value, mode); 1371 // In 64-bit mode, need to zero extend the operand to 8 bytes. 1372 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 1373 // Developer's Manual Volume 2. 1374 emitl(0); 1375 } 1376 } 1377 1378 1379 void Assembler::load_rax(ExternalReference ref) { 1380 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 1381 } 1382 1383 1384 void Assembler::leave() { 1385 EnsureSpace ensure_space(this); 1386 emit(0xC9); 1387 } 1388 1389 1390 void Assembler::movb(Register dst, const Operand& src) { 1391 EnsureSpace ensure_space(this); 1392 if (!dst.is_byte_register()) { 1393 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1394 emit_rex_32(dst, src); 1395 } else { 1396 emit_optional_rex_32(dst, src); 1397 } 1398 emit(0x8A); 1399 emit_operand(dst, src); 1400 } 1401 1402 1403 void Assembler::movb(Register dst, Immediate imm) { 1404 EnsureSpace ensure_space(this); 1405 if (!dst.is_byte_register()) { 1406 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1407 emit_rex_32(dst); 1408 } 1409 emit(0xB0 + dst.low_bits()); 1410 emit(imm.value_); 1411 } 1412 1413 1414 void Assembler::movb(const Operand& dst, Register src) { 1415 EnsureSpace ensure_space(this); 1416 if (!src.is_byte_register()) { 1417 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1418 emit_rex_32(src, dst); 1419 } else { 1420 emit_optional_rex_32(src, dst); 1421 } 1422 emit(0x88); 1423 emit_operand(src, dst); 1424 } 1425 1426 1427 void Assembler::movb(const Operand& dst, Immediate imm) { 1428 EnsureSpace ensure_space(this); 1429 emit_optional_rex_32(dst); 1430 emit(0xC6); 1431 emit_operand(0x0, dst); 1432 emit(static_cast<byte>(imm.value_)); 1433 } 1434 1435 1436 void Assembler::movw(Register dst, const Operand& src) { 1437 EnsureSpace ensure_space(this); 1438 emit(0x66); 1439 emit_optional_rex_32(dst, src); 1440 emit(0x8B); 1441 emit_operand(dst, src); 1442 } 1443 1444 1445 void Assembler::movw(const Operand& dst, Register src) { 1446 EnsureSpace ensure_space(this); 1447 emit(0x66); 1448 emit_optional_rex_32(src, dst); 1449 emit(0x89); 1450 emit_operand(src, dst); 1451 } 1452 1453 1454 void Assembler::movw(const Operand& dst, Immediate imm) { 1455 EnsureSpace ensure_space(this); 1456 emit(0x66); 1457 emit_optional_rex_32(dst); 1458 emit(0xC7); 1459 emit_operand(0x0, dst); 1460 emit(static_cast<byte>(imm.value_ & 0xff)); 1461 emit(static_cast<byte>(imm.value_ >> 8)); 1462 } 1463 1464 1465 void Assembler::emit_mov(Register dst, const Operand& src, int size) { 1466 EnsureSpace ensure_space(this); 1467 emit_rex(dst, src, size); 1468 emit(0x8B); 1469 emit_operand(dst, src); 1470 } 1471 1472 1473 void Assembler::emit_mov(Register dst, Register src, int size) { 1474 EnsureSpace ensure_space(this); 1475 if (src.low_bits() == 4) { 1476 emit_rex(src, dst, size); 1477 emit(0x89); 1478 emit_modrm(src, dst); 1479 } else { 1480 emit_rex(dst, src, size); 1481 emit(0x8B); 1482 emit_modrm(dst, src); 1483 } 1484 } 1485 1486 1487 void Assembler::emit_mov(const Operand& dst, Register src, int size) { 1488 EnsureSpace ensure_space(this); 1489 emit_rex(src, dst, size); 1490 emit(0x89); 1491 emit_operand(src, dst); 1492 } 1493 1494 1495 void Assembler::emit_mov(Register dst, Immediate value, int size) { 1496 EnsureSpace ensure_space(this); 1497 emit_rex(dst, size); 1498 if (size == kInt64Size) { 1499 emit(0xC7); 1500 emit_modrm(0x0, dst); 1501 } else { 1502 DCHECK(size == kInt32Size); 1503 emit(0xB8 + dst.low_bits()); 1504 } 1505 emit(value); 1506 } 1507 1508 1509 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) { 1510 EnsureSpace ensure_space(this); 1511 emit_rex(dst, size); 1512 emit(0xC7); 1513 emit_operand(0x0, dst); 1514 emit(value); 1515 } 1516 1517 1518 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) { 1519 EnsureSpace ensure_space(this); 1520 emit_rex(dst, kPointerSize); 1521 emit(0xB8 | dst.low_bits()); 1522 emitp(value, rmode); 1523 } 1524 1525 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) { 1526 EnsureSpace ensure_space(this); 1527 emit_rex_64(dst); 1528 emit(0xB8 | dst.low_bits()); 1529 if (!RelocInfo::IsNone(rmode)) { 1530 RecordRelocInfo(rmode, value); 1531 } 1532 emitq(value); 1533 } 1534 1535 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) { 1536 movq(dst, static_cast<int64_t>(value), rmode); 1537 } 1538 1539 // Loads the ip-relative location of the src label into the target location 1540 // (as a 32-bit offset sign extended to 64-bit). 1541 void Assembler::movl(const Operand& dst, Label* src) { 1542 EnsureSpace ensure_space(this); 1543 emit_optional_rex_32(dst); 1544 emit(0xC7); 1545 emit_operand(0, dst); 1546 if (src->is_bound()) { 1547 int offset = src->pos() - pc_offset() - sizeof(int32_t); 1548 DCHECK(offset <= 0); 1549 emitl(offset); 1550 } else if (src->is_linked()) { 1551 emitl(src->pos()); 1552 src->link_to(pc_offset() - sizeof(int32_t)); 1553 } else { 1554 DCHECK(src->is_unused()); 1555 int32_t current = pc_offset(); 1556 emitl(current); 1557 src->link_to(current); 1558 } 1559 } 1560 1561 1562 void Assembler::movsxbl(Register dst, Register src) { 1563 EnsureSpace ensure_space(this); 1564 if (!src.is_byte_register()) { 1565 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1566 emit_rex_32(dst, src); 1567 } else { 1568 emit_optional_rex_32(dst, src); 1569 } 1570 emit(0x0F); 1571 emit(0xBE); 1572 emit_modrm(dst, src); 1573 } 1574 1575 1576 void Assembler::movsxbl(Register dst, const Operand& src) { 1577 EnsureSpace ensure_space(this); 1578 emit_optional_rex_32(dst, src); 1579 emit(0x0F); 1580 emit(0xBE); 1581 emit_operand(dst, src); 1582 } 1583 1584 1585 void Assembler::movsxbq(Register dst, const Operand& src) { 1586 EnsureSpace ensure_space(this); 1587 emit_rex_64(dst, src); 1588 emit(0x0F); 1589 emit(0xBE); 1590 emit_operand(dst, src); 1591 } 1592 1593 void Assembler::movsxbq(Register dst, Register src) { 1594 EnsureSpace ensure_space(this); 1595 emit_rex_64(dst, src); 1596 emit(0x0F); 1597 emit(0xBE); 1598 emit_modrm(dst, src); 1599 } 1600 1601 void Assembler::movsxwl(Register dst, Register src) { 1602 EnsureSpace ensure_space(this); 1603 emit_optional_rex_32(dst, src); 1604 emit(0x0F); 1605 emit(0xBF); 1606 emit_modrm(dst, src); 1607 } 1608 1609 1610 void Assembler::movsxwl(Register dst, const Operand& src) { 1611 EnsureSpace ensure_space(this); 1612 emit_optional_rex_32(dst, src); 1613 emit(0x0F); 1614 emit(0xBF); 1615 emit_operand(dst, src); 1616 } 1617 1618 1619 void Assembler::movsxwq(Register dst, const Operand& src) { 1620 EnsureSpace ensure_space(this); 1621 emit_rex_64(dst, src); 1622 emit(0x0F); 1623 emit(0xBF); 1624 emit_operand(dst, src); 1625 } 1626 1627 void Assembler::movsxwq(Register dst, Register src) { 1628 EnsureSpace ensure_space(this); 1629 emit_rex_64(dst, src); 1630 emit(0x0F); 1631 emit(0xBF); 1632 emit_modrm(dst, src); 1633 } 1634 1635 void Assembler::movsxlq(Register dst, Register src) { 1636 EnsureSpace ensure_space(this); 1637 emit_rex_64(dst, src); 1638 emit(0x63); 1639 emit_modrm(dst, src); 1640 } 1641 1642 1643 void Assembler::movsxlq(Register dst, const Operand& src) { 1644 EnsureSpace ensure_space(this); 1645 emit_rex_64(dst, src); 1646 emit(0x63); 1647 emit_operand(dst, src); 1648 } 1649 1650 1651 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) { 1652 EnsureSpace ensure_space(this); 1653 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1654 // there is no need to make this a 64 bit operation. 1655 emit_optional_rex_32(dst, src); 1656 emit(0x0F); 1657 emit(0xB6); 1658 emit_operand(dst, src); 1659 } 1660 1661 1662 void Assembler::emit_movzxb(Register dst, Register src, int size) { 1663 EnsureSpace ensure_space(this); 1664 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1665 // there is no need to make this a 64 bit operation. 1666 if (!src.is_byte_register()) { 1667 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1668 emit_rex_32(dst, src); 1669 } else { 1670 emit_optional_rex_32(dst, src); 1671 } 1672 emit(0x0F); 1673 emit(0xB6); 1674 emit_modrm(dst, src); 1675 } 1676 1677 1678 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) { 1679 EnsureSpace ensure_space(this); 1680 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1681 // there is no need to make this a 64 bit operation. 1682 emit_optional_rex_32(dst, src); 1683 emit(0x0F); 1684 emit(0xB7); 1685 emit_operand(dst, src); 1686 } 1687 1688 1689 void Assembler::emit_movzxw(Register dst, Register src, int size) { 1690 EnsureSpace ensure_space(this); 1691 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1692 // there is no need to make this a 64 bit operation. 1693 emit_optional_rex_32(dst, src); 1694 emit(0x0F); 1695 emit(0xB7); 1696 emit_modrm(dst, src); 1697 } 1698 1699 1700 void Assembler::repmovsb() { 1701 EnsureSpace ensure_space(this); 1702 emit(0xF3); 1703 emit(0xA4); 1704 } 1705 1706 1707 void Assembler::repmovsw() { 1708 EnsureSpace ensure_space(this); 1709 emit(0x66); // Operand size override. 1710 emit(0xF3); 1711 emit(0xA4); 1712 } 1713 1714 1715 void Assembler::emit_repmovs(int size) { 1716 EnsureSpace ensure_space(this); 1717 emit(0xF3); 1718 emit_rex(size); 1719 emit(0xA5); 1720 } 1721 1722 1723 void Assembler::mull(Register src) { 1724 EnsureSpace ensure_space(this); 1725 emit_optional_rex_32(src); 1726 emit(0xF7); 1727 emit_modrm(0x4, src); 1728 } 1729 1730 1731 void Assembler::mull(const Operand& src) { 1732 EnsureSpace ensure_space(this); 1733 emit_optional_rex_32(src); 1734 emit(0xF7); 1735 emit_operand(0x4, src); 1736 } 1737 1738 1739 void Assembler::mulq(Register src) { 1740 EnsureSpace ensure_space(this); 1741 emit_rex_64(src); 1742 emit(0xF7); 1743 emit_modrm(0x4, src); 1744 } 1745 1746 1747 void Assembler::emit_neg(Register dst, int size) { 1748 EnsureSpace ensure_space(this); 1749 emit_rex(dst, size); 1750 emit(0xF7); 1751 emit_modrm(0x3, dst); 1752 } 1753 1754 1755 void Assembler::emit_neg(const Operand& dst, int size) { 1756 EnsureSpace ensure_space(this); 1757 emit_rex_64(dst); 1758 emit(0xF7); 1759 emit_operand(3, dst); 1760 } 1761 1762 1763 void Assembler::nop() { 1764 EnsureSpace ensure_space(this); 1765 emit(0x90); 1766 } 1767 1768 1769 void Assembler::emit_not(Register dst, int size) { 1770 EnsureSpace ensure_space(this); 1771 emit_rex(dst, size); 1772 emit(0xF7); 1773 emit_modrm(0x2, dst); 1774 } 1775 1776 1777 void Assembler::emit_not(const Operand& dst, int size) { 1778 EnsureSpace ensure_space(this); 1779 emit_rex(dst, size); 1780 emit(0xF7); 1781 emit_operand(2, dst); 1782 } 1783 1784 1785 void Assembler::Nop(int n) { 1786 // The recommended muti-byte sequences of NOP instructions from the Intel 64 1787 // and IA-32 Architectures Software Developer's Manual. 1788 // 1789 // Length Assembly Byte Sequence 1790 // 2 bytes 66 NOP 66 90H 1791 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H 1792 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H 1793 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H 1794 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H 1795 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H 1796 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H 1797 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00 1798 // 00000000H] 00H 1799 1800 EnsureSpace ensure_space(this); 1801 while (n > 0) { 1802 switch (n) { 1803 case 2: 1804 emit(0x66); 1805 case 1: 1806 emit(0x90); 1807 return; 1808 case 3: 1809 emit(0x0f); 1810 emit(0x1f); 1811 emit(0x00); 1812 return; 1813 case 4: 1814 emit(0x0f); 1815 emit(0x1f); 1816 emit(0x40); 1817 emit(0x00); 1818 return; 1819 case 6: 1820 emit(0x66); 1821 case 5: 1822 emit(0x0f); 1823 emit(0x1f); 1824 emit(0x44); 1825 emit(0x00); 1826 emit(0x00); 1827 return; 1828 case 7: 1829 emit(0x0f); 1830 emit(0x1f); 1831 emit(0x80); 1832 emit(0x00); 1833 emit(0x00); 1834 emit(0x00); 1835 emit(0x00); 1836 return; 1837 default: 1838 case 11: 1839 emit(0x66); 1840 n--; 1841 case 10: 1842 emit(0x66); 1843 n--; 1844 case 9: 1845 emit(0x66); 1846 n--; 1847 case 8: 1848 emit(0x0f); 1849 emit(0x1f); 1850 emit(0x84); 1851 emit(0x00); 1852 emit(0x00); 1853 emit(0x00); 1854 emit(0x00); 1855 emit(0x00); 1856 n -= 8; 1857 } 1858 } 1859 } 1860 1861 1862 void Assembler::popq(Register dst) { 1863 EnsureSpace ensure_space(this); 1864 emit_optional_rex_32(dst); 1865 emit(0x58 | dst.low_bits()); 1866 } 1867 1868 1869 void Assembler::popq(const Operand& dst) { 1870 EnsureSpace ensure_space(this); 1871 emit_optional_rex_32(dst); 1872 emit(0x8F); 1873 emit_operand(0, dst); 1874 } 1875 1876 1877 void Assembler::popfq() { 1878 EnsureSpace ensure_space(this); 1879 emit(0x9D); 1880 } 1881 1882 1883 void Assembler::pushq(Register src) { 1884 EnsureSpace ensure_space(this); 1885 emit_optional_rex_32(src); 1886 emit(0x50 | src.low_bits()); 1887 } 1888 1889 1890 void Assembler::pushq(const Operand& src) { 1891 EnsureSpace ensure_space(this); 1892 emit_optional_rex_32(src); 1893 emit(0xFF); 1894 emit_operand(6, src); 1895 } 1896 1897 1898 void Assembler::pushq(Immediate value) { 1899 EnsureSpace ensure_space(this); 1900 if (is_int8(value.value_)) { 1901 emit(0x6A); 1902 emit(value.value_); // Emit low byte of value. 1903 } else { 1904 emit(0x68); 1905 emitl(value.value_); 1906 } 1907 } 1908 1909 1910 void Assembler::pushq_imm32(int32_t imm32) { 1911 EnsureSpace ensure_space(this); 1912 emit(0x68); 1913 emitl(imm32); 1914 } 1915 1916 1917 void Assembler::pushfq() { 1918 EnsureSpace ensure_space(this); 1919 emit(0x9C); 1920 } 1921 1922 1923 void Assembler::ret(int imm16) { 1924 EnsureSpace ensure_space(this); 1925 DCHECK(is_uint16(imm16)); 1926 if (imm16 == 0) { 1927 emit(0xC3); 1928 } else { 1929 emit(0xC2); 1930 emit(imm16 & 0xFF); 1931 emit((imm16 >> 8) & 0xFF); 1932 } 1933 } 1934 1935 1936 void Assembler::ud2() { 1937 EnsureSpace ensure_space(this); 1938 emit(0x0F); 1939 emit(0x0B); 1940 } 1941 1942 1943 void Assembler::setcc(Condition cc, Register reg) { 1944 if (cc > last_condition) { 1945 movb(reg, Immediate(cc == always ? 1 : 0)); 1946 return; 1947 } 1948 EnsureSpace ensure_space(this); 1949 DCHECK(is_uint4(cc)); 1950 if (!reg.is_byte_register()) { 1951 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1952 emit_rex_32(reg); 1953 } 1954 emit(0x0F); 1955 emit(0x90 | cc); 1956 emit_modrm(0x0, reg); 1957 } 1958 1959 1960 void Assembler::shld(Register dst, Register src) { 1961 EnsureSpace ensure_space(this); 1962 emit_rex_64(src, dst); 1963 emit(0x0F); 1964 emit(0xA5); 1965 emit_modrm(src, dst); 1966 } 1967 1968 1969 void Assembler::shrd(Register dst, Register src) { 1970 EnsureSpace ensure_space(this); 1971 emit_rex_64(src, dst); 1972 emit(0x0F); 1973 emit(0xAD); 1974 emit_modrm(src, dst); 1975 } 1976 1977 void Assembler::xchgb(Register reg, const Operand& op) { 1978 EnsureSpace ensure_space(this); 1979 if (!reg.is_byte_register()) { 1980 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1981 emit_rex_32(reg, op); 1982 } else { 1983 emit_optional_rex_32(reg, op); 1984 } 1985 emit(0x86); 1986 emit_operand(reg, op); 1987 } 1988 1989 void Assembler::xchgw(Register reg, const Operand& op) { 1990 EnsureSpace ensure_space(this); 1991 emit(0x66); 1992 emit_optional_rex_32(reg, op); 1993 emit(0x87); 1994 emit_operand(reg, op); 1995 } 1996 1997 void Assembler::emit_xchg(Register dst, Register src, int size) { 1998 EnsureSpace ensure_space(this); 1999 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding 2000 Register other = src.is(rax) ? dst : src; 2001 emit_rex(other, size); 2002 emit(0x90 | other.low_bits()); 2003 } else if (dst.low_bits() == 4) { 2004 emit_rex(dst, src, size); 2005 emit(0x87); 2006 emit_modrm(dst, src); 2007 } else { 2008 emit_rex(src, dst, size); 2009 emit(0x87); 2010 emit_modrm(src, dst); 2011 } 2012 } 2013 2014 2015 void Assembler::emit_xchg(Register dst, const Operand& src, int size) { 2016 EnsureSpace ensure_space(this); 2017 emit_rex(dst, src, size); 2018 emit(0x87); 2019 emit_operand(dst, src); 2020 } 2021 2022 2023 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) { 2024 EnsureSpace ensure_space(this); 2025 if (kPointerSize == kInt64Size) { 2026 emit(0x48); // REX.W 2027 emit(0xA3); 2028 emitp(dst, mode); 2029 } else { 2030 DCHECK(kPointerSize == kInt32Size); 2031 emit(0xA3); 2032 emitp(dst, mode); 2033 // In 64-bit mode, need to zero extend the operand to 8 bytes. 2034 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 2035 // Developer's Manual Volume 2. 2036 emitl(0); 2037 } 2038 } 2039 2040 2041 void Assembler::store_rax(ExternalReference ref) { 2042 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 2043 } 2044 2045 2046 void Assembler::testb(Register dst, Register src) { 2047 EnsureSpace ensure_space(this); 2048 if (src.low_bits() == 4) { 2049 emit_rex_32(src, dst); 2050 emit(0x84); 2051 emit_modrm(src, dst); 2052 } else { 2053 if (!dst.is_byte_register() || !src.is_byte_register()) { 2054 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2055 emit_rex_32(dst, src); 2056 } 2057 emit(0x84); 2058 emit_modrm(dst, src); 2059 } 2060 } 2061 2062 2063 void Assembler::testb(Register reg, Immediate mask) { 2064 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2065 EnsureSpace ensure_space(this); 2066 if (reg.is(rax)) { 2067 emit(0xA8); 2068 emit(mask.value_); // Low byte emitted. 2069 } else { 2070 if (!reg.is_byte_register()) { 2071 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2072 emit_rex_32(reg); 2073 } 2074 emit(0xF6); 2075 emit_modrm(0x0, reg); 2076 emit(mask.value_); // Low byte emitted. 2077 } 2078 } 2079 2080 2081 void Assembler::testb(const Operand& op, Immediate mask) { 2082 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2083 EnsureSpace ensure_space(this); 2084 emit_optional_rex_32(rax, op); 2085 emit(0xF6); 2086 emit_operand(rax, op); // Operation code 0 2087 emit(mask.value_); // Low byte emitted. 2088 } 2089 2090 2091 void Assembler::testb(const Operand& op, Register reg) { 2092 EnsureSpace ensure_space(this); 2093 if (!reg.is_byte_register()) { 2094 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2095 emit_rex_32(reg, op); 2096 } else { 2097 emit_optional_rex_32(reg, op); 2098 } 2099 emit(0x84); 2100 emit_operand(reg, op); 2101 } 2102 2103 void Assembler::testw(Register dst, Register src) { 2104 EnsureSpace ensure_space(this); 2105 emit(0x66); 2106 if (src.low_bits() == 4) { 2107 emit_rex_32(src, dst); 2108 } 2109 emit(0x85); 2110 emit_modrm(src, dst); 2111 } 2112 2113 void Assembler::testw(Register reg, Immediate mask) { 2114 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_)); 2115 EnsureSpace ensure_space(this); 2116 emit(0x66); 2117 if (reg.is(rax)) { 2118 emit(0xA9); 2119 emitw(mask.value_); 2120 } else { 2121 if (reg.low_bits() == 4) { 2122 emit_rex_32(reg); 2123 } 2124 emit(0xF7); 2125 emit_modrm(0x0, reg); 2126 emitw(mask.value_); 2127 } 2128 } 2129 2130 void Assembler::testw(const Operand& op, Immediate mask) { 2131 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_)); 2132 EnsureSpace ensure_space(this); 2133 emit(0x66); 2134 emit_optional_rex_32(rax, op); 2135 emit(0xF7); 2136 emit_operand(rax, op); 2137 emitw(mask.value_); 2138 } 2139 2140 void Assembler::testw(const Operand& op, Register reg) { 2141 EnsureSpace ensure_space(this); 2142 emit(0x66); 2143 emit_optional_rex_32(reg, op); 2144 emit(0x85); 2145 emit_operand(rax, op); 2146 } 2147 2148 void Assembler::emit_test(Register dst, Register src, int size) { 2149 EnsureSpace ensure_space(this); 2150 if (src.low_bits() == 4) { 2151 emit_rex(src, dst, size); 2152 emit(0x85); 2153 emit_modrm(src, dst); 2154 } else { 2155 emit_rex(dst, src, size); 2156 emit(0x85); 2157 emit_modrm(dst, src); 2158 } 2159 } 2160 2161 2162 void Assembler::emit_test(Register reg, Immediate mask, int size) { 2163 // testl with a mask that fits in the low byte is exactly testb. 2164 if (is_uint8(mask.value_)) { 2165 testb(reg, mask); 2166 return; 2167 } 2168 EnsureSpace ensure_space(this); 2169 if (reg.is(rax)) { 2170 emit_rex(rax, size); 2171 emit(0xA9); 2172 emit(mask); 2173 } else { 2174 emit_rex(reg, size); 2175 emit(0xF7); 2176 emit_modrm(0x0, reg); 2177 emit(mask); 2178 } 2179 } 2180 2181 2182 void Assembler::emit_test(const Operand& op, Immediate mask, int size) { 2183 // testl with a mask that fits in the low byte is exactly testb. 2184 if (is_uint8(mask.value_)) { 2185 testb(op, mask); 2186 return; 2187 } 2188 EnsureSpace ensure_space(this); 2189 emit_rex(rax, op, size); 2190 emit(0xF7); 2191 emit_operand(rax, op); // Operation code 0 2192 emit(mask); 2193 } 2194 2195 2196 void Assembler::emit_test(const Operand& op, Register reg, int size) { 2197 EnsureSpace ensure_space(this); 2198 emit_rex(reg, op, size); 2199 emit(0x85); 2200 emit_operand(reg, op); 2201 } 2202 2203 2204 // FPU instructions. 2205 2206 2207 void Assembler::fld(int i) { 2208 EnsureSpace ensure_space(this); 2209 emit_farith(0xD9, 0xC0, i); 2210 } 2211 2212 2213 void Assembler::fld1() { 2214 EnsureSpace ensure_space(this); 2215 emit(0xD9); 2216 emit(0xE8); 2217 } 2218 2219 2220 void Assembler::fldz() { 2221 EnsureSpace ensure_space(this); 2222 emit(0xD9); 2223 emit(0xEE); 2224 } 2225 2226 2227 void Assembler::fldpi() { 2228 EnsureSpace ensure_space(this); 2229 emit(0xD9); 2230 emit(0xEB); 2231 } 2232 2233 2234 void Assembler::fldln2() { 2235 EnsureSpace ensure_space(this); 2236 emit(0xD9); 2237 emit(0xED); 2238 } 2239 2240 2241 void Assembler::fld_s(const Operand& adr) { 2242 EnsureSpace ensure_space(this); 2243 emit_optional_rex_32(adr); 2244 emit(0xD9); 2245 emit_operand(0, adr); 2246 } 2247 2248 2249 void Assembler::fld_d(const Operand& adr) { 2250 EnsureSpace ensure_space(this); 2251 emit_optional_rex_32(adr); 2252 emit(0xDD); 2253 emit_operand(0, adr); 2254 } 2255 2256 2257 void Assembler::fstp_s(const Operand& adr) { 2258 EnsureSpace ensure_space(this); 2259 emit_optional_rex_32(adr); 2260 emit(0xD9); 2261 emit_operand(3, adr); 2262 } 2263 2264 2265 void Assembler::fstp_d(const Operand& adr) { 2266 EnsureSpace ensure_space(this); 2267 emit_optional_rex_32(adr); 2268 emit(0xDD); 2269 emit_operand(3, adr); 2270 } 2271 2272 2273 void Assembler::fstp(int index) { 2274 DCHECK(is_uint3(index)); 2275 EnsureSpace ensure_space(this); 2276 emit_farith(0xDD, 0xD8, index); 2277 } 2278 2279 2280 void Assembler::fild_s(const Operand& adr) { 2281 EnsureSpace ensure_space(this); 2282 emit_optional_rex_32(adr); 2283 emit(0xDB); 2284 emit_operand(0, adr); 2285 } 2286 2287 2288 void Assembler::fild_d(const Operand& adr) { 2289 EnsureSpace ensure_space(this); 2290 emit_optional_rex_32(adr); 2291 emit(0xDF); 2292 emit_operand(5, adr); 2293 } 2294 2295 2296 void Assembler::fistp_s(const Operand& adr) { 2297 EnsureSpace ensure_space(this); 2298 emit_optional_rex_32(adr); 2299 emit(0xDB); 2300 emit_operand(3, adr); 2301 } 2302 2303 2304 void Assembler::fisttp_s(const Operand& adr) { 2305 DCHECK(IsEnabled(SSE3)); 2306 EnsureSpace ensure_space(this); 2307 emit_optional_rex_32(adr); 2308 emit(0xDB); 2309 emit_operand(1, adr); 2310 } 2311 2312 2313 void Assembler::fisttp_d(const Operand& adr) { 2314 DCHECK(IsEnabled(SSE3)); 2315 EnsureSpace ensure_space(this); 2316 emit_optional_rex_32(adr); 2317 emit(0xDD); 2318 emit_operand(1, adr); 2319 } 2320 2321 2322 void Assembler::fist_s(const Operand& adr) { 2323 EnsureSpace ensure_space(this); 2324 emit_optional_rex_32(adr); 2325 emit(0xDB); 2326 emit_operand(2, adr); 2327 } 2328 2329 2330 void Assembler::fistp_d(const Operand& adr) { 2331 EnsureSpace ensure_space(this); 2332 emit_optional_rex_32(adr); 2333 emit(0xDF); 2334 emit_operand(7, adr); 2335 } 2336 2337 2338 void Assembler::fabs() { 2339 EnsureSpace ensure_space(this); 2340 emit(0xD9); 2341 emit(0xE1); 2342 } 2343 2344 2345 void Assembler::fchs() { 2346 EnsureSpace ensure_space(this); 2347 emit(0xD9); 2348 emit(0xE0); 2349 } 2350 2351 2352 void Assembler::fcos() { 2353 EnsureSpace ensure_space(this); 2354 emit(0xD9); 2355 emit(0xFF); 2356 } 2357 2358 2359 void Assembler::fsin() { 2360 EnsureSpace ensure_space(this); 2361 emit(0xD9); 2362 emit(0xFE); 2363 } 2364 2365 2366 void Assembler::fptan() { 2367 EnsureSpace ensure_space(this); 2368 emit(0xD9); 2369 emit(0xF2); 2370 } 2371 2372 2373 void Assembler::fyl2x() { 2374 EnsureSpace ensure_space(this); 2375 emit(0xD9); 2376 emit(0xF1); 2377 } 2378 2379 2380 void Assembler::f2xm1() { 2381 EnsureSpace ensure_space(this); 2382 emit(0xD9); 2383 emit(0xF0); 2384 } 2385 2386 2387 void Assembler::fscale() { 2388 EnsureSpace ensure_space(this); 2389 emit(0xD9); 2390 emit(0xFD); 2391 } 2392 2393 2394 void Assembler::fninit() { 2395 EnsureSpace ensure_space(this); 2396 emit(0xDB); 2397 emit(0xE3); 2398 } 2399 2400 2401 void Assembler::fadd(int i) { 2402 EnsureSpace ensure_space(this); 2403 emit_farith(0xDC, 0xC0, i); 2404 } 2405 2406 2407 void Assembler::fsub(int i) { 2408 EnsureSpace ensure_space(this); 2409 emit_farith(0xDC, 0xE8, i); 2410 } 2411 2412 2413 void Assembler::fisub_s(const Operand& adr) { 2414 EnsureSpace ensure_space(this); 2415 emit_optional_rex_32(adr); 2416 emit(0xDA); 2417 emit_operand(4, adr); 2418 } 2419 2420 2421 void Assembler::fmul(int i) { 2422 EnsureSpace ensure_space(this); 2423 emit_farith(0xDC, 0xC8, i); 2424 } 2425 2426 2427 void Assembler::fdiv(int i) { 2428 EnsureSpace ensure_space(this); 2429 emit_farith(0xDC, 0xF8, i); 2430 } 2431 2432 2433 void Assembler::faddp(int i) { 2434 EnsureSpace ensure_space(this); 2435 emit_farith(0xDE, 0xC0, i); 2436 } 2437 2438 2439 void Assembler::fsubp(int i) { 2440 EnsureSpace ensure_space(this); 2441 emit_farith(0xDE, 0xE8, i); 2442 } 2443 2444 2445 void Assembler::fsubrp(int i) { 2446 EnsureSpace ensure_space(this); 2447 emit_farith(0xDE, 0xE0, i); 2448 } 2449 2450 2451 void Assembler::fmulp(int i) { 2452 EnsureSpace ensure_space(this); 2453 emit_farith(0xDE, 0xC8, i); 2454 } 2455 2456 2457 void Assembler::fdivp(int i) { 2458 EnsureSpace ensure_space(this); 2459 emit_farith(0xDE, 0xF8, i); 2460 } 2461 2462 2463 void Assembler::fprem() { 2464 EnsureSpace ensure_space(this); 2465 emit(0xD9); 2466 emit(0xF8); 2467 } 2468 2469 2470 void Assembler::fprem1() { 2471 EnsureSpace ensure_space(this); 2472 emit(0xD9); 2473 emit(0xF5); 2474 } 2475 2476 2477 void Assembler::fxch(int i) { 2478 EnsureSpace ensure_space(this); 2479 emit_farith(0xD9, 0xC8, i); 2480 } 2481 2482 2483 void Assembler::fincstp() { 2484 EnsureSpace ensure_space(this); 2485 emit(0xD9); 2486 emit(0xF7); 2487 } 2488 2489 2490 void Assembler::ffree(int i) { 2491 EnsureSpace ensure_space(this); 2492 emit_farith(0xDD, 0xC0, i); 2493 } 2494 2495 2496 void Assembler::ftst() { 2497 EnsureSpace ensure_space(this); 2498 emit(0xD9); 2499 emit(0xE4); 2500 } 2501 2502 2503 void Assembler::fucomp(int i) { 2504 EnsureSpace ensure_space(this); 2505 emit_farith(0xDD, 0xE8, i); 2506 } 2507 2508 2509 void Assembler::fucompp() { 2510 EnsureSpace ensure_space(this); 2511 emit(0xDA); 2512 emit(0xE9); 2513 } 2514 2515 2516 void Assembler::fucomi(int i) { 2517 EnsureSpace ensure_space(this); 2518 emit(0xDB); 2519 emit(0xE8 + i); 2520 } 2521 2522 2523 void Assembler::fucomip() { 2524 EnsureSpace ensure_space(this); 2525 emit(0xDF); 2526 emit(0xE9); 2527 } 2528 2529 2530 void Assembler::fcompp() { 2531 EnsureSpace ensure_space(this); 2532 emit(0xDE); 2533 emit(0xD9); 2534 } 2535 2536 2537 void Assembler::fnstsw_ax() { 2538 EnsureSpace ensure_space(this); 2539 emit(0xDF); 2540 emit(0xE0); 2541 } 2542 2543 2544 void Assembler::fwait() { 2545 EnsureSpace ensure_space(this); 2546 emit(0x9B); 2547 } 2548 2549 2550 void Assembler::frndint() { 2551 EnsureSpace ensure_space(this); 2552 emit(0xD9); 2553 emit(0xFC); 2554 } 2555 2556 2557 void Assembler::fnclex() { 2558 EnsureSpace ensure_space(this); 2559 emit(0xDB); 2560 emit(0xE2); 2561 } 2562 2563 2564 void Assembler::sahf() { 2565 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf 2566 // in 64-bit mode. Test CpuID. 2567 DCHECK(IsEnabled(SAHF)); 2568 EnsureSpace ensure_space(this); 2569 emit(0x9E); 2570 } 2571 2572 2573 void Assembler::emit_farith(int b1, int b2, int i) { 2574 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode 2575 DCHECK(is_uint3(i)); // illegal stack offset 2576 emit(b1); 2577 emit(b2 + i); 2578 } 2579 2580 2581 // SSE operations. 2582 2583 void Assembler::andps(XMMRegister dst, XMMRegister src) { 2584 EnsureSpace ensure_space(this); 2585 emit_optional_rex_32(dst, src); 2586 emit(0x0F); 2587 emit(0x54); 2588 emit_sse_operand(dst, src); 2589 } 2590 2591 2592 void Assembler::andps(XMMRegister dst, const Operand& src) { 2593 EnsureSpace ensure_space(this); 2594 emit_optional_rex_32(dst, src); 2595 emit(0x0F); 2596 emit(0x54); 2597 emit_sse_operand(dst, src); 2598 } 2599 2600 2601 void Assembler::orps(XMMRegister dst, XMMRegister src) { 2602 EnsureSpace ensure_space(this); 2603 emit_optional_rex_32(dst, src); 2604 emit(0x0F); 2605 emit(0x56); 2606 emit_sse_operand(dst, src); 2607 } 2608 2609 2610 void Assembler::orps(XMMRegister dst, const Operand& src) { 2611 EnsureSpace ensure_space(this); 2612 emit_optional_rex_32(dst, src); 2613 emit(0x0F); 2614 emit(0x56); 2615 emit_sse_operand(dst, src); 2616 } 2617 2618 2619 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 2620 DCHECK(!IsEnabled(AVX)); 2621 EnsureSpace ensure_space(this); 2622 emit_optional_rex_32(dst, src); 2623 emit(0x0F); 2624 emit(0x57); 2625 emit_sse_operand(dst, src); 2626 } 2627 2628 2629 void Assembler::xorps(XMMRegister dst, const Operand& src) { 2630 DCHECK(!IsEnabled(AVX)); 2631 EnsureSpace ensure_space(this); 2632 emit_optional_rex_32(dst, src); 2633 emit(0x0F); 2634 emit(0x57); 2635 emit_sse_operand(dst, src); 2636 } 2637 2638 2639 void Assembler::addps(XMMRegister dst, XMMRegister src) { 2640 EnsureSpace ensure_space(this); 2641 emit_optional_rex_32(dst, src); 2642 emit(0x0F); 2643 emit(0x58); 2644 emit_sse_operand(dst, src); 2645 } 2646 2647 2648 void Assembler::addps(XMMRegister dst, const Operand& src) { 2649 EnsureSpace ensure_space(this); 2650 emit_optional_rex_32(dst, src); 2651 emit(0x0F); 2652 emit(0x58); 2653 emit_sse_operand(dst, src); 2654 } 2655 2656 2657 void Assembler::subps(XMMRegister dst, XMMRegister src) { 2658 EnsureSpace ensure_space(this); 2659 emit_optional_rex_32(dst, src); 2660 emit(0x0F); 2661 emit(0x5C); 2662 emit_sse_operand(dst, src); 2663 } 2664 2665 2666 void Assembler::subps(XMMRegister dst, const Operand& src) { 2667 EnsureSpace ensure_space(this); 2668 emit_optional_rex_32(dst, src); 2669 emit(0x0F); 2670 emit(0x5C); 2671 emit_sse_operand(dst, src); 2672 } 2673 2674 2675 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 2676 EnsureSpace ensure_space(this); 2677 emit_optional_rex_32(dst, src); 2678 emit(0x0F); 2679 emit(0x59); 2680 emit_sse_operand(dst, src); 2681 } 2682 2683 2684 void Assembler::mulps(XMMRegister dst, const Operand& src) { 2685 EnsureSpace ensure_space(this); 2686 emit_optional_rex_32(dst, src); 2687 emit(0x0F); 2688 emit(0x59); 2689 emit_sse_operand(dst, src); 2690 } 2691 2692 2693 void Assembler::divps(XMMRegister dst, XMMRegister src) { 2694 EnsureSpace ensure_space(this); 2695 emit_optional_rex_32(dst, src); 2696 emit(0x0F); 2697 emit(0x5E); 2698 emit_sse_operand(dst, src); 2699 } 2700 2701 2702 void Assembler::divps(XMMRegister dst, const Operand& src) { 2703 EnsureSpace ensure_space(this); 2704 emit_optional_rex_32(dst, src); 2705 emit(0x0F); 2706 emit(0x5E); 2707 emit_sse_operand(dst, src); 2708 } 2709 2710 2711 // SSE 2 operations. 2712 2713 void Assembler::movd(XMMRegister dst, Register src) { 2714 DCHECK(!IsEnabled(AVX)); 2715 EnsureSpace ensure_space(this); 2716 emit(0x66); 2717 emit_optional_rex_32(dst, src); 2718 emit(0x0F); 2719 emit(0x6E); 2720 emit_sse_operand(dst, src); 2721 } 2722 2723 2724 void Assembler::movd(XMMRegister dst, const Operand& src) { 2725 DCHECK(!IsEnabled(AVX)); 2726 EnsureSpace ensure_space(this); 2727 emit(0x66); 2728 emit_optional_rex_32(dst, src); 2729 emit(0x0F); 2730 emit(0x6E); 2731 emit_sse_operand(dst, src); 2732 } 2733 2734 2735 void Assembler::movd(Register dst, XMMRegister src) { 2736 DCHECK(!IsEnabled(AVX)); 2737 EnsureSpace ensure_space(this); 2738 emit(0x66); 2739 emit_optional_rex_32(src, dst); 2740 emit(0x0F); 2741 emit(0x7E); 2742 emit_sse_operand(src, dst); 2743 } 2744 2745 2746 void Assembler::movq(XMMRegister dst, Register src) { 2747 DCHECK(!IsEnabled(AVX)); 2748 EnsureSpace ensure_space(this); 2749 emit(0x66); 2750 emit_rex_64(dst, src); 2751 emit(0x0F); 2752 emit(0x6E); 2753 emit_sse_operand(dst, src); 2754 } 2755 2756 2757 void Assembler::movq(Register dst, XMMRegister src) { 2758 DCHECK(!IsEnabled(AVX)); 2759 EnsureSpace ensure_space(this); 2760 emit(0x66); 2761 emit_rex_64(src, dst); 2762 emit(0x0F); 2763 emit(0x7E); 2764 emit_sse_operand(src, dst); 2765 } 2766 2767 2768 void Assembler::movq(XMMRegister dst, XMMRegister src) { 2769 DCHECK(!IsEnabled(AVX)); 2770 EnsureSpace ensure_space(this); 2771 if (dst.low_bits() == 4) { 2772 // Avoid unnecessary SIB byte. 2773 emit(0xf3); 2774 emit_optional_rex_32(dst, src); 2775 emit(0x0F); 2776 emit(0x7e); 2777 emit_sse_operand(dst, src); 2778 } else { 2779 emit(0x66); 2780 emit_optional_rex_32(src, dst); 2781 emit(0x0F); 2782 emit(0xD6); 2783 emit_sse_operand(src, dst); 2784 } 2785 } 2786 2787 2788 void Assembler::movdqa(const Operand& dst, XMMRegister src) { 2789 EnsureSpace ensure_space(this); 2790 emit(0x66); 2791 emit_rex_64(src, dst); 2792 emit(0x0F); 2793 emit(0x7F); 2794 emit_sse_operand(src, dst); 2795 } 2796 2797 2798 void Assembler::movdqa(XMMRegister dst, const Operand& src) { 2799 EnsureSpace ensure_space(this); 2800 emit(0x66); 2801 emit_rex_64(dst, src); 2802 emit(0x0F); 2803 emit(0x6F); 2804 emit_sse_operand(dst, src); 2805 } 2806 2807 2808 void Assembler::movdqu(const Operand& dst, XMMRegister src) { 2809 EnsureSpace ensure_space(this); 2810 emit(0xF3); 2811 emit_rex_64(src, dst); 2812 emit(0x0F); 2813 emit(0x7F); 2814 emit_sse_operand(src, dst); 2815 } 2816 2817 2818 void Assembler::movdqu(XMMRegister dst, const Operand& src) { 2819 EnsureSpace ensure_space(this); 2820 emit(0xF3); 2821 emit_rex_64(dst, src); 2822 emit(0x0F); 2823 emit(0x6F); 2824 emit_sse_operand(dst, src); 2825 } 2826 2827 2828 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) { 2829 DCHECK(IsEnabled(SSE4_1)); 2830 DCHECK(is_uint8(imm8)); 2831 EnsureSpace ensure_space(this); 2832 emit(0x66); 2833 emit_optional_rex_32(src, dst); 2834 emit(0x0F); 2835 emit(0x3A); 2836 emit(0x17); 2837 emit_sse_operand(src, dst); 2838 emit(imm8); 2839 } 2840 2841 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) { 2842 DCHECK(IsEnabled(SSE4_1)); 2843 DCHECK(is_uint8(imm8)); 2844 EnsureSpace ensure_space(this); 2845 emit(0x66); 2846 emit_optional_rex_32(src, dst); 2847 emit(0x0F); 2848 emit(0x3A); 2849 emit(0x14); 2850 emit_sse_operand(src, dst); 2851 emit(imm8); 2852 } 2853 2854 void Assembler::pextrb(const Operand& dst, XMMRegister src, int8_t imm8) { 2855 DCHECK(IsEnabled(SSE4_1)); 2856 DCHECK(is_uint8(imm8)); 2857 EnsureSpace ensure_space(this); 2858 emit(0x66); 2859 emit_optional_rex_32(src, dst); 2860 emit(0x0F); 2861 emit(0x3A); 2862 emit(0x14); 2863 emit_sse_operand(src, dst); 2864 emit(imm8); 2865 } 2866 2867 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) { 2868 DCHECK(is_uint8(imm8)); 2869 EnsureSpace ensure_space(this); 2870 emit(0x66); 2871 emit_optional_rex_32(dst, src); 2872 emit(0x0F); 2873 emit(0xC4); 2874 emit_sse_operand(dst, src); 2875 emit(imm8); 2876 } 2877 2878 void Assembler::pinsrw(XMMRegister dst, const Operand& src, int8_t imm8) { 2879 DCHECK(is_uint8(imm8)); 2880 EnsureSpace ensure_space(this); 2881 emit(0x66); 2882 emit_optional_rex_32(dst, src); 2883 emit(0x0F); 2884 emit(0xC4); 2885 emit_sse_operand(dst, src); 2886 emit(imm8); 2887 } 2888 2889 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) { 2890 DCHECK(is_uint8(imm8)); 2891 EnsureSpace ensure_space(this); 2892 emit(0x66); 2893 emit_optional_rex_32(src, dst); 2894 emit(0x0F); 2895 emit(0xC5); 2896 emit_sse_operand(src, dst); 2897 emit(imm8); 2898 } 2899 2900 void Assembler::pextrw(const Operand& dst, XMMRegister src, int8_t imm8) { 2901 DCHECK(IsEnabled(SSE4_1)); 2902 DCHECK(is_uint8(imm8)); 2903 EnsureSpace ensure_space(this); 2904 emit(0x66); 2905 emit_optional_rex_32(src, dst); 2906 emit(0x0F); 2907 emit(0x3A); 2908 emit(0x15); 2909 emit_sse_operand(src, dst); 2910 emit(imm8); 2911 } 2912 2913 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) { 2914 DCHECK(IsEnabled(SSE4_1)); 2915 EnsureSpace ensure_space(this); 2916 emit(0x66); 2917 emit_optional_rex_32(src, dst); 2918 emit(0x0F); 2919 emit(0x3A); 2920 emit(0x16); 2921 emit_sse_operand(src, dst); 2922 emit(imm8); 2923 } 2924 2925 void Assembler::pextrd(const Operand& dst, XMMRegister src, int8_t imm8) { 2926 DCHECK(IsEnabled(SSE4_1)); 2927 EnsureSpace ensure_space(this); 2928 emit(0x66); 2929 emit_optional_rex_32(src, dst); 2930 emit(0x0F); 2931 emit(0x3A); 2932 emit(0x16); 2933 emit_sse_operand(src, dst); 2934 emit(imm8); 2935 } 2936 2937 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) { 2938 DCHECK(IsEnabled(SSE4_1)); 2939 EnsureSpace ensure_space(this); 2940 emit(0x66); 2941 emit_optional_rex_32(dst, src); 2942 emit(0x0F); 2943 emit(0x3A); 2944 emit(0x22); 2945 emit_sse_operand(dst, src); 2946 emit(imm8); 2947 } 2948 2949 2950 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) { 2951 DCHECK(IsEnabled(SSE4_1)); 2952 EnsureSpace ensure_space(this); 2953 emit(0x66); 2954 emit_optional_rex_32(dst, src); 2955 emit(0x0F); 2956 emit(0x3A); 2957 emit(0x22); 2958 emit_sse_operand(dst, src); 2959 emit(imm8); 2960 } 2961 2962 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) { 2963 DCHECK(IsEnabled(SSE4_1)); 2964 EnsureSpace ensure_space(this); 2965 emit(0x66); 2966 emit_optional_rex_32(dst, src); 2967 emit(0x0F); 2968 emit(0x3A); 2969 emit(0x20); 2970 emit_sse_operand(dst, src); 2971 emit(imm8); 2972 } 2973 2974 void Assembler::pinsrb(XMMRegister dst, const Operand& src, int8_t imm8) { 2975 DCHECK(IsEnabled(SSE4_1)); 2976 EnsureSpace ensure_space(this); 2977 emit(0x66); 2978 emit_optional_rex_32(dst, src); 2979 emit(0x0F); 2980 emit(0x3A); 2981 emit(0x20); 2982 emit_sse_operand(dst, src); 2983 emit(imm8); 2984 } 2985 2986 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) { 2987 DCHECK(CpuFeatures::IsSupported(SSE4_1)); 2988 DCHECK(is_uint8(imm8)); 2989 EnsureSpace ensure_space(this); 2990 emit(0x66); 2991 emit_optional_rex_32(dst, src); 2992 emit(0x0F); 2993 emit(0x3A); 2994 emit(0x21); 2995 emit_sse_operand(dst, src); 2996 emit(imm8); 2997 } 2998 2999 void Assembler::movsd(const Operand& dst, XMMRegister src) { 3000 DCHECK(!IsEnabled(AVX)); 3001 EnsureSpace ensure_space(this); 3002 emit(0xF2); // double 3003 emit_optional_rex_32(src, dst); 3004 emit(0x0F); 3005 emit(0x11); // store 3006 emit_sse_operand(src, dst); 3007 } 3008 3009 3010 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 3011 DCHECK(!IsEnabled(AVX)); 3012 EnsureSpace ensure_space(this); 3013 emit(0xF2); // double 3014 emit_optional_rex_32(dst, src); 3015 emit(0x0F); 3016 emit(0x10); // load 3017 emit_sse_operand(dst, src); 3018 } 3019 3020 3021 void Assembler::movsd(XMMRegister dst, const Operand& src) { 3022 DCHECK(!IsEnabled(AVX)); 3023 EnsureSpace ensure_space(this); 3024 emit(0xF2); // double 3025 emit_optional_rex_32(dst, src); 3026 emit(0x0F); 3027 emit(0x10); // load 3028 emit_sse_operand(dst, src); 3029 } 3030 3031 3032 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 3033 DCHECK(!IsEnabled(AVX)); 3034 EnsureSpace ensure_space(this); 3035 if (src.low_bits() == 4) { 3036 // Try to avoid an unnecessary SIB byte. 3037 emit_optional_rex_32(src, dst); 3038 emit(0x0F); 3039 emit(0x29); 3040 emit_sse_operand(src, dst); 3041 } else { 3042 emit_optional_rex_32(dst, src); 3043 emit(0x0F); 3044 emit(0x28); 3045 emit_sse_operand(dst, src); 3046 } 3047 } 3048 3049 3050 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) { 3051 DCHECK(is_uint8(imm8)); 3052 EnsureSpace ensure_space(this); 3053 emit_optional_rex_32(src, dst); 3054 emit(0x0F); 3055 emit(0xC6); 3056 emit_sse_operand(dst, src); 3057 emit(imm8); 3058 } 3059 3060 3061 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 3062 DCHECK(!IsEnabled(AVX)); 3063 EnsureSpace ensure_space(this); 3064 if (src.low_bits() == 4) { 3065 // Try to avoid an unnecessary SIB byte. 3066 emit(0x66); 3067 emit_optional_rex_32(src, dst); 3068 emit(0x0F); 3069 emit(0x29); 3070 emit_sse_operand(src, dst); 3071 } else { 3072 emit(0x66); 3073 emit_optional_rex_32(dst, src); 3074 emit(0x0F); 3075 emit(0x28); 3076 emit_sse_operand(dst, src); 3077 } 3078 } 3079 3080 3081 void Assembler::movupd(XMMRegister dst, const Operand& src) { 3082 EnsureSpace ensure_space(this); 3083 emit(0x66); 3084 emit_optional_rex_32(dst, src); 3085 emit(0x0F); 3086 emit(0x10); 3087 emit_sse_operand(dst, src); 3088 } 3089 3090 void Assembler::movupd(const Operand& dst, XMMRegister src) { 3091 EnsureSpace ensure_space(this); 3092 emit(0x66); 3093 emit_optional_rex_32(src, dst); 3094 emit(0x0F); 3095 emit(0x11); 3096 emit_sse_operand(src, dst); 3097 } 3098 3099 void Assembler::addss(XMMRegister dst, XMMRegister src) { 3100 EnsureSpace ensure_space(this); 3101 emit(0xF3); 3102 emit_optional_rex_32(dst, src); 3103 emit(0x0F); 3104 emit(0x58); 3105 emit_sse_operand(dst, src); 3106 } 3107 3108 3109 void Assembler::addss(XMMRegister dst, const Operand& src) { 3110 EnsureSpace ensure_space(this); 3111 emit(0xF3); 3112 emit_optional_rex_32(dst, src); 3113 emit(0x0F); 3114 emit(0x58); 3115 emit_sse_operand(dst, src); 3116 } 3117 3118 3119 void Assembler::subss(XMMRegister dst, XMMRegister src) { 3120 EnsureSpace ensure_space(this); 3121 emit(0xF3); 3122 emit_optional_rex_32(dst, src); 3123 emit(0x0F); 3124 emit(0x5C); 3125 emit_sse_operand(dst, src); 3126 } 3127 3128 3129 void Assembler::subss(XMMRegister dst, const Operand& src) { 3130 EnsureSpace ensure_space(this); 3131 emit(0xF3); 3132 emit_optional_rex_32(dst, src); 3133 emit(0x0F); 3134 emit(0x5C); 3135 emit_sse_operand(dst, src); 3136 } 3137 3138 3139 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 3140 EnsureSpace ensure_space(this); 3141 emit(0xF3); 3142 emit_optional_rex_32(dst, src); 3143 emit(0x0F); 3144 emit(0x59); 3145 emit_sse_operand(dst, src); 3146 } 3147 3148 3149 void Assembler::mulss(XMMRegister dst, const Operand& src) { 3150 EnsureSpace ensure_space(this); 3151 emit(0xF3); 3152 emit_optional_rex_32(dst, src); 3153 emit(0x0F); 3154 emit(0x59); 3155 emit_sse_operand(dst, src); 3156 } 3157 3158 3159 void Assembler::divss(XMMRegister dst, XMMRegister src) { 3160 EnsureSpace ensure_space(this); 3161 emit(0xF3); 3162 emit_optional_rex_32(dst, src); 3163 emit(0x0F); 3164 emit(0x5E); 3165 emit_sse_operand(dst, src); 3166 } 3167 3168 3169 void Assembler::divss(XMMRegister dst, const Operand& src) { 3170 EnsureSpace ensure_space(this); 3171 emit(0xF3); 3172 emit_optional_rex_32(dst, src); 3173 emit(0x0F); 3174 emit(0x5E); 3175 emit_sse_operand(dst, src); 3176 } 3177 3178 3179 void Assembler::maxss(XMMRegister dst, XMMRegister src) { 3180 EnsureSpace ensure_space(this); 3181 emit(0xF3); 3182 emit_optional_rex_32(dst, src); 3183 emit(0x0F); 3184 emit(0x5F); 3185 emit_sse_operand(dst, src); 3186 } 3187 3188 3189 void Assembler::maxss(XMMRegister dst, const Operand& src) { 3190 EnsureSpace ensure_space(this); 3191 emit(0xF3); 3192 emit_optional_rex_32(dst, src); 3193 emit(0x0F); 3194 emit(0x5F); 3195 emit_sse_operand(dst, src); 3196 } 3197 3198 3199 void Assembler::minss(XMMRegister dst, XMMRegister src) { 3200 EnsureSpace ensure_space(this); 3201 emit(0xF3); 3202 emit_optional_rex_32(dst, src); 3203 emit(0x0F); 3204 emit(0x5D); 3205 emit_sse_operand(dst, src); 3206 } 3207 3208 3209 void Assembler::minss(XMMRegister dst, const Operand& src) { 3210 EnsureSpace ensure_space(this); 3211 emit(0xF3); 3212 emit_optional_rex_32(dst, src); 3213 emit(0x0F); 3214 emit(0x5D); 3215 emit_sse_operand(dst, src); 3216 } 3217 3218 3219 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 3220 EnsureSpace ensure_space(this); 3221 emit(0xF3); 3222 emit_optional_rex_32(dst, src); 3223 emit(0x0F); 3224 emit(0x51); 3225 emit_sse_operand(dst, src); 3226 } 3227 3228 3229 void Assembler::sqrtss(XMMRegister dst, const Operand& src) { 3230 EnsureSpace ensure_space(this); 3231 emit(0xF3); 3232 emit_optional_rex_32(dst, src); 3233 emit(0x0F); 3234 emit(0x51); 3235 emit_sse_operand(dst, src); 3236 } 3237 3238 3239 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 3240 DCHECK(!IsEnabled(AVX)); 3241 EnsureSpace ensure_space(this); 3242 emit_optional_rex_32(dst, src); 3243 emit(0x0f); 3244 emit(0x2e); 3245 emit_sse_operand(dst, src); 3246 } 3247 3248 3249 void Assembler::ucomiss(XMMRegister dst, const Operand& src) { 3250 DCHECK(!IsEnabled(AVX)); 3251 EnsureSpace ensure_space(this); 3252 emit_optional_rex_32(dst, src); 3253 emit(0x0f); 3254 emit(0x2e); 3255 emit_sse_operand(dst, src); 3256 } 3257 3258 3259 void Assembler::movss(XMMRegister dst, XMMRegister src) { 3260 DCHECK(!IsEnabled(AVX)); 3261 EnsureSpace ensure_space(this); 3262 emit(0xF3); // single 3263 emit_optional_rex_32(dst, src); 3264 emit(0x0F); 3265 emit(0x10); // load 3266 emit_sse_operand(dst, src); 3267 } 3268 3269 3270 void Assembler::movss(XMMRegister dst, const Operand& src) { 3271 DCHECK(!IsEnabled(AVX)); 3272 EnsureSpace ensure_space(this); 3273 emit(0xF3); // single 3274 emit_optional_rex_32(dst, src); 3275 emit(0x0F); 3276 emit(0x10); // load 3277 emit_sse_operand(dst, src); 3278 } 3279 3280 3281 void Assembler::movss(const Operand& src, XMMRegister dst) { 3282 DCHECK(!IsEnabled(AVX)); 3283 EnsureSpace ensure_space(this); 3284 emit(0xF3); // single 3285 emit_optional_rex_32(dst, src); 3286 emit(0x0F); 3287 emit(0x11); // store 3288 emit_sse_operand(dst, src); 3289 } 3290 3291 3292 void Assembler::psllq(XMMRegister reg, byte imm8) { 3293 DCHECK(!IsEnabled(AVX)); 3294 EnsureSpace ensure_space(this); 3295 emit(0x66); 3296 emit_optional_rex_32(reg); 3297 emit(0x0F); 3298 emit(0x73); 3299 emit_sse_operand(rsi, reg); // rsi == 6 3300 emit(imm8); 3301 } 3302 3303 3304 void Assembler::psrlq(XMMRegister reg, byte imm8) { 3305 DCHECK(!IsEnabled(AVX)); 3306 EnsureSpace ensure_space(this); 3307 emit(0x66); 3308 emit_optional_rex_32(reg); 3309 emit(0x0F); 3310 emit(0x73); 3311 emit_sse_operand(rdx, reg); // rdx == 2 3312 emit(imm8); 3313 } 3314 3315 void Assembler::psllw(XMMRegister reg, byte imm8) { 3316 EnsureSpace ensure_space(this); 3317 emit(0x66); 3318 emit_optional_rex_32(reg); 3319 emit(0x0F); 3320 emit(0x71); 3321 emit_sse_operand(rsi, reg); // rsi == 6 3322 emit(imm8); 3323 } 3324 3325 void Assembler::pslld(XMMRegister reg, byte imm8) { 3326 EnsureSpace ensure_space(this); 3327 emit(0x66); 3328 emit_optional_rex_32(reg); 3329 emit(0x0F); 3330 emit(0x72); 3331 emit_sse_operand(rsi, reg); // rsi == 6 3332 emit(imm8); 3333 } 3334 3335 void Assembler::psrlw(XMMRegister reg, byte imm8) { 3336 EnsureSpace ensure_space(this); 3337 emit(0x66); 3338 emit_optional_rex_32(reg); 3339 emit(0x0F); 3340 emit(0x71); 3341 emit_sse_operand(rdx, reg); // rdx == 2 3342 emit(imm8); 3343 } 3344 3345 void Assembler::psrld(XMMRegister reg, byte imm8) { 3346 EnsureSpace ensure_space(this); 3347 emit(0x66); 3348 emit_optional_rex_32(reg); 3349 emit(0x0F); 3350 emit(0x72); 3351 emit_sse_operand(rdx, reg); // rdx == 2 3352 emit(imm8); 3353 } 3354 3355 void Assembler::psraw(XMMRegister reg, byte imm8) { 3356 EnsureSpace ensure_space(this); 3357 emit(0x66); 3358 emit_optional_rex_32(reg); 3359 emit(0x0F); 3360 emit(0x71); 3361 emit_sse_operand(rsp, reg); // rsp == 4 3362 emit(imm8); 3363 } 3364 3365 void Assembler::psrad(XMMRegister reg, byte imm8) { 3366 EnsureSpace ensure_space(this); 3367 emit(0x66); 3368 emit_optional_rex_32(reg); 3369 emit(0x0F); 3370 emit(0x72); 3371 emit_sse_operand(rsp, reg); // rsp == 4 3372 emit(imm8); 3373 } 3374 3375 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) { 3376 EnsureSpace ensure_space(this); 3377 emit_optional_rex_32(dst, src); 3378 emit(0x0F); 3379 emit(0xC2); 3380 emit_sse_operand(dst, src); 3381 emit(cmp); 3382 } 3383 3384 void Assembler::cmpps(XMMRegister dst, const Operand& src, int8_t cmp) { 3385 EnsureSpace ensure_space(this); 3386 emit_optional_rex_32(dst, src); 3387 emit(0x0F); 3388 emit(0xC2); 3389 emit_sse_operand(dst, src); 3390 emit(cmp); 3391 } 3392 3393 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) { 3394 EnsureSpace ensure_space(this); 3395 emit_optional_rex_32(dst, src); 3396 emit(0x66); 3397 emit(0x0F); 3398 emit(0xC2); 3399 emit_sse_operand(dst, src); 3400 emit(cmp); 3401 } 3402 3403 void Assembler::cmppd(XMMRegister dst, const Operand& src, int8_t cmp) { 3404 EnsureSpace ensure_space(this); 3405 emit_optional_rex_32(dst, src); 3406 emit(0x66); 3407 emit(0x0F); 3408 emit(0xC2); 3409 emit_sse_operand(dst, src); 3410 emit(cmp); 3411 } 3412 3413 void Assembler::cvttss2si(Register dst, const Operand& src) { 3414 DCHECK(!IsEnabled(AVX)); 3415 EnsureSpace ensure_space(this); 3416 emit(0xF3); 3417 emit_optional_rex_32(dst, src); 3418 emit(0x0F); 3419 emit(0x2C); 3420 emit_operand(dst, src); 3421 } 3422 3423 3424 void Assembler::cvttss2si(Register dst, XMMRegister src) { 3425 DCHECK(!IsEnabled(AVX)); 3426 EnsureSpace ensure_space(this); 3427 emit(0xF3); 3428 emit_optional_rex_32(dst, src); 3429 emit(0x0F); 3430 emit(0x2C); 3431 emit_sse_operand(dst, src); 3432 } 3433 3434 3435 void Assembler::cvttsd2si(Register dst, const Operand& src) { 3436 DCHECK(!IsEnabled(AVX)); 3437 EnsureSpace ensure_space(this); 3438 emit(0xF2); 3439 emit_optional_rex_32(dst, src); 3440 emit(0x0F); 3441 emit(0x2C); 3442 emit_operand(dst, src); 3443 } 3444 3445 3446 void Assembler::cvttsd2si(Register dst, XMMRegister src) { 3447 DCHECK(!IsEnabled(AVX)); 3448 EnsureSpace ensure_space(this); 3449 emit(0xF2); 3450 emit_optional_rex_32(dst, src); 3451 emit(0x0F); 3452 emit(0x2C); 3453 emit_sse_operand(dst, src); 3454 } 3455 3456 3457 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 3458 DCHECK(!IsEnabled(AVX)); 3459 EnsureSpace ensure_space(this); 3460 emit(0xF3); 3461 emit_rex_64(dst, src); 3462 emit(0x0F); 3463 emit(0x2C); 3464 emit_sse_operand(dst, src); 3465 } 3466 3467 3468 void Assembler::cvttss2siq(Register dst, const Operand& src) { 3469 DCHECK(!IsEnabled(AVX)); 3470 EnsureSpace ensure_space(this); 3471 emit(0xF3); 3472 emit_rex_64(dst, src); 3473 emit(0x0F); 3474 emit(0x2C); 3475 emit_sse_operand(dst, src); 3476 } 3477 3478 3479 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 3480 DCHECK(!IsEnabled(AVX)); 3481 EnsureSpace ensure_space(this); 3482 emit(0xF2); 3483 emit_rex_64(dst, src); 3484 emit(0x0F); 3485 emit(0x2C); 3486 emit_sse_operand(dst, src); 3487 } 3488 3489 3490 void Assembler::cvttsd2siq(Register dst, const Operand& src) { 3491 DCHECK(!IsEnabled(AVX)); 3492 EnsureSpace ensure_space(this); 3493 emit(0xF2); 3494 emit_rex_64(dst, src); 3495 emit(0x0F); 3496 emit(0x2C); 3497 emit_sse_operand(dst, src); 3498 } 3499 3500 3501 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) { 3502 DCHECK(!IsEnabled(AVX)); 3503 EnsureSpace ensure_space(this); 3504 emit(0xF2); 3505 emit_optional_rex_32(dst, src); 3506 emit(0x0F); 3507 emit(0x2A); 3508 emit_sse_operand(dst, src); 3509 } 3510 3511 3512 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) { 3513 DCHECK(!IsEnabled(AVX)); 3514 EnsureSpace ensure_space(this); 3515 emit(0xF2); 3516 emit_optional_rex_32(dst, src); 3517 emit(0x0F); 3518 emit(0x2A); 3519 emit_sse_operand(dst, src); 3520 } 3521 3522 3523 void Assembler::cvtlsi2ss(XMMRegister dst, const Operand& src) { 3524 DCHECK(!IsEnabled(AVX)); 3525 EnsureSpace ensure_space(this); 3526 emit(0xF3); 3527 emit_optional_rex_32(dst, src); 3528 emit(0x0F); 3529 emit(0x2A); 3530 emit_sse_operand(dst, src); 3531 } 3532 3533 3534 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) { 3535 EnsureSpace ensure_space(this); 3536 emit(0xF3); 3537 emit_optional_rex_32(dst, src); 3538 emit(0x0F); 3539 emit(0x2A); 3540 emit_sse_operand(dst, src); 3541 } 3542 3543 3544 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) { 3545 DCHECK(!IsEnabled(AVX)); 3546 EnsureSpace ensure_space(this); 3547 emit(0xF3); 3548 emit_rex_64(dst, src); 3549 emit(0x0F); 3550 emit(0x2A); 3551 emit_sse_operand(dst, src); 3552 } 3553 3554 3555 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) { 3556 DCHECK(!IsEnabled(AVX)); 3557 EnsureSpace ensure_space(this); 3558 emit(0xF3); 3559 emit_rex_64(dst, src); 3560 emit(0x0F); 3561 emit(0x2A); 3562 emit_sse_operand(dst, src); 3563 } 3564 3565 3566 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) { 3567 DCHECK(!IsEnabled(AVX)); 3568 EnsureSpace ensure_space(this); 3569 emit(0xF2); 3570 emit_rex_64(dst, src); 3571 emit(0x0F); 3572 emit(0x2A); 3573 emit_sse_operand(dst, src); 3574 } 3575 3576 3577 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) { 3578 DCHECK(!IsEnabled(AVX)); 3579 EnsureSpace ensure_space(this); 3580 emit(0xF2); 3581 emit_rex_64(dst, src); 3582 emit(0x0F); 3583 emit(0x2A); 3584 emit_sse_operand(dst, src); 3585 } 3586 3587 3588 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 3589 DCHECK(!IsEnabled(AVX)); 3590 EnsureSpace ensure_space(this); 3591 emit(0xF3); 3592 emit_optional_rex_32(dst, src); 3593 emit(0x0F); 3594 emit(0x5A); 3595 emit_sse_operand(dst, src); 3596 } 3597 3598 3599 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) { 3600 DCHECK(!IsEnabled(AVX)); 3601 EnsureSpace ensure_space(this); 3602 emit(0xF3); 3603 emit_optional_rex_32(dst, src); 3604 emit(0x0F); 3605 emit(0x5A); 3606 emit_sse_operand(dst, src); 3607 } 3608 3609 3610 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 3611 DCHECK(!IsEnabled(AVX)); 3612 EnsureSpace ensure_space(this); 3613 emit(0xF2); 3614 emit_optional_rex_32(dst, src); 3615 emit(0x0F); 3616 emit(0x5A); 3617 emit_sse_operand(dst, src); 3618 } 3619 3620 3621 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) { 3622 DCHECK(!IsEnabled(AVX)); 3623 EnsureSpace ensure_space(this); 3624 emit(0xF2); 3625 emit_optional_rex_32(dst, src); 3626 emit(0x0F); 3627 emit(0x5A); 3628 emit_sse_operand(dst, src); 3629 } 3630 3631 3632 void Assembler::cvtsd2si(Register dst, XMMRegister src) { 3633 DCHECK(!IsEnabled(AVX)); 3634 EnsureSpace ensure_space(this); 3635 emit(0xF2); 3636 emit_optional_rex_32(dst, src); 3637 emit(0x0F); 3638 emit(0x2D); 3639 emit_sse_operand(dst, src); 3640 } 3641 3642 3643 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 3644 DCHECK(!IsEnabled(AVX)); 3645 EnsureSpace ensure_space(this); 3646 emit(0xF2); 3647 emit_rex_64(dst, src); 3648 emit(0x0F); 3649 emit(0x2D); 3650 emit_sse_operand(dst, src); 3651 } 3652 3653 3654 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 3655 EnsureSpace ensure_space(this); 3656 emit(0xF2); 3657 emit_optional_rex_32(dst, src); 3658 emit(0x0F); 3659 emit(0x58); 3660 emit_sse_operand(dst, src); 3661 } 3662 3663 3664 void Assembler::addsd(XMMRegister dst, const Operand& src) { 3665 EnsureSpace ensure_space(this); 3666 emit(0xF2); 3667 emit_optional_rex_32(dst, src); 3668 emit(0x0F); 3669 emit(0x58); 3670 emit_sse_operand(dst, src); 3671 } 3672 3673 3674 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 3675 EnsureSpace ensure_space(this); 3676 emit(0xF2); 3677 emit_optional_rex_32(dst, src); 3678 emit(0x0F); 3679 emit(0x59); 3680 emit_sse_operand(dst, src); 3681 } 3682 3683 3684 void Assembler::mulsd(XMMRegister dst, const Operand& src) { 3685 EnsureSpace ensure_space(this); 3686 emit(0xF2); 3687 emit_optional_rex_32(dst, src); 3688 emit(0x0F); 3689 emit(0x59); 3690 emit_sse_operand(dst, src); 3691 } 3692 3693 3694 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 3695 EnsureSpace ensure_space(this); 3696 emit(0xF2); 3697 emit_optional_rex_32(dst, src); 3698 emit(0x0F); 3699 emit(0x5C); 3700 emit_sse_operand(dst, src); 3701 } 3702 3703 3704 void Assembler::subsd(XMMRegister dst, const Operand& src) { 3705 EnsureSpace ensure_space(this); 3706 emit(0xF2); 3707 emit_optional_rex_32(dst, src); 3708 emit(0x0F); 3709 emit(0x5C); 3710 emit_sse_operand(dst, src); 3711 } 3712 3713 3714 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 3715 EnsureSpace ensure_space(this); 3716 emit(0xF2); 3717 emit_optional_rex_32(dst, src); 3718 emit(0x0F); 3719 emit(0x5E); 3720 emit_sse_operand(dst, src); 3721 } 3722 3723 3724 void Assembler::divsd(XMMRegister dst, const Operand& src) { 3725 EnsureSpace ensure_space(this); 3726 emit(0xF2); 3727 emit_optional_rex_32(dst, src); 3728 emit(0x0F); 3729 emit(0x5E); 3730 emit_sse_operand(dst, src); 3731 } 3732 3733 3734 void Assembler::maxsd(XMMRegister dst, XMMRegister src) { 3735 EnsureSpace ensure_space(this); 3736 emit(0xF2); 3737 emit_optional_rex_32(dst, src); 3738 emit(0x0F); 3739 emit(0x5F); 3740 emit_sse_operand(dst, src); 3741 } 3742 3743 3744 void Assembler::maxsd(XMMRegister dst, const Operand& src) { 3745 EnsureSpace ensure_space(this); 3746 emit(0xF2); 3747 emit_optional_rex_32(dst, src); 3748 emit(0x0F); 3749 emit(0x5F); 3750 emit_sse_operand(dst, src); 3751 } 3752 3753 3754 void Assembler::minsd(XMMRegister dst, XMMRegister src) { 3755 EnsureSpace ensure_space(this); 3756 emit(0xF2); 3757 emit_optional_rex_32(dst, src); 3758 emit(0x0F); 3759 emit(0x5D); 3760 emit_sse_operand(dst, src); 3761 } 3762 3763 3764 void Assembler::minsd(XMMRegister dst, const Operand& src) { 3765 EnsureSpace ensure_space(this); 3766 emit(0xF2); 3767 emit_optional_rex_32(dst, src); 3768 emit(0x0F); 3769 emit(0x5D); 3770 emit_sse_operand(dst, src); 3771 } 3772 3773 3774 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 3775 EnsureSpace ensure_space(this); 3776 emit(0x66); 3777 emit_optional_rex_32(dst, src); 3778 emit(0x0F); 3779 emit(0x54); 3780 emit_sse_operand(dst, src); 3781 } 3782 3783 3784 void Assembler::andpd(XMMRegister dst, const Operand& src) { 3785 EnsureSpace ensure_space(this); 3786 emit(0x66); 3787 emit_optional_rex_32(dst, src); 3788 emit(0x0F); 3789 emit(0x54); 3790 emit_sse_operand(dst, src); 3791 } 3792 3793 3794 void Assembler::orpd(XMMRegister dst, XMMRegister src) { 3795 EnsureSpace ensure_space(this); 3796 emit(0x66); 3797 emit_optional_rex_32(dst, src); 3798 emit(0x0F); 3799 emit(0x56); 3800 emit_sse_operand(dst, src); 3801 } 3802 3803 3804 void Assembler::orpd(XMMRegister dst, const Operand& src) { 3805 EnsureSpace ensure_space(this); 3806 emit(0x66); 3807 emit_optional_rex_32(dst, src); 3808 emit(0x0F); 3809 emit(0x56); 3810 emit_sse_operand(dst, src); 3811 } 3812 3813 3814 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 3815 DCHECK(!IsEnabled(AVX)); 3816 EnsureSpace ensure_space(this); 3817 emit(0x66); 3818 emit_optional_rex_32(dst, src); 3819 emit(0x0F); 3820 emit(0x57); 3821 emit_sse_operand(dst, src); 3822 } 3823 3824 3825 void Assembler::xorpd(XMMRegister dst, const Operand& src) { 3826 DCHECK(!IsEnabled(AVX)); 3827 EnsureSpace ensure_space(this); 3828 emit(0x66); 3829 emit_optional_rex_32(dst, src); 3830 emit(0x0F); 3831 emit(0x57); 3832 emit_sse_operand(dst, src); 3833 } 3834 3835 3836 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 3837 DCHECK(!IsEnabled(AVX)); 3838 EnsureSpace ensure_space(this); 3839 emit(0xF2); 3840 emit_optional_rex_32(dst, src); 3841 emit(0x0F); 3842 emit(0x51); 3843 emit_sse_operand(dst, src); 3844 } 3845 3846 3847 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) { 3848 DCHECK(!IsEnabled(AVX)); 3849 EnsureSpace ensure_space(this); 3850 emit(0xF2); 3851 emit_optional_rex_32(dst, src); 3852 emit(0x0F); 3853 emit(0x51); 3854 emit_sse_operand(dst, src); 3855 } 3856 3857 3858 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 3859 DCHECK(!IsEnabled(AVX)); 3860 EnsureSpace ensure_space(this); 3861 emit(0x66); 3862 emit_optional_rex_32(dst, src); 3863 emit(0x0f); 3864 emit(0x2e); 3865 emit_sse_operand(dst, src); 3866 } 3867 3868 3869 void Assembler::ucomisd(XMMRegister dst, const Operand& src) { 3870 DCHECK(!IsEnabled(AVX)); 3871 EnsureSpace ensure_space(this); 3872 emit(0x66); 3873 emit_optional_rex_32(dst, src); 3874 emit(0x0f); 3875 emit(0x2e); 3876 emit_sse_operand(dst, src); 3877 } 3878 3879 3880 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) { 3881 EnsureSpace ensure_space(this); 3882 emit(0xF2); 3883 emit_optional_rex_32(dst, src); 3884 emit(0x0F); 3885 emit(0xC2); 3886 emit_sse_operand(dst, src); 3887 emit(0x01); // LT == 1 3888 } 3889 3890 3891 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3892 DCHECK(!IsEnabled(AVX)); 3893 DCHECK(IsEnabled(SSE4_1)); 3894 EnsureSpace ensure_space(this); 3895 emit(0x66); 3896 emit_optional_rex_32(dst, src); 3897 emit(0x0f); 3898 emit(0x3a); 3899 emit(0x0a); 3900 emit_sse_operand(dst, src); 3901 // Mask precision exception. 3902 emit(static_cast<byte>(mode) | 0x8); 3903 } 3904 3905 3906 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3907 DCHECK(!IsEnabled(AVX)); 3908 DCHECK(IsEnabled(SSE4_1)); 3909 EnsureSpace ensure_space(this); 3910 emit(0x66); 3911 emit_optional_rex_32(dst, src); 3912 emit(0x0f); 3913 emit(0x3a); 3914 emit(0x0b); 3915 emit_sse_operand(dst, src); 3916 // Mask precision exception. 3917 emit(static_cast<byte>(mode) | 0x8); 3918 } 3919 3920 3921 void Assembler::movmskpd(Register dst, XMMRegister src) { 3922 EnsureSpace ensure_space(this); 3923 emit(0x66); 3924 emit_optional_rex_32(dst, src); 3925 emit(0x0f); 3926 emit(0x50); 3927 emit_sse_operand(dst, src); 3928 } 3929 3930 3931 void Assembler::movmskps(Register dst, XMMRegister src) { 3932 EnsureSpace ensure_space(this); 3933 emit_optional_rex_32(dst, src); 3934 emit(0x0f); 3935 emit(0x50); 3936 emit_sse_operand(dst, src); 3937 } 3938 3939 3940 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 3941 EnsureSpace ensure_space(this); 3942 emit(0x66); 3943 emit_optional_rex_32(dst, src); 3944 emit(0x0F); 3945 emit(0x62); 3946 emit_sse_operand(dst, src); 3947 } 3948 3949 void Assembler::punpckldq(XMMRegister dst, const Operand& src) { 3950 EnsureSpace ensure_space(this); 3951 emit(0x66); 3952 emit_optional_rex_32(dst, src); 3953 emit(0x0F); 3954 emit(0x62); 3955 emit_sse_operand(dst, src); 3956 } 3957 3958 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) { 3959 EnsureSpace ensure_space(this); 3960 emit(0x66); 3961 emit_optional_rex_32(dst, src); 3962 emit(0x0F); 3963 emit(0x6A); 3964 emit_sse_operand(dst, src); 3965 } 3966 3967 3968 // AVX instructions 3969 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3970 XMMRegister src2) { 3971 DCHECK(IsEnabled(FMA3)); 3972 EnsureSpace ensure_space(this); 3973 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3974 emit(op); 3975 emit_sse_operand(dst, src2); 3976 } 3977 3978 3979 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3980 const Operand& src2) { 3981 DCHECK(IsEnabled(FMA3)); 3982 EnsureSpace ensure_space(this); 3983 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3984 emit(op); 3985 emit_sse_operand(dst, src2); 3986 } 3987 3988 3989 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3990 XMMRegister src2) { 3991 DCHECK(IsEnabled(FMA3)); 3992 EnsureSpace ensure_space(this); 3993 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3994 emit(op); 3995 emit_sse_operand(dst, src2); 3996 } 3997 3998 3999 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 4000 const Operand& src2) { 4001 DCHECK(IsEnabled(FMA3)); 4002 EnsureSpace ensure_space(this); 4003 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 4004 emit(op); 4005 emit_sse_operand(dst, src2); 4006 } 4007 4008 4009 void Assembler::vmovd(XMMRegister dst, Register src) { 4010 DCHECK(IsEnabled(AVX)); 4011 EnsureSpace ensure_space(this); 4012 XMMRegister isrc = {src.code()}; 4013 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0); 4014 emit(0x6e); 4015 emit_sse_operand(dst, src); 4016 } 4017 4018 4019 void Assembler::vmovd(XMMRegister dst, const Operand& src) { 4020 DCHECK(IsEnabled(AVX)); 4021 EnsureSpace ensure_space(this); 4022 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0); 4023 emit(0x6e); 4024 emit_sse_operand(dst, src); 4025 } 4026 4027 4028 void Assembler::vmovd(Register dst, XMMRegister src) { 4029 DCHECK(IsEnabled(AVX)); 4030 EnsureSpace ensure_space(this); 4031 XMMRegister idst = {dst.code()}; 4032 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0); 4033 emit(0x7e); 4034 emit_sse_operand(src, dst); 4035 } 4036 4037 4038 void Assembler::vmovq(XMMRegister dst, Register src) { 4039 DCHECK(IsEnabled(AVX)); 4040 EnsureSpace ensure_space(this); 4041 XMMRegister isrc = {src.code()}; 4042 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1); 4043 emit(0x6e); 4044 emit_sse_operand(dst, src); 4045 } 4046 4047 4048 void Assembler::vmovq(XMMRegister dst, const Operand& src) { 4049 DCHECK(IsEnabled(AVX)); 4050 EnsureSpace ensure_space(this); 4051 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1); 4052 emit(0x6e); 4053 emit_sse_operand(dst, src); 4054 } 4055 4056 4057 void Assembler::vmovq(Register dst, XMMRegister src) { 4058 DCHECK(IsEnabled(AVX)); 4059 EnsureSpace ensure_space(this); 4060 XMMRegister idst = {dst.code()}; 4061 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1); 4062 emit(0x7e); 4063 emit_sse_operand(src, dst); 4064 } 4065 4066 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, 4067 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, 4068 VexW w) { 4069 DCHECK(IsEnabled(AVX)); 4070 EnsureSpace ensure_space(this); 4071 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 4072 emit(op); 4073 emit_sse_operand(dst, src2); 4074 } 4075 4076 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, 4077 const Operand& src2, SIMDPrefix pp, LeadingOpcode m, 4078 VexW w) { 4079 DCHECK(IsEnabled(AVX)); 4080 EnsureSpace ensure_space(this); 4081 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 4082 emit(op); 4083 emit_sse_operand(dst, src2); 4084 } 4085 4086 4087 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 4088 XMMRegister src2) { 4089 DCHECK(IsEnabled(AVX)); 4090 EnsureSpace ensure_space(this); 4091 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 4092 emit(op); 4093 emit_sse_operand(dst, src2); 4094 } 4095 4096 4097 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 4098 const Operand& src2) { 4099 DCHECK(IsEnabled(AVX)); 4100 EnsureSpace ensure_space(this); 4101 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 4102 emit(op); 4103 emit_sse_operand(dst, src2); 4104 } 4105 4106 4107 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 4108 XMMRegister src2) { 4109 DCHECK(IsEnabled(AVX)); 4110 EnsureSpace ensure_space(this); 4111 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 4112 emit(op); 4113 emit_sse_operand(dst, src2); 4114 } 4115 4116 4117 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 4118 const Operand& src2) { 4119 DCHECK(IsEnabled(AVX)); 4120 EnsureSpace ensure_space(this); 4121 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 4122 emit(op); 4123 emit_sse_operand(dst, src2); 4124 } 4125 4126 4127 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) { 4128 DCHECK(IsEnabled(AVX)); 4129 EnsureSpace ensure_space(this); 4130 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 4131 emit(0x2e); 4132 emit_sse_operand(dst, src); 4133 } 4134 4135 4136 void Assembler::vucomiss(XMMRegister dst, const Operand& src) { 4137 DCHECK(IsEnabled(AVX)); 4138 EnsureSpace ensure_space(this); 4139 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 4140 emit(0x2e); 4141 emit_sse_operand(dst, src); 4142 } 4143 4144 4145 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 4146 XMMRegister src2) { 4147 DCHECK(IsEnabled(AVX)); 4148 EnsureSpace ensure_space(this); 4149 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 4150 emit(op); 4151 emit_sse_operand(dst, src2); 4152 } 4153 4154 4155 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 4156 const Operand& src2) { 4157 DCHECK(IsEnabled(AVX)); 4158 EnsureSpace ensure_space(this); 4159 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 4160 emit(op); 4161 emit_sse_operand(dst, src2); 4162 } 4163 4164 4165 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) { 4166 DCHECK(IsEnabled(BMI1)); 4167 EnsureSpace ensure_space(this); 4168 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 4169 emit(op); 4170 emit_modrm(reg, rm); 4171 } 4172 4173 4174 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) { 4175 DCHECK(IsEnabled(BMI1)); 4176 EnsureSpace ensure_space(this); 4177 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 4178 emit(op); 4179 emit_operand(reg, rm); 4180 } 4181 4182 4183 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) { 4184 DCHECK(IsEnabled(BMI1)); 4185 EnsureSpace ensure_space(this); 4186 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 4187 emit(op); 4188 emit_modrm(reg, rm); 4189 } 4190 4191 4192 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) { 4193 DCHECK(IsEnabled(BMI1)); 4194 EnsureSpace ensure_space(this); 4195 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 4196 emit(op); 4197 emit_operand(reg, rm); 4198 } 4199 4200 4201 void Assembler::tzcntq(Register dst, Register src) { 4202 DCHECK(IsEnabled(BMI1)); 4203 EnsureSpace ensure_space(this); 4204 emit(0xF3); 4205 emit_rex_64(dst, src); 4206 emit(0x0F); 4207 emit(0xBC); 4208 emit_modrm(dst, src); 4209 } 4210 4211 4212 void Assembler::tzcntq(Register dst, const Operand& src) { 4213 DCHECK(IsEnabled(BMI1)); 4214 EnsureSpace ensure_space(this); 4215 emit(0xF3); 4216 emit_rex_64(dst, src); 4217 emit(0x0F); 4218 emit(0xBC); 4219 emit_operand(dst, src); 4220 } 4221 4222 4223 void Assembler::tzcntl(Register dst, Register src) { 4224 DCHECK(IsEnabled(BMI1)); 4225 EnsureSpace ensure_space(this); 4226 emit(0xF3); 4227 emit_optional_rex_32(dst, src); 4228 emit(0x0F); 4229 emit(0xBC); 4230 emit_modrm(dst, src); 4231 } 4232 4233 4234 void Assembler::tzcntl(Register dst, const Operand& src) { 4235 DCHECK(IsEnabled(BMI1)); 4236 EnsureSpace ensure_space(this); 4237 emit(0xF3); 4238 emit_optional_rex_32(dst, src); 4239 emit(0x0F); 4240 emit(0xBC); 4241 emit_operand(dst, src); 4242 } 4243 4244 4245 void Assembler::lzcntq(Register dst, Register src) { 4246 DCHECK(IsEnabled(LZCNT)); 4247 EnsureSpace ensure_space(this); 4248 emit(0xF3); 4249 emit_rex_64(dst, src); 4250 emit(0x0F); 4251 emit(0xBD); 4252 emit_modrm(dst, src); 4253 } 4254 4255 4256 void Assembler::lzcntq(Register dst, const Operand& src) { 4257 DCHECK(IsEnabled(LZCNT)); 4258 EnsureSpace ensure_space(this); 4259 emit(0xF3); 4260 emit_rex_64(dst, src); 4261 emit(0x0F); 4262 emit(0xBD); 4263 emit_operand(dst, src); 4264 } 4265 4266 4267 void Assembler::lzcntl(Register dst, Register src) { 4268 DCHECK(IsEnabled(LZCNT)); 4269 EnsureSpace ensure_space(this); 4270 emit(0xF3); 4271 emit_optional_rex_32(dst, src); 4272 emit(0x0F); 4273 emit(0xBD); 4274 emit_modrm(dst, src); 4275 } 4276 4277 4278 void Assembler::lzcntl(Register dst, const Operand& src) { 4279 DCHECK(IsEnabled(LZCNT)); 4280 EnsureSpace ensure_space(this); 4281 emit(0xF3); 4282 emit_optional_rex_32(dst, src); 4283 emit(0x0F); 4284 emit(0xBD); 4285 emit_operand(dst, src); 4286 } 4287 4288 4289 void Assembler::popcntq(Register dst, Register src) { 4290 DCHECK(IsEnabled(POPCNT)); 4291 EnsureSpace ensure_space(this); 4292 emit(0xF3); 4293 emit_rex_64(dst, src); 4294 emit(0x0F); 4295 emit(0xB8); 4296 emit_modrm(dst, src); 4297 } 4298 4299 4300 void Assembler::popcntq(Register dst, const Operand& src) { 4301 DCHECK(IsEnabled(POPCNT)); 4302 EnsureSpace ensure_space(this); 4303 emit(0xF3); 4304 emit_rex_64(dst, src); 4305 emit(0x0F); 4306 emit(0xB8); 4307 emit_operand(dst, src); 4308 } 4309 4310 4311 void Assembler::popcntl(Register dst, Register src) { 4312 DCHECK(IsEnabled(POPCNT)); 4313 EnsureSpace ensure_space(this); 4314 emit(0xF3); 4315 emit_optional_rex_32(dst, src); 4316 emit(0x0F); 4317 emit(0xB8); 4318 emit_modrm(dst, src); 4319 } 4320 4321 4322 void Assembler::popcntl(Register dst, const Operand& src) { 4323 DCHECK(IsEnabled(POPCNT)); 4324 EnsureSpace ensure_space(this); 4325 emit(0xF3); 4326 emit_optional_rex_32(dst, src); 4327 emit(0x0F); 4328 emit(0xB8); 4329 emit_operand(dst, src); 4330 } 4331 4332 4333 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4334 Register rm) { 4335 DCHECK(IsEnabled(BMI2)); 4336 EnsureSpace ensure_space(this); 4337 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4338 emit(op); 4339 emit_modrm(reg, rm); 4340 } 4341 4342 4343 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4344 const Operand& rm) { 4345 DCHECK(IsEnabled(BMI2)); 4346 EnsureSpace ensure_space(this); 4347 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4348 emit(op); 4349 emit_operand(reg, rm); 4350 } 4351 4352 4353 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4354 Register rm) { 4355 DCHECK(IsEnabled(BMI2)); 4356 EnsureSpace ensure_space(this); 4357 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4358 emit(op); 4359 emit_modrm(reg, rm); 4360 } 4361 4362 4363 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4364 const Operand& rm) { 4365 DCHECK(IsEnabled(BMI2)); 4366 EnsureSpace ensure_space(this); 4367 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4368 emit(op); 4369 emit_operand(reg, rm); 4370 } 4371 4372 4373 void Assembler::rorxq(Register dst, Register src, byte imm8) { 4374 DCHECK(IsEnabled(BMI2)); 4375 DCHECK(is_uint8(imm8)); 4376 Register vreg = {0}; // VEX.vvvv unused 4377 EnsureSpace ensure_space(this); 4378 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4379 emit(0xF0); 4380 emit_modrm(dst, src); 4381 emit(imm8); 4382 } 4383 4384 4385 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) { 4386 DCHECK(IsEnabled(BMI2)); 4387 DCHECK(is_uint8(imm8)); 4388 Register vreg = {0}; // VEX.vvvv unused 4389 EnsureSpace ensure_space(this); 4390 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4391 emit(0xF0); 4392 emit_operand(dst, src); 4393 emit(imm8); 4394 } 4395 4396 4397 void Assembler::rorxl(Register dst, Register src, byte imm8) { 4398 DCHECK(IsEnabled(BMI2)); 4399 DCHECK(is_uint8(imm8)); 4400 Register vreg = {0}; // VEX.vvvv unused 4401 EnsureSpace ensure_space(this); 4402 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4403 emit(0xF0); 4404 emit_modrm(dst, src); 4405 emit(imm8); 4406 } 4407 4408 4409 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) { 4410 DCHECK(IsEnabled(BMI2)); 4411 DCHECK(is_uint8(imm8)); 4412 Register vreg = {0}; // VEX.vvvv unused 4413 EnsureSpace ensure_space(this); 4414 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4415 emit(0xF0); 4416 emit_operand(dst, src); 4417 emit(imm8); 4418 } 4419 4420 void Assembler::minps(XMMRegister dst, XMMRegister src) { 4421 EnsureSpace ensure_space(this); 4422 emit_optional_rex_32(dst, src); 4423 emit(0x0F); 4424 emit(0x5D); 4425 emit_sse_operand(dst, src); 4426 } 4427 4428 void Assembler::minps(XMMRegister dst, const Operand& src) { 4429 EnsureSpace ensure_space(this); 4430 emit_optional_rex_32(dst, src); 4431 emit(0x0F); 4432 emit(0x5D); 4433 emit_sse_operand(dst, src); 4434 } 4435 4436 void Assembler::maxps(XMMRegister dst, XMMRegister src) { 4437 EnsureSpace ensure_space(this); 4438 emit_optional_rex_32(dst, src); 4439 emit(0x0F); 4440 emit(0x5F); 4441 emit_sse_operand(dst, src); 4442 } 4443 4444 void Assembler::maxps(XMMRegister dst, const Operand& src) { 4445 EnsureSpace ensure_space(this); 4446 emit_optional_rex_32(dst, src); 4447 emit(0x0F); 4448 emit(0x5F); 4449 emit_sse_operand(dst, src); 4450 } 4451 4452 void Assembler::rcpps(XMMRegister dst, XMMRegister src) { 4453 EnsureSpace ensure_space(this); 4454 emit_optional_rex_32(dst, src); 4455 emit(0x0F); 4456 emit(0x53); 4457 emit_sse_operand(dst, src); 4458 } 4459 4460 void Assembler::rcpps(XMMRegister dst, const Operand& src) { 4461 EnsureSpace ensure_space(this); 4462 emit_optional_rex_32(dst, src); 4463 emit(0x0F); 4464 emit(0x53); 4465 emit_sse_operand(dst, src); 4466 } 4467 4468 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) { 4469 EnsureSpace ensure_space(this); 4470 emit_optional_rex_32(dst, src); 4471 emit(0x0F); 4472 emit(0x52); 4473 emit_sse_operand(dst, src); 4474 } 4475 4476 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) { 4477 EnsureSpace ensure_space(this); 4478 emit_optional_rex_32(dst, src); 4479 emit(0x0F); 4480 emit(0x52); 4481 emit_sse_operand(dst, src); 4482 } 4483 4484 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) { 4485 EnsureSpace ensure_space(this); 4486 emit_optional_rex_32(dst, src); 4487 emit(0x0F); 4488 emit(0x51); 4489 emit_sse_operand(dst, src); 4490 } 4491 4492 void Assembler::sqrtps(XMMRegister dst, const Operand& src) { 4493 EnsureSpace ensure_space(this); 4494 emit_optional_rex_32(dst, src); 4495 emit(0x0F); 4496 emit(0x51); 4497 emit_sse_operand(dst, src); 4498 } 4499 4500 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) { 4501 EnsureSpace ensure_space(this); 4502 emit_optional_rex_32(dst, src); 4503 emit(0x0F); 4504 emit(0x5B); 4505 emit_sse_operand(dst, src); 4506 } 4507 4508 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) { 4509 EnsureSpace ensure_space(this); 4510 emit_optional_rex_32(dst, src); 4511 emit(0x0F); 4512 emit(0x5B); 4513 emit_sse_operand(dst, src); 4514 } 4515 4516 void Assembler::movups(XMMRegister dst, XMMRegister src) { 4517 EnsureSpace ensure_space(this); 4518 if (src.low_bits() == 4) { 4519 // Try to avoid an unnecessary SIB byte. 4520 emit_optional_rex_32(src, dst); 4521 emit(0x0F); 4522 emit(0x11); 4523 emit_sse_operand(src, dst); 4524 } else { 4525 emit_optional_rex_32(dst, src); 4526 emit(0x0F); 4527 emit(0x10); 4528 emit_sse_operand(dst, src); 4529 } 4530 } 4531 4532 void Assembler::movups(XMMRegister dst, const Operand& src) { 4533 EnsureSpace ensure_space(this); 4534 emit_optional_rex_32(dst, src); 4535 emit(0x0F); 4536 emit(0x10); 4537 emit_sse_operand(dst, src); 4538 } 4539 4540 void Assembler::movups(const Operand& dst, XMMRegister src) { 4541 EnsureSpace ensure_space(this); 4542 emit_optional_rex_32(src, dst); 4543 emit(0x0F); 4544 emit(0x11); 4545 emit_sse_operand(src, dst); 4546 } 4547 4548 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix, 4549 byte escape, byte opcode) { 4550 EnsureSpace ensure_space(this); 4551 emit(prefix); 4552 emit_optional_rex_32(dst, src); 4553 emit(escape); 4554 emit(opcode); 4555 emit_sse_operand(dst, src); 4556 } 4557 4558 void Assembler::sse2_instr(XMMRegister dst, const Operand& src, byte prefix, 4559 byte escape, byte opcode) { 4560 EnsureSpace ensure_space(this); 4561 emit(prefix); 4562 emit_optional_rex_32(dst, src); 4563 emit(escape); 4564 emit(opcode); 4565 emit_sse_operand(dst, src); 4566 } 4567 4568 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix, 4569 byte escape1, byte escape2, byte opcode) { 4570 DCHECK(IsEnabled(SSSE3)); 4571 EnsureSpace ensure_space(this); 4572 emit(prefix); 4573 emit_optional_rex_32(dst, src); 4574 emit(escape1); 4575 emit(escape2); 4576 emit(opcode); 4577 emit_sse_operand(dst, src); 4578 } 4579 4580 void Assembler::ssse3_instr(XMMRegister dst, const Operand& src, byte prefix, 4581 byte escape1, byte escape2, byte opcode) { 4582 DCHECK(IsEnabled(SSSE3)); 4583 EnsureSpace ensure_space(this); 4584 emit(prefix); 4585 emit_optional_rex_32(dst, src); 4586 emit(escape1); 4587 emit(escape2); 4588 emit(opcode); 4589 emit_sse_operand(dst, src); 4590 } 4591 4592 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix, 4593 byte escape1, byte escape2, byte opcode) { 4594 DCHECK(IsEnabled(SSE4_1)); 4595 EnsureSpace ensure_space(this); 4596 emit(prefix); 4597 emit_optional_rex_32(dst, src); 4598 emit(escape1); 4599 emit(escape2); 4600 emit(opcode); 4601 emit_sse_operand(dst, src); 4602 } 4603 4604 void Assembler::sse4_instr(XMMRegister dst, const Operand& src, byte prefix, 4605 byte escape1, byte escape2, byte opcode) { 4606 DCHECK(IsEnabled(SSE4_1)); 4607 EnsureSpace ensure_space(this); 4608 emit(prefix); 4609 emit_optional_rex_32(dst, src); 4610 emit(escape1); 4611 emit(escape2); 4612 emit(opcode); 4613 emit_sse_operand(dst, src); 4614 } 4615 4616 void Assembler::lddqu(XMMRegister dst, const Operand& src) { 4617 DCHECK(IsEnabled(SSE3)); 4618 EnsureSpace ensure_space(this); 4619 emit(0xF2); 4620 emit_optional_rex_32(dst, src); 4621 emit(0x0F); 4622 emit(0xF0); 4623 emit_sse_operand(dst, src); 4624 } 4625 4626 void Assembler::psrldq(XMMRegister dst, uint8_t shift) { 4627 EnsureSpace ensure_space(this); 4628 emit(0x66); 4629 emit_optional_rex_32(dst); 4630 emit(0x0F); 4631 emit(0x73); 4632 emit_sse_operand(dst); 4633 emit(shift); 4634 } 4635 4636 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) { 4637 EnsureSpace ensure_space(this); 4638 emit(0x66); 4639 emit_optional_rex_32(dst, src); 4640 emit(0x0F); 4641 emit(0x70); 4642 emit_sse_operand(dst, src); 4643 emit(shuffle); 4644 } 4645 4646 void Assembler::pshufd(XMMRegister dst, const Operand& src, uint8_t shuffle) { 4647 EnsureSpace ensure_space(this); 4648 emit(0x66); 4649 emit_optional_rex_32(dst, src); 4650 emit(0x0F); 4651 emit(0x70); 4652 emit_sse_operand(dst, src); 4653 emit(shuffle); 4654 } 4655 4656 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) { 4657 Register ireg = { reg.code() }; 4658 emit_operand(ireg, adr); 4659 } 4660 4661 4662 void Assembler::emit_sse_operand(Register reg, const Operand& adr) { 4663 Register ireg = {reg.code()}; 4664 emit_operand(ireg, adr); 4665 } 4666 4667 4668 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) { 4669 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4670 } 4671 4672 4673 void Assembler::emit_sse_operand(XMMRegister dst, Register src) { 4674 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4675 } 4676 4677 4678 void Assembler::emit_sse_operand(Register dst, XMMRegister src) { 4679 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4680 } 4681 4682 void Assembler::emit_sse_operand(XMMRegister dst) { 4683 emit(0xD8 | dst.low_bits()); 4684 } 4685 4686 4687 void Assembler::db(uint8_t data) { 4688 EnsureSpace ensure_space(this); 4689 emit(data); 4690 } 4691 4692 4693 void Assembler::dd(uint32_t data) { 4694 EnsureSpace ensure_space(this); 4695 emitl(data); 4696 } 4697 4698 4699 void Assembler::dq(uint64_t data) { 4700 EnsureSpace ensure_space(this); 4701 emitq(data); 4702 } 4703 4704 4705 void Assembler::dq(Label* label) { 4706 EnsureSpace ensure_space(this); 4707 if (label->is_bound()) { 4708 internal_reference_positions_.push_back(pc_offset()); 4709 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE); 4710 } else { 4711 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 4712 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address. 4713 if (label->is_linked()) { 4714 emitl(label->pos()); 4715 label->link_to(pc_offset() - sizeof(int32_t)); 4716 } else { 4717 DCHECK(label->is_unused()); 4718 int32_t current = pc_offset(); 4719 emitl(current); 4720 label->link_to(current); 4721 } 4722 } 4723 } 4724 4725 4726 // Relocation information implementations. 4727 4728 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 4729 DCHECK(!RelocInfo::IsNone(rmode)); 4730 // Don't record external references unless the heap will be serialized. 4731 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 4732 !serializer_enabled() && !emit_debug_code()) { 4733 return; 4734 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) { 4735 // Don't record psuedo relocation info for code age sequence mode. 4736 return; 4737 } 4738 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL); 4739 reloc_info_writer.Write(&rinfo); 4740 } 4741 4742 4743 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask | 4744 1 << RelocInfo::RUNTIME_ENTRY | 4745 1 << RelocInfo::INTERNAL_REFERENCE | 4746 1 << RelocInfo::CODE_AGE_SEQUENCE; 4747 4748 4749 bool RelocInfo::IsCodedSpecially() { 4750 // The deserializer needs to know whether a pointer is specially coded. Being 4751 // specially coded on x64 means that it is a relative 32 bit address, as used 4752 // by branch instructions. 4753 return (1 << rmode_) & kApplyMask; 4754 } 4755 4756 4757 bool RelocInfo::IsInConstantPool() { 4758 return false; 4759 } 4760 4761 4762 } // namespace internal 4763 } // namespace v8 4764 4765 #endif // V8_TARGET_ARCH_X64 4766
