1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/x64/assembler-x64.h" 6 7 #include <cstring> 8 9 #if V8_TARGET_ARCH_X64 10 11 #if V8_LIBC_MSVCRT 12 #include <intrin.h> // _xgetbv() 13 #endif 14 #if V8_OS_MACOSX 15 #include <sys/sysctl.h> 16 #endif 17 18 #include "src/assembler-inl.h" 19 #include "src/base/bits.h" 20 #include "src/base/cpu.h" 21 #include "src/macro-assembler.h" 22 #include "src/v8.h" 23 24 namespace v8 { 25 namespace internal { 26 27 // ----------------------------------------------------------------------------- 28 // Implementation of CpuFeatures 29 30 namespace { 31 32 #if !V8_LIBC_MSVCRT 33 34 V8_INLINE uint64_t _xgetbv(unsigned int xcr) { 35 unsigned eax, edx; 36 // Check xgetbv; this uses a .byte sequence instead of the instruction 37 // directly because older assemblers do not include support for xgetbv and 38 // there is no easy way to conditionally compile based on the assembler 39 // used. 40 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 41 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32); 42 } 43 44 #define _XCR_XFEATURE_ENABLED_MASK 0 45 46 #endif // !V8_LIBC_MSVCRT 47 48 49 bool OSHasAVXSupport() { 50 #if V8_OS_MACOSX 51 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being 52 // caused by ISRs, so we detect that here and disable AVX in that case. 53 char buffer[128]; 54 size_t buffer_size = arraysize(buffer); 55 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE}; 56 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) { 57 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version"); 58 } 59 // The buffer now contains a string of the form XX.YY.ZZ, where 60 // XX is the major kernel version component. 61 char* period_pos = strchr(buffer, '.'); 62 DCHECK_NOT_NULL(period_pos); 63 *period_pos = '\0'; 64 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT 65 if (kernel_version_major <= 13) return false; 66 #endif // V8_OS_MACOSX 67 // Check whether OS claims to support AVX. 68 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); 69 return (feature_mask & 0x6) == 0x6; 70 } 71 72 } // namespace 73 74 75 void CpuFeatures::ProbeImpl(bool cross_compile) { 76 base::CPU cpu; 77 CHECK(cpu.has_sse2()); // SSE2 support is mandatory. 78 CHECK(cpu.has_cmov()); // CMOV support is mandatory. 79 80 // Only use statically determined features for cross compile (snapshot). 81 if (cross_compile) return; 82 83 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1; 84 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3; 85 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; 86 // SAHF is not generally available in long mode. 87 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; 88 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && 89 OSHasAVXSupport()) { 90 supported_ |= 1u << AVX; 91 } 92 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() && 93 OSHasAVXSupport()) { 94 supported_ |= 1u << FMA3; 95 } 96 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; 97 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; 98 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; 99 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT; 100 if (strcmp(FLAG_mcpu, "auto") == 0) { 101 if (cpu.is_atom()) supported_ |= 1u << ATOM; 102 } else if (strcmp(FLAG_mcpu, "atom") == 0) { 103 supported_ |= 1u << ATOM; 104 } 105 } 106 107 108 void CpuFeatures::PrintTarget() { } 109 void CpuFeatures::PrintFeatures() { 110 printf( 111 "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d " 112 "LZCNT=%d " 113 "POPCNT=%d ATOM=%d\n", 114 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3), 115 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF), 116 CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3), 117 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2), 118 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT), 119 CpuFeatures::IsSupported(ATOM)); 120 } 121 122 // ----------------------------------------------------------------------------- 123 // Implementation of RelocInfo 124 125 Address RelocInfo::wasm_memory_reference() { 126 DCHECK(IsWasmMemoryReference(rmode_)); 127 return Memory::Address_at(pc_); 128 } 129 130 Address RelocInfo::wasm_global_reference() { 131 DCHECK(IsWasmGlobalReference(rmode_)); 132 return Memory::Address_at(pc_); 133 } 134 135 uint32_t RelocInfo::wasm_memory_size_reference() { 136 DCHECK(IsWasmMemorySizeReference(rmode_)); 137 return Memory::uint32_at(pc_); 138 } 139 140 uint32_t RelocInfo::wasm_function_table_size_reference() { 141 DCHECK(IsWasmFunctionTableSizeReference(rmode_)); 142 return Memory::uint32_at(pc_); 143 } 144 145 void RelocInfo::unchecked_update_wasm_memory_reference( 146 Address address, ICacheFlushMode flush_mode) { 147 Memory::Address_at(pc_) = address; 148 } 149 150 void RelocInfo::unchecked_update_wasm_size(uint32_t size, 151 ICacheFlushMode flush_mode) { 152 Memory::uint32_at(pc_) = size; 153 } 154 155 // ----------------------------------------------------------------------------- 156 // Implementation of Operand 157 158 Operand::Operand(Register base, int32_t disp) : rex_(0) { 159 len_ = 1; 160 if (base.is(rsp) || base.is(r12)) { 161 // SIB byte is needed to encode (rsp + offset) or (r12 + offset). 162 set_sib(times_1, rsp, base); 163 } 164 165 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 166 set_modrm(0, base); 167 } else if (is_int8(disp)) { 168 set_modrm(1, base); 169 set_disp8(disp); 170 } else { 171 set_modrm(2, base); 172 set_disp32(disp); 173 } 174 } 175 176 177 Operand::Operand(Register base, 178 Register index, 179 ScaleFactor scale, 180 int32_t disp) : rex_(0) { 181 DCHECK(!index.is(rsp)); 182 len_ = 1; 183 set_sib(scale, index, base); 184 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 185 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits 186 // possibly set by set_sib. 187 set_modrm(0, rsp); 188 } else if (is_int8(disp)) { 189 set_modrm(1, rsp); 190 set_disp8(disp); 191 } else { 192 set_modrm(2, rsp); 193 set_disp32(disp); 194 } 195 } 196 197 198 Operand::Operand(Register index, 199 ScaleFactor scale, 200 int32_t disp) : rex_(0) { 201 DCHECK(!index.is(rsp)); 202 len_ = 1; 203 set_modrm(0, rsp); 204 set_sib(scale, index, rbp); 205 set_disp32(disp); 206 } 207 208 209 Operand::Operand(Label* label) : rex_(0), len_(1) { 210 DCHECK_NOT_NULL(label); 211 set_modrm(0, rbp); 212 set_disp64(reinterpret_cast<intptr_t>(label)); 213 } 214 215 216 Operand::Operand(const Operand& operand, int32_t offset) { 217 DCHECK(operand.len_ >= 1); 218 // Operand encodes REX ModR/M [SIB] [Disp]. 219 byte modrm = operand.buf_[0]; 220 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target). 221 bool has_sib = ((modrm & 0x07) == 0x04); 222 byte mode = modrm & 0xC0; 223 int disp_offset = has_sib ? 2 : 1; 224 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07; 225 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit 226 // displacement. 227 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base. 228 int32_t disp_value = 0; 229 if (mode == 0x80 || is_baseless) { 230 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement. 231 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]); 232 } else if (mode == 0x40) { 233 // Mode 1: Byte displacement. 234 disp_value = static_cast<signed char>(operand.buf_[disp_offset]); 235 } 236 237 // Write new operand with same registers, but with modified displacement. 238 DCHECK(offset >= 0 ? disp_value + offset > disp_value 239 : disp_value + offset < disp_value); // No overflow. 240 disp_value += offset; 241 rex_ = operand.rex_; 242 if (!is_int8(disp_value) || is_baseless) { 243 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13. 244 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80); 245 len_ = disp_offset + 4; 246 Memory::int32_at(&buf_[disp_offset]) = disp_value; 247 } else if (disp_value != 0 || (base_reg == 0x05)) { 248 // Need 8 bits of displacement. 249 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1. 250 len_ = disp_offset + 1; 251 buf_[disp_offset] = static_cast<byte>(disp_value); 252 } else { 253 // Need no displacement. 254 buf_[0] = (modrm & 0x3f); // Mode 0. 255 len_ = disp_offset; 256 } 257 if (has_sib) { 258 buf_[1] = operand.buf_[1]; 259 } 260 } 261 262 263 bool Operand::AddressUsesRegister(Register reg) const { 264 int code = reg.code(); 265 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand. 266 // Start with only low three bits of base register. Initial decoding doesn't 267 // distinguish on the REX.B bit. 268 int base_code = buf_[0] & 0x07; 269 if (base_code == rsp.code()) { 270 // SIB byte present in buf_[1]. 271 // Check the index register from the SIB byte + REX.X prefix. 272 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2); 273 // Index code (including REX.X) of 0x04 (rsp) means no index register. 274 if (index_code != rsp.code() && index_code == code) return true; 275 // Add REX.B to get the full base register code. 276 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3); 277 // A base register of 0x05 (rbp) with mod = 0 means no base register. 278 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 279 return code == base_code; 280 } else { 281 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means 282 // no base register. 283 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 284 base_code |= ((rex_ & 0x01) << 3); 285 return code == base_code; 286 } 287 } 288 289 290 // ----------------------------------------------------------------------------- 291 // Implementation of Assembler. 292 293 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 294 : AssemblerBase(isolate, buffer, buffer_size), code_targets_(100) { 295 // Clear the buffer in debug mode unless it was provided by the 296 // caller in which case we can't be sure it's okay to overwrite 297 // existing code in it. 298 #ifdef DEBUG 299 if (own_buffer_) { 300 memset(buffer_, 0xCC, buffer_size_); // int3 301 } 302 #endif 303 304 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_); 305 } 306 307 308 void Assembler::GetCode(CodeDesc* desc) { 309 // Finalize code (at this point overflow() may be true, but the gap ensures 310 // that we are still not overlapping instructions and relocation info). 311 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap. 312 // Set up code descriptor. 313 desc->buffer = buffer_; 314 desc->buffer_size = buffer_size_; 315 desc->instr_size = pc_offset(); 316 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system. 317 desc->reloc_size = 318 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos()); 319 desc->origin = this; 320 desc->constant_pool_size = 0; 321 desc->unwinding_info_size = 0; 322 desc->unwinding_info = nullptr; 323 } 324 325 326 void Assembler::Align(int m) { 327 DCHECK(base::bits::IsPowerOfTwo32(m)); 328 int delta = (m - (pc_offset() & (m - 1))) & (m - 1); 329 Nop(delta); 330 } 331 332 333 void Assembler::CodeTargetAlign() { 334 Align(16); // Preferred alignment of jump targets on x64. 335 } 336 337 338 bool Assembler::IsNop(Address addr) { 339 Address a = addr; 340 while (*a == 0x66) a++; 341 if (*a == 0x90) return true; 342 if (a[0] == 0xf && a[1] == 0x1f) return true; 343 return false; 344 } 345 346 347 void Assembler::bind_to(Label* L, int pos) { 348 DCHECK(!L->is_bound()); // Label may only be bound once. 349 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid. 350 if (L->is_linked()) { 351 int current = L->pos(); 352 int next = long_at(current); 353 while (next != current) { 354 if (current >= 4 && long_at(current - 4) == 0) { 355 // Absolute address. 356 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 357 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 358 internal_reference_positions_.push_back(current - 4); 359 } else { 360 // Relative address, relative to point after address. 361 int imm32 = pos - (current + sizeof(int32_t)); 362 long_at_put(current, imm32); 363 } 364 current = next; 365 next = long_at(next); 366 } 367 // Fix up last fixup on linked list. 368 if (current >= 4 && long_at(current - 4) == 0) { 369 // Absolute address. 370 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 371 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 372 internal_reference_positions_.push_back(current - 4); 373 } else { 374 // Relative address, relative to point after address. 375 int imm32 = pos - (current + sizeof(int32_t)); 376 long_at_put(current, imm32); 377 } 378 } 379 while (L->is_near_linked()) { 380 int fixup_pos = L->near_link_pos(); 381 int offset_to_next = 382 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos))); 383 DCHECK(offset_to_next <= 0); 384 int disp = pos - (fixup_pos + sizeof(int8_t)); 385 CHECK(is_int8(disp)); 386 set_byte_at(fixup_pos, disp); 387 if (offset_to_next < 0) { 388 L->link_to(fixup_pos + offset_to_next, Label::kNear); 389 } else { 390 L->UnuseNear(); 391 } 392 } 393 L->bind_to(pos); 394 } 395 396 397 void Assembler::bind(Label* L) { 398 bind_to(L, pc_offset()); 399 } 400 401 402 void Assembler::GrowBuffer() { 403 DCHECK(buffer_overflow()); 404 if (!own_buffer_) FATAL("external code buffer is too small"); 405 406 // Compute new buffer size. 407 CodeDesc desc; // the new buffer 408 desc.buffer_size = 2 * buffer_size_; 409 410 // Some internal data structures overflow for very large buffers, 411 // they must ensure that kMaximalBufferSize is not too large. 412 if (desc.buffer_size > kMaximalBufferSize || 413 static_cast<size_t>(desc.buffer_size) > 414 isolate()->heap()->MaxOldGenerationSize()) { 415 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer"); 416 } 417 418 // Set up new buffer. 419 desc.buffer = NewArray<byte>(desc.buffer_size); 420 desc.origin = this; 421 desc.instr_size = pc_offset(); 422 desc.reloc_size = 423 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos())); 424 425 // Clear the buffer in debug mode. Use 'int3' instructions to make 426 // sure to get into problems if we ever run uninitialized code. 427 #ifdef DEBUG 428 memset(desc.buffer, 0xCC, desc.buffer_size); 429 #endif 430 431 // Copy the data. 432 intptr_t pc_delta = desc.buffer - buffer_; 433 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 434 (buffer_ + buffer_size_); 435 MemMove(desc.buffer, buffer_, desc.instr_size); 436 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(), 437 desc.reloc_size); 438 439 // Switch buffers. 440 DeleteArray(buffer_); 441 buffer_ = desc.buffer; 442 buffer_size_ = desc.buffer_size; 443 pc_ += pc_delta; 444 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 445 reloc_info_writer.last_pc() + pc_delta); 446 447 // Relocate internal references. 448 for (auto pos : internal_reference_positions_) { 449 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 450 *p += pc_delta; 451 } 452 453 DCHECK(!buffer_overflow()); 454 } 455 456 457 void Assembler::emit_operand(int code, const Operand& adr) { 458 DCHECK(is_uint3(code)); 459 const unsigned length = adr.len_; 460 DCHECK(length > 0); 461 462 // Emit updated ModR/M byte containing the given register. 463 DCHECK((adr.buf_[0] & 0x38) == 0); 464 *pc_++ = adr.buf_[0] | code << 3; 465 466 // Recognize RIP relative addressing. 467 if (adr.buf_[0] == 5) { 468 DCHECK_EQ(9u, length); 469 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]); 470 if (label->is_bound()) { 471 int offset = label->pos() - pc_offset() - sizeof(int32_t); 472 DCHECK_GE(0, offset); 473 emitl(offset); 474 } else if (label->is_linked()) { 475 emitl(label->pos()); 476 label->link_to(pc_offset() - sizeof(int32_t)); 477 } else { 478 DCHECK(label->is_unused()); 479 int32_t current = pc_offset(); 480 emitl(current); 481 label->link_to(current); 482 } 483 } else { 484 // Emit the rest of the encoded operand. 485 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i]; 486 } 487 } 488 489 490 // Assembler Instruction implementations. 491 492 void Assembler::arithmetic_op(byte opcode, 493 Register reg, 494 const Operand& op, 495 int size) { 496 EnsureSpace ensure_space(this); 497 emit_rex(reg, op, size); 498 emit(opcode); 499 emit_operand(reg, op); 500 } 501 502 503 void Assembler::arithmetic_op(byte opcode, 504 Register reg, 505 Register rm_reg, 506 int size) { 507 EnsureSpace ensure_space(this); 508 DCHECK((opcode & 0xC6) == 2); 509 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 510 // Swap reg and rm_reg and change opcode operand order. 511 emit_rex(rm_reg, reg, size); 512 emit(opcode ^ 0x02); 513 emit_modrm(rm_reg, reg); 514 } else { 515 emit_rex(reg, rm_reg, size); 516 emit(opcode); 517 emit_modrm(reg, rm_reg); 518 } 519 } 520 521 522 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) { 523 EnsureSpace ensure_space(this); 524 DCHECK((opcode & 0xC6) == 2); 525 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 526 // Swap reg and rm_reg and change opcode operand order. 527 emit(0x66); 528 emit_optional_rex_32(rm_reg, reg); 529 emit(opcode ^ 0x02); 530 emit_modrm(rm_reg, reg); 531 } else { 532 emit(0x66); 533 emit_optional_rex_32(reg, rm_reg); 534 emit(opcode); 535 emit_modrm(reg, rm_reg); 536 } 537 } 538 539 540 void Assembler::arithmetic_op_16(byte opcode, 541 Register reg, 542 const Operand& rm_reg) { 543 EnsureSpace ensure_space(this); 544 emit(0x66); 545 emit_optional_rex_32(reg, rm_reg); 546 emit(opcode); 547 emit_operand(reg, rm_reg); 548 } 549 550 551 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) { 552 EnsureSpace ensure_space(this); 553 if (!reg.is_byte_register()) { 554 emit_rex_32(reg, op); 555 } else { 556 emit_optional_rex_32(reg, op); 557 } 558 emit(opcode); 559 emit_operand(reg, op); 560 } 561 562 563 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) { 564 EnsureSpace ensure_space(this); 565 DCHECK((opcode & 0xC6) == 2); 566 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 567 // Swap reg and rm_reg and change opcode operand order. 568 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) { 569 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 570 emit_rex_32(rm_reg, reg); 571 } 572 emit(opcode ^ 0x02); 573 emit_modrm(rm_reg, reg); 574 } else { 575 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) { 576 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 577 emit_rex_32(reg, rm_reg); 578 } 579 emit(opcode); 580 emit_modrm(reg, rm_reg); 581 } 582 } 583 584 585 void Assembler::immediate_arithmetic_op(byte subcode, 586 Register dst, 587 Immediate src, 588 int size) { 589 EnsureSpace ensure_space(this); 590 emit_rex(dst, size); 591 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) { 592 emit(0x83); 593 emit_modrm(subcode, dst); 594 emit(src.value_); 595 } else if (dst.is(rax)) { 596 emit(0x05 | (subcode << 3)); 597 emit(src); 598 } else { 599 emit(0x81); 600 emit_modrm(subcode, dst); 601 emit(src); 602 } 603 } 604 605 void Assembler::immediate_arithmetic_op(byte subcode, 606 const Operand& dst, 607 Immediate src, 608 int size) { 609 EnsureSpace ensure_space(this); 610 emit_rex(dst, size); 611 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) { 612 emit(0x83); 613 emit_operand(subcode, dst); 614 emit(src.value_); 615 } else { 616 emit(0x81); 617 emit_operand(subcode, dst); 618 emit(src); 619 } 620 } 621 622 623 void Assembler::immediate_arithmetic_op_16(byte subcode, 624 Register dst, 625 Immediate src) { 626 EnsureSpace ensure_space(this); 627 emit(0x66); // Operand size override prefix. 628 emit_optional_rex_32(dst); 629 if (is_int8(src.value_)) { 630 emit(0x83); 631 emit_modrm(subcode, dst); 632 emit(src.value_); 633 } else if (dst.is(rax)) { 634 emit(0x05 | (subcode << 3)); 635 emitw(src.value_); 636 } else { 637 emit(0x81); 638 emit_modrm(subcode, dst); 639 emitw(src.value_); 640 } 641 } 642 643 644 void Assembler::immediate_arithmetic_op_16(byte subcode, 645 const Operand& dst, 646 Immediate src) { 647 EnsureSpace ensure_space(this); 648 emit(0x66); // Operand size override prefix. 649 emit_optional_rex_32(dst); 650 if (is_int8(src.value_)) { 651 emit(0x83); 652 emit_operand(subcode, dst); 653 emit(src.value_); 654 } else { 655 emit(0x81); 656 emit_operand(subcode, dst); 657 emitw(src.value_); 658 } 659 } 660 661 662 void Assembler::immediate_arithmetic_op_8(byte subcode, 663 const Operand& dst, 664 Immediate src) { 665 EnsureSpace ensure_space(this); 666 emit_optional_rex_32(dst); 667 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 668 emit(0x80); 669 emit_operand(subcode, dst); 670 emit(src.value_); 671 } 672 673 674 void Assembler::immediate_arithmetic_op_8(byte subcode, 675 Register dst, 676 Immediate src) { 677 EnsureSpace ensure_space(this); 678 if (!dst.is_byte_register()) { 679 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 680 emit_rex_32(dst); 681 } 682 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 683 emit(0x80); 684 emit_modrm(subcode, dst); 685 emit(src.value_); 686 } 687 688 689 void Assembler::shift(Register dst, 690 Immediate shift_amount, 691 int subcode, 692 int size) { 693 EnsureSpace ensure_space(this); 694 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 695 : is_uint5(shift_amount.value_)); 696 if (shift_amount.value_ == 1) { 697 emit_rex(dst, size); 698 emit(0xD1); 699 emit_modrm(subcode, dst); 700 } else { 701 emit_rex(dst, size); 702 emit(0xC1); 703 emit_modrm(subcode, dst); 704 emit(shift_amount.value_); 705 } 706 } 707 708 709 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode, 710 int size) { 711 EnsureSpace ensure_space(this); 712 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 713 : is_uint5(shift_amount.value_)); 714 if (shift_amount.value_ == 1) { 715 emit_rex(dst, size); 716 emit(0xD1); 717 emit_operand(subcode, dst); 718 } else { 719 emit_rex(dst, size); 720 emit(0xC1); 721 emit_operand(subcode, dst); 722 emit(shift_amount.value_); 723 } 724 } 725 726 727 void Assembler::shift(Register dst, int subcode, int size) { 728 EnsureSpace ensure_space(this); 729 emit_rex(dst, size); 730 emit(0xD3); 731 emit_modrm(subcode, dst); 732 } 733 734 735 void Assembler::shift(Operand dst, int subcode, int size) { 736 EnsureSpace ensure_space(this); 737 emit_rex(dst, size); 738 emit(0xD3); 739 emit_operand(subcode, dst); 740 } 741 742 743 void Assembler::bt(const Operand& dst, Register src) { 744 EnsureSpace ensure_space(this); 745 emit_rex_64(src, dst); 746 emit(0x0F); 747 emit(0xA3); 748 emit_operand(src, dst); 749 } 750 751 752 void Assembler::bts(const Operand& dst, Register src) { 753 EnsureSpace ensure_space(this); 754 emit_rex_64(src, dst); 755 emit(0x0F); 756 emit(0xAB); 757 emit_operand(src, dst); 758 } 759 760 761 void Assembler::bsrl(Register dst, Register src) { 762 EnsureSpace ensure_space(this); 763 emit_optional_rex_32(dst, src); 764 emit(0x0F); 765 emit(0xBD); 766 emit_modrm(dst, src); 767 } 768 769 770 void Assembler::bsrl(Register dst, const Operand& src) { 771 EnsureSpace ensure_space(this); 772 emit_optional_rex_32(dst, src); 773 emit(0x0F); 774 emit(0xBD); 775 emit_operand(dst, src); 776 } 777 778 779 void Assembler::bsrq(Register dst, Register src) { 780 EnsureSpace ensure_space(this); 781 emit_rex_64(dst, src); 782 emit(0x0F); 783 emit(0xBD); 784 emit_modrm(dst, src); 785 } 786 787 788 void Assembler::bsrq(Register dst, const Operand& src) { 789 EnsureSpace ensure_space(this); 790 emit_rex_64(dst, src); 791 emit(0x0F); 792 emit(0xBD); 793 emit_operand(dst, src); 794 } 795 796 797 void Assembler::bsfl(Register dst, Register src) { 798 EnsureSpace ensure_space(this); 799 emit_optional_rex_32(dst, src); 800 emit(0x0F); 801 emit(0xBC); 802 emit_modrm(dst, src); 803 } 804 805 806 void Assembler::bsfl(Register dst, const Operand& src) { 807 EnsureSpace ensure_space(this); 808 emit_optional_rex_32(dst, src); 809 emit(0x0F); 810 emit(0xBC); 811 emit_operand(dst, src); 812 } 813 814 815 void Assembler::bsfq(Register dst, Register src) { 816 EnsureSpace ensure_space(this); 817 emit_rex_64(dst, src); 818 emit(0x0F); 819 emit(0xBC); 820 emit_modrm(dst, src); 821 } 822 823 824 void Assembler::bsfq(Register dst, const Operand& src) { 825 EnsureSpace ensure_space(this); 826 emit_rex_64(dst, src); 827 emit(0x0F); 828 emit(0xBC); 829 emit_operand(dst, src); 830 } 831 832 833 void Assembler::call(Label* L) { 834 EnsureSpace ensure_space(this); 835 // 1110 1000 #32-bit disp. 836 emit(0xE8); 837 if (L->is_bound()) { 838 int offset = L->pos() - pc_offset() - sizeof(int32_t); 839 DCHECK(offset <= 0); 840 emitl(offset); 841 } else if (L->is_linked()) { 842 emitl(L->pos()); 843 L->link_to(pc_offset() - sizeof(int32_t)); 844 } else { 845 DCHECK(L->is_unused()); 846 int32_t current = pc_offset(); 847 emitl(current); 848 L->link_to(current); 849 } 850 } 851 852 853 void Assembler::call(Address entry, RelocInfo::Mode rmode) { 854 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 855 EnsureSpace ensure_space(this); 856 // 1110 1000 #32-bit disp. 857 emit(0xE8); 858 emit_runtime_entry(entry, rmode); 859 } 860 861 862 void Assembler::call(Handle<Code> target, 863 RelocInfo::Mode rmode, 864 TypeFeedbackId ast_id) { 865 EnsureSpace ensure_space(this); 866 // 1110 1000 #32-bit disp. 867 emit(0xE8); 868 emit_code_target(target, rmode, ast_id); 869 } 870 871 872 void Assembler::call(Register adr) { 873 EnsureSpace ensure_space(this); 874 // Opcode: FF /2 r64. 875 emit_optional_rex_32(adr); 876 emit(0xFF); 877 emit_modrm(0x2, adr); 878 } 879 880 881 void Assembler::call(const Operand& op) { 882 EnsureSpace ensure_space(this); 883 // Opcode: FF /2 m64. 884 emit_optional_rex_32(op); 885 emit(0xFF); 886 emit_operand(0x2, op); 887 } 888 889 890 // Calls directly to the given address using a relative offset. 891 // Should only ever be used in Code objects for calls within the 892 // same Code object. Should not be used when generating new code (use labels), 893 // but only when patching existing code. 894 void Assembler::call(Address target) { 895 EnsureSpace ensure_space(this); 896 // 1110 1000 #32-bit disp. 897 emit(0xE8); 898 Address source = pc_ + 4; 899 intptr_t displacement = target - source; 900 DCHECK(is_int32(displacement)); 901 emitl(static_cast<int32_t>(displacement)); 902 } 903 904 905 void Assembler::clc() { 906 EnsureSpace ensure_space(this); 907 emit(0xF8); 908 } 909 910 911 void Assembler::cld() { 912 EnsureSpace ensure_space(this); 913 emit(0xFC); 914 } 915 916 917 void Assembler::cdq() { 918 EnsureSpace ensure_space(this); 919 emit(0x99); 920 } 921 922 923 void Assembler::cmovq(Condition cc, Register dst, Register src) { 924 if (cc == always) { 925 movq(dst, src); 926 } else if (cc == never) { 927 return; 928 } 929 // No need to check CpuInfo for CMOV support, it's a required part of the 930 // 64-bit architecture. 931 DCHECK(cc >= 0); // Use mov for unconditional moves. 932 EnsureSpace ensure_space(this); 933 // Opcode: REX.W 0f 40 + cc /r. 934 emit_rex_64(dst, src); 935 emit(0x0f); 936 emit(0x40 + cc); 937 emit_modrm(dst, src); 938 } 939 940 941 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) { 942 if (cc == always) { 943 movq(dst, src); 944 } else if (cc == never) { 945 return; 946 } 947 DCHECK(cc >= 0); 948 EnsureSpace ensure_space(this); 949 // Opcode: REX.W 0f 40 + cc /r. 950 emit_rex_64(dst, src); 951 emit(0x0f); 952 emit(0x40 + cc); 953 emit_operand(dst, src); 954 } 955 956 957 void Assembler::cmovl(Condition cc, Register dst, Register src) { 958 if (cc == always) { 959 movl(dst, src); 960 } else if (cc == never) { 961 return; 962 } 963 DCHECK(cc >= 0); 964 EnsureSpace ensure_space(this); 965 // Opcode: 0f 40 + cc /r. 966 emit_optional_rex_32(dst, src); 967 emit(0x0f); 968 emit(0x40 + cc); 969 emit_modrm(dst, src); 970 } 971 972 973 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) { 974 if (cc == always) { 975 movl(dst, src); 976 } else if (cc == never) { 977 return; 978 } 979 DCHECK(cc >= 0); 980 EnsureSpace ensure_space(this); 981 // Opcode: 0f 40 + cc /r. 982 emit_optional_rex_32(dst, src); 983 emit(0x0f); 984 emit(0x40 + cc); 985 emit_operand(dst, src); 986 } 987 988 989 void Assembler::cmpb_al(Immediate imm8) { 990 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_)); 991 EnsureSpace ensure_space(this); 992 emit(0x3c); 993 emit(imm8.value_); 994 } 995 996 void Assembler::lock() { 997 EnsureSpace ensure_space(this); 998 emit(0xf0); 999 } 1000 1001 void Assembler::cmpxchgb(const Operand& dst, Register src) { 1002 EnsureSpace ensure_space(this); 1003 if (!src.is_byte_register()) { 1004 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1005 emit_rex_32(src, dst); 1006 } else { 1007 emit_optional_rex_32(src, dst); 1008 } 1009 emit(0x0f); 1010 emit(0xb0); 1011 emit_operand(src, dst); 1012 } 1013 1014 void Assembler::cmpxchgw(const Operand& dst, Register src) { 1015 EnsureSpace ensure_space(this); 1016 emit(0x66); 1017 emit_optional_rex_32(src, dst); 1018 emit(0x0f); 1019 emit(0xb1); 1020 emit_operand(src, dst); 1021 } 1022 1023 void Assembler::emit_cmpxchg(const Operand& dst, Register src, int size) { 1024 EnsureSpace ensure_space(this); 1025 emit_rex(src, dst, size); 1026 emit(0x0f); 1027 emit(0xb1); 1028 emit_operand(src, dst); 1029 } 1030 1031 void Assembler::cpuid() { 1032 EnsureSpace ensure_space(this); 1033 emit(0x0F); 1034 emit(0xA2); 1035 } 1036 1037 1038 void Assembler::cqo() { 1039 EnsureSpace ensure_space(this); 1040 emit_rex_64(); 1041 emit(0x99); 1042 } 1043 1044 1045 void Assembler::emit_dec(Register dst, int size) { 1046 EnsureSpace ensure_space(this); 1047 emit_rex(dst, size); 1048 emit(0xFF); 1049 emit_modrm(0x1, dst); 1050 } 1051 1052 1053 void Assembler::emit_dec(const Operand& dst, int size) { 1054 EnsureSpace ensure_space(this); 1055 emit_rex(dst, size); 1056 emit(0xFF); 1057 emit_operand(1, dst); 1058 } 1059 1060 1061 void Assembler::decb(Register dst) { 1062 EnsureSpace ensure_space(this); 1063 if (!dst.is_byte_register()) { 1064 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1065 emit_rex_32(dst); 1066 } 1067 emit(0xFE); 1068 emit_modrm(0x1, dst); 1069 } 1070 1071 1072 void Assembler::decb(const Operand& dst) { 1073 EnsureSpace ensure_space(this); 1074 emit_optional_rex_32(dst); 1075 emit(0xFE); 1076 emit_operand(1, dst); 1077 } 1078 1079 1080 void Assembler::enter(Immediate size) { 1081 EnsureSpace ensure_space(this); 1082 emit(0xC8); 1083 emitw(size.value_); // 16 bit operand, always. 1084 emit(0); 1085 } 1086 1087 1088 void Assembler::hlt() { 1089 EnsureSpace ensure_space(this); 1090 emit(0xF4); 1091 } 1092 1093 1094 void Assembler::emit_idiv(Register src, int size) { 1095 EnsureSpace ensure_space(this); 1096 emit_rex(src, size); 1097 emit(0xF7); 1098 emit_modrm(0x7, src); 1099 } 1100 1101 1102 void Assembler::emit_div(Register src, int size) { 1103 EnsureSpace ensure_space(this); 1104 emit_rex(src, size); 1105 emit(0xF7); 1106 emit_modrm(0x6, src); 1107 } 1108 1109 1110 void Assembler::emit_imul(Register src, int size) { 1111 EnsureSpace ensure_space(this); 1112 emit_rex(src, size); 1113 emit(0xF7); 1114 emit_modrm(0x5, src); 1115 } 1116 1117 1118 void Assembler::emit_imul(const Operand& src, int size) { 1119 EnsureSpace ensure_space(this); 1120 emit_rex(src, size); 1121 emit(0xF7); 1122 emit_operand(0x5, src); 1123 } 1124 1125 1126 void Assembler::emit_imul(Register dst, Register src, int size) { 1127 EnsureSpace ensure_space(this); 1128 emit_rex(dst, src, size); 1129 emit(0x0F); 1130 emit(0xAF); 1131 emit_modrm(dst, src); 1132 } 1133 1134 1135 void Assembler::emit_imul(Register dst, const Operand& src, int size) { 1136 EnsureSpace ensure_space(this); 1137 emit_rex(dst, src, size); 1138 emit(0x0F); 1139 emit(0xAF); 1140 emit_operand(dst, src); 1141 } 1142 1143 1144 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) { 1145 EnsureSpace ensure_space(this); 1146 emit_rex(dst, src, size); 1147 if (is_int8(imm.value_)) { 1148 emit(0x6B); 1149 emit_modrm(dst, src); 1150 emit(imm.value_); 1151 } else { 1152 emit(0x69); 1153 emit_modrm(dst, src); 1154 emitl(imm.value_); 1155 } 1156 } 1157 1158 1159 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm, 1160 int size) { 1161 EnsureSpace ensure_space(this); 1162 emit_rex(dst, src, size); 1163 if (is_int8(imm.value_)) { 1164 emit(0x6B); 1165 emit_operand(dst, src); 1166 emit(imm.value_); 1167 } else { 1168 emit(0x69); 1169 emit_operand(dst, src); 1170 emitl(imm.value_); 1171 } 1172 } 1173 1174 1175 void Assembler::emit_inc(Register dst, int size) { 1176 EnsureSpace ensure_space(this); 1177 emit_rex(dst, size); 1178 emit(0xFF); 1179 emit_modrm(0x0, dst); 1180 } 1181 1182 1183 void Assembler::emit_inc(const Operand& dst, int size) { 1184 EnsureSpace ensure_space(this); 1185 emit_rex(dst, size); 1186 emit(0xFF); 1187 emit_operand(0, dst); 1188 } 1189 1190 1191 void Assembler::int3() { 1192 EnsureSpace ensure_space(this); 1193 emit(0xCC); 1194 } 1195 1196 1197 void Assembler::j(Condition cc, Label* L, Label::Distance distance) { 1198 if (cc == always) { 1199 jmp(L); 1200 return; 1201 } else if (cc == never) { 1202 return; 1203 } 1204 EnsureSpace ensure_space(this); 1205 DCHECK(is_uint4(cc)); 1206 if (L->is_bound()) { 1207 const int short_size = 2; 1208 const int long_size = 6; 1209 int offs = L->pos() - pc_offset(); 1210 DCHECK(offs <= 0); 1211 // Determine whether we can use 1-byte offsets for backwards branches, 1212 // which have a max range of 128 bytes. 1213 1214 // We also need to check predictable_code_size() flag here, because on x64, 1215 // when the full code generator recompiles code for debugging, some places 1216 // need to be padded out to a certain size. The debugger is keeping track of 1217 // how often it did this so that it can adjust return addresses on the 1218 // stack, but if the size of jump instructions can also change, that's not 1219 // enough and the calculated offsets would be incorrect. 1220 if (is_int8(offs - short_size) && !predictable_code_size()) { 1221 // 0111 tttn #8-bit disp. 1222 emit(0x70 | cc); 1223 emit((offs - short_size) & 0xFF); 1224 } else { 1225 // 0000 1111 1000 tttn #32-bit disp. 1226 emit(0x0F); 1227 emit(0x80 | cc); 1228 emitl(offs - long_size); 1229 } 1230 } else if (distance == Label::kNear) { 1231 // 0111 tttn #8-bit disp 1232 emit(0x70 | cc); 1233 byte disp = 0x00; 1234 if (L->is_near_linked()) { 1235 int offset = L->near_link_pos() - pc_offset(); 1236 DCHECK(is_int8(offset)); 1237 disp = static_cast<byte>(offset & 0xFF); 1238 } 1239 L->link_to(pc_offset(), Label::kNear); 1240 emit(disp); 1241 } else if (L->is_linked()) { 1242 // 0000 1111 1000 tttn #32-bit disp. 1243 emit(0x0F); 1244 emit(0x80 | cc); 1245 emitl(L->pos()); 1246 L->link_to(pc_offset() - sizeof(int32_t)); 1247 } else { 1248 DCHECK(L->is_unused()); 1249 emit(0x0F); 1250 emit(0x80 | cc); 1251 int32_t current = pc_offset(); 1252 emitl(current); 1253 L->link_to(current); 1254 } 1255 } 1256 1257 1258 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) { 1259 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1260 EnsureSpace ensure_space(this); 1261 DCHECK(is_uint4(cc)); 1262 emit(0x0F); 1263 emit(0x80 | cc); 1264 emit_runtime_entry(entry, rmode); 1265 } 1266 1267 1268 void Assembler::j(Condition cc, 1269 Handle<Code> target, 1270 RelocInfo::Mode rmode) { 1271 EnsureSpace ensure_space(this); 1272 DCHECK(is_uint4(cc)); 1273 // 0000 1111 1000 tttn #32-bit disp. 1274 emit(0x0F); 1275 emit(0x80 | cc); 1276 emit_code_target(target, rmode); 1277 } 1278 1279 1280 void Assembler::jmp(Label* L, Label::Distance distance) { 1281 EnsureSpace ensure_space(this); 1282 const int short_size = sizeof(int8_t); 1283 const int long_size = sizeof(int32_t); 1284 if (L->is_bound()) { 1285 int offs = L->pos() - pc_offset() - 1; 1286 DCHECK(offs <= 0); 1287 if (is_int8(offs - short_size) && !predictable_code_size()) { 1288 // 1110 1011 #8-bit disp. 1289 emit(0xEB); 1290 emit((offs - short_size) & 0xFF); 1291 } else { 1292 // 1110 1001 #32-bit disp. 1293 emit(0xE9); 1294 emitl(offs - long_size); 1295 } 1296 } else if (distance == Label::kNear) { 1297 emit(0xEB); 1298 byte disp = 0x00; 1299 if (L->is_near_linked()) { 1300 int offset = L->near_link_pos() - pc_offset(); 1301 DCHECK(is_int8(offset)); 1302 disp = static_cast<byte>(offset & 0xFF); 1303 } 1304 L->link_to(pc_offset(), Label::kNear); 1305 emit(disp); 1306 } else if (L->is_linked()) { 1307 // 1110 1001 #32-bit disp. 1308 emit(0xE9); 1309 emitl(L->pos()); 1310 L->link_to(pc_offset() - long_size); 1311 } else { 1312 // 1110 1001 #32-bit disp. 1313 DCHECK(L->is_unused()); 1314 emit(0xE9); 1315 int32_t current = pc_offset(); 1316 emitl(current); 1317 L->link_to(current); 1318 } 1319 } 1320 1321 1322 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) { 1323 EnsureSpace ensure_space(this); 1324 // 1110 1001 #32-bit disp. 1325 emit(0xE9); 1326 emit_code_target(target, rmode); 1327 } 1328 1329 1330 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { 1331 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1332 EnsureSpace ensure_space(this); 1333 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1334 emit(0xE9); 1335 emit_runtime_entry(entry, rmode); 1336 } 1337 1338 1339 void Assembler::jmp(Register target) { 1340 EnsureSpace ensure_space(this); 1341 // Opcode FF/4 r64. 1342 emit_optional_rex_32(target); 1343 emit(0xFF); 1344 emit_modrm(0x4, target); 1345 } 1346 1347 1348 void Assembler::jmp(const Operand& src) { 1349 EnsureSpace ensure_space(this); 1350 // Opcode FF/4 m64. 1351 emit_optional_rex_32(src); 1352 emit(0xFF); 1353 emit_operand(0x4, src); 1354 } 1355 1356 1357 void Assembler::emit_lea(Register dst, const Operand& src, int size) { 1358 EnsureSpace ensure_space(this); 1359 emit_rex(dst, src, size); 1360 emit(0x8D); 1361 emit_operand(dst, src); 1362 } 1363 1364 1365 void Assembler::load_rax(void* value, RelocInfo::Mode mode) { 1366 EnsureSpace ensure_space(this); 1367 if (kPointerSize == kInt64Size) { 1368 emit(0x48); // REX.W 1369 emit(0xA1); 1370 emitp(value, mode); 1371 } else { 1372 DCHECK(kPointerSize == kInt32Size); 1373 emit(0xA1); 1374 emitp(value, mode); 1375 // In 64-bit mode, need to zero extend the operand to 8 bytes. 1376 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 1377 // Developer's Manual Volume 2. 1378 emitl(0); 1379 } 1380 } 1381 1382 1383 void Assembler::load_rax(ExternalReference ref) { 1384 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 1385 } 1386 1387 1388 void Assembler::leave() { 1389 EnsureSpace ensure_space(this); 1390 emit(0xC9); 1391 } 1392 1393 1394 void Assembler::movb(Register dst, const Operand& src) { 1395 EnsureSpace ensure_space(this); 1396 if (!dst.is_byte_register()) { 1397 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1398 emit_rex_32(dst, src); 1399 } else { 1400 emit_optional_rex_32(dst, src); 1401 } 1402 emit(0x8A); 1403 emit_operand(dst, src); 1404 } 1405 1406 1407 void Assembler::movb(Register dst, Immediate imm) { 1408 EnsureSpace ensure_space(this); 1409 if (!dst.is_byte_register()) { 1410 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1411 emit_rex_32(dst); 1412 } 1413 emit(0xB0 + dst.low_bits()); 1414 emit(imm.value_); 1415 } 1416 1417 1418 void Assembler::movb(const Operand& dst, Register src) { 1419 EnsureSpace ensure_space(this); 1420 if (!src.is_byte_register()) { 1421 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1422 emit_rex_32(src, dst); 1423 } else { 1424 emit_optional_rex_32(src, dst); 1425 } 1426 emit(0x88); 1427 emit_operand(src, dst); 1428 } 1429 1430 1431 void Assembler::movb(const Operand& dst, Immediate imm) { 1432 EnsureSpace ensure_space(this); 1433 emit_optional_rex_32(dst); 1434 emit(0xC6); 1435 emit_operand(0x0, dst); 1436 emit(static_cast<byte>(imm.value_)); 1437 } 1438 1439 1440 void Assembler::movw(Register dst, const Operand& src) { 1441 EnsureSpace ensure_space(this); 1442 emit(0x66); 1443 emit_optional_rex_32(dst, src); 1444 emit(0x8B); 1445 emit_operand(dst, src); 1446 } 1447 1448 1449 void Assembler::movw(const Operand& dst, Register src) { 1450 EnsureSpace ensure_space(this); 1451 emit(0x66); 1452 emit_optional_rex_32(src, dst); 1453 emit(0x89); 1454 emit_operand(src, dst); 1455 } 1456 1457 1458 void Assembler::movw(const Operand& dst, Immediate imm) { 1459 EnsureSpace ensure_space(this); 1460 emit(0x66); 1461 emit_optional_rex_32(dst); 1462 emit(0xC7); 1463 emit_operand(0x0, dst); 1464 emit(static_cast<byte>(imm.value_ & 0xff)); 1465 emit(static_cast<byte>(imm.value_ >> 8)); 1466 } 1467 1468 1469 void Assembler::emit_mov(Register dst, const Operand& src, int size) { 1470 EnsureSpace ensure_space(this); 1471 emit_rex(dst, src, size); 1472 emit(0x8B); 1473 emit_operand(dst, src); 1474 } 1475 1476 1477 void Assembler::emit_mov(Register dst, Register src, int size) { 1478 EnsureSpace ensure_space(this); 1479 if (src.low_bits() == 4) { 1480 emit_rex(src, dst, size); 1481 emit(0x89); 1482 emit_modrm(src, dst); 1483 } else { 1484 emit_rex(dst, src, size); 1485 emit(0x8B); 1486 emit_modrm(dst, src); 1487 } 1488 } 1489 1490 1491 void Assembler::emit_mov(const Operand& dst, Register src, int size) { 1492 EnsureSpace ensure_space(this); 1493 emit_rex(src, dst, size); 1494 emit(0x89); 1495 emit_operand(src, dst); 1496 } 1497 1498 1499 void Assembler::emit_mov(Register dst, Immediate value, int size) { 1500 EnsureSpace ensure_space(this); 1501 emit_rex(dst, size); 1502 if (size == kInt64Size) { 1503 emit(0xC7); 1504 emit_modrm(0x0, dst); 1505 } else { 1506 DCHECK(size == kInt32Size); 1507 emit(0xB8 + dst.low_bits()); 1508 } 1509 emit(value); 1510 } 1511 1512 1513 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) { 1514 EnsureSpace ensure_space(this); 1515 emit_rex(dst, size); 1516 emit(0xC7); 1517 emit_operand(0x0, dst); 1518 emit(value); 1519 } 1520 1521 1522 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) { 1523 EnsureSpace ensure_space(this); 1524 emit_rex(dst, kPointerSize); 1525 emit(0xB8 | dst.low_bits()); 1526 emitp(value, rmode); 1527 } 1528 1529 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) { 1530 EnsureSpace ensure_space(this); 1531 emit_rex_64(dst); 1532 emit(0xB8 | dst.low_bits()); 1533 if (!RelocInfo::IsNone(rmode)) { 1534 RecordRelocInfo(rmode, value); 1535 } 1536 emitq(value); 1537 } 1538 1539 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) { 1540 movq(dst, static_cast<int64_t>(value), rmode); 1541 } 1542 1543 // Loads the ip-relative location of the src label into the target location 1544 // (as a 32-bit offset sign extended to 64-bit). 1545 void Assembler::movl(const Operand& dst, Label* src) { 1546 EnsureSpace ensure_space(this); 1547 emit_optional_rex_32(dst); 1548 emit(0xC7); 1549 emit_operand(0, dst); 1550 if (src->is_bound()) { 1551 int offset = src->pos() - pc_offset() - sizeof(int32_t); 1552 DCHECK(offset <= 0); 1553 emitl(offset); 1554 } else if (src->is_linked()) { 1555 emitl(src->pos()); 1556 src->link_to(pc_offset() - sizeof(int32_t)); 1557 } else { 1558 DCHECK(src->is_unused()); 1559 int32_t current = pc_offset(); 1560 emitl(current); 1561 src->link_to(current); 1562 } 1563 } 1564 1565 1566 void Assembler::movsxbl(Register dst, Register src) { 1567 EnsureSpace ensure_space(this); 1568 if (!src.is_byte_register()) { 1569 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1570 emit_rex_32(dst, src); 1571 } else { 1572 emit_optional_rex_32(dst, src); 1573 } 1574 emit(0x0F); 1575 emit(0xBE); 1576 emit_modrm(dst, src); 1577 } 1578 1579 1580 void Assembler::movsxbl(Register dst, const Operand& src) { 1581 EnsureSpace ensure_space(this); 1582 emit_optional_rex_32(dst, src); 1583 emit(0x0F); 1584 emit(0xBE); 1585 emit_operand(dst, src); 1586 } 1587 1588 1589 void Assembler::movsxbq(Register dst, const Operand& src) { 1590 EnsureSpace ensure_space(this); 1591 emit_rex_64(dst, src); 1592 emit(0x0F); 1593 emit(0xBE); 1594 emit_operand(dst, src); 1595 } 1596 1597 void Assembler::movsxbq(Register dst, Register src) { 1598 EnsureSpace ensure_space(this); 1599 emit_rex_64(dst, src); 1600 emit(0x0F); 1601 emit(0xBE); 1602 emit_modrm(dst, src); 1603 } 1604 1605 void Assembler::movsxwl(Register dst, Register src) { 1606 EnsureSpace ensure_space(this); 1607 emit_optional_rex_32(dst, src); 1608 emit(0x0F); 1609 emit(0xBF); 1610 emit_modrm(dst, src); 1611 } 1612 1613 1614 void Assembler::movsxwl(Register dst, const Operand& src) { 1615 EnsureSpace ensure_space(this); 1616 emit_optional_rex_32(dst, src); 1617 emit(0x0F); 1618 emit(0xBF); 1619 emit_operand(dst, src); 1620 } 1621 1622 1623 void Assembler::movsxwq(Register dst, const Operand& src) { 1624 EnsureSpace ensure_space(this); 1625 emit_rex_64(dst, src); 1626 emit(0x0F); 1627 emit(0xBF); 1628 emit_operand(dst, src); 1629 } 1630 1631 void Assembler::movsxwq(Register dst, Register src) { 1632 EnsureSpace ensure_space(this); 1633 emit_rex_64(dst, src); 1634 emit(0x0F); 1635 emit(0xBF); 1636 emit_modrm(dst, src); 1637 } 1638 1639 void Assembler::movsxlq(Register dst, Register src) { 1640 EnsureSpace ensure_space(this); 1641 emit_rex_64(dst, src); 1642 emit(0x63); 1643 emit_modrm(dst, src); 1644 } 1645 1646 1647 void Assembler::movsxlq(Register dst, const Operand& src) { 1648 EnsureSpace ensure_space(this); 1649 emit_rex_64(dst, src); 1650 emit(0x63); 1651 emit_operand(dst, src); 1652 } 1653 1654 1655 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) { 1656 EnsureSpace ensure_space(this); 1657 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1658 // there is no need to make this a 64 bit operation. 1659 emit_optional_rex_32(dst, src); 1660 emit(0x0F); 1661 emit(0xB6); 1662 emit_operand(dst, src); 1663 } 1664 1665 1666 void Assembler::emit_movzxb(Register dst, Register src, int size) { 1667 EnsureSpace ensure_space(this); 1668 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1669 // there is no need to make this a 64 bit operation. 1670 if (!src.is_byte_register()) { 1671 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1672 emit_rex_32(dst, src); 1673 } else { 1674 emit_optional_rex_32(dst, src); 1675 } 1676 emit(0x0F); 1677 emit(0xB6); 1678 emit_modrm(dst, src); 1679 } 1680 1681 1682 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) { 1683 EnsureSpace ensure_space(this); 1684 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1685 // there is no need to make this a 64 bit operation. 1686 emit_optional_rex_32(dst, src); 1687 emit(0x0F); 1688 emit(0xB7); 1689 emit_operand(dst, src); 1690 } 1691 1692 1693 void Assembler::emit_movzxw(Register dst, Register src, int size) { 1694 EnsureSpace ensure_space(this); 1695 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1696 // there is no need to make this a 64 bit operation. 1697 emit_optional_rex_32(dst, src); 1698 emit(0x0F); 1699 emit(0xB7); 1700 emit_modrm(dst, src); 1701 } 1702 1703 1704 void Assembler::repmovsb() { 1705 EnsureSpace ensure_space(this); 1706 emit(0xF3); 1707 emit(0xA4); 1708 } 1709 1710 1711 void Assembler::repmovsw() { 1712 EnsureSpace ensure_space(this); 1713 emit(0x66); // Operand size override. 1714 emit(0xF3); 1715 emit(0xA4); 1716 } 1717 1718 1719 void Assembler::emit_repmovs(int size) { 1720 EnsureSpace ensure_space(this); 1721 emit(0xF3); 1722 emit_rex(size); 1723 emit(0xA5); 1724 } 1725 1726 1727 void Assembler::mull(Register src) { 1728 EnsureSpace ensure_space(this); 1729 emit_optional_rex_32(src); 1730 emit(0xF7); 1731 emit_modrm(0x4, src); 1732 } 1733 1734 1735 void Assembler::mull(const Operand& src) { 1736 EnsureSpace ensure_space(this); 1737 emit_optional_rex_32(src); 1738 emit(0xF7); 1739 emit_operand(0x4, src); 1740 } 1741 1742 1743 void Assembler::mulq(Register src) { 1744 EnsureSpace ensure_space(this); 1745 emit_rex_64(src); 1746 emit(0xF7); 1747 emit_modrm(0x4, src); 1748 } 1749 1750 1751 void Assembler::emit_neg(Register dst, int size) { 1752 EnsureSpace ensure_space(this); 1753 emit_rex(dst, size); 1754 emit(0xF7); 1755 emit_modrm(0x3, dst); 1756 } 1757 1758 1759 void Assembler::emit_neg(const Operand& dst, int size) { 1760 EnsureSpace ensure_space(this); 1761 emit_rex_64(dst); 1762 emit(0xF7); 1763 emit_operand(3, dst); 1764 } 1765 1766 1767 void Assembler::nop() { 1768 EnsureSpace ensure_space(this); 1769 emit(0x90); 1770 } 1771 1772 1773 void Assembler::emit_not(Register dst, int size) { 1774 EnsureSpace ensure_space(this); 1775 emit_rex(dst, size); 1776 emit(0xF7); 1777 emit_modrm(0x2, dst); 1778 } 1779 1780 1781 void Assembler::emit_not(const Operand& dst, int size) { 1782 EnsureSpace ensure_space(this); 1783 emit_rex(dst, size); 1784 emit(0xF7); 1785 emit_operand(2, dst); 1786 } 1787 1788 1789 void Assembler::Nop(int n) { 1790 // The recommended muti-byte sequences of NOP instructions from the Intel 64 1791 // and IA-32 Architectures Software Developer's Manual. 1792 // 1793 // Length Assembly Byte Sequence 1794 // 2 bytes 66 NOP 66 90H 1795 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H 1796 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H 1797 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H 1798 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H 1799 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H 1800 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H 1801 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00 1802 // 00000000H] 00H 1803 1804 EnsureSpace ensure_space(this); 1805 while (n > 0) { 1806 switch (n) { 1807 case 2: 1808 emit(0x66); 1809 case 1: 1810 emit(0x90); 1811 return; 1812 case 3: 1813 emit(0x0f); 1814 emit(0x1f); 1815 emit(0x00); 1816 return; 1817 case 4: 1818 emit(0x0f); 1819 emit(0x1f); 1820 emit(0x40); 1821 emit(0x00); 1822 return; 1823 case 6: 1824 emit(0x66); 1825 case 5: 1826 emit(0x0f); 1827 emit(0x1f); 1828 emit(0x44); 1829 emit(0x00); 1830 emit(0x00); 1831 return; 1832 case 7: 1833 emit(0x0f); 1834 emit(0x1f); 1835 emit(0x80); 1836 emit(0x00); 1837 emit(0x00); 1838 emit(0x00); 1839 emit(0x00); 1840 return; 1841 default: 1842 case 11: 1843 emit(0x66); 1844 n--; 1845 case 10: 1846 emit(0x66); 1847 n--; 1848 case 9: 1849 emit(0x66); 1850 n--; 1851 case 8: 1852 emit(0x0f); 1853 emit(0x1f); 1854 emit(0x84); 1855 emit(0x00); 1856 emit(0x00); 1857 emit(0x00); 1858 emit(0x00); 1859 emit(0x00); 1860 n -= 8; 1861 } 1862 } 1863 } 1864 1865 1866 void Assembler::popq(Register dst) { 1867 EnsureSpace ensure_space(this); 1868 emit_optional_rex_32(dst); 1869 emit(0x58 | dst.low_bits()); 1870 } 1871 1872 1873 void Assembler::popq(const Operand& dst) { 1874 EnsureSpace ensure_space(this); 1875 emit_optional_rex_32(dst); 1876 emit(0x8F); 1877 emit_operand(0, dst); 1878 } 1879 1880 1881 void Assembler::popfq() { 1882 EnsureSpace ensure_space(this); 1883 emit(0x9D); 1884 } 1885 1886 1887 void Assembler::pushq(Register src) { 1888 EnsureSpace ensure_space(this); 1889 emit_optional_rex_32(src); 1890 emit(0x50 | src.low_bits()); 1891 } 1892 1893 1894 void Assembler::pushq(const Operand& src) { 1895 EnsureSpace ensure_space(this); 1896 emit_optional_rex_32(src); 1897 emit(0xFF); 1898 emit_operand(6, src); 1899 } 1900 1901 1902 void Assembler::pushq(Immediate value) { 1903 EnsureSpace ensure_space(this); 1904 if (is_int8(value.value_)) { 1905 emit(0x6A); 1906 emit(value.value_); // Emit low byte of value. 1907 } else { 1908 emit(0x68); 1909 emitl(value.value_); 1910 } 1911 } 1912 1913 1914 void Assembler::pushq_imm32(int32_t imm32) { 1915 EnsureSpace ensure_space(this); 1916 emit(0x68); 1917 emitl(imm32); 1918 } 1919 1920 1921 void Assembler::pushfq() { 1922 EnsureSpace ensure_space(this); 1923 emit(0x9C); 1924 } 1925 1926 1927 void Assembler::ret(int imm16) { 1928 EnsureSpace ensure_space(this); 1929 DCHECK(is_uint16(imm16)); 1930 if (imm16 == 0) { 1931 emit(0xC3); 1932 } else { 1933 emit(0xC2); 1934 emit(imm16 & 0xFF); 1935 emit((imm16 >> 8) & 0xFF); 1936 } 1937 } 1938 1939 1940 void Assembler::ud2() { 1941 EnsureSpace ensure_space(this); 1942 emit(0x0F); 1943 emit(0x0B); 1944 } 1945 1946 1947 void Assembler::setcc(Condition cc, Register reg) { 1948 if (cc > last_condition) { 1949 movb(reg, Immediate(cc == always ? 1 : 0)); 1950 return; 1951 } 1952 EnsureSpace ensure_space(this); 1953 DCHECK(is_uint4(cc)); 1954 if (!reg.is_byte_register()) { 1955 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1956 emit_rex_32(reg); 1957 } 1958 emit(0x0F); 1959 emit(0x90 | cc); 1960 emit_modrm(0x0, reg); 1961 } 1962 1963 1964 void Assembler::shld(Register dst, Register src) { 1965 EnsureSpace ensure_space(this); 1966 emit_rex_64(src, dst); 1967 emit(0x0F); 1968 emit(0xA5); 1969 emit_modrm(src, dst); 1970 } 1971 1972 1973 void Assembler::shrd(Register dst, Register src) { 1974 EnsureSpace ensure_space(this); 1975 emit_rex_64(src, dst); 1976 emit(0x0F); 1977 emit(0xAD); 1978 emit_modrm(src, dst); 1979 } 1980 1981 void Assembler::xchgb(Register reg, const Operand& op) { 1982 EnsureSpace ensure_space(this); 1983 if (!reg.is_byte_register()) { 1984 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1985 emit_rex_32(reg, op); 1986 } else { 1987 emit_optional_rex_32(reg, op); 1988 } 1989 emit(0x86); 1990 emit_operand(reg, op); 1991 } 1992 1993 void Assembler::xchgw(Register reg, const Operand& op) { 1994 EnsureSpace ensure_space(this); 1995 emit(0x66); 1996 emit_optional_rex_32(reg, op); 1997 emit(0x87); 1998 emit_operand(reg, op); 1999 } 2000 2001 void Assembler::emit_xchg(Register dst, Register src, int size) { 2002 EnsureSpace ensure_space(this); 2003 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding 2004 Register other = src.is(rax) ? dst : src; 2005 emit_rex(other, size); 2006 emit(0x90 | other.low_bits()); 2007 } else if (dst.low_bits() == 4) { 2008 emit_rex(dst, src, size); 2009 emit(0x87); 2010 emit_modrm(dst, src); 2011 } else { 2012 emit_rex(src, dst, size); 2013 emit(0x87); 2014 emit_modrm(src, dst); 2015 } 2016 } 2017 2018 2019 void Assembler::emit_xchg(Register dst, const Operand& src, int size) { 2020 EnsureSpace ensure_space(this); 2021 emit_rex(dst, src, size); 2022 emit(0x87); 2023 emit_operand(dst, src); 2024 } 2025 2026 2027 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) { 2028 EnsureSpace ensure_space(this); 2029 if (kPointerSize == kInt64Size) { 2030 emit(0x48); // REX.W 2031 emit(0xA3); 2032 emitp(dst, mode); 2033 } else { 2034 DCHECK(kPointerSize == kInt32Size); 2035 emit(0xA3); 2036 emitp(dst, mode); 2037 // In 64-bit mode, need to zero extend the operand to 8 bytes. 2038 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 2039 // Developer's Manual Volume 2. 2040 emitl(0); 2041 } 2042 } 2043 2044 2045 void Assembler::store_rax(ExternalReference ref) { 2046 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 2047 } 2048 2049 2050 void Assembler::testb(Register dst, Register src) { 2051 EnsureSpace ensure_space(this); 2052 emit_test(dst, src, sizeof(int8_t)); 2053 } 2054 2055 void Assembler::testb(Register reg, Immediate mask) { 2056 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2057 emit_test(reg, mask, sizeof(int8_t)); 2058 } 2059 2060 void Assembler::testb(const Operand& op, Immediate mask) { 2061 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2062 emit_test(op, mask, sizeof(int8_t)); 2063 } 2064 2065 2066 void Assembler::testb(const Operand& op, Register reg) { 2067 emit_test(op, reg, sizeof(int8_t)); 2068 } 2069 2070 void Assembler::testw(Register dst, Register src) { 2071 emit_test(dst, src, sizeof(uint16_t)); 2072 } 2073 2074 void Assembler::testw(Register reg, Immediate mask) { 2075 emit_test(reg, mask, sizeof(int16_t)); 2076 } 2077 2078 void Assembler::testw(const Operand& op, Immediate mask) { 2079 emit_test(op, mask, sizeof(int16_t)); 2080 } 2081 2082 void Assembler::testw(const Operand& op, Register reg) { 2083 emit_test(op, reg, sizeof(int16_t)); 2084 } 2085 2086 void Assembler::emit_test(Register dst, Register src, int size) { 2087 EnsureSpace ensure_space(this); 2088 if (src.low_bits() == 4) std::swap(dst, src); 2089 if (size == sizeof(int16_t)) { 2090 emit(0x66); 2091 size = sizeof(int32_t); 2092 } 2093 bool byte_operand = size == sizeof(int8_t); 2094 if (byte_operand) { 2095 size = sizeof(int32_t); 2096 if (!src.is_byte_register() || !dst.is_byte_register()) { 2097 emit_rex_32(dst, src); 2098 } 2099 } else { 2100 emit_rex(dst, src, size); 2101 } 2102 emit(byte_operand ? 0x84 : 0x85); 2103 emit_modrm(dst, src); 2104 } 2105 2106 2107 void Assembler::emit_test(Register reg, Immediate mask, int size) { 2108 if (is_uint8(mask.value_)) { 2109 size = sizeof(int8_t); 2110 } else if (is_uint16(mask.value_)) { 2111 size = sizeof(int16_t); 2112 } 2113 EnsureSpace ensure_space(this); 2114 bool half_word = size == sizeof(int16_t); 2115 if (half_word) { 2116 emit(0x66); 2117 size = sizeof(int32_t); 2118 } 2119 bool byte_operand = size == sizeof(int8_t); 2120 if (byte_operand) { 2121 size = sizeof(int32_t); 2122 if (!reg.is_byte_register()) emit_rex_32(reg); 2123 } else { 2124 emit_rex(reg, size); 2125 } 2126 if (reg.is(rax)) { 2127 emit(byte_operand ? 0xA8 : 0xA9); 2128 } else { 2129 emit(byte_operand ? 0xF6 : 0xF7); 2130 emit_modrm(0x0, reg); 2131 } 2132 if (byte_operand) { 2133 emit(mask.value_); 2134 } else if (half_word) { 2135 emitw(mask.value_); 2136 } else { 2137 emit(mask); 2138 } 2139 } 2140 2141 void Assembler::emit_test(const Operand& op, Immediate mask, int size) { 2142 if (is_uint8(mask.value_)) { 2143 size = sizeof(int8_t); 2144 } else if (is_uint16(mask.value_)) { 2145 size = sizeof(int16_t); 2146 } 2147 EnsureSpace ensure_space(this); 2148 bool half_word = size == sizeof(int16_t); 2149 if (half_word) { 2150 emit(0x66); 2151 size = sizeof(int32_t); 2152 } 2153 bool byte_operand = size == sizeof(int8_t); 2154 if (byte_operand) { 2155 size = sizeof(int32_t); 2156 } 2157 emit_rex(rax, op, size); 2158 emit(byte_operand ? 0xF6 : 0xF7); 2159 emit_operand(rax, op); // Operation code 0 2160 if (byte_operand) { 2161 emit(mask.value_); 2162 } else if (half_word) { 2163 emitw(mask.value_); 2164 } else { 2165 emit(mask); 2166 } 2167 } 2168 2169 void Assembler::emit_test(const Operand& op, Register reg, int size) { 2170 EnsureSpace ensure_space(this); 2171 if (size == sizeof(int16_t)) { 2172 emit(0x66); 2173 size = sizeof(int32_t); 2174 } 2175 bool byte_operand = size == sizeof(int8_t); 2176 if (byte_operand) { 2177 size = sizeof(int32_t); 2178 if (!reg.is_byte_register()) { 2179 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2180 emit_rex_32(reg, op); 2181 } else { 2182 emit_optional_rex_32(reg, op); 2183 } 2184 } else { 2185 emit_rex(reg, op, size); 2186 } 2187 emit(byte_operand ? 0x84 : 0x85); 2188 emit_operand(reg, op); 2189 } 2190 2191 2192 // FPU instructions. 2193 2194 2195 void Assembler::fld(int i) { 2196 EnsureSpace ensure_space(this); 2197 emit_farith(0xD9, 0xC0, i); 2198 } 2199 2200 2201 void Assembler::fld1() { 2202 EnsureSpace ensure_space(this); 2203 emit(0xD9); 2204 emit(0xE8); 2205 } 2206 2207 2208 void Assembler::fldz() { 2209 EnsureSpace ensure_space(this); 2210 emit(0xD9); 2211 emit(0xEE); 2212 } 2213 2214 2215 void Assembler::fldpi() { 2216 EnsureSpace ensure_space(this); 2217 emit(0xD9); 2218 emit(0xEB); 2219 } 2220 2221 2222 void Assembler::fldln2() { 2223 EnsureSpace ensure_space(this); 2224 emit(0xD9); 2225 emit(0xED); 2226 } 2227 2228 2229 void Assembler::fld_s(const Operand& adr) { 2230 EnsureSpace ensure_space(this); 2231 emit_optional_rex_32(adr); 2232 emit(0xD9); 2233 emit_operand(0, adr); 2234 } 2235 2236 2237 void Assembler::fld_d(const Operand& adr) { 2238 EnsureSpace ensure_space(this); 2239 emit_optional_rex_32(adr); 2240 emit(0xDD); 2241 emit_operand(0, adr); 2242 } 2243 2244 2245 void Assembler::fstp_s(const Operand& adr) { 2246 EnsureSpace ensure_space(this); 2247 emit_optional_rex_32(adr); 2248 emit(0xD9); 2249 emit_operand(3, adr); 2250 } 2251 2252 2253 void Assembler::fstp_d(const Operand& adr) { 2254 EnsureSpace ensure_space(this); 2255 emit_optional_rex_32(adr); 2256 emit(0xDD); 2257 emit_operand(3, adr); 2258 } 2259 2260 2261 void Assembler::fstp(int index) { 2262 DCHECK(is_uint3(index)); 2263 EnsureSpace ensure_space(this); 2264 emit_farith(0xDD, 0xD8, index); 2265 } 2266 2267 2268 void Assembler::fild_s(const Operand& adr) { 2269 EnsureSpace ensure_space(this); 2270 emit_optional_rex_32(adr); 2271 emit(0xDB); 2272 emit_operand(0, adr); 2273 } 2274 2275 2276 void Assembler::fild_d(const Operand& adr) { 2277 EnsureSpace ensure_space(this); 2278 emit_optional_rex_32(adr); 2279 emit(0xDF); 2280 emit_operand(5, adr); 2281 } 2282 2283 2284 void Assembler::fistp_s(const Operand& adr) { 2285 EnsureSpace ensure_space(this); 2286 emit_optional_rex_32(adr); 2287 emit(0xDB); 2288 emit_operand(3, adr); 2289 } 2290 2291 2292 void Assembler::fisttp_s(const Operand& adr) { 2293 DCHECK(IsEnabled(SSE3)); 2294 EnsureSpace ensure_space(this); 2295 emit_optional_rex_32(adr); 2296 emit(0xDB); 2297 emit_operand(1, adr); 2298 } 2299 2300 2301 void Assembler::fisttp_d(const Operand& adr) { 2302 DCHECK(IsEnabled(SSE3)); 2303 EnsureSpace ensure_space(this); 2304 emit_optional_rex_32(adr); 2305 emit(0xDD); 2306 emit_operand(1, adr); 2307 } 2308 2309 2310 void Assembler::fist_s(const Operand& adr) { 2311 EnsureSpace ensure_space(this); 2312 emit_optional_rex_32(adr); 2313 emit(0xDB); 2314 emit_operand(2, adr); 2315 } 2316 2317 2318 void Assembler::fistp_d(const Operand& adr) { 2319 EnsureSpace ensure_space(this); 2320 emit_optional_rex_32(adr); 2321 emit(0xDF); 2322 emit_operand(7, adr); 2323 } 2324 2325 2326 void Assembler::fabs() { 2327 EnsureSpace ensure_space(this); 2328 emit(0xD9); 2329 emit(0xE1); 2330 } 2331 2332 2333 void Assembler::fchs() { 2334 EnsureSpace ensure_space(this); 2335 emit(0xD9); 2336 emit(0xE0); 2337 } 2338 2339 2340 void Assembler::fcos() { 2341 EnsureSpace ensure_space(this); 2342 emit(0xD9); 2343 emit(0xFF); 2344 } 2345 2346 2347 void Assembler::fsin() { 2348 EnsureSpace ensure_space(this); 2349 emit(0xD9); 2350 emit(0xFE); 2351 } 2352 2353 2354 void Assembler::fptan() { 2355 EnsureSpace ensure_space(this); 2356 emit(0xD9); 2357 emit(0xF2); 2358 } 2359 2360 2361 void Assembler::fyl2x() { 2362 EnsureSpace ensure_space(this); 2363 emit(0xD9); 2364 emit(0xF1); 2365 } 2366 2367 2368 void Assembler::f2xm1() { 2369 EnsureSpace ensure_space(this); 2370 emit(0xD9); 2371 emit(0xF0); 2372 } 2373 2374 2375 void Assembler::fscale() { 2376 EnsureSpace ensure_space(this); 2377 emit(0xD9); 2378 emit(0xFD); 2379 } 2380 2381 2382 void Assembler::fninit() { 2383 EnsureSpace ensure_space(this); 2384 emit(0xDB); 2385 emit(0xE3); 2386 } 2387 2388 2389 void Assembler::fadd(int i) { 2390 EnsureSpace ensure_space(this); 2391 emit_farith(0xDC, 0xC0, i); 2392 } 2393 2394 2395 void Assembler::fsub(int i) { 2396 EnsureSpace ensure_space(this); 2397 emit_farith(0xDC, 0xE8, i); 2398 } 2399 2400 2401 void Assembler::fisub_s(const Operand& adr) { 2402 EnsureSpace ensure_space(this); 2403 emit_optional_rex_32(adr); 2404 emit(0xDA); 2405 emit_operand(4, adr); 2406 } 2407 2408 2409 void Assembler::fmul(int i) { 2410 EnsureSpace ensure_space(this); 2411 emit_farith(0xDC, 0xC8, i); 2412 } 2413 2414 2415 void Assembler::fdiv(int i) { 2416 EnsureSpace ensure_space(this); 2417 emit_farith(0xDC, 0xF8, i); 2418 } 2419 2420 2421 void Assembler::faddp(int i) { 2422 EnsureSpace ensure_space(this); 2423 emit_farith(0xDE, 0xC0, i); 2424 } 2425 2426 2427 void Assembler::fsubp(int i) { 2428 EnsureSpace ensure_space(this); 2429 emit_farith(0xDE, 0xE8, i); 2430 } 2431 2432 2433 void Assembler::fsubrp(int i) { 2434 EnsureSpace ensure_space(this); 2435 emit_farith(0xDE, 0xE0, i); 2436 } 2437 2438 2439 void Assembler::fmulp(int i) { 2440 EnsureSpace ensure_space(this); 2441 emit_farith(0xDE, 0xC8, i); 2442 } 2443 2444 2445 void Assembler::fdivp(int i) { 2446 EnsureSpace ensure_space(this); 2447 emit_farith(0xDE, 0xF8, i); 2448 } 2449 2450 2451 void Assembler::fprem() { 2452 EnsureSpace ensure_space(this); 2453 emit(0xD9); 2454 emit(0xF8); 2455 } 2456 2457 2458 void Assembler::fprem1() { 2459 EnsureSpace ensure_space(this); 2460 emit(0xD9); 2461 emit(0xF5); 2462 } 2463 2464 2465 void Assembler::fxch(int i) { 2466 EnsureSpace ensure_space(this); 2467 emit_farith(0xD9, 0xC8, i); 2468 } 2469 2470 2471 void Assembler::fincstp() { 2472 EnsureSpace ensure_space(this); 2473 emit(0xD9); 2474 emit(0xF7); 2475 } 2476 2477 2478 void Assembler::ffree(int i) { 2479 EnsureSpace ensure_space(this); 2480 emit_farith(0xDD, 0xC0, i); 2481 } 2482 2483 2484 void Assembler::ftst() { 2485 EnsureSpace ensure_space(this); 2486 emit(0xD9); 2487 emit(0xE4); 2488 } 2489 2490 2491 void Assembler::fucomp(int i) { 2492 EnsureSpace ensure_space(this); 2493 emit_farith(0xDD, 0xE8, i); 2494 } 2495 2496 2497 void Assembler::fucompp() { 2498 EnsureSpace ensure_space(this); 2499 emit(0xDA); 2500 emit(0xE9); 2501 } 2502 2503 2504 void Assembler::fucomi(int i) { 2505 EnsureSpace ensure_space(this); 2506 emit(0xDB); 2507 emit(0xE8 + i); 2508 } 2509 2510 2511 void Assembler::fucomip() { 2512 EnsureSpace ensure_space(this); 2513 emit(0xDF); 2514 emit(0xE9); 2515 } 2516 2517 2518 void Assembler::fcompp() { 2519 EnsureSpace ensure_space(this); 2520 emit(0xDE); 2521 emit(0xD9); 2522 } 2523 2524 2525 void Assembler::fnstsw_ax() { 2526 EnsureSpace ensure_space(this); 2527 emit(0xDF); 2528 emit(0xE0); 2529 } 2530 2531 2532 void Assembler::fwait() { 2533 EnsureSpace ensure_space(this); 2534 emit(0x9B); 2535 } 2536 2537 2538 void Assembler::frndint() { 2539 EnsureSpace ensure_space(this); 2540 emit(0xD9); 2541 emit(0xFC); 2542 } 2543 2544 2545 void Assembler::fnclex() { 2546 EnsureSpace ensure_space(this); 2547 emit(0xDB); 2548 emit(0xE2); 2549 } 2550 2551 2552 void Assembler::sahf() { 2553 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf 2554 // in 64-bit mode. Test CpuID. 2555 DCHECK(IsEnabled(SAHF)); 2556 EnsureSpace ensure_space(this); 2557 emit(0x9E); 2558 } 2559 2560 2561 void Assembler::emit_farith(int b1, int b2, int i) { 2562 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode 2563 DCHECK(is_uint3(i)); // illegal stack offset 2564 emit(b1); 2565 emit(b2 + i); 2566 } 2567 2568 2569 // SSE operations. 2570 2571 void Assembler::andps(XMMRegister dst, XMMRegister src) { 2572 EnsureSpace ensure_space(this); 2573 emit_optional_rex_32(dst, src); 2574 emit(0x0F); 2575 emit(0x54); 2576 emit_sse_operand(dst, src); 2577 } 2578 2579 2580 void Assembler::andps(XMMRegister dst, const Operand& src) { 2581 EnsureSpace ensure_space(this); 2582 emit_optional_rex_32(dst, src); 2583 emit(0x0F); 2584 emit(0x54); 2585 emit_sse_operand(dst, src); 2586 } 2587 2588 2589 void Assembler::orps(XMMRegister dst, XMMRegister src) { 2590 EnsureSpace ensure_space(this); 2591 emit_optional_rex_32(dst, src); 2592 emit(0x0F); 2593 emit(0x56); 2594 emit_sse_operand(dst, src); 2595 } 2596 2597 2598 void Assembler::orps(XMMRegister dst, const Operand& src) { 2599 EnsureSpace ensure_space(this); 2600 emit_optional_rex_32(dst, src); 2601 emit(0x0F); 2602 emit(0x56); 2603 emit_sse_operand(dst, src); 2604 } 2605 2606 2607 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 2608 DCHECK(!IsEnabled(AVX)); 2609 EnsureSpace ensure_space(this); 2610 emit_optional_rex_32(dst, src); 2611 emit(0x0F); 2612 emit(0x57); 2613 emit_sse_operand(dst, src); 2614 } 2615 2616 2617 void Assembler::xorps(XMMRegister dst, const Operand& src) { 2618 DCHECK(!IsEnabled(AVX)); 2619 EnsureSpace ensure_space(this); 2620 emit_optional_rex_32(dst, src); 2621 emit(0x0F); 2622 emit(0x57); 2623 emit_sse_operand(dst, src); 2624 } 2625 2626 2627 void Assembler::addps(XMMRegister dst, XMMRegister src) { 2628 EnsureSpace ensure_space(this); 2629 emit_optional_rex_32(dst, src); 2630 emit(0x0F); 2631 emit(0x58); 2632 emit_sse_operand(dst, src); 2633 } 2634 2635 2636 void Assembler::addps(XMMRegister dst, const Operand& src) { 2637 EnsureSpace ensure_space(this); 2638 emit_optional_rex_32(dst, src); 2639 emit(0x0F); 2640 emit(0x58); 2641 emit_sse_operand(dst, src); 2642 } 2643 2644 2645 void Assembler::subps(XMMRegister dst, XMMRegister src) { 2646 EnsureSpace ensure_space(this); 2647 emit_optional_rex_32(dst, src); 2648 emit(0x0F); 2649 emit(0x5C); 2650 emit_sse_operand(dst, src); 2651 } 2652 2653 2654 void Assembler::subps(XMMRegister dst, const Operand& src) { 2655 EnsureSpace ensure_space(this); 2656 emit_optional_rex_32(dst, src); 2657 emit(0x0F); 2658 emit(0x5C); 2659 emit_sse_operand(dst, src); 2660 } 2661 2662 2663 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 2664 EnsureSpace ensure_space(this); 2665 emit_optional_rex_32(dst, src); 2666 emit(0x0F); 2667 emit(0x59); 2668 emit_sse_operand(dst, src); 2669 } 2670 2671 2672 void Assembler::mulps(XMMRegister dst, const Operand& src) { 2673 EnsureSpace ensure_space(this); 2674 emit_optional_rex_32(dst, src); 2675 emit(0x0F); 2676 emit(0x59); 2677 emit_sse_operand(dst, src); 2678 } 2679 2680 2681 void Assembler::divps(XMMRegister dst, XMMRegister src) { 2682 EnsureSpace ensure_space(this); 2683 emit_optional_rex_32(dst, src); 2684 emit(0x0F); 2685 emit(0x5E); 2686 emit_sse_operand(dst, src); 2687 } 2688 2689 2690 void Assembler::divps(XMMRegister dst, const Operand& src) { 2691 EnsureSpace ensure_space(this); 2692 emit_optional_rex_32(dst, src); 2693 emit(0x0F); 2694 emit(0x5E); 2695 emit_sse_operand(dst, src); 2696 } 2697 2698 2699 // SSE 2 operations. 2700 2701 void Assembler::movd(XMMRegister dst, Register src) { 2702 DCHECK(!IsEnabled(AVX)); 2703 EnsureSpace ensure_space(this); 2704 emit(0x66); 2705 emit_optional_rex_32(dst, src); 2706 emit(0x0F); 2707 emit(0x6E); 2708 emit_sse_operand(dst, src); 2709 } 2710 2711 2712 void Assembler::movd(XMMRegister dst, const Operand& src) { 2713 DCHECK(!IsEnabled(AVX)); 2714 EnsureSpace ensure_space(this); 2715 emit(0x66); 2716 emit_optional_rex_32(dst, src); 2717 emit(0x0F); 2718 emit(0x6E); 2719 emit_sse_operand(dst, src); 2720 } 2721 2722 2723 void Assembler::movd(Register dst, XMMRegister src) { 2724 DCHECK(!IsEnabled(AVX)); 2725 EnsureSpace ensure_space(this); 2726 emit(0x66); 2727 emit_optional_rex_32(src, dst); 2728 emit(0x0F); 2729 emit(0x7E); 2730 emit_sse_operand(src, dst); 2731 } 2732 2733 2734 void Assembler::movq(XMMRegister dst, Register src) { 2735 DCHECK(!IsEnabled(AVX)); 2736 EnsureSpace ensure_space(this); 2737 emit(0x66); 2738 emit_rex_64(dst, src); 2739 emit(0x0F); 2740 emit(0x6E); 2741 emit_sse_operand(dst, src); 2742 } 2743 2744 2745 void Assembler::movq(Register dst, XMMRegister src) { 2746 DCHECK(!IsEnabled(AVX)); 2747 EnsureSpace ensure_space(this); 2748 emit(0x66); 2749 emit_rex_64(src, dst); 2750 emit(0x0F); 2751 emit(0x7E); 2752 emit_sse_operand(src, dst); 2753 } 2754 2755 2756 void Assembler::movq(XMMRegister dst, XMMRegister src) { 2757 DCHECK(!IsEnabled(AVX)); 2758 EnsureSpace ensure_space(this); 2759 if (dst.low_bits() == 4) { 2760 // Avoid unnecessary SIB byte. 2761 emit(0xf3); 2762 emit_optional_rex_32(dst, src); 2763 emit(0x0F); 2764 emit(0x7e); 2765 emit_sse_operand(dst, src); 2766 } else { 2767 emit(0x66); 2768 emit_optional_rex_32(src, dst); 2769 emit(0x0F); 2770 emit(0xD6); 2771 emit_sse_operand(src, dst); 2772 } 2773 } 2774 2775 2776 void Assembler::movdqa(const Operand& dst, XMMRegister src) { 2777 EnsureSpace ensure_space(this); 2778 emit(0x66); 2779 emit_rex_64(src, dst); 2780 emit(0x0F); 2781 emit(0x7F); 2782 emit_sse_operand(src, dst); 2783 } 2784 2785 2786 void Assembler::movdqa(XMMRegister dst, const Operand& src) { 2787 EnsureSpace ensure_space(this); 2788 emit(0x66); 2789 emit_rex_64(dst, src); 2790 emit(0x0F); 2791 emit(0x6F); 2792 emit_sse_operand(dst, src); 2793 } 2794 2795 2796 void Assembler::movdqu(const Operand& dst, XMMRegister src) { 2797 EnsureSpace ensure_space(this); 2798 emit(0xF3); 2799 emit_rex_64(src, dst); 2800 emit(0x0F); 2801 emit(0x7F); 2802 emit_sse_operand(src, dst); 2803 } 2804 2805 2806 void Assembler::movdqu(XMMRegister dst, const Operand& src) { 2807 EnsureSpace ensure_space(this); 2808 emit(0xF3); 2809 emit_rex_64(dst, src); 2810 emit(0x0F); 2811 emit(0x6F); 2812 emit_sse_operand(dst, src); 2813 } 2814 2815 2816 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) { 2817 DCHECK(IsEnabled(SSE4_1)); 2818 DCHECK(is_uint8(imm8)); 2819 EnsureSpace ensure_space(this); 2820 emit(0x66); 2821 emit_optional_rex_32(src, dst); 2822 emit(0x0F); 2823 emit(0x3A); 2824 emit(0x17); 2825 emit_sse_operand(src, dst); 2826 emit(imm8); 2827 } 2828 2829 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) { 2830 DCHECK(IsEnabled(SSE4_1)); 2831 DCHECK(is_uint8(imm8)); 2832 EnsureSpace ensure_space(this); 2833 emit(0x66); 2834 emit_optional_rex_32(src, dst); 2835 emit(0x0F); 2836 emit(0x3A); 2837 emit(0x14); 2838 emit_sse_operand(src, dst); 2839 emit(imm8); 2840 } 2841 2842 void Assembler::pextrb(const Operand& dst, XMMRegister src, int8_t imm8) { 2843 DCHECK(IsEnabled(SSE4_1)); 2844 DCHECK(is_uint8(imm8)); 2845 EnsureSpace ensure_space(this); 2846 emit(0x66); 2847 emit_optional_rex_32(src, dst); 2848 emit(0x0F); 2849 emit(0x3A); 2850 emit(0x14); 2851 emit_sse_operand(src, dst); 2852 emit(imm8); 2853 } 2854 2855 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) { 2856 DCHECK(is_uint8(imm8)); 2857 EnsureSpace ensure_space(this); 2858 emit(0x66); 2859 emit_optional_rex_32(dst, src); 2860 emit(0x0F); 2861 emit(0xC4); 2862 emit_sse_operand(dst, src); 2863 emit(imm8); 2864 } 2865 2866 void Assembler::pinsrw(XMMRegister dst, const Operand& src, int8_t imm8) { 2867 DCHECK(is_uint8(imm8)); 2868 EnsureSpace ensure_space(this); 2869 emit(0x66); 2870 emit_optional_rex_32(dst, src); 2871 emit(0x0F); 2872 emit(0xC4); 2873 emit_sse_operand(dst, src); 2874 emit(imm8); 2875 } 2876 2877 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) { 2878 DCHECK(is_uint8(imm8)); 2879 EnsureSpace ensure_space(this); 2880 emit(0x66); 2881 emit_optional_rex_32(src, dst); 2882 emit(0x0F); 2883 emit(0xC5); 2884 emit_sse_operand(src, dst); 2885 emit(imm8); 2886 } 2887 2888 void Assembler::pextrw(const Operand& dst, XMMRegister src, int8_t imm8) { 2889 DCHECK(IsEnabled(SSE4_1)); 2890 DCHECK(is_uint8(imm8)); 2891 EnsureSpace ensure_space(this); 2892 emit(0x66); 2893 emit_optional_rex_32(src, dst); 2894 emit(0x0F); 2895 emit(0x3A); 2896 emit(0x15); 2897 emit_sse_operand(src, dst); 2898 emit(imm8); 2899 } 2900 2901 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) { 2902 DCHECK(IsEnabled(SSE4_1)); 2903 EnsureSpace ensure_space(this); 2904 emit(0x66); 2905 emit_optional_rex_32(src, dst); 2906 emit(0x0F); 2907 emit(0x3A); 2908 emit(0x16); 2909 emit_sse_operand(src, dst); 2910 emit(imm8); 2911 } 2912 2913 void Assembler::pextrd(const Operand& dst, XMMRegister src, int8_t imm8) { 2914 DCHECK(IsEnabled(SSE4_1)); 2915 EnsureSpace ensure_space(this); 2916 emit(0x66); 2917 emit_optional_rex_32(src, dst); 2918 emit(0x0F); 2919 emit(0x3A); 2920 emit(0x16); 2921 emit_sse_operand(src, dst); 2922 emit(imm8); 2923 } 2924 2925 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) { 2926 DCHECK(IsEnabled(SSE4_1)); 2927 EnsureSpace ensure_space(this); 2928 emit(0x66); 2929 emit_optional_rex_32(dst, src); 2930 emit(0x0F); 2931 emit(0x3A); 2932 emit(0x22); 2933 emit_sse_operand(dst, src); 2934 emit(imm8); 2935 } 2936 2937 2938 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) { 2939 DCHECK(IsEnabled(SSE4_1)); 2940 EnsureSpace ensure_space(this); 2941 emit(0x66); 2942 emit_optional_rex_32(dst, src); 2943 emit(0x0F); 2944 emit(0x3A); 2945 emit(0x22); 2946 emit_sse_operand(dst, src); 2947 emit(imm8); 2948 } 2949 2950 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) { 2951 DCHECK(IsEnabled(SSE4_1)); 2952 EnsureSpace ensure_space(this); 2953 emit(0x66); 2954 emit_optional_rex_32(dst, src); 2955 emit(0x0F); 2956 emit(0x3A); 2957 emit(0x20); 2958 emit_sse_operand(dst, src); 2959 emit(imm8); 2960 } 2961 2962 void Assembler::pinsrb(XMMRegister dst, const Operand& src, int8_t imm8) { 2963 DCHECK(IsEnabled(SSE4_1)); 2964 EnsureSpace ensure_space(this); 2965 emit(0x66); 2966 emit_optional_rex_32(dst, src); 2967 emit(0x0F); 2968 emit(0x3A); 2969 emit(0x20); 2970 emit_sse_operand(dst, src); 2971 emit(imm8); 2972 } 2973 2974 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) { 2975 DCHECK(CpuFeatures::IsSupported(SSE4_1)); 2976 DCHECK(is_uint8(imm8)); 2977 EnsureSpace ensure_space(this); 2978 emit(0x66); 2979 emit_optional_rex_32(dst, src); 2980 emit(0x0F); 2981 emit(0x3A); 2982 emit(0x21); 2983 emit_sse_operand(dst, src); 2984 emit(imm8); 2985 } 2986 2987 void Assembler::movsd(const Operand& dst, XMMRegister src) { 2988 DCHECK(!IsEnabled(AVX)); 2989 EnsureSpace ensure_space(this); 2990 emit(0xF2); // double 2991 emit_optional_rex_32(src, dst); 2992 emit(0x0F); 2993 emit(0x11); // store 2994 emit_sse_operand(src, dst); 2995 } 2996 2997 2998 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 2999 DCHECK(!IsEnabled(AVX)); 3000 EnsureSpace ensure_space(this); 3001 emit(0xF2); // double 3002 emit_optional_rex_32(dst, src); 3003 emit(0x0F); 3004 emit(0x10); // load 3005 emit_sse_operand(dst, src); 3006 } 3007 3008 3009 void Assembler::movsd(XMMRegister dst, const Operand& src) { 3010 DCHECK(!IsEnabled(AVX)); 3011 EnsureSpace ensure_space(this); 3012 emit(0xF2); // double 3013 emit_optional_rex_32(dst, src); 3014 emit(0x0F); 3015 emit(0x10); // load 3016 emit_sse_operand(dst, src); 3017 } 3018 3019 3020 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 3021 DCHECK(!IsEnabled(AVX)); 3022 EnsureSpace ensure_space(this); 3023 if (src.low_bits() == 4) { 3024 // Try to avoid an unnecessary SIB byte. 3025 emit_optional_rex_32(src, dst); 3026 emit(0x0F); 3027 emit(0x29); 3028 emit_sse_operand(src, dst); 3029 } else { 3030 emit_optional_rex_32(dst, src); 3031 emit(0x0F); 3032 emit(0x28); 3033 emit_sse_operand(dst, src); 3034 } 3035 } 3036 3037 3038 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) { 3039 DCHECK(is_uint8(imm8)); 3040 EnsureSpace ensure_space(this); 3041 emit_optional_rex_32(dst, src); 3042 emit(0x0F); 3043 emit(0xC6); 3044 emit_sse_operand(dst, src); 3045 emit(imm8); 3046 } 3047 3048 3049 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 3050 DCHECK(!IsEnabled(AVX)); 3051 EnsureSpace ensure_space(this); 3052 if (src.low_bits() == 4) { 3053 // Try to avoid an unnecessary SIB byte. 3054 emit(0x66); 3055 emit_optional_rex_32(src, dst); 3056 emit(0x0F); 3057 emit(0x29); 3058 emit_sse_operand(src, dst); 3059 } else { 3060 emit(0x66); 3061 emit_optional_rex_32(dst, src); 3062 emit(0x0F); 3063 emit(0x28); 3064 emit_sse_operand(dst, src); 3065 } 3066 } 3067 3068 3069 void Assembler::movupd(XMMRegister dst, const Operand& src) { 3070 EnsureSpace ensure_space(this); 3071 emit(0x66); 3072 emit_optional_rex_32(dst, src); 3073 emit(0x0F); 3074 emit(0x10); 3075 emit_sse_operand(dst, src); 3076 } 3077 3078 void Assembler::movupd(const Operand& dst, XMMRegister src) { 3079 EnsureSpace ensure_space(this); 3080 emit(0x66); 3081 emit_optional_rex_32(src, dst); 3082 emit(0x0F); 3083 emit(0x11); 3084 emit_sse_operand(src, dst); 3085 } 3086 3087 void Assembler::addss(XMMRegister dst, XMMRegister src) { 3088 EnsureSpace ensure_space(this); 3089 emit(0xF3); 3090 emit_optional_rex_32(dst, src); 3091 emit(0x0F); 3092 emit(0x58); 3093 emit_sse_operand(dst, src); 3094 } 3095 3096 3097 void Assembler::addss(XMMRegister dst, const Operand& src) { 3098 EnsureSpace ensure_space(this); 3099 emit(0xF3); 3100 emit_optional_rex_32(dst, src); 3101 emit(0x0F); 3102 emit(0x58); 3103 emit_sse_operand(dst, src); 3104 } 3105 3106 3107 void Assembler::subss(XMMRegister dst, XMMRegister src) { 3108 EnsureSpace ensure_space(this); 3109 emit(0xF3); 3110 emit_optional_rex_32(dst, src); 3111 emit(0x0F); 3112 emit(0x5C); 3113 emit_sse_operand(dst, src); 3114 } 3115 3116 3117 void Assembler::subss(XMMRegister dst, const Operand& src) { 3118 EnsureSpace ensure_space(this); 3119 emit(0xF3); 3120 emit_optional_rex_32(dst, src); 3121 emit(0x0F); 3122 emit(0x5C); 3123 emit_sse_operand(dst, src); 3124 } 3125 3126 3127 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 3128 EnsureSpace ensure_space(this); 3129 emit(0xF3); 3130 emit_optional_rex_32(dst, src); 3131 emit(0x0F); 3132 emit(0x59); 3133 emit_sse_operand(dst, src); 3134 } 3135 3136 3137 void Assembler::mulss(XMMRegister dst, const Operand& src) { 3138 EnsureSpace ensure_space(this); 3139 emit(0xF3); 3140 emit_optional_rex_32(dst, src); 3141 emit(0x0F); 3142 emit(0x59); 3143 emit_sse_operand(dst, src); 3144 } 3145 3146 3147 void Assembler::divss(XMMRegister dst, XMMRegister src) { 3148 EnsureSpace ensure_space(this); 3149 emit(0xF3); 3150 emit_optional_rex_32(dst, src); 3151 emit(0x0F); 3152 emit(0x5E); 3153 emit_sse_operand(dst, src); 3154 } 3155 3156 3157 void Assembler::divss(XMMRegister dst, const Operand& src) { 3158 EnsureSpace ensure_space(this); 3159 emit(0xF3); 3160 emit_optional_rex_32(dst, src); 3161 emit(0x0F); 3162 emit(0x5E); 3163 emit_sse_operand(dst, src); 3164 } 3165 3166 3167 void Assembler::maxss(XMMRegister dst, XMMRegister src) { 3168 EnsureSpace ensure_space(this); 3169 emit(0xF3); 3170 emit_optional_rex_32(dst, src); 3171 emit(0x0F); 3172 emit(0x5F); 3173 emit_sse_operand(dst, src); 3174 } 3175 3176 3177 void Assembler::maxss(XMMRegister dst, const Operand& src) { 3178 EnsureSpace ensure_space(this); 3179 emit(0xF3); 3180 emit_optional_rex_32(dst, src); 3181 emit(0x0F); 3182 emit(0x5F); 3183 emit_sse_operand(dst, src); 3184 } 3185 3186 3187 void Assembler::minss(XMMRegister dst, XMMRegister src) { 3188 EnsureSpace ensure_space(this); 3189 emit(0xF3); 3190 emit_optional_rex_32(dst, src); 3191 emit(0x0F); 3192 emit(0x5D); 3193 emit_sse_operand(dst, src); 3194 } 3195 3196 3197 void Assembler::minss(XMMRegister dst, const Operand& src) { 3198 EnsureSpace ensure_space(this); 3199 emit(0xF3); 3200 emit_optional_rex_32(dst, src); 3201 emit(0x0F); 3202 emit(0x5D); 3203 emit_sse_operand(dst, src); 3204 } 3205 3206 3207 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 3208 EnsureSpace ensure_space(this); 3209 emit(0xF3); 3210 emit_optional_rex_32(dst, src); 3211 emit(0x0F); 3212 emit(0x51); 3213 emit_sse_operand(dst, src); 3214 } 3215 3216 3217 void Assembler::sqrtss(XMMRegister dst, const Operand& src) { 3218 EnsureSpace ensure_space(this); 3219 emit(0xF3); 3220 emit_optional_rex_32(dst, src); 3221 emit(0x0F); 3222 emit(0x51); 3223 emit_sse_operand(dst, src); 3224 } 3225 3226 3227 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 3228 DCHECK(!IsEnabled(AVX)); 3229 EnsureSpace ensure_space(this); 3230 emit_optional_rex_32(dst, src); 3231 emit(0x0f); 3232 emit(0x2e); 3233 emit_sse_operand(dst, src); 3234 } 3235 3236 3237 void Assembler::ucomiss(XMMRegister dst, const Operand& src) { 3238 DCHECK(!IsEnabled(AVX)); 3239 EnsureSpace ensure_space(this); 3240 emit_optional_rex_32(dst, src); 3241 emit(0x0f); 3242 emit(0x2e); 3243 emit_sse_operand(dst, src); 3244 } 3245 3246 3247 void Assembler::movss(XMMRegister dst, XMMRegister src) { 3248 DCHECK(!IsEnabled(AVX)); 3249 EnsureSpace ensure_space(this); 3250 emit(0xF3); // single 3251 emit_optional_rex_32(dst, src); 3252 emit(0x0F); 3253 emit(0x10); // load 3254 emit_sse_operand(dst, src); 3255 } 3256 3257 3258 void Assembler::movss(XMMRegister dst, const Operand& src) { 3259 DCHECK(!IsEnabled(AVX)); 3260 EnsureSpace ensure_space(this); 3261 emit(0xF3); // single 3262 emit_optional_rex_32(dst, src); 3263 emit(0x0F); 3264 emit(0x10); // load 3265 emit_sse_operand(dst, src); 3266 } 3267 3268 3269 void Assembler::movss(const Operand& src, XMMRegister dst) { 3270 DCHECK(!IsEnabled(AVX)); 3271 EnsureSpace ensure_space(this); 3272 emit(0xF3); // single 3273 emit_optional_rex_32(dst, src); 3274 emit(0x0F); 3275 emit(0x11); // store 3276 emit_sse_operand(dst, src); 3277 } 3278 3279 3280 void Assembler::psllq(XMMRegister reg, byte imm8) { 3281 DCHECK(!IsEnabled(AVX)); 3282 EnsureSpace ensure_space(this); 3283 emit(0x66); 3284 emit_optional_rex_32(reg); 3285 emit(0x0F); 3286 emit(0x73); 3287 emit_sse_operand(rsi, reg); // rsi == 6 3288 emit(imm8); 3289 } 3290 3291 3292 void Assembler::psrlq(XMMRegister reg, byte imm8) { 3293 DCHECK(!IsEnabled(AVX)); 3294 EnsureSpace ensure_space(this); 3295 emit(0x66); 3296 emit_optional_rex_32(reg); 3297 emit(0x0F); 3298 emit(0x73); 3299 emit_sse_operand(rdx, reg); // rdx == 2 3300 emit(imm8); 3301 } 3302 3303 void Assembler::psllw(XMMRegister reg, byte imm8) { 3304 EnsureSpace ensure_space(this); 3305 emit(0x66); 3306 emit_optional_rex_32(reg); 3307 emit(0x0F); 3308 emit(0x71); 3309 emit_sse_operand(rsi, reg); // rsi == 6 3310 emit(imm8); 3311 } 3312 3313 void Assembler::pslld(XMMRegister reg, byte imm8) { 3314 EnsureSpace ensure_space(this); 3315 emit(0x66); 3316 emit_optional_rex_32(reg); 3317 emit(0x0F); 3318 emit(0x72); 3319 emit_sse_operand(rsi, reg); // rsi == 6 3320 emit(imm8); 3321 } 3322 3323 void Assembler::psrlw(XMMRegister reg, byte imm8) { 3324 EnsureSpace ensure_space(this); 3325 emit(0x66); 3326 emit_optional_rex_32(reg); 3327 emit(0x0F); 3328 emit(0x71); 3329 emit_sse_operand(rdx, reg); // rdx == 2 3330 emit(imm8); 3331 } 3332 3333 void Assembler::psrld(XMMRegister reg, byte imm8) { 3334 EnsureSpace ensure_space(this); 3335 emit(0x66); 3336 emit_optional_rex_32(reg); 3337 emit(0x0F); 3338 emit(0x72); 3339 emit_sse_operand(rdx, reg); // rdx == 2 3340 emit(imm8); 3341 } 3342 3343 void Assembler::psraw(XMMRegister reg, byte imm8) { 3344 EnsureSpace ensure_space(this); 3345 emit(0x66); 3346 emit_optional_rex_32(reg); 3347 emit(0x0F); 3348 emit(0x71); 3349 emit_sse_operand(rsp, reg); // rsp == 4 3350 emit(imm8); 3351 } 3352 3353 void Assembler::psrad(XMMRegister reg, byte imm8) { 3354 EnsureSpace ensure_space(this); 3355 emit(0x66); 3356 emit_optional_rex_32(reg); 3357 emit(0x0F); 3358 emit(0x72); 3359 emit_sse_operand(rsp, reg); // rsp == 4 3360 emit(imm8); 3361 } 3362 3363 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) { 3364 EnsureSpace ensure_space(this); 3365 emit_optional_rex_32(dst, src); 3366 emit(0x0F); 3367 emit(0xC2); 3368 emit_sse_operand(dst, src); 3369 emit(cmp); 3370 } 3371 3372 void Assembler::cmpps(XMMRegister dst, const Operand& src, int8_t cmp) { 3373 EnsureSpace ensure_space(this); 3374 emit_optional_rex_32(dst, src); 3375 emit(0x0F); 3376 emit(0xC2); 3377 emit_sse_operand(dst, src); 3378 emit(cmp); 3379 } 3380 3381 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) { 3382 EnsureSpace ensure_space(this); 3383 emit_optional_rex_32(dst, src); 3384 emit(0x66); 3385 emit(0x0F); 3386 emit(0xC2); 3387 emit_sse_operand(dst, src); 3388 emit(cmp); 3389 } 3390 3391 void Assembler::cmppd(XMMRegister dst, const Operand& src, int8_t cmp) { 3392 EnsureSpace ensure_space(this); 3393 emit_optional_rex_32(dst, src); 3394 emit(0x66); 3395 emit(0x0F); 3396 emit(0xC2); 3397 emit_sse_operand(dst, src); 3398 emit(cmp); 3399 } 3400 3401 void Assembler::cvttss2si(Register dst, const Operand& src) { 3402 DCHECK(!IsEnabled(AVX)); 3403 EnsureSpace ensure_space(this); 3404 emit(0xF3); 3405 emit_optional_rex_32(dst, src); 3406 emit(0x0F); 3407 emit(0x2C); 3408 emit_operand(dst, src); 3409 } 3410 3411 3412 void Assembler::cvttss2si(Register dst, XMMRegister src) { 3413 DCHECK(!IsEnabled(AVX)); 3414 EnsureSpace ensure_space(this); 3415 emit(0xF3); 3416 emit_optional_rex_32(dst, src); 3417 emit(0x0F); 3418 emit(0x2C); 3419 emit_sse_operand(dst, src); 3420 } 3421 3422 3423 void Assembler::cvttsd2si(Register dst, const Operand& src) { 3424 DCHECK(!IsEnabled(AVX)); 3425 EnsureSpace ensure_space(this); 3426 emit(0xF2); 3427 emit_optional_rex_32(dst, src); 3428 emit(0x0F); 3429 emit(0x2C); 3430 emit_operand(dst, src); 3431 } 3432 3433 3434 void Assembler::cvttsd2si(Register dst, XMMRegister src) { 3435 DCHECK(!IsEnabled(AVX)); 3436 EnsureSpace ensure_space(this); 3437 emit(0xF2); 3438 emit_optional_rex_32(dst, src); 3439 emit(0x0F); 3440 emit(0x2C); 3441 emit_sse_operand(dst, src); 3442 } 3443 3444 3445 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 3446 DCHECK(!IsEnabled(AVX)); 3447 EnsureSpace ensure_space(this); 3448 emit(0xF3); 3449 emit_rex_64(dst, src); 3450 emit(0x0F); 3451 emit(0x2C); 3452 emit_sse_operand(dst, src); 3453 } 3454 3455 3456 void Assembler::cvttss2siq(Register dst, const Operand& src) { 3457 DCHECK(!IsEnabled(AVX)); 3458 EnsureSpace ensure_space(this); 3459 emit(0xF3); 3460 emit_rex_64(dst, src); 3461 emit(0x0F); 3462 emit(0x2C); 3463 emit_sse_operand(dst, src); 3464 } 3465 3466 3467 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 3468 DCHECK(!IsEnabled(AVX)); 3469 EnsureSpace ensure_space(this); 3470 emit(0xF2); 3471 emit_rex_64(dst, src); 3472 emit(0x0F); 3473 emit(0x2C); 3474 emit_sse_operand(dst, src); 3475 } 3476 3477 3478 void Assembler::cvttsd2siq(Register dst, const Operand& src) { 3479 DCHECK(!IsEnabled(AVX)); 3480 EnsureSpace ensure_space(this); 3481 emit(0xF2); 3482 emit_rex_64(dst, src); 3483 emit(0x0F); 3484 emit(0x2C); 3485 emit_sse_operand(dst, src); 3486 } 3487 3488 3489 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) { 3490 DCHECK(!IsEnabled(AVX)); 3491 EnsureSpace ensure_space(this); 3492 emit(0xF2); 3493 emit_optional_rex_32(dst, src); 3494 emit(0x0F); 3495 emit(0x2A); 3496 emit_sse_operand(dst, src); 3497 } 3498 3499 3500 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) { 3501 DCHECK(!IsEnabled(AVX)); 3502 EnsureSpace ensure_space(this); 3503 emit(0xF2); 3504 emit_optional_rex_32(dst, src); 3505 emit(0x0F); 3506 emit(0x2A); 3507 emit_sse_operand(dst, src); 3508 } 3509 3510 3511 void Assembler::cvtlsi2ss(XMMRegister dst, const Operand& src) { 3512 DCHECK(!IsEnabled(AVX)); 3513 EnsureSpace ensure_space(this); 3514 emit(0xF3); 3515 emit_optional_rex_32(dst, src); 3516 emit(0x0F); 3517 emit(0x2A); 3518 emit_sse_operand(dst, src); 3519 } 3520 3521 3522 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) { 3523 EnsureSpace ensure_space(this); 3524 emit(0xF3); 3525 emit_optional_rex_32(dst, src); 3526 emit(0x0F); 3527 emit(0x2A); 3528 emit_sse_operand(dst, src); 3529 } 3530 3531 3532 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) { 3533 DCHECK(!IsEnabled(AVX)); 3534 EnsureSpace ensure_space(this); 3535 emit(0xF3); 3536 emit_rex_64(dst, src); 3537 emit(0x0F); 3538 emit(0x2A); 3539 emit_sse_operand(dst, src); 3540 } 3541 3542 3543 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) { 3544 DCHECK(!IsEnabled(AVX)); 3545 EnsureSpace ensure_space(this); 3546 emit(0xF3); 3547 emit_rex_64(dst, src); 3548 emit(0x0F); 3549 emit(0x2A); 3550 emit_sse_operand(dst, src); 3551 } 3552 3553 3554 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) { 3555 DCHECK(!IsEnabled(AVX)); 3556 EnsureSpace ensure_space(this); 3557 emit(0xF2); 3558 emit_rex_64(dst, src); 3559 emit(0x0F); 3560 emit(0x2A); 3561 emit_sse_operand(dst, src); 3562 } 3563 3564 3565 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) { 3566 DCHECK(!IsEnabled(AVX)); 3567 EnsureSpace ensure_space(this); 3568 emit(0xF2); 3569 emit_rex_64(dst, src); 3570 emit(0x0F); 3571 emit(0x2A); 3572 emit_sse_operand(dst, src); 3573 } 3574 3575 3576 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 3577 DCHECK(!IsEnabled(AVX)); 3578 EnsureSpace ensure_space(this); 3579 emit(0xF3); 3580 emit_optional_rex_32(dst, src); 3581 emit(0x0F); 3582 emit(0x5A); 3583 emit_sse_operand(dst, src); 3584 } 3585 3586 3587 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) { 3588 DCHECK(!IsEnabled(AVX)); 3589 EnsureSpace ensure_space(this); 3590 emit(0xF3); 3591 emit_optional_rex_32(dst, src); 3592 emit(0x0F); 3593 emit(0x5A); 3594 emit_sse_operand(dst, src); 3595 } 3596 3597 3598 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 3599 DCHECK(!IsEnabled(AVX)); 3600 EnsureSpace ensure_space(this); 3601 emit(0xF2); 3602 emit_optional_rex_32(dst, src); 3603 emit(0x0F); 3604 emit(0x5A); 3605 emit_sse_operand(dst, src); 3606 } 3607 3608 3609 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) { 3610 DCHECK(!IsEnabled(AVX)); 3611 EnsureSpace ensure_space(this); 3612 emit(0xF2); 3613 emit_optional_rex_32(dst, src); 3614 emit(0x0F); 3615 emit(0x5A); 3616 emit_sse_operand(dst, src); 3617 } 3618 3619 3620 void Assembler::cvtsd2si(Register dst, XMMRegister src) { 3621 DCHECK(!IsEnabled(AVX)); 3622 EnsureSpace ensure_space(this); 3623 emit(0xF2); 3624 emit_optional_rex_32(dst, src); 3625 emit(0x0F); 3626 emit(0x2D); 3627 emit_sse_operand(dst, src); 3628 } 3629 3630 3631 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 3632 DCHECK(!IsEnabled(AVX)); 3633 EnsureSpace ensure_space(this); 3634 emit(0xF2); 3635 emit_rex_64(dst, src); 3636 emit(0x0F); 3637 emit(0x2D); 3638 emit_sse_operand(dst, src); 3639 } 3640 3641 3642 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 3643 EnsureSpace ensure_space(this); 3644 emit(0xF2); 3645 emit_optional_rex_32(dst, src); 3646 emit(0x0F); 3647 emit(0x58); 3648 emit_sse_operand(dst, src); 3649 } 3650 3651 3652 void Assembler::addsd(XMMRegister dst, const Operand& src) { 3653 EnsureSpace ensure_space(this); 3654 emit(0xF2); 3655 emit_optional_rex_32(dst, src); 3656 emit(0x0F); 3657 emit(0x58); 3658 emit_sse_operand(dst, src); 3659 } 3660 3661 3662 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 3663 EnsureSpace ensure_space(this); 3664 emit(0xF2); 3665 emit_optional_rex_32(dst, src); 3666 emit(0x0F); 3667 emit(0x59); 3668 emit_sse_operand(dst, src); 3669 } 3670 3671 3672 void Assembler::mulsd(XMMRegister dst, const Operand& src) { 3673 EnsureSpace ensure_space(this); 3674 emit(0xF2); 3675 emit_optional_rex_32(dst, src); 3676 emit(0x0F); 3677 emit(0x59); 3678 emit_sse_operand(dst, src); 3679 } 3680 3681 3682 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 3683 EnsureSpace ensure_space(this); 3684 emit(0xF2); 3685 emit_optional_rex_32(dst, src); 3686 emit(0x0F); 3687 emit(0x5C); 3688 emit_sse_operand(dst, src); 3689 } 3690 3691 3692 void Assembler::subsd(XMMRegister dst, const Operand& src) { 3693 EnsureSpace ensure_space(this); 3694 emit(0xF2); 3695 emit_optional_rex_32(dst, src); 3696 emit(0x0F); 3697 emit(0x5C); 3698 emit_sse_operand(dst, src); 3699 } 3700 3701 3702 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 3703 EnsureSpace ensure_space(this); 3704 emit(0xF2); 3705 emit_optional_rex_32(dst, src); 3706 emit(0x0F); 3707 emit(0x5E); 3708 emit_sse_operand(dst, src); 3709 } 3710 3711 3712 void Assembler::divsd(XMMRegister dst, const Operand& src) { 3713 EnsureSpace ensure_space(this); 3714 emit(0xF2); 3715 emit_optional_rex_32(dst, src); 3716 emit(0x0F); 3717 emit(0x5E); 3718 emit_sse_operand(dst, src); 3719 } 3720 3721 3722 void Assembler::maxsd(XMMRegister dst, XMMRegister src) { 3723 EnsureSpace ensure_space(this); 3724 emit(0xF2); 3725 emit_optional_rex_32(dst, src); 3726 emit(0x0F); 3727 emit(0x5F); 3728 emit_sse_operand(dst, src); 3729 } 3730 3731 3732 void Assembler::maxsd(XMMRegister dst, const Operand& src) { 3733 EnsureSpace ensure_space(this); 3734 emit(0xF2); 3735 emit_optional_rex_32(dst, src); 3736 emit(0x0F); 3737 emit(0x5F); 3738 emit_sse_operand(dst, src); 3739 } 3740 3741 3742 void Assembler::minsd(XMMRegister dst, XMMRegister src) { 3743 EnsureSpace ensure_space(this); 3744 emit(0xF2); 3745 emit_optional_rex_32(dst, src); 3746 emit(0x0F); 3747 emit(0x5D); 3748 emit_sse_operand(dst, src); 3749 } 3750 3751 3752 void Assembler::minsd(XMMRegister dst, const Operand& src) { 3753 EnsureSpace ensure_space(this); 3754 emit(0xF2); 3755 emit_optional_rex_32(dst, src); 3756 emit(0x0F); 3757 emit(0x5D); 3758 emit_sse_operand(dst, src); 3759 } 3760 3761 3762 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 3763 EnsureSpace ensure_space(this); 3764 emit(0x66); 3765 emit_optional_rex_32(dst, src); 3766 emit(0x0F); 3767 emit(0x54); 3768 emit_sse_operand(dst, src); 3769 } 3770 3771 3772 void Assembler::andpd(XMMRegister dst, const Operand& src) { 3773 EnsureSpace ensure_space(this); 3774 emit(0x66); 3775 emit_optional_rex_32(dst, src); 3776 emit(0x0F); 3777 emit(0x54); 3778 emit_sse_operand(dst, src); 3779 } 3780 3781 3782 void Assembler::orpd(XMMRegister dst, XMMRegister src) { 3783 EnsureSpace ensure_space(this); 3784 emit(0x66); 3785 emit_optional_rex_32(dst, src); 3786 emit(0x0F); 3787 emit(0x56); 3788 emit_sse_operand(dst, src); 3789 } 3790 3791 3792 void Assembler::orpd(XMMRegister dst, const Operand& src) { 3793 EnsureSpace ensure_space(this); 3794 emit(0x66); 3795 emit_optional_rex_32(dst, src); 3796 emit(0x0F); 3797 emit(0x56); 3798 emit_sse_operand(dst, src); 3799 } 3800 3801 3802 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 3803 DCHECK(!IsEnabled(AVX)); 3804 EnsureSpace ensure_space(this); 3805 emit(0x66); 3806 emit_optional_rex_32(dst, src); 3807 emit(0x0F); 3808 emit(0x57); 3809 emit_sse_operand(dst, src); 3810 } 3811 3812 3813 void Assembler::xorpd(XMMRegister dst, const Operand& src) { 3814 DCHECK(!IsEnabled(AVX)); 3815 EnsureSpace ensure_space(this); 3816 emit(0x66); 3817 emit_optional_rex_32(dst, src); 3818 emit(0x0F); 3819 emit(0x57); 3820 emit_sse_operand(dst, src); 3821 } 3822 3823 3824 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 3825 DCHECK(!IsEnabled(AVX)); 3826 EnsureSpace ensure_space(this); 3827 emit(0xF2); 3828 emit_optional_rex_32(dst, src); 3829 emit(0x0F); 3830 emit(0x51); 3831 emit_sse_operand(dst, src); 3832 } 3833 3834 3835 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) { 3836 DCHECK(!IsEnabled(AVX)); 3837 EnsureSpace ensure_space(this); 3838 emit(0xF2); 3839 emit_optional_rex_32(dst, src); 3840 emit(0x0F); 3841 emit(0x51); 3842 emit_sse_operand(dst, src); 3843 } 3844 3845 3846 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 3847 DCHECK(!IsEnabled(AVX)); 3848 EnsureSpace ensure_space(this); 3849 emit(0x66); 3850 emit_optional_rex_32(dst, src); 3851 emit(0x0f); 3852 emit(0x2e); 3853 emit_sse_operand(dst, src); 3854 } 3855 3856 3857 void Assembler::ucomisd(XMMRegister dst, const Operand& src) { 3858 DCHECK(!IsEnabled(AVX)); 3859 EnsureSpace ensure_space(this); 3860 emit(0x66); 3861 emit_optional_rex_32(dst, src); 3862 emit(0x0f); 3863 emit(0x2e); 3864 emit_sse_operand(dst, src); 3865 } 3866 3867 3868 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) { 3869 EnsureSpace ensure_space(this); 3870 emit(0xF2); 3871 emit_optional_rex_32(dst, src); 3872 emit(0x0F); 3873 emit(0xC2); 3874 emit_sse_operand(dst, src); 3875 emit(0x01); // LT == 1 3876 } 3877 3878 3879 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3880 DCHECK(!IsEnabled(AVX)); 3881 DCHECK(IsEnabled(SSE4_1)); 3882 EnsureSpace ensure_space(this); 3883 emit(0x66); 3884 emit_optional_rex_32(dst, src); 3885 emit(0x0f); 3886 emit(0x3a); 3887 emit(0x0a); 3888 emit_sse_operand(dst, src); 3889 // Mask precision exception. 3890 emit(static_cast<byte>(mode) | 0x8); 3891 } 3892 3893 3894 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3895 DCHECK(!IsEnabled(AVX)); 3896 DCHECK(IsEnabled(SSE4_1)); 3897 EnsureSpace ensure_space(this); 3898 emit(0x66); 3899 emit_optional_rex_32(dst, src); 3900 emit(0x0f); 3901 emit(0x3a); 3902 emit(0x0b); 3903 emit_sse_operand(dst, src); 3904 // Mask precision exception. 3905 emit(static_cast<byte>(mode) | 0x8); 3906 } 3907 3908 3909 void Assembler::movmskpd(Register dst, XMMRegister src) { 3910 EnsureSpace ensure_space(this); 3911 emit(0x66); 3912 emit_optional_rex_32(dst, src); 3913 emit(0x0f); 3914 emit(0x50); 3915 emit_sse_operand(dst, src); 3916 } 3917 3918 3919 void Assembler::movmskps(Register dst, XMMRegister src) { 3920 EnsureSpace ensure_space(this); 3921 emit_optional_rex_32(dst, src); 3922 emit(0x0f); 3923 emit(0x50); 3924 emit_sse_operand(dst, src); 3925 } 3926 3927 3928 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 3929 EnsureSpace ensure_space(this); 3930 emit(0x66); 3931 emit_optional_rex_32(dst, src); 3932 emit(0x0F); 3933 emit(0x62); 3934 emit_sse_operand(dst, src); 3935 } 3936 3937 void Assembler::punpckldq(XMMRegister dst, const Operand& src) { 3938 EnsureSpace ensure_space(this); 3939 emit(0x66); 3940 emit_optional_rex_32(dst, src); 3941 emit(0x0F); 3942 emit(0x62); 3943 emit_sse_operand(dst, src); 3944 } 3945 3946 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) { 3947 EnsureSpace ensure_space(this); 3948 emit(0x66); 3949 emit_optional_rex_32(dst, src); 3950 emit(0x0F); 3951 emit(0x6A); 3952 emit_sse_operand(dst, src); 3953 } 3954 3955 3956 // AVX instructions 3957 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3958 XMMRegister src2) { 3959 DCHECK(IsEnabled(FMA3)); 3960 EnsureSpace ensure_space(this); 3961 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3962 emit(op); 3963 emit_sse_operand(dst, src2); 3964 } 3965 3966 3967 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3968 const Operand& src2) { 3969 DCHECK(IsEnabled(FMA3)); 3970 EnsureSpace ensure_space(this); 3971 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3972 emit(op); 3973 emit_sse_operand(dst, src2); 3974 } 3975 3976 3977 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3978 XMMRegister src2) { 3979 DCHECK(IsEnabled(FMA3)); 3980 EnsureSpace ensure_space(this); 3981 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3982 emit(op); 3983 emit_sse_operand(dst, src2); 3984 } 3985 3986 3987 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3988 const Operand& src2) { 3989 DCHECK(IsEnabled(FMA3)); 3990 EnsureSpace ensure_space(this); 3991 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3992 emit(op); 3993 emit_sse_operand(dst, src2); 3994 } 3995 3996 3997 void Assembler::vmovd(XMMRegister dst, Register src) { 3998 DCHECK(IsEnabled(AVX)); 3999 EnsureSpace ensure_space(this); 4000 XMMRegister isrc = {src.code()}; 4001 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0); 4002 emit(0x6e); 4003 emit_sse_operand(dst, src); 4004 } 4005 4006 4007 void Assembler::vmovd(XMMRegister dst, const Operand& src) { 4008 DCHECK(IsEnabled(AVX)); 4009 EnsureSpace ensure_space(this); 4010 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0); 4011 emit(0x6e); 4012 emit_sse_operand(dst, src); 4013 } 4014 4015 4016 void Assembler::vmovd(Register dst, XMMRegister src) { 4017 DCHECK(IsEnabled(AVX)); 4018 EnsureSpace ensure_space(this); 4019 XMMRegister idst = {dst.code()}; 4020 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0); 4021 emit(0x7e); 4022 emit_sse_operand(src, dst); 4023 } 4024 4025 4026 void Assembler::vmovq(XMMRegister dst, Register src) { 4027 DCHECK(IsEnabled(AVX)); 4028 EnsureSpace ensure_space(this); 4029 XMMRegister isrc = {src.code()}; 4030 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1); 4031 emit(0x6e); 4032 emit_sse_operand(dst, src); 4033 } 4034 4035 4036 void Assembler::vmovq(XMMRegister dst, const Operand& src) { 4037 DCHECK(IsEnabled(AVX)); 4038 EnsureSpace ensure_space(this); 4039 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1); 4040 emit(0x6e); 4041 emit_sse_operand(dst, src); 4042 } 4043 4044 4045 void Assembler::vmovq(Register dst, XMMRegister src) { 4046 DCHECK(IsEnabled(AVX)); 4047 EnsureSpace ensure_space(this); 4048 XMMRegister idst = {dst.code()}; 4049 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1); 4050 emit(0x7e); 4051 emit_sse_operand(src, dst); 4052 } 4053 4054 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, 4055 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, 4056 VexW w) { 4057 DCHECK(IsEnabled(AVX)); 4058 EnsureSpace ensure_space(this); 4059 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 4060 emit(op); 4061 emit_sse_operand(dst, src2); 4062 } 4063 4064 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, 4065 const Operand& src2, SIMDPrefix pp, LeadingOpcode m, 4066 VexW w) { 4067 DCHECK(IsEnabled(AVX)); 4068 EnsureSpace ensure_space(this); 4069 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 4070 emit(op); 4071 emit_sse_operand(dst, src2); 4072 } 4073 4074 4075 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 4076 XMMRegister src2) { 4077 DCHECK(IsEnabled(AVX)); 4078 EnsureSpace ensure_space(this); 4079 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 4080 emit(op); 4081 emit_sse_operand(dst, src2); 4082 } 4083 4084 4085 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 4086 const Operand& src2) { 4087 DCHECK(IsEnabled(AVX)); 4088 EnsureSpace ensure_space(this); 4089 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 4090 emit(op); 4091 emit_sse_operand(dst, src2); 4092 } 4093 4094 4095 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 4096 XMMRegister src2) { 4097 DCHECK(IsEnabled(AVX)); 4098 EnsureSpace ensure_space(this); 4099 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 4100 emit(op); 4101 emit_sse_operand(dst, src2); 4102 } 4103 4104 4105 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 4106 const Operand& src2) { 4107 DCHECK(IsEnabled(AVX)); 4108 EnsureSpace ensure_space(this); 4109 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 4110 emit(op); 4111 emit_sse_operand(dst, src2); 4112 } 4113 4114 4115 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) { 4116 DCHECK(IsEnabled(AVX)); 4117 EnsureSpace ensure_space(this); 4118 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 4119 emit(0x2e); 4120 emit_sse_operand(dst, src); 4121 } 4122 4123 4124 void Assembler::vucomiss(XMMRegister dst, const Operand& src) { 4125 DCHECK(IsEnabled(AVX)); 4126 EnsureSpace ensure_space(this); 4127 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 4128 emit(0x2e); 4129 emit_sse_operand(dst, src); 4130 } 4131 4132 4133 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 4134 XMMRegister src2) { 4135 DCHECK(IsEnabled(AVX)); 4136 EnsureSpace ensure_space(this); 4137 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 4138 emit(op); 4139 emit_sse_operand(dst, src2); 4140 } 4141 4142 4143 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 4144 const Operand& src2) { 4145 DCHECK(IsEnabled(AVX)); 4146 EnsureSpace ensure_space(this); 4147 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 4148 emit(op); 4149 emit_sse_operand(dst, src2); 4150 } 4151 4152 4153 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) { 4154 DCHECK(IsEnabled(BMI1)); 4155 EnsureSpace ensure_space(this); 4156 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 4157 emit(op); 4158 emit_modrm(reg, rm); 4159 } 4160 4161 4162 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) { 4163 DCHECK(IsEnabled(BMI1)); 4164 EnsureSpace ensure_space(this); 4165 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 4166 emit(op); 4167 emit_operand(reg, rm); 4168 } 4169 4170 4171 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) { 4172 DCHECK(IsEnabled(BMI1)); 4173 EnsureSpace ensure_space(this); 4174 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 4175 emit(op); 4176 emit_modrm(reg, rm); 4177 } 4178 4179 4180 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) { 4181 DCHECK(IsEnabled(BMI1)); 4182 EnsureSpace ensure_space(this); 4183 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 4184 emit(op); 4185 emit_operand(reg, rm); 4186 } 4187 4188 4189 void Assembler::tzcntq(Register dst, Register src) { 4190 DCHECK(IsEnabled(BMI1)); 4191 EnsureSpace ensure_space(this); 4192 emit(0xF3); 4193 emit_rex_64(dst, src); 4194 emit(0x0F); 4195 emit(0xBC); 4196 emit_modrm(dst, src); 4197 } 4198 4199 4200 void Assembler::tzcntq(Register dst, const Operand& src) { 4201 DCHECK(IsEnabled(BMI1)); 4202 EnsureSpace ensure_space(this); 4203 emit(0xF3); 4204 emit_rex_64(dst, src); 4205 emit(0x0F); 4206 emit(0xBC); 4207 emit_operand(dst, src); 4208 } 4209 4210 4211 void Assembler::tzcntl(Register dst, Register src) { 4212 DCHECK(IsEnabled(BMI1)); 4213 EnsureSpace ensure_space(this); 4214 emit(0xF3); 4215 emit_optional_rex_32(dst, src); 4216 emit(0x0F); 4217 emit(0xBC); 4218 emit_modrm(dst, src); 4219 } 4220 4221 4222 void Assembler::tzcntl(Register dst, const Operand& src) { 4223 DCHECK(IsEnabled(BMI1)); 4224 EnsureSpace ensure_space(this); 4225 emit(0xF3); 4226 emit_optional_rex_32(dst, src); 4227 emit(0x0F); 4228 emit(0xBC); 4229 emit_operand(dst, src); 4230 } 4231 4232 4233 void Assembler::lzcntq(Register dst, Register src) { 4234 DCHECK(IsEnabled(LZCNT)); 4235 EnsureSpace ensure_space(this); 4236 emit(0xF3); 4237 emit_rex_64(dst, src); 4238 emit(0x0F); 4239 emit(0xBD); 4240 emit_modrm(dst, src); 4241 } 4242 4243 4244 void Assembler::lzcntq(Register dst, const Operand& src) { 4245 DCHECK(IsEnabled(LZCNT)); 4246 EnsureSpace ensure_space(this); 4247 emit(0xF3); 4248 emit_rex_64(dst, src); 4249 emit(0x0F); 4250 emit(0xBD); 4251 emit_operand(dst, src); 4252 } 4253 4254 4255 void Assembler::lzcntl(Register dst, Register src) { 4256 DCHECK(IsEnabled(LZCNT)); 4257 EnsureSpace ensure_space(this); 4258 emit(0xF3); 4259 emit_optional_rex_32(dst, src); 4260 emit(0x0F); 4261 emit(0xBD); 4262 emit_modrm(dst, src); 4263 } 4264 4265 4266 void Assembler::lzcntl(Register dst, const Operand& src) { 4267 DCHECK(IsEnabled(LZCNT)); 4268 EnsureSpace ensure_space(this); 4269 emit(0xF3); 4270 emit_optional_rex_32(dst, src); 4271 emit(0x0F); 4272 emit(0xBD); 4273 emit_operand(dst, src); 4274 } 4275 4276 4277 void Assembler::popcntq(Register dst, Register src) { 4278 DCHECK(IsEnabled(POPCNT)); 4279 EnsureSpace ensure_space(this); 4280 emit(0xF3); 4281 emit_rex_64(dst, src); 4282 emit(0x0F); 4283 emit(0xB8); 4284 emit_modrm(dst, src); 4285 } 4286 4287 4288 void Assembler::popcntq(Register dst, const Operand& src) { 4289 DCHECK(IsEnabled(POPCNT)); 4290 EnsureSpace ensure_space(this); 4291 emit(0xF3); 4292 emit_rex_64(dst, src); 4293 emit(0x0F); 4294 emit(0xB8); 4295 emit_operand(dst, src); 4296 } 4297 4298 4299 void Assembler::popcntl(Register dst, Register src) { 4300 DCHECK(IsEnabled(POPCNT)); 4301 EnsureSpace ensure_space(this); 4302 emit(0xF3); 4303 emit_optional_rex_32(dst, src); 4304 emit(0x0F); 4305 emit(0xB8); 4306 emit_modrm(dst, src); 4307 } 4308 4309 4310 void Assembler::popcntl(Register dst, const Operand& src) { 4311 DCHECK(IsEnabled(POPCNT)); 4312 EnsureSpace ensure_space(this); 4313 emit(0xF3); 4314 emit_optional_rex_32(dst, src); 4315 emit(0x0F); 4316 emit(0xB8); 4317 emit_operand(dst, src); 4318 } 4319 4320 4321 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4322 Register rm) { 4323 DCHECK(IsEnabled(BMI2)); 4324 EnsureSpace ensure_space(this); 4325 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4326 emit(op); 4327 emit_modrm(reg, rm); 4328 } 4329 4330 4331 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4332 const Operand& rm) { 4333 DCHECK(IsEnabled(BMI2)); 4334 EnsureSpace ensure_space(this); 4335 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4336 emit(op); 4337 emit_operand(reg, rm); 4338 } 4339 4340 4341 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4342 Register rm) { 4343 DCHECK(IsEnabled(BMI2)); 4344 EnsureSpace ensure_space(this); 4345 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4346 emit(op); 4347 emit_modrm(reg, rm); 4348 } 4349 4350 4351 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4352 const Operand& rm) { 4353 DCHECK(IsEnabled(BMI2)); 4354 EnsureSpace ensure_space(this); 4355 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4356 emit(op); 4357 emit_operand(reg, rm); 4358 } 4359 4360 4361 void Assembler::rorxq(Register dst, Register src, byte imm8) { 4362 DCHECK(IsEnabled(BMI2)); 4363 DCHECK(is_uint8(imm8)); 4364 Register vreg = {0}; // VEX.vvvv unused 4365 EnsureSpace ensure_space(this); 4366 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4367 emit(0xF0); 4368 emit_modrm(dst, src); 4369 emit(imm8); 4370 } 4371 4372 4373 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) { 4374 DCHECK(IsEnabled(BMI2)); 4375 DCHECK(is_uint8(imm8)); 4376 Register vreg = {0}; // VEX.vvvv unused 4377 EnsureSpace ensure_space(this); 4378 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4379 emit(0xF0); 4380 emit_operand(dst, src); 4381 emit(imm8); 4382 } 4383 4384 4385 void Assembler::rorxl(Register dst, Register src, byte imm8) { 4386 DCHECK(IsEnabled(BMI2)); 4387 DCHECK(is_uint8(imm8)); 4388 Register vreg = {0}; // VEX.vvvv unused 4389 EnsureSpace ensure_space(this); 4390 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4391 emit(0xF0); 4392 emit_modrm(dst, src); 4393 emit(imm8); 4394 } 4395 4396 4397 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) { 4398 DCHECK(IsEnabled(BMI2)); 4399 DCHECK(is_uint8(imm8)); 4400 Register vreg = {0}; // VEX.vvvv unused 4401 EnsureSpace ensure_space(this); 4402 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4403 emit(0xF0); 4404 emit_operand(dst, src); 4405 emit(imm8); 4406 } 4407 4408 void Assembler::minps(XMMRegister dst, XMMRegister src) { 4409 EnsureSpace ensure_space(this); 4410 emit_optional_rex_32(dst, src); 4411 emit(0x0F); 4412 emit(0x5D); 4413 emit_sse_operand(dst, src); 4414 } 4415 4416 void Assembler::minps(XMMRegister dst, const Operand& src) { 4417 EnsureSpace ensure_space(this); 4418 emit_optional_rex_32(dst, src); 4419 emit(0x0F); 4420 emit(0x5D); 4421 emit_sse_operand(dst, src); 4422 } 4423 4424 void Assembler::maxps(XMMRegister dst, XMMRegister src) { 4425 EnsureSpace ensure_space(this); 4426 emit_optional_rex_32(dst, src); 4427 emit(0x0F); 4428 emit(0x5F); 4429 emit_sse_operand(dst, src); 4430 } 4431 4432 void Assembler::maxps(XMMRegister dst, const Operand& src) { 4433 EnsureSpace ensure_space(this); 4434 emit_optional_rex_32(dst, src); 4435 emit(0x0F); 4436 emit(0x5F); 4437 emit_sse_operand(dst, src); 4438 } 4439 4440 void Assembler::rcpps(XMMRegister dst, XMMRegister src) { 4441 EnsureSpace ensure_space(this); 4442 emit_optional_rex_32(dst, src); 4443 emit(0x0F); 4444 emit(0x53); 4445 emit_sse_operand(dst, src); 4446 } 4447 4448 void Assembler::rcpps(XMMRegister dst, const Operand& src) { 4449 EnsureSpace ensure_space(this); 4450 emit_optional_rex_32(dst, src); 4451 emit(0x0F); 4452 emit(0x53); 4453 emit_sse_operand(dst, src); 4454 } 4455 4456 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) { 4457 EnsureSpace ensure_space(this); 4458 emit_optional_rex_32(dst, src); 4459 emit(0x0F); 4460 emit(0x52); 4461 emit_sse_operand(dst, src); 4462 } 4463 4464 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) { 4465 EnsureSpace ensure_space(this); 4466 emit_optional_rex_32(dst, src); 4467 emit(0x0F); 4468 emit(0x52); 4469 emit_sse_operand(dst, src); 4470 } 4471 4472 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) { 4473 EnsureSpace ensure_space(this); 4474 emit_optional_rex_32(dst, src); 4475 emit(0x0F); 4476 emit(0x51); 4477 emit_sse_operand(dst, src); 4478 } 4479 4480 void Assembler::sqrtps(XMMRegister dst, const Operand& src) { 4481 EnsureSpace ensure_space(this); 4482 emit_optional_rex_32(dst, src); 4483 emit(0x0F); 4484 emit(0x51); 4485 emit_sse_operand(dst, src); 4486 } 4487 4488 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) { 4489 EnsureSpace ensure_space(this); 4490 emit_optional_rex_32(dst, src); 4491 emit(0x0F); 4492 emit(0x5B); 4493 emit_sse_operand(dst, src); 4494 } 4495 4496 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) { 4497 EnsureSpace ensure_space(this); 4498 emit_optional_rex_32(dst, src); 4499 emit(0x0F); 4500 emit(0x5B); 4501 emit_sse_operand(dst, src); 4502 } 4503 4504 void Assembler::movups(XMMRegister dst, XMMRegister src) { 4505 EnsureSpace ensure_space(this); 4506 if (src.low_bits() == 4) { 4507 // Try to avoid an unnecessary SIB byte. 4508 emit_optional_rex_32(src, dst); 4509 emit(0x0F); 4510 emit(0x11); 4511 emit_sse_operand(src, dst); 4512 } else { 4513 emit_optional_rex_32(dst, src); 4514 emit(0x0F); 4515 emit(0x10); 4516 emit_sse_operand(dst, src); 4517 } 4518 } 4519 4520 void Assembler::movups(XMMRegister dst, const Operand& src) { 4521 EnsureSpace ensure_space(this); 4522 emit_optional_rex_32(dst, src); 4523 emit(0x0F); 4524 emit(0x10); 4525 emit_sse_operand(dst, src); 4526 } 4527 4528 void Assembler::movups(const Operand& dst, XMMRegister src) { 4529 EnsureSpace ensure_space(this); 4530 emit_optional_rex_32(src, dst); 4531 emit(0x0F); 4532 emit(0x11); 4533 emit_sse_operand(src, dst); 4534 } 4535 4536 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix, 4537 byte escape, byte opcode) { 4538 EnsureSpace ensure_space(this); 4539 emit(prefix); 4540 emit_optional_rex_32(dst, src); 4541 emit(escape); 4542 emit(opcode); 4543 emit_sse_operand(dst, src); 4544 } 4545 4546 void Assembler::sse2_instr(XMMRegister dst, const Operand& src, byte prefix, 4547 byte escape, byte opcode) { 4548 EnsureSpace ensure_space(this); 4549 emit(prefix); 4550 emit_optional_rex_32(dst, src); 4551 emit(escape); 4552 emit(opcode); 4553 emit_sse_operand(dst, src); 4554 } 4555 4556 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix, 4557 byte escape1, byte escape2, byte opcode) { 4558 DCHECK(IsEnabled(SSSE3)); 4559 EnsureSpace ensure_space(this); 4560 emit(prefix); 4561 emit_optional_rex_32(dst, src); 4562 emit(escape1); 4563 emit(escape2); 4564 emit(opcode); 4565 emit_sse_operand(dst, src); 4566 } 4567 4568 void Assembler::ssse3_instr(XMMRegister dst, const Operand& src, byte prefix, 4569 byte escape1, byte escape2, byte opcode) { 4570 DCHECK(IsEnabled(SSSE3)); 4571 EnsureSpace ensure_space(this); 4572 emit(prefix); 4573 emit_optional_rex_32(dst, src); 4574 emit(escape1); 4575 emit(escape2); 4576 emit(opcode); 4577 emit_sse_operand(dst, src); 4578 } 4579 4580 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix, 4581 byte escape1, byte escape2, byte opcode) { 4582 DCHECK(IsEnabled(SSE4_1)); 4583 EnsureSpace ensure_space(this); 4584 emit(prefix); 4585 emit_optional_rex_32(dst, src); 4586 emit(escape1); 4587 emit(escape2); 4588 emit(opcode); 4589 emit_sse_operand(dst, src); 4590 } 4591 4592 void Assembler::sse4_instr(XMMRegister dst, const Operand& src, byte prefix, 4593 byte escape1, byte escape2, byte opcode) { 4594 DCHECK(IsEnabled(SSE4_1)); 4595 EnsureSpace ensure_space(this); 4596 emit(prefix); 4597 emit_optional_rex_32(dst, src); 4598 emit(escape1); 4599 emit(escape2); 4600 emit(opcode); 4601 emit_sse_operand(dst, src); 4602 } 4603 4604 void Assembler::lddqu(XMMRegister dst, const Operand& src) { 4605 DCHECK(IsEnabled(SSE3)); 4606 EnsureSpace ensure_space(this); 4607 emit(0xF2); 4608 emit_optional_rex_32(dst, src); 4609 emit(0x0F); 4610 emit(0xF0); 4611 emit_sse_operand(dst, src); 4612 } 4613 4614 void Assembler::psrldq(XMMRegister dst, uint8_t shift) { 4615 EnsureSpace ensure_space(this); 4616 emit(0x66); 4617 emit_optional_rex_32(dst); 4618 emit(0x0F); 4619 emit(0x73); 4620 emit_sse_operand(dst); 4621 emit(shift); 4622 } 4623 4624 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) { 4625 EnsureSpace ensure_space(this); 4626 emit(0x66); 4627 emit_optional_rex_32(dst, src); 4628 emit(0x0F); 4629 emit(0x70); 4630 emit_sse_operand(dst, src); 4631 emit(shuffle); 4632 } 4633 4634 void Assembler::pshufd(XMMRegister dst, const Operand& src, uint8_t shuffle) { 4635 EnsureSpace ensure_space(this); 4636 emit(0x66); 4637 emit_optional_rex_32(dst, src); 4638 emit(0x0F); 4639 emit(0x70); 4640 emit_sse_operand(dst, src); 4641 emit(shuffle); 4642 } 4643 4644 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) { 4645 Register ireg = { reg.code() }; 4646 emit_operand(ireg, adr); 4647 } 4648 4649 4650 void Assembler::emit_sse_operand(Register reg, const Operand& adr) { 4651 Register ireg = {reg.code()}; 4652 emit_operand(ireg, adr); 4653 } 4654 4655 4656 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) { 4657 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4658 } 4659 4660 4661 void Assembler::emit_sse_operand(XMMRegister dst, Register src) { 4662 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4663 } 4664 4665 4666 void Assembler::emit_sse_operand(Register dst, XMMRegister src) { 4667 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4668 } 4669 4670 void Assembler::emit_sse_operand(XMMRegister dst) { 4671 emit(0xD8 | dst.low_bits()); 4672 } 4673 4674 void Assembler::RecordProtectedInstructionLanding(int pc_offset) { 4675 EnsureSpace ensure_space(this); 4676 RelocInfo rinfo(isolate(), pc(), 4677 RelocInfo::WASM_PROTECTED_INSTRUCTION_LANDING, pc_offset, 4678 nullptr); 4679 reloc_info_writer.Write(&rinfo); 4680 } 4681 4682 4683 void Assembler::db(uint8_t data) { 4684 EnsureSpace ensure_space(this); 4685 emit(data); 4686 } 4687 4688 4689 void Assembler::dd(uint32_t data) { 4690 EnsureSpace ensure_space(this); 4691 emitl(data); 4692 } 4693 4694 4695 void Assembler::dq(uint64_t data) { 4696 EnsureSpace ensure_space(this); 4697 emitq(data); 4698 } 4699 4700 4701 void Assembler::dq(Label* label) { 4702 EnsureSpace ensure_space(this); 4703 if (label->is_bound()) { 4704 internal_reference_positions_.push_back(pc_offset()); 4705 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE); 4706 } else { 4707 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 4708 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address. 4709 if (label->is_linked()) { 4710 emitl(label->pos()); 4711 label->link_to(pc_offset() - sizeof(int32_t)); 4712 } else { 4713 DCHECK(label->is_unused()); 4714 int32_t current = pc_offset(); 4715 emitl(current); 4716 label->link_to(current); 4717 } 4718 } 4719 } 4720 4721 4722 // Relocation information implementations. 4723 4724 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 4725 DCHECK(!RelocInfo::IsNone(rmode)); 4726 // Don't record external references unless the heap will be serialized. 4727 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 4728 !serializer_enabled() && !emit_debug_code()) { 4729 return; 4730 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) { 4731 // Don't record psuedo relocation info for code age sequence mode. 4732 return; 4733 } 4734 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL); 4735 reloc_info_writer.Write(&rinfo); 4736 } 4737 4738 4739 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask | 4740 1 << RelocInfo::RUNTIME_ENTRY | 4741 1 << RelocInfo::INTERNAL_REFERENCE | 4742 1 << RelocInfo::CODE_AGE_SEQUENCE; 4743 4744 4745 bool RelocInfo::IsCodedSpecially() { 4746 // The deserializer needs to know whether a pointer is specially coded. Being 4747 // specially coded on x64 means that it is a relative 32 bit address, as used 4748 // by branch instructions. 4749 return (1 << rmode_) & kApplyMask; 4750 } 4751 4752 4753 bool RelocInfo::IsInConstantPool() { 4754 return false; 4755 } 4756 4757 4758 } // namespace internal 4759 } // namespace v8 4760 4761 #endif // V8_TARGET_ARCH_X64 4762
