1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "src/x64/assembler-x64.h" 6 7 #include <cstring> 8 9 #if V8_TARGET_ARCH_X64 10 11 #if V8_LIBC_MSVCRT 12 #include <intrin.h> // _xgetbv() 13 #endif 14 #if V8_OS_MACOSX 15 #include <sys/sysctl.h> 16 #endif 17 18 #include "src/base/bits.h" 19 #include "src/macro-assembler.h" 20 #include "src/v8.h" 21 22 namespace v8 { 23 namespace internal { 24 25 // ----------------------------------------------------------------------------- 26 // Implementation of CpuFeatures 27 28 namespace { 29 30 #if !V8_LIBC_MSVCRT 31 32 V8_INLINE uint64_t _xgetbv(unsigned int xcr) { 33 unsigned eax, edx; 34 // Check xgetbv; this uses a .byte sequence instead of the instruction 35 // directly because older assemblers do not include support for xgetbv and 36 // there is no easy way to conditionally compile based on the assembler 37 // used. 38 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 39 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32); 40 } 41 42 #define _XCR_XFEATURE_ENABLED_MASK 0 43 44 #endif // !V8_LIBC_MSVCRT 45 46 47 bool OSHasAVXSupport() { 48 #if V8_OS_MACOSX 49 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being 50 // caused by ISRs, so we detect that here and disable AVX in that case. 51 char buffer[128]; 52 size_t buffer_size = arraysize(buffer); 53 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE}; 54 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) { 55 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version"); 56 } 57 // The buffer now contains a string of the form XX.YY.ZZ, where 58 // XX is the major kernel version component. 59 char* period_pos = strchr(buffer, '.'); 60 DCHECK_NOT_NULL(period_pos); 61 *period_pos = '\0'; 62 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT 63 if (kernel_version_major <= 13) return false; 64 #endif // V8_OS_MACOSX 65 // Check whether OS claims to support AVX. 66 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); 67 return (feature_mask & 0x6) == 0x6; 68 } 69 70 } // namespace 71 72 73 void CpuFeatures::ProbeImpl(bool cross_compile) { 74 base::CPU cpu; 75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory. 76 CHECK(cpu.has_cmov()); // CMOV support is mandatory. 77 78 // Only use statically determined features for cross compile (snapshot). 79 if (cross_compile) return; 80 81 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1; 82 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; 83 // SAHF is not generally available in long mode. 84 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; 85 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && 86 OSHasAVXSupport()) { 87 supported_ |= 1u << AVX; 88 } 89 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() && 90 OSHasAVXSupport()) { 91 supported_ |= 1u << FMA3; 92 } 93 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; 94 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; 95 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; 96 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT; 97 if (strcmp(FLAG_mcpu, "auto") == 0) { 98 if (cpu.is_atom()) supported_ |= 1u << ATOM; 99 } else if (strcmp(FLAG_mcpu, "atom") == 0) { 100 supported_ |= 1u << ATOM; 101 } 102 } 103 104 105 void CpuFeatures::PrintTarget() { } 106 void CpuFeatures::PrintFeatures() { 107 printf( 108 "SSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d " 109 "POPCNT=%d ATOM=%d\n", 110 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSE4_1), 111 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX), 112 CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1), 113 CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT), 114 CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM)); 115 } 116 117 // ----------------------------------------------------------------------------- 118 // Implementation of RelocInfo 119 120 Address RelocInfo::wasm_memory_reference() { 121 DCHECK(IsWasmMemoryReference(rmode_)); 122 return Memory::Address_at(pc_); 123 } 124 125 Address RelocInfo::wasm_global_reference() { 126 DCHECK(IsWasmGlobalReference(rmode_)); 127 return Memory::Address_at(pc_); 128 } 129 130 uint32_t RelocInfo::wasm_memory_size_reference() { 131 DCHECK(IsWasmMemorySizeReference(rmode_)); 132 return Memory::uint32_at(pc_); 133 } 134 135 void RelocInfo::unchecked_update_wasm_memory_reference( 136 Address address, ICacheFlushMode flush_mode) { 137 Memory::Address_at(pc_) = address; 138 } 139 140 void RelocInfo::unchecked_update_wasm_memory_size(uint32_t size, 141 ICacheFlushMode flush_mode) { 142 Memory::uint32_at(pc_) = size; 143 } 144 145 // ----------------------------------------------------------------------------- 146 // Implementation of Operand 147 148 Operand::Operand(Register base, int32_t disp) : rex_(0) { 149 len_ = 1; 150 if (base.is(rsp) || base.is(r12)) { 151 // SIB byte is needed to encode (rsp + offset) or (r12 + offset). 152 set_sib(times_1, rsp, base); 153 } 154 155 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 156 set_modrm(0, base); 157 } else if (is_int8(disp)) { 158 set_modrm(1, base); 159 set_disp8(disp); 160 } else { 161 set_modrm(2, base); 162 set_disp32(disp); 163 } 164 } 165 166 167 Operand::Operand(Register base, 168 Register index, 169 ScaleFactor scale, 170 int32_t disp) : rex_(0) { 171 DCHECK(!index.is(rsp)); 172 len_ = 1; 173 set_sib(scale, index, base); 174 if (disp == 0 && !base.is(rbp) && !base.is(r13)) { 175 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits 176 // possibly set by set_sib. 177 set_modrm(0, rsp); 178 } else if (is_int8(disp)) { 179 set_modrm(1, rsp); 180 set_disp8(disp); 181 } else { 182 set_modrm(2, rsp); 183 set_disp32(disp); 184 } 185 } 186 187 188 Operand::Operand(Register index, 189 ScaleFactor scale, 190 int32_t disp) : rex_(0) { 191 DCHECK(!index.is(rsp)); 192 len_ = 1; 193 set_modrm(0, rsp); 194 set_sib(scale, index, rbp); 195 set_disp32(disp); 196 } 197 198 199 Operand::Operand(Label* label) : rex_(0), len_(1) { 200 DCHECK_NOT_NULL(label); 201 set_modrm(0, rbp); 202 set_disp64(reinterpret_cast<intptr_t>(label)); 203 } 204 205 206 Operand::Operand(const Operand& operand, int32_t offset) { 207 DCHECK(operand.len_ >= 1); 208 // Operand encodes REX ModR/M [SIB] [Disp]. 209 byte modrm = operand.buf_[0]; 210 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target). 211 bool has_sib = ((modrm & 0x07) == 0x04); 212 byte mode = modrm & 0xC0; 213 int disp_offset = has_sib ? 2 : 1; 214 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07; 215 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit 216 // displacement. 217 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base. 218 int32_t disp_value = 0; 219 if (mode == 0x80 || is_baseless) { 220 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement. 221 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]); 222 } else if (mode == 0x40) { 223 // Mode 1: Byte displacement. 224 disp_value = static_cast<signed char>(operand.buf_[disp_offset]); 225 } 226 227 // Write new operand with same registers, but with modified displacement. 228 DCHECK(offset >= 0 ? disp_value + offset > disp_value 229 : disp_value + offset < disp_value); // No overflow. 230 disp_value += offset; 231 rex_ = operand.rex_; 232 if (!is_int8(disp_value) || is_baseless) { 233 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13. 234 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80); 235 len_ = disp_offset + 4; 236 Memory::int32_at(&buf_[disp_offset]) = disp_value; 237 } else if (disp_value != 0 || (base_reg == 0x05)) { 238 // Need 8 bits of displacement. 239 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1. 240 len_ = disp_offset + 1; 241 buf_[disp_offset] = static_cast<byte>(disp_value); 242 } else { 243 // Need no displacement. 244 buf_[0] = (modrm & 0x3f); // Mode 0. 245 len_ = disp_offset; 246 } 247 if (has_sib) { 248 buf_[1] = operand.buf_[1]; 249 } 250 } 251 252 253 bool Operand::AddressUsesRegister(Register reg) const { 254 int code = reg.code(); 255 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand. 256 // Start with only low three bits of base register. Initial decoding doesn't 257 // distinguish on the REX.B bit. 258 int base_code = buf_[0] & 0x07; 259 if (base_code == rsp.code()) { 260 // SIB byte present in buf_[1]. 261 // Check the index register from the SIB byte + REX.X prefix. 262 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2); 263 // Index code (including REX.X) of 0x04 (rsp) means no index register. 264 if (index_code != rsp.code() && index_code == code) return true; 265 // Add REX.B to get the full base register code. 266 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3); 267 // A base register of 0x05 (rbp) with mod = 0 means no base register. 268 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 269 return code == base_code; 270 } else { 271 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means 272 // no base register. 273 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false; 274 base_code |= ((rex_ & 0x01) << 3); 275 return code == base_code; 276 } 277 } 278 279 280 // ----------------------------------------------------------------------------- 281 // Implementation of Assembler. 282 283 #ifdef GENERATED_CODE_COVERAGE 284 static void InitCoverageLog(); 285 #endif 286 287 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size) 288 : AssemblerBase(isolate, buffer, buffer_size), 289 code_targets_(100), 290 positions_recorder_(this) { 291 // Clear the buffer in debug mode unless it was provided by the 292 // caller in which case we can't be sure it's okay to overwrite 293 // existing code in it. 294 #ifdef DEBUG 295 if (own_buffer_) { 296 memset(buffer_, 0xCC, buffer_size_); // int3 297 } 298 #endif 299 300 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_); 301 302 303 #ifdef GENERATED_CODE_COVERAGE 304 InitCoverageLog(); 305 #endif 306 } 307 308 309 void Assembler::GetCode(CodeDesc* desc) { 310 // Finalize code (at this point overflow() may be true, but the gap ensures 311 // that we are still not overlapping instructions and relocation info). 312 reloc_info_writer.Finish(); 313 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap. 314 // Set up code descriptor. 315 desc->buffer = buffer_; 316 desc->buffer_size = buffer_size_; 317 desc->instr_size = pc_offset(); 318 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system. 319 desc->reloc_size = 320 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos()); 321 desc->origin = this; 322 desc->constant_pool_size = 0; 323 desc->unwinding_info_size = 0; 324 desc->unwinding_info = nullptr; 325 } 326 327 328 void Assembler::Align(int m) { 329 DCHECK(base::bits::IsPowerOfTwo32(m)); 330 int delta = (m - (pc_offset() & (m - 1))) & (m - 1); 331 Nop(delta); 332 } 333 334 335 void Assembler::CodeTargetAlign() { 336 Align(16); // Preferred alignment of jump targets on x64. 337 } 338 339 340 bool Assembler::IsNop(Address addr) { 341 Address a = addr; 342 while (*a == 0x66) a++; 343 if (*a == 0x90) return true; 344 if (a[0] == 0xf && a[1] == 0x1f) return true; 345 return false; 346 } 347 348 349 void Assembler::bind_to(Label* L, int pos) { 350 DCHECK(!L->is_bound()); // Label may only be bound once. 351 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid. 352 if (L->is_linked()) { 353 int current = L->pos(); 354 int next = long_at(current); 355 while (next != current) { 356 if (current >= 4 && long_at(current - 4) == 0) { 357 // Absolute address. 358 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 359 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 360 internal_reference_positions_.push_back(current - 4); 361 } else { 362 // Relative address, relative to point after address. 363 int imm32 = pos - (current + sizeof(int32_t)); 364 long_at_put(current, imm32); 365 } 366 current = next; 367 next = long_at(next); 368 } 369 // Fix up last fixup on linked list. 370 if (current >= 4 && long_at(current - 4) == 0) { 371 // Absolute address. 372 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos); 373 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64; 374 internal_reference_positions_.push_back(current - 4); 375 } else { 376 // Relative address, relative to point after address. 377 int imm32 = pos - (current + sizeof(int32_t)); 378 long_at_put(current, imm32); 379 } 380 } 381 while (L->is_near_linked()) { 382 int fixup_pos = L->near_link_pos(); 383 int offset_to_next = 384 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos))); 385 DCHECK(offset_to_next <= 0); 386 int disp = pos - (fixup_pos + sizeof(int8_t)); 387 CHECK(is_int8(disp)); 388 set_byte_at(fixup_pos, disp); 389 if (offset_to_next < 0) { 390 L->link_to(fixup_pos + offset_to_next, Label::kNear); 391 } else { 392 L->UnuseNear(); 393 } 394 } 395 L->bind_to(pos); 396 } 397 398 399 void Assembler::bind(Label* L) { 400 bind_to(L, pc_offset()); 401 } 402 403 404 void Assembler::GrowBuffer() { 405 DCHECK(buffer_overflow()); 406 if (!own_buffer_) FATAL("external code buffer is too small"); 407 408 // Compute new buffer size. 409 CodeDesc desc; // the new buffer 410 desc.buffer_size = 2 * buffer_size_; 411 412 // Some internal data structures overflow for very large buffers, 413 // they must ensure that kMaximalBufferSize is not too large. 414 if ((desc.buffer_size > kMaximalBufferSize) || 415 (desc.buffer_size > isolate()->heap()->MaxOldGenerationSize())) { 416 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer"); 417 } 418 419 // Set up new buffer. 420 desc.buffer = NewArray<byte>(desc.buffer_size); 421 desc.origin = this; 422 desc.instr_size = pc_offset(); 423 desc.reloc_size = 424 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos())); 425 426 // Clear the buffer in debug mode. Use 'int3' instructions to make 427 // sure to get into problems if we ever run uninitialized code. 428 #ifdef DEBUG 429 memset(desc.buffer, 0xCC, desc.buffer_size); 430 #endif 431 432 // Copy the data. 433 intptr_t pc_delta = desc.buffer - buffer_; 434 intptr_t rc_delta = (desc.buffer + desc.buffer_size) - 435 (buffer_ + buffer_size_); 436 MemMove(desc.buffer, buffer_, desc.instr_size); 437 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(), 438 desc.reloc_size); 439 440 // Switch buffers. 441 DeleteArray(buffer_); 442 buffer_ = desc.buffer; 443 buffer_size_ = desc.buffer_size; 444 pc_ += pc_delta; 445 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, 446 reloc_info_writer.last_pc() + pc_delta); 447 448 // Relocate internal references. 449 for (auto pos : internal_reference_positions_) { 450 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos); 451 *p += pc_delta; 452 } 453 454 DCHECK(!buffer_overflow()); 455 } 456 457 458 void Assembler::emit_operand(int code, const Operand& adr) { 459 DCHECK(is_uint3(code)); 460 const unsigned length = adr.len_; 461 DCHECK(length > 0); 462 463 // Emit updated ModR/M byte containing the given register. 464 DCHECK((adr.buf_[0] & 0x38) == 0); 465 *pc_++ = adr.buf_[0] | code << 3; 466 467 // Recognize RIP relative addressing. 468 if (adr.buf_[0] == 5) { 469 DCHECK_EQ(9u, length); 470 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]); 471 if (label->is_bound()) { 472 int offset = label->pos() - pc_offset() - sizeof(int32_t); 473 DCHECK_GE(0, offset); 474 emitl(offset); 475 } else if (label->is_linked()) { 476 emitl(label->pos()); 477 label->link_to(pc_offset() - sizeof(int32_t)); 478 } else { 479 DCHECK(label->is_unused()); 480 int32_t current = pc_offset(); 481 emitl(current); 482 label->link_to(current); 483 } 484 } else { 485 // Emit the rest of the encoded operand. 486 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i]; 487 } 488 } 489 490 491 // Assembler Instruction implementations. 492 493 void Assembler::arithmetic_op(byte opcode, 494 Register reg, 495 const Operand& op, 496 int size) { 497 EnsureSpace ensure_space(this); 498 emit_rex(reg, op, size); 499 emit(opcode); 500 emit_operand(reg, op); 501 } 502 503 504 void Assembler::arithmetic_op(byte opcode, 505 Register reg, 506 Register rm_reg, 507 int size) { 508 EnsureSpace ensure_space(this); 509 DCHECK((opcode & 0xC6) == 2); 510 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 511 // Swap reg and rm_reg and change opcode operand order. 512 emit_rex(rm_reg, reg, size); 513 emit(opcode ^ 0x02); 514 emit_modrm(rm_reg, reg); 515 } else { 516 emit_rex(reg, rm_reg, size); 517 emit(opcode); 518 emit_modrm(reg, rm_reg); 519 } 520 } 521 522 523 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) { 524 EnsureSpace ensure_space(this); 525 DCHECK((opcode & 0xC6) == 2); 526 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 527 // Swap reg and rm_reg and change opcode operand order. 528 emit(0x66); 529 emit_optional_rex_32(rm_reg, reg); 530 emit(opcode ^ 0x02); 531 emit_modrm(rm_reg, reg); 532 } else { 533 emit(0x66); 534 emit_optional_rex_32(reg, rm_reg); 535 emit(opcode); 536 emit_modrm(reg, rm_reg); 537 } 538 } 539 540 541 void Assembler::arithmetic_op_16(byte opcode, 542 Register reg, 543 const Operand& rm_reg) { 544 EnsureSpace ensure_space(this); 545 emit(0x66); 546 emit_optional_rex_32(reg, rm_reg); 547 emit(opcode); 548 emit_operand(reg, rm_reg); 549 } 550 551 552 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) { 553 EnsureSpace ensure_space(this); 554 if (!reg.is_byte_register()) { 555 emit_rex_32(reg, op); 556 } else { 557 emit_optional_rex_32(reg, op); 558 } 559 emit(opcode); 560 emit_operand(reg, op); 561 } 562 563 564 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) { 565 EnsureSpace ensure_space(this); 566 DCHECK((opcode & 0xC6) == 2); 567 if (rm_reg.low_bits() == 4) { // Forces SIB byte. 568 // Swap reg and rm_reg and change opcode operand order. 569 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) { 570 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 571 emit_rex_32(rm_reg, reg); 572 } 573 emit(opcode ^ 0x02); 574 emit_modrm(rm_reg, reg); 575 } else { 576 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) { 577 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 578 emit_rex_32(reg, rm_reg); 579 } 580 emit(opcode); 581 emit_modrm(reg, rm_reg); 582 } 583 } 584 585 586 void Assembler::immediate_arithmetic_op(byte subcode, 587 Register dst, 588 Immediate src, 589 int size) { 590 EnsureSpace ensure_space(this); 591 emit_rex(dst, size); 592 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) { 593 emit(0x83); 594 emit_modrm(subcode, dst); 595 emit(src.value_); 596 } else if (dst.is(rax)) { 597 emit(0x05 | (subcode << 3)); 598 emit(src); 599 } else { 600 emit(0x81); 601 emit_modrm(subcode, dst); 602 emit(src); 603 } 604 } 605 606 void Assembler::immediate_arithmetic_op(byte subcode, 607 const Operand& dst, 608 Immediate src, 609 int size) { 610 EnsureSpace ensure_space(this); 611 emit_rex(dst, size); 612 if (is_int8(src.value_)) { 613 emit(0x83); 614 emit_operand(subcode, dst); 615 if (!RelocInfo::IsNone(src.rmode_)) { 616 RecordRelocInfo(src.rmode_); 617 } 618 emit(src.value_); 619 } else { 620 emit(0x81); 621 emit_operand(subcode, dst); 622 emit(src); 623 } 624 } 625 626 627 void Assembler::immediate_arithmetic_op_16(byte subcode, 628 Register dst, 629 Immediate src) { 630 EnsureSpace ensure_space(this); 631 emit(0x66); // Operand size override prefix. 632 emit_optional_rex_32(dst); 633 if (is_int8(src.value_)) { 634 emit(0x83); 635 emit_modrm(subcode, dst); 636 emit(src.value_); 637 } else if (dst.is(rax)) { 638 emit(0x05 | (subcode << 3)); 639 emitw(src.value_); 640 } else { 641 emit(0x81); 642 emit_modrm(subcode, dst); 643 emitw(src.value_); 644 } 645 } 646 647 648 void Assembler::immediate_arithmetic_op_16(byte subcode, 649 const Operand& dst, 650 Immediate src) { 651 EnsureSpace ensure_space(this); 652 emit(0x66); // Operand size override prefix. 653 emit_optional_rex_32(dst); 654 if (is_int8(src.value_)) { 655 emit(0x83); 656 emit_operand(subcode, dst); 657 emit(src.value_); 658 } else { 659 emit(0x81); 660 emit_operand(subcode, dst); 661 emitw(src.value_); 662 } 663 } 664 665 666 void Assembler::immediate_arithmetic_op_8(byte subcode, 667 const Operand& dst, 668 Immediate src) { 669 EnsureSpace ensure_space(this); 670 emit_optional_rex_32(dst); 671 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 672 emit(0x80); 673 emit_operand(subcode, dst); 674 emit(src.value_); 675 } 676 677 678 void Assembler::immediate_arithmetic_op_8(byte subcode, 679 Register dst, 680 Immediate src) { 681 EnsureSpace ensure_space(this); 682 if (!dst.is_byte_register()) { 683 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 684 emit_rex_32(dst); 685 } 686 DCHECK(is_int8(src.value_) || is_uint8(src.value_)); 687 emit(0x80); 688 emit_modrm(subcode, dst); 689 emit(src.value_); 690 } 691 692 693 void Assembler::shift(Register dst, 694 Immediate shift_amount, 695 int subcode, 696 int size) { 697 EnsureSpace ensure_space(this); 698 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 699 : is_uint5(shift_amount.value_)); 700 if (shift_amount.value_ == 1) { 701 emit_rex(dst, size); 702 emit(0xD1); 703 emit_modrm(subcode, dst); 704 } else { 705 emit_rex(dst, size); 706 emit(0xC1); 707 emit_modrm(subcode, dst); 708 emit(shift_amount.value_); 709 } 710 } 711 712 713 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode, 714 int size) { 715 EnsureSpace ensure_space(this); 716 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_) 717 : is_uint5(shift_amount.value_)); 718 if (shift_amount.value_ == 1) { 719 emit_rex(dst, size); 720 emit(0xD1); 721 emit_operand(subcode, dst); 722 } else { 723 emit_rex(dst, size); 724 emit(0xC1); 725 emit_operand(subcode, dst); 726 emit(shift_amount.value_); 727 } 728 } 729 730 731 void Assembler::shift(Register dst, int subcode, int size) { 732 EnsureSpace ensure_space(this); 733 emit_rex(dst, size); 734 emit(0xD3); 735 emit_modrm(subcode, dst); 736 } 737 738 739 void Assembler::shift(Operand dst, int subcode, int size) { 740 EnsureSpace ensure_space(this); 741 emit_rex(dst, size); 742 emit(0xD3); 743 emit_operand(subcode, dst); 744 } 745 746 747 void Assembler::bt(const Operand& dst, Register src) { 748 EnsureSpace ensure_space(this); 749 emit_rex_64(src, dst); 750 emit(0x0F); 751 emit(0xA3); 752 emit_operand(src, dst); 753 } 754 755 756 void Assembler::bts(const Operand& dst, Register src) { 757 EnsureSpace ensure_space(this); 758 emit_rex_64(src, dst); 759 emit(0x0F); 760 emit(0xAB); 761 emit_operand(src, dst); 762 } 763 764 765 void Assembler::bsrl(Register dst, Register src) { 766 EnsureSpace ensure_space(this); 767 emit_optional_rex_32(dst, src); 768 emit(0x0F); 769 emit(0xBD); 770 emit_modrm(dst, src); 771 } 772 773 774 void Assembler::bsrl(Register dst, const Operand& src) { 775 EnsureSpace ensure_space(this); 776 emit_optional_rex_32(dst, src); 777 emit(0x0F); 778 emit(0xBD); 779 emit_operand(dst, src); 780 } 781 782 783 void Assembler::bsrq(Register dst, Register src) { 784 EnsureSpace ensure_space(this); 785 emit_rex_64(dst, src); 786 emit(0x0F); 787 emit(0xBD); 788 emit_modrm(dst, src); 789 } 790 791 792 void Assembler::bsrq(Register dst, const Operand& src) { 793 EnsureSpace ensure_space(this); 794 emit_rex_64(dst, src); 795 emit(0x0F); 796 emit(0xBD); 797 emit_operand(dst, src); 798 } 799 800 801 void Assembler::bsfl(Register dst, Register src) { 802 EnsureSpace ensure_space(this); 803 emit_optional_rex_32(dst, src); 804 emit(0x0F); 805 emit(0xBC); 806 emit_modrm(dst, src); 807 } 808 809 810 void Assembler::bsfl(Register dst, const Operand& src) { 811 EnsureSpace ensure_space(this); 812 emit_optional_rex_32(dst, src); 813 emit(0x0F); 814 emit(0xBC); 815 emit_operand(dst, src); 816 } 817 818 819 void Assembler::bsfq(Register dst, Register src) { 820 EnsureSpace ensure_space(this); 821 emit_rex_64(dst, src); 822 emit(0x0F); 823 emit(0xBC); 824 emit_modrm(dst, src); 825 } 826 827 828 void Assembler::bsfq(Register dst, const Operand& src) { 829 EnsureSpace ensure_space(this); 830 emit_rex_64(dst, src); 831 emit(0x0F); 832 emit(0xBC); 833 emit_operand(dst, src); 834 } 835 836 837 void Assembler::call(Label* L) { 838 EnsureSpace ensure_space(this); 839 // 1110 1000 #32-bit disp. 840 emit(0xE8); 841 if (L->is_bound()) { 842 int offset = L->pos() - pc_offset() - sizeof(int32_t); 843 DCHECK(offset <= 0); 844 emitl(offset); 845 } else if (L->is_linked()) { 846 emitl(L->pos()); 847 L->link_to(pc_offset() - sizeof(int32_t)); 848 } else { 849 DCHECK(L->is_unused()); 850 int32_t current = pc_offset(); 851 emitl(current); 852 L->link_to(current); 853 } 854 } 855 856 857 void Assembler::call(Address entry, RelocInfo::Mode rmode) { 858 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 859 EnsureSpace ensure_space(this); 860 // 1110 1000 #32-bit disp. 861 emit(0xE8); 862 emit_runtime_entry(entry, rmode); 863 } 864 865 866 void Assembler::call(Handle<Code> target, 867 RelocInfo::Mode rmode, 868 TypeFeedbackId ast_id) { 869 EnsureSpace ensure_space(this); 870 // 1110 1000 #32-bit disp. 871 emit(0xE8); 872 emit_code_target(target, rmode, ast_id); 873 } 874 875 876 void Assembler::call(Register adr) { 877 EnsureSpace ensure_space(this); 878 // Opcode: FF /2 r64. 879 emit_optional_rex_32(adr); 880 emit(0xFF); 881 emit_modrm(0x2, adr); 882 } 883 884 885 void Assembler::call(const Operand& op) { 886 EnsureSpace ensure_space(this); 887 // Opcode: FF /2 m64. 888 emit_optional_rex_32(op); 889 emit(0xFF); 890 emit_operand(0x2, op); 891 } 892 893 894 // Calls directly to the given address using a relative offset. 895 // Should only ever be used in Code objects for calls within the 896 // same Code object. Should not be used when generating new code (use labels), 897 // but only when patching existing code. 898 void Assembler::call(Address target) { 899 EnsureSpace ensure_space(this); 900 // 1110 1000 #32-bit disp. 901 emit(0xE8); 902 Address source = pc_ + 4; 903 intptr_t displacement = target - source; 904 DCHECK(is_int32(displacement)); 905 emitl(static_cast<int32_t>(displacement)); 906 } 907 908 909 void Assembler::clc() { 910 EnsureSpace ensure_space(this); 911 emit(0xF8); 912 } 913 914 915 void Assembler::cld() { 916 EnsureSpace ensure_space(this); 917 emit(0xFC); 918 } 919 920 921 void Assembler::cdq() { 922 EnsureSpace ensure_space(this); 923 emit(0x99); 924 } 925 926 927 void Assembler::cmovq(Condition cc, Register dst, Register src) { 928 if (cc == always) { 929 movq(dst, src); 930 } else if (cc == never) { 931 return; 932 } 933 // No need to check CpuInfo for CMOV support, it's a required part of the 934 // 64-bit architecture. 935 DCHECK(cc >= 0); // Use mov for unconditional moves. 936 EnsureSpace ensure_space(this); 937 // Opcode: REX.W 0f 40 + cc /r. 938 emit_rex_64(dst, src); 939 emit(0x0f); 940 emit(0x40 + cc); 941 emit_modrm(dst, src); 942 } 943 944 945 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) { 946 if (cc == always) { 947 movq(dst, src); 948 } else if (cc == never) { 949 return; 950 } 951 DCHECK(cc >= 0); 952 EnsureSpace ensure_space(this); 953 // Opcode: REX.W 0f 40 + cc /r. 954 emit_rex_64(dst, src); 955 emit(0x0f); 956 emit(0x40 + cc); 957 emit_operand(dst, src); 958 } 959 960 961 void Assembler::cmovl(Condition cc, Register dst, Register src) { 962 if (cc == always) { 963 movl(dst, src); 964 } else if (cc == never) { 965 return; 966 } 967 DCHECK(cc >= 0); 968 EnsureSpace ensure_space(this); 969 // Opcode: 0f 40 + cc /r. 970 emit_optional_rex_32(dst, src); 971 emit(0x0f); 972 emit(0x40 + cc); 973 emit_modrm(dst, src); 974 } 975 976 977 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) { 978 if (cc == always) { 979 movl(dst, src); 980 } else if (cc == never) { 981 return; 982 } 983 DCHECK(cc >= 0); 984 EnsureSpace ensure_space(this); 985 // Opcode: 0f 40 + cc /r. 986 emit_optional_rex_32(dst, src); 987 emit(0x0f); 988 emit(0x40 + cc); 989 emit_operand(dst, src); 990 } 991 992 993 void Assembler::cmpb_al(Immediate imm8) { 994 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_)); 995 EnsureSpace ensure_space(this); 996 emit(0x3c); 997 emit(imm8.value_); 998 } 999 1000 void Assembler::lock() { 1001 EnsureSpace ensure_space(this); 1002 emit(0xf0); 1003 } 1004 1005 void Assembler::cmpxchgb(const Operand& dst, Register src) { 1006 EnsureSpace ensure_space(this); 1007 if (!src.is_byte_register()) { 1008 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1009 emit_rex_32(src, dst); 1010 } else { 1011 emit_optional_rex_32(src, dst); 1012 } 1013 emit(0x0f); 1014 emit(0xb0); 1015 emit_operand(src, dst); 1016 } 1017 1018 void Assembler::cmpxchgw(const Operand& dst, Register src) { 1019 EnsureSpace ensure_space(this); 1020 emit(0x66); 1021 emit_optional_rex_32(src, dst); 1022 emit(0x0f); 1023 emit(0xb1); 1024 emit_operand(src, dst); 1025 } 1026 1027 void Assembler::emit_cmpxchg(const Operand& dst, Register src, int size) { 1028 EnsureSpace ensure_space(this); 1029 emit_rex(src, dst, size); 1030 emit(0x0f); 1031 emit(0xb1); 1032 emit_operand(src, dst); 1033 } 1034 1035 void Assembler::cpuid() { 1036 EnsureSpace ensure_space(this); 1037 emit(0x0F); 1038 emit(0xA2); 1039 } 1040 1041 1042 void Assembler::cqo() { 1043 EnsureSpace ensure_space(this); 1044 emit_rex_64(); 1045 emit(0x99); 1046 } 1047 1048 1049 void Assembler::emit_dec(Register dst, int size) { 1050 EnsureSpace ensure_space(this); 1051 emit_rex(dst, size); 1052 emit(0xFF); 1053 emit_modrm(0x1, dst); 1054 } 1055 1056 1057 void Assembler::emit_dec(const Operand& dst, int size) { 1058 EnsureSpace ensure_space(this); 1059 emit_rex(dst, size); 1060 emit(0xFF); 1061 emit_operand(1, dst); 1062 } 1063 1064 1065 void Assembler::decb(Register dst) { 1066 EnsureSpace ensure_space(this); 1067 if (!dst.is_byte_register()) { 1068 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1069 emit_rex_32(dst); 1070 } 1071 emit(0xFE); 1072 emit_modrm(0x1, dst); 1073 } 1074 1075 1076 void Assembler::decb(const Operand& dst) { 1077 EnsureSpace ensure_space(this); 1078 emit_optional_rex_32(dst); 1079 emit(0xFE); 1080 emit_operand(1, dst); 1081 } 1082 1083 1084 void Assembler::enter(Immediate size) { 1085 EnsureSpace ensure_space(this); 1086 emit(0xC8); 1087 emitw(size.value_); // 16 bit operand, always. 1088 emit(0); 1089 } 1090 1091 1092 void Assembler::hlt() { 1093 EnsureSpace ensure_space(this); 1094 emit(0xF4); 1095 } 1096 1097 1098 void Assembler::emit_idiv(Register src, int size) { 1099 EnsureSpace ensure_space(this); 1100 emit_rex(src, size); 1101 emit(0xF7); 1102 emit_modrm(0x7, src); 1103 } 1104 1105 1106 void Assembler::emit_div(Register src, int size) { 1107 EnsureSpace ensure_space(this); 1108 emit_rex(src, size); 1109 emit(0xF7); 1110 emit_modrm(0x6, src); 1111 } 1112 1113 1114 void Assembler::emit_imul(Register src, int size) { 1115 EnsureSpace ensure_space(this); 1116 emit_rex(src, size); 1117 emit(0xF7); 1118 emit_modrm(0x5, src); 1119 } 1120 1121 1122 void Assembler::emit_imul(const Operand& src, int size) { 1123 EnsureSpace ensure_space(this); 1124 emit_rex(src, size); 1125 emit(0xF7); 1126 emit_operand(0x5, src); 1127 } 1128 1129 1130 void Assembler::emit_imul(Register dst, Register src, int size) { 1131 EnsureSpace ensure_space(this); 1132 emit_rex(dst, src, size); 1133 emit(0x0F); 1134 emit(0xAF); 1135 emit_modrm(dst, src); 1136 } 1137 1138 1139 void Assembler::emit_imul(Register dst, const Operand& src, int size) { 1140 EnsureSpace ensure_space(this); 1141 emit_rex(dst, src, size); 1142 emit(0x0F); 1143 emit(0xAF); 1144 emit_operand(dst, src); 1145 } 1146 1147 1148 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) { 1149 EnsureSpace ensure_space(this); 1150 emit_rex(dst, src, size); 1151 if (is_int8(imm.value_)) { 1152 emit(0x6B); 1153 emit_modrm(dst, src); 1154 emit(imm.value_); 1155 } else { 1156 emit(0x69); 1157 emit_modrm(dst, src); 1158 emitl(imm.value_); 1159 } 1160 } 1161 1162 1163 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm, 1164 int size) { 1165 EnsureSpace ensure_space(this); 1166 emit_rex(dst, src, size); 1167 if (is_int8(imm.value_)) { 1168 emit(0x6B); 1169 emit_operand(dst, src); 1170 emit(imm.value_); 1171 } else { 1172 emit(0x69); 1173 emit_operand(dst, src); 1174 emitl(imm.value_); 1175 } 1176 } 1177 1178 1179 void Assembler::emit_inc(Register dst, int size) { 1180 EnsureSpace ensure_space(this); 1181 emit_rex(dst, size); 1182 emit(0xFF); 1183 emit_modrm(0x0, dst); 1184 } 1185 1186 1187 void Assembler::emit_inc(const Operand& dst, int size) { 1188 EnsureSpace ensure_space(this); 1189 emit_rex(dst, size); 1190 emit(0xFF); 1191 emit_operand(0, dst); 1192 } 1193 1194 1195 void Assembler::int3() { 1196 EnsureSpace ensure_space(this); 1197 emit(0xCC); 1198 } 1199 1200 1201 void Assembler::j(Condition cc, Label* L, Label::Distance distance) { 1202 if (cc == always) { 1203 jmp(L); 1204 return; 1205 } else if (cc == never) { 1206 return; 1207 } 1208 EnsureSpace ensure_space(this); 1209 DCHECK(is_uint4(cc)); 1210 if (L->is_bound()) { 1211 const int short_size = 2; 1212 const int long_size = 6; 1213 int offs = L->pos() - pc_offset(); 1214 DCHECK(offs <= 0); 1215 // Determine whether we can use 1-byte offsets for backwards branches, 1216 // which have a max range of 128 bytes. 1217 1218 // We also need to check predictable_code_size() flag here, because on x64, 1219 // when the full code generator recompiles code for debugging, some places 1220 // need to be padded out to a certain size. The debugger is keeping track of 1221 // how often it did this so that it can adjust return addresses on the 1222 // stack, but if the size of jump instructions can also change, that's not 1223 // enough and the calculated offsets would be incorrect. 1224 if (is_int8(offs - short_size) && !predictable_code_size()) { 1225 // 0111 tttn #8-bit disp. 1226 emit(0x70 | cc); 1227 emit((offs - short_size) & 0xFF); 1228 } else { 1229 // 0000 1111 1000 tttn #32-bit disp. 1230 emit(0x0F); 1231 emit(0x80 | cc); 1232 emitl(offs - long_size); 1233 } 1234 } else if (distance == Label::kNear) { 1235 // 0111 tttn #8-bit disp 1236 emit(0x70 | cc); 1237 byte disp = 0x00; 1238 if (L->is_near_linked()) { 1239 int offset = L->near_link_pos() - pc_offset(); 1240 DCHECK(is_int8(offset)); 1241 disp = static_cast<byte>(offset & 0xFF); 1242 } 1243 L->link_to(pc_offset(), Label::kNear); 1244 emit(disp); 1245 } else if (L->is_linked()) { 1246 // 0000 1111 1000 tttn #32-bit disp. 1247 emit(0x0F); 1248 emit(0x80 | cc); 1249 emitl(L->pos()); 1250 L->link_to(pc_offset() - sizeof(int32_t)); 1251 } else { 1252 DCHECK(L->is_unused()); 1253 emit(0x0F); 1254 emit(0x80 | cc); 1255 int32_t current = pc_offset(); 1256 emitl(current); 1257 L->link_to(current); 1258 } 1259 } 1260 1261 1262 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) { 1263 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1264 EnsureSpace ensure_space(this); 1265 DCHECK(is_uint4(cc)); 1266 emit(0x0F); 1267 emit(0x80 | cc); 1268 emit_runtime_entry(entry, rmode); 1269 } 1270 1271 1272 void Assembler::j(Condition cc, 1273 Handle<Code> target, 1274 RelocInfo::Mode rmode) { 1275 EnsureSpace ensure_space(this); 1276 DCHECK(is_uint4(cc)); 1277 // 0000 1111 1000 tttn #32-bit disp. 1278 emit(0x0F); 1279 emit(0x80 | cc); 1280 emit_code_target(target, rmode); 1281 } 1282 1283 1284 void Assembler::jmp(Label* L, Label::Distance distance) { 1285 EnsureSpace ensure_space(this); 1286 const int short_size = sizeof(int8_t); 1287 const int long_size = sizeof(int32_t); 1288 if (L->is_bound()) { 1289 int offs = L->pos() - pc_offset() - 1; 1290 DCHECK(offs <= 0); 1291 if (is_int8(offs - short_size) && !predictable_code_size()) { 1292 // 1110 1011 #8-bit disp. 1293 emit(0xEB); 1294 emit((offs - short_size) & 0xFF); 1295 } else { 1296 // 1110 1001 #32-bit disp. 1297 emit(0xE9); 1298 emitl(offs - long_size); 1299 } 1300 } else if (distance == Label::kNear) { 1301 emit(0xEB); 1302 byte disp = 0x00; 1303 if (L->is_near_linked()) { 1304 int offset = L->near_link_pos() - pc_offset(); 1305 DCHECK(is_int8(offset)); 1306 disp = static_cast<byte>(offset & 0xFF); 1307 } 1308 L->link_to(pc_offset(), Label::kNear); 1309 emit(disp); 1310 } else if (L->is_linked()) { 1311 // 1110 1001 #32-bit disp. 1312 emit(0xE9); 1313 emitl(L->pos()); 1314 L->link_to(pc_offset() - long_size); 1315 } else { 1316 // 1110 1001 #32-bit disp. 1317 DCHECK(L->is_unused()); 1318 emit(0xE9); 1319 int32_t current = pc_offset(); 1320 emitl(current); 1321 L->link_to(current); 1322 } 1323 } 1324 1325 1326 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) { 1327 EnsureSpace ensure_space(this); 1328 // 1110 1001 #32-bit disp. 1329 emit(0xE9); 1330 emit_code_target(target, rmode); 1331 } 1332 1333 1334 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { 1335 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1336 EnsureSpace ensure_space(this); 1337 DCHECK(RelocInfo::IsRuntimeEntry(rmode)); 1338 emit(0xE9); 1339 emit_runtime_entry(entry, rmode); 1340 } 1341 1342 1343 void Assembler::jmp(Register target) { 1344 EnsureSpace ensure_space(this); 1345 // Opcode FF/4 r64. 1346 emit_optional_rex_32(target); 1347 emit(0xFF); 1348 emit_modrm(0x4, target); 1349 } 1350 1351 1352 void Assembler::jmp(const Operand& src) { 1353 EnsureSpace ensure_space(this); 1354 // Opcode FF/4 m64. 1355 emit_optional_rex_32(src); 1356 emit(0xFF); 1357 emit_operand(0x4, src); 1358 } 1359 1360 1361 void Assembler::emit_lea(Register dst, const Operand& src, int size) { 1362 EnsureSpace ensure_space(this); 1363 emit_rex(dst, src, size); 1364 emit(0x8D); 1365 emit_operand(dst, src); 1366 } 1367 1368 1369 void Assembler::load_rax(void* value, RelocInfo::Mode mode) { 1370 EnsureSpace ensure_space(this); 1371 if (kPointerSize == kInt64Size) { 1372 emit(0x48); // REX.W 1373 emit(0xA1); 1374 emitp(value, mode); 1375 } else { 1376 DCHECK(kPointerSize == kInt32Size); 1377 emit(0xA1); 1378 emitp(value, mode); 1379 // In 64-bit mode, need to zero extend the operand to 8 bytes. 1380 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 1381 // Developer's Manual Volume 2. 1382 emitl(0); 1383 } 1384 } 1385 1386 1387 void Assembler::load_rax(ExternalReference ref) { 1388 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 1389 } 1390 1391 1392 void Assembler::leave() { 1393 EnsureSpace ensure_space(this); 1394 emit(0xC9); 1395 } 1396 1397 1398 void Assembler::movb(Register dst, const Operand& src) { 1399 EnsureSpace ensure_space(this); 1400 if (!dst.is_byte_register()) { 1401 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1402 emit_rex_32(dst, src); 1403 } else { 1404 emit_optional_rex_32(dst, src); 1405 } 1406 emit(0x8A); 1407 emit_operand(dst, src); 1408 } 1409 1410 1411 void Assembler::movb(Register dst, Immediate imm) { 1412 EnsureSpace ensure_space(this); 1413 if (!dst.is_byte_register()) { 1414 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1415 emit_rex_32(dst); 1416 } 1417 emit(0xB0 + dst.low_bits()); 1418 emit(imm.value_); 1419 } 1420 1421 1422 void Assembler::movb(const Operand& dst, Register src) { 1423 EnsureSpace ensure_space(this); 1424 if (!src.is_byte_register()) { 1425 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1426 emit_rex_32(src, dst); 1427 } else { 1428 emit_optional_rex_32(src, dst); 1429 } 1430 emit(0x88); 1431 emit_operand(src, dst); 1432 } 1433 1434 1435 void Assembler::movb(const Operand& dst, Immediate imm) { 1436 EnsureSpace ensure_space(this); 1437 emit_optional_rex_32(dst); 1438 emit(0xC6); 1439 emit_operand(0x0, dst); 1440 emit(static_cast<byte>(imm.value_)); 1441 } 1442 1443 1444 void Assembler::movw(Register dst, const Operand& src) { 1445 EnsureSpace ensure_space(this); 1446 emit(0x66); 1447 emit_optional_rex_32(dst, src); 1448 emit(0x8B); 1449 emit_operand(dst, src); 1450 } 1451 1452 1453 void Assembler::movw(const Operand& dst, Register src) { 1454 EnsureSpace ensure_space(this); 1455 emit(0x66); 1456 emit_optional_rex_32(src, dst); 1457 emit(0x89); 1458 emit_operand(src, dst); 1459 } 1460 1461 1462 void Assembler::movw(const Operand& dst, Immediate imm) { 1463 EnsureSpace ensure_space(this); 1464 emit(0x66); 1465 emit_optional_rex_32(dst); 1466 emit(0xC7); 1467 emit_operand(0x0, dst); 1468 emit(static_cast<byte>(imm.value_ & 0xff)); 1469 emit(static_cast<byte>(imm.value_ >> 8)); 1470 } 1471 1472 1473 void Assembler::emit_mov(Register dst, const Operand& src, int size) { 1474 EnsureSpace ensure_space(this); 1475 emit_rex(dst, src, size); 1476 emit(0x8B); 1477 emit_operand(dst, src); 1478 } 1479 1480 1481 void Assembler::emit_mov(Register dst, Register src, int size) { 1482 EnsureSpace ensure_space(this); 1483 if (src.low_bits() == 4) { 1484 emit_rex(src, dst, size); 1485 emit(0x89); 1486 emit_modrm(src, dst); 1487 } else { 1488 emit_rex(dst, src, size); 1489 emit(0x8B); 1490 emit_modrm(dst, src); 1491 } 1492 } 1493 1494 1495 void Assembler::emit_mov(const Operand& dst, Register src, int size) { 1496 EnsureSpace ensure_space(this); 1497 emit_rex(src, dst, size); 1498 emit(0x89); 1499 emit_operand(src, dst); 1500 } 1501 1502 1503 void Assembler::emit_mov(Register dst, Immediate value, int size) { 1504 EnsureSpace ensure_space(this); 1505 emit_rex(dst, size); 1506 if (size == kInt64Size) { 1507 emit(0xC7); 1508 emit_modrm(0x0, dst); 1509 } else { 1510 DCHECK(size == kInt32Size); 1511 emit(0xB8 + dst.low_bits()); 1512 } 1513 emit(value); 1514 } 1515 1516 1517 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) { 1518 EnsureSpace ensure_space(this); 1519 emit_rex(dst, size); 1520 emit(0xC7); 1521 emit_operand(0x0, dst); 1522 emit(value); 1523 } 1524 1525 1526 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) { 1527 EnsureSpace ensure_space(this); 1528 emit_rex(dst, kPointerSize); 1529 emit(0xB8 | dst.low_bits()); 1530 emitp(value, rmode); 1531 } 1532 1533 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) { 1534 EnsureSpace ensure_space(this); 1535 emit_rex_64(dst); 1536 emit(0xB8 | dst.low_bits()); 1537 if (!RelocInfo::IsNone(rmode)) { 1538 RecordRelocInfo(rmode, value); 1539 } 1540 emitq(value); 1541 } 1542 1543 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) { 1544 movq(dst, static_cast<int64_t>(value), rmode); 1545 } 1546 1547 // Loads the ip-relative location of the src label into the target location 1548 // (as a 32-bit offset sign extended to 64-bit). 1549 void Assembler::movl(const Operand& dst, Label* src) { 1550 EnsureSpace ensure_space(this); 1551 emit_optional_rex_32(dst); 1552 emit(0xC7); 1553 emit_operand(0, dst); 1554 if (src->is_bound()) { 1555 int offset = src->pos() - pc_offset() - sizeof(int32_t); 1556 DCHECK(offset <= 0); 1557 emitl(offset); 1558 } else if (src->is_linked()) { 1559 emitl(src->pos()); 1560 src->link_to(pc_offset() - sizeof(int32_t)); 1561 } else { 1562 DCHECK(src->is_unused()); 1563 int32_t current = pc_offset(); 1564 emitl(current); 1565 src->link_to(current); 1566 } 1567 } 1568 1569 1570 void Assembler::movsxbl(Register dst, Register src) { 1571 EnsureSpace ensure_space(this); 1572 if (!src.is_byte_register()) { 1573 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1574 emit_rex_32(dst, src); 1575 } else { 1576 emit_optional_rex_32(dst, src); 1577 } 1578 emit(0x0F); 1579 emit(0xBE); 1580 emit_modrm(dst, src); 1581 } 1582 1583 1584 void Assembler::movsxbl(Register dst, const Operand& src) { 1585 EnsureSpace ensure_space(this); 1586 emit_optional_rex_32(dst, src); 1587 emit(0x0F); 1588 emit(0xBE); 1589 emit_operand(dst, src); 1590 } 1591 1592 1593 void Assembler::movsxbq(Register dst, const Operand& src) { 1594 EnsureSpace ensure_space(this); 1595 emit_rex_64(dst, src); 1596 emit(0x0F); 1597 emit(0xBE); 1598 emit_operand(dst, src); 1599 } 1600 1601 1602 void Assembler::movsxwl(Register dst, Register src) { 1603 EnsureSpace ensure_space(this); 1604 emit_optional_rex_32(dst, src); 1605 emit(0x0F); 1606 emit(0xBF); 1607 emit_modrm(dst, src); 1608 } 1609 1610 1611 void Assembler::movsxwl(Register dst, const Operand& src) { 1612 EnsureSpace ensure_space(this); 1613 emit_optional_rex_32(dst, src); 1614 emit(0x0F); 1615 emit(0xBF); 1616 emit_operand(dst, src); 1617 } 1618 1619 1620 void Assembler::movsxwq(Register dst, const Operand& src) { 1621 EnsureSpace ensure_space(this); 1622 emit_rex_64(dst, src); 1623 emit(0x0F); 1624 emit(0xBF); 1625 emit_operand(dst, src); 1626 } 1627 1628 1629 void Assembler::movsxlq(Register dst, Register src) { 1630 EnsureSpace ensure_space(this); 1631 emit_rex_64(dst, src); 1632 emit(0x63); 1633 emit_modrm(dst, src); 1634 } 1635 1636 1637 void Assembler::movsxlq(Register dst, const Operand& src) { 1638 EnsureSpace ensure_space(this); 1639 emit_rex_64(dst, src); 1640 emit(0x63); 1641 emit_operand(dst, src); 1642 } 1643 1644 1645 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) { 1646 EnsureSpace ensure_space(this); 1647 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1648 // there is no need to make this a 64 bit operation. 1649 emit_optional_rex_32(dst, src); 1650 emit(0x0F); 1651 emit(0xB6); 1652 emit_operand(dst, src); 1653 } 1654 1655 1656 void Assembler::emit_movzxb(Register dst, Register src, int size) { 1657 EnsureSpace ensure_space(this); 1658 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1659 // there is no need to make this a 64 bit operation. 1660 if (!src.is_byte_register()) { 1661 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1662 emit_rex_32(dst, src); 1663 } else { 1664 emit_optional_rex_32(dst, src); 1665 } 1666 emit(0x0F); 1667 emit(0xB6); 1668 emit_modrm(dst, src); 1669 } 1670 1671 1672 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) { 1673 EnsureSpace ensure_space(this); 1674 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1675 // there is no need to make this a 64 bit operation. 1676 emit_optional_rex_32(dst, src); 1677 emit(0x0F); 1678 emit(0xB7); 1679 emit_operand(dst, src); 1680 } 1681 1682 1683 void Assembler::emit_movzxw(Register dst, Register src, int size) { 1684 EnsureSpace ensure_space(this); 1685 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore 1686 // there is no need to make this a 64 bit operation. 1687 emit_optional_rex_32(dst, src); 1688 emit(0x0F); 1689 emit(0xB7); 1690 emit_modrm(dst, src); 1691 } 1692 1693 1694 void Assembler::repmovsb() { 1695 EnsureSpace ensure_space(this); 1696 emit(0xF3); 1697 emit(0xA4); 1698 } 1699 1700 1701 void Assembler::repmovsw() { 1702 EnsureSpace ensure_space(this); 1703 emit(0x66); // Operand size override. 1704 emit(0xF3); 1705 emit(0xA4); 1706 } 1707 1708 1709 void Assembler::emit_repmovs(int size) { 1710 EnsureSpace ensure_space(this); 1711 emit(0xF3); 1712 emit_rex(size); 1713 emit(0xA5); 1714 } 1715 1716 1717 void Assembler::mull(Register src) { 1718 EnsureSpace ensure_space(this); 1719 emit_optional_rex_32(src); 1720 emit(0xF7); 1721 emit_modrm(0x4, src); 1722 } 1723 1724 1725 void Assembler::mull(const Operand& src) { 1726 EnsureSpace ensure_space(this); 1727 emit_optional_rex_32(src); 1728 emit(0xF7); 1729 emit_operand(0x4, src); 1730 } 1731 1732 1733 void Assembler::mulq(Register src) { 1734 EnsureSpace ensure_space(this); 1735 emit_rex_64(src); 1736 emit(0xF7); 1737 emit_modrm(0x4, src); 1738 } 1739 1740 1741 void Assembler::emit_neg(Register dst, int size) { 1742 EnsureSpace ensure_space(this); 1743 emit_rex(dst, size); 1744 emit(0xF7); 1745 emit_modrm(0x3, dst); 1746 } 1747 1748 1749 void Assembler::emit_neg(const Operand& dst, int size) { 1750 EnsureSpace ensure_space(this); 1751 emit_rex_64(dst); 1752 emit(0xF7); 1753 emit_operand(3, dst); 1754 } 1755 1756 1757 void Assembler::nop() { 1758 EnsureSpace ensure_space(this); 1759 emit(0x90); 1760 } 1761 1762 1763 void Assembler::emit_not(Register dst, int size) { 1764 EnsureSpace ensure_space(this); 1765 emit_rex(dst, size); 1766 emit(0xF7); 1767 emit_modrm(0x2, dst); 1768 } 1769 1770 1771 void Assembler::emit_not(const Operand& dst, int size) { 1772 EnsureSpace ensure_space(this); 1773 emit_rex(dst, size); 1774 emit(0xF7); 1775 emit_operand(2, dst); 1776 } 1777 1778 1779 void Assembler::Nop(int n) { 1780 // The recommended muti-byte sequences of NOP instructions from the Intel 64 1781 // and IA-32 Architectures Software Developer's Manual. 1782 // 1783 // Length Assembly Byte Sequence 1784 // 2 bytes 66 NOP 66 90H 1785 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H 1786 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H 1787 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H 1788 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H 1789 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H 1790 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H 1791 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00 1792 // 00000000H] 00H 1793 1794 EnsureSpace ensure_space(this); 1795 while (n > 0) { 1796 switch (n) { 1797 case 2: 1798 emit(0x66); 1799 case 1: 1800 emit(0x90); 1801 return; 1802 case 3: 1803 emit(0x0f); 1804 emit(0x1f); 1805 emit(0x00); 1806 return; 1807 case 4: 1808 emit(0x0f); 1809 emit(0x1f); 1810 emit(0x40); 1811 emit(0x00); 1812 return; 1813 case 6: 1814 emit(0x66); 1815 case 5: 1816 emit(0x0f); 1817 emit(0x1f); 1818 emit(0x44); 1819 emit(0x00); 1820 emit(0x00); 1821 return; 1822 case 7: 1823 emit(0x0f); 1824 emit(0x1f); 1825 emit(0x80); 1826 emit(0x00); 1827 emit(0x00); 1828 emit(0x00); 1829 emit(0x00); 1830 return; 1831 default: 1832 case 11: 1833 emit(0x66); 1834 n--; 1835 case 10: 1836 emit(0x66); 1837 n--; 1838 case 9: 1839 emit(0x66); 1840 n--; 1841 case 8: 1842 emit(0x0f); 1843 emit(0x1f); 1844 emit(0x84); 1845 emit(0x00); 1846 emit(0x00); 1847 emit(0x00); 1848 emit(0x00); 1849 emit(0x00); 1850 n -= 8; 1851 } 1852 } 1853 } 1854 1855 1856 void Assembler::popq(Register dst) { 1857 EnsureSpace ensure_space(this); 1858 emit_optional_rex_32(dst); 1859 emit(0x58 | dst.low_bits()); 1860 } 1861 1862 1863 void Assembler::popq(const Operand& dst) { 1864 EnsureSpace ensure_space(this); 1865 emit_optional_rex_32(dst); 1866 emit(0x8F); 1867 emit_operand(0, dst); 1868 } 1869 1870 1871 void Assembler::popfq() { 1872 EnsureSpace ensure_space(this); 1873 emit(0x9D); 1874 } 1875 1876 1877 void Assembler::pushq(Register src) { 1878 EnsureSpace ensure_space(this); 1879 emit_optional_rex_32(src); 1880 emit(0x50 | src.low_bits()); 1881 } 1882 1883 1884 void Assembler::pushq(const Operand& src) { 1885 EnsureSpace ensure_space(this); 1886 emit_optional_rex_32(src); 1887 emit(0xFF); 1888 emit_operand(6, src); 1889 } 1890 1891 1892 void Assembler::pushq(Immediate value) { 1893 EnsureSpace ensure_space(this); 1894 if (is_int8(value.value_)) { 1895 emit(0x6A); 1896 emit(value.value_); // Emit low byte of value. 1897 } else { 1898 emit(0x68); 1899 emitl(value.value_); 1900 } 1901 } 1902 1903 1904 void Assembler::pushq_imm32(int32_t imm32) { 1905 EnsureSpace ensure_space(this); 1906 emit(0x68); 1907 emitl(imm32); 1908 } 1909 1910 1911 void Assembler::pushfq() { 1912 EnsureSpace ensure_space(this); 1913 emit(0x9C); 1914 } 1915 1916 1917 void Assembler::ret(int imm16) { 1918 EnsureSpace ensure_space(this); 1919 DCHECK(is_uint16(imm16)); 1920 if (imm16 == 0) { 1921 emit(0xC3); 1922 } else { 1923 emit(0xC2); 1924 emit(imm16 & 0xFF); 1925 emit((imm16 >> 8) & 0xFF); 1926 } 1927 } 1928 1929 1930 void Assembler::ud2() { 1931 EnsureSpace ensure_space(this); 1932 emit(0x0F); 1933 emit(0x0B); 1934 } 1935 1936 1937 void Assembler::setcc(Condition cc, Register reg) { 1938 if (cc > last_condition) { 1939 movb(reg, Immediate(cc == always ? 1 : 0)); 1940 return; 1941 } 1942 EnsureSpace ensure_space(this); 1943 DCHECK(is_uint4(cc)); 1944 if (!reg.is_byte_register()) { 1945 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1946 emit_rex_32(reg); 1947 } 1948 emit(0x0F); 1949 emit(0x90 | cc); 1950 emit_modrm(0x0, reg); 1951 } 1952 1953 1954 void Assembler::shld(Register dst, Register src) { 1955 EnsureSpace ensure_space(this); 1956 emit_rex_64(src, dst); 1957 emit(0x0F); 1958 emit(0xA5); 1959 emit_modrm(src, dst); 1960 } 1961 1962 1963 void Assembler::shrd(Register dst, Register src) { 1964 EnsureSpace ensure_space(this); 1965 emit_rex_64(src, dst); 1966 emit(0x0F); 1967 emit(0xAD); 1968 emit_modrm(src, dst); 1969 } 1970 1971 void Assembler::xchgb(Register reg, const Operand& op) { 1972 EnsureSpace ensure_space(this); 1973 if (!reg.is_byte_register()) { 1974 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 1975 emit_rex_32(reg, op); 1976 } else { 1977 emit_optional_rex_32(reg, op); 1978 } 1979 emit(0x86); 1980 emit_operand(reg, op); 1981 } 1982 1983 void Assembler::xchgw(Register reg, const Operand& op) { 1984 EnsureSpace ensure_space(this); 1985 emit(0x66); 1986 emit_optional_rex_32(reg, op); 1987 emit(0x87); 1988 emit_operand(reg, op); 1989 } 1990 1991 void Assembler::emit_xchg(Register dst, Register src, int size) { 1992 EnsureSpace ensure_space(this); 1993 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding 1994 Register other = src.is(rax) ? dst : src; 1995 emit_rex(other, size); 1996 emit(0x90 | other.low_bits()); 1997 } else if (dst.low_bits() == 4) { 1998 emit_rex(dst, src, size); 1999 emit(0x87); 2000 emit_modrm(dst, src); 2001 } else { 2002 emit_rex(src, dst, size); 2003 emit(0x87); 2004 emit_modrm(src, dst); 2005 } 2006 } 2007 2008 2009 void Assembler::emit_xchg(Register dst, const Operand& src, int size) { 2010 EnsureSpace ensure_space(this); 2011 emit_rex(dst, src, size); 2012 emit(0x87); 2013 emit_operand(dst, src); 2014 } 2015 2016 2017 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) { 2018 EnsureSpace ensure_space(this); 2019 if (kPointerSize == kInt64Size) { 2020 emit(0x48); // REX.W 2021 emit(0xA3); 2022 emitp(dst, mode); 2023 } else { 2024 DCHECK(kPointerSize == kInt32Size); 2025 emit(0xA3); 2026 emitp(dst, mode); 2027 // In 64-bit mode, need to zero extend the operand to 8 bytes. 2028 // See 2.2.1.4 in Intel64 and IA32 Architectures Software 2029 // Developer's Manual Volume 2. 2030 emitl(0); 2031 } 2032 } 2033 2034 2035 void Assembler::store_rax(ExternalReference ref) { 2036 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE); 2037 } 2038 2039 2040 void Assembler::testb(Register dst, Register src) { 2041 EnsureSpace ensure_space(this); 2042 if (src.low_bits() == 4) { 2043 emit_rex_32(src, dst); 2044 emit(0x84); 2045 emit_modrm(src, dst); 2046 } else { 2047 if (!dst.is_byte_register() || !src.is_byte_register()) { 2048 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2049 emit_rex_32(dst, src); 2050 } 2051 emit(0x84); 2052 emit_modrm(dst, src); 2053 } 2054 } 2055 2056 2057 void Assembler::testb(Register reg, Immediate mask) { 2058 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2059 EnsureSpace ensure_space(this); 2060 if (reg.is(rax)) { 2061 emit(0xA8); 2062 emit(mask.value_); // Low byte emitted. 2063 } else { 2064 if (!reg.is_byte_register()) { 2065 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2066 emit_rex_32(reg); 2067 } 2068 emit(0xF6); 2069 emit_modrm(0x0, reg); 2070 emit(mask.value_); // Low byte emitted. 2071 } 2072 } 2073 2074 2075 void Assembler::testb(const Operand& op, Immediate mask) { 2076 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_)); 2077 EnsureSpace ensure_space(this); 2078 emit_optional_rex_32(rax, op); 2079 emit(0xF6); 2080 emit_operand(rax, op); // Operation code 0 2081 emit(mask.value_); // Low byte emitted. 2082 } 2083 2084 2085 void Assembler::testb(const Operand& op, Register reg) { 2086 EnsureSpace ensure_space(this); 2087 if (!reg.is_byte_register()) { 2088 // Register is not one of al, bl, cl, dl. Its encoding needs REX. 2089 emit_rex_32(reg, op); 2090 } else { 2091 emit_optional_rex_32(reg, op); 2092 } 2093 emit(0x84); 2094 emit_operand(reg, op); 2095 } 2096 2097 void Assembler::testw(Register dst, Register src) { 2098 EnsureSpace ensure_space(this); 2099 emit(0x66); 2100 if (src.low_bits() == 4) { 2101 emit_rex_32(src, dst); 2102 } 2103 emit(0x85); 2104 emit_modrm(src, dst); 2105 } 2106 2107 void Assembler::testw(Register reg, Immediate mask) { 2108 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_)); 2109 EnsureSpace ensure_space(this); 2110 emit(0x66); 2111 if (reg.is(rax)) { 2112 emit(0xA9); 2113 emitw(mask.value_); 2114 } else { 2115 if (reg.low_bits() == 4) { 2116 emit_rex_32(reg); 2117 } 2118 emit(0xF7); 2119 emit_modrm(0x0, reg); 2120 emitw(mask.value_); 2121 } 2122 } 2123 2124 void Assembler::testw(const Operand& op, Immediate mask) { 2125 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_)); 2126 EnsureSpace ensure_space(this); 2127 emit(0x66); 2128 emit_optional_rex_32(rax, op); 2129 emit(0xF7); 2130 emit_operand(rax, op); 2131 emitw(mask.value_); 2132 } 2133 2134 void Assembler::testw(const Operand& op, Register reg) { 2135 EnsureSpace ensure_space(this); 2136 emit(0x66); 2137 emit_optional_rex_32(reg, op); 2138 emit(0x85); 2139 emit_operand(rax, op); 2140 } 2141 2142 void Assembler::emit_test(Register dst, Register src, int size) { 2143 EnsureSpace ensure_space(this); 2144 if (src.low_bits() == 4) { 2145 emit_rex(src, dst, size); 2146 emit(0x85); 2147 emit_modrm(src, dst); 2148 } else { 2149 emit_rex(dst, src, size); 2150 emit(0x85); 2151 emit_modrm(dst, src); 2152 } 2153 } 2154 2155 2156 void Assembler::emit_test(Register reg, Immediate mask, int size) { 2157 // testl with a mask that fits in the low byte is exactly testb. 2158 if (is_uint8(mask.value_)) { 2159 testb(reg, mask); 2160 return; 2161 } 2162 EnsureSpace ensure_space(this); 2163 if (reg.is(rax)) { 2164 emit_rex(rax, size); 2165 emit(0xA9); 2166 emit(mask); 2167 } else { 2168 emit_rex(reg, size); 2169 emit(0xF7); 2170 emit_modrm(0x0, reg); 2171 emit(mask); 2172 } 2173 } 2174 2175 2176 void Assembler::emit_test(const Operand& op, Immediate mask, int size) { 2177 // testl with a mask that fits in the low byte is exactly testb. 2178 if (is_uint8(mask.value_)) { 2179 testb(op, mask); 2180 return; 2181 } 2182 EnsureSpace ensure_space(this); 2183 emit_rex(rax, op, size); 2184 emit(0xF7); 2185 emit_operand(rax, op); // Operation code 0 2186 emit(mask); 2187 } 2188 2189 2190 void Assembler::emit_test(const Operand& op, Register reg, int size) { 2191 EnsureSpace ensure_space(this); 2192 emit_rex(reg, op, size); 2193 emit(0x85); 2194 emit_operand(reg, op); 2195 } 2196 2197 2198 // FPU instructions. 2199 2200 2201 void Assembler::fld(int i) { 2202 EnsureSpace ensure_space(this); 2203 emit_farith(0xD9, 0xC0, i); 2204 } 2205 2206 2207 void Assembler::fld1() { 2208 EnsureSpace ensure_space(this); 2209 emit(0xD9); 2210 emit(0xE8); 2211 } 2212 2213 2214 void Assembler::fldz() { 2215 EnsureSpace ensure_space(this); 2216 emit(0xD9); 2217 emit(0xEE); 2218 } 2219 2220 2221 void Assembler::fldpi() { 2222 EnsureSpace ensure_space(this); 2223 emit(0xD9); 2224 emit(0xEB); 2225 } 2226 2227 2228 void Assembler::fldln2() { 2229 EnsureSpace ensure_space(this); 2230 emit(0xD9); 2231 emit(0xED); 2232 } 2233 2234 2235 void Assembler::fld_s(const Operand& adr) { 2236 EnsureSpace ensure_space(this); 2237 emit_optional_rex_32(adr); 2238 emit(0xD9); 2239 emit_operand(0, adr); 2240 } 2241 2242 2243 void Assembler::fld_d(const Operand& adr) { 2244 EnsureSpace ensure_space(this); 2245 emit_optional_rex_32(adr); 2246 emit(0xDD); 2247 emit_operand(0, adr); 2248 } 2249 2250 2251 void Assembler::fstp_s(const Operand& adr) { 2252 EnsureSpace ensure_space(this); 2253 emit_optional_rex_32(adr); 2254 emit(0xD9); 2255 emit_operand(3, adr); 2256 } 2257 2258 2259 void Assembler::fstp_d(const Operand& adr) { 2260 EnsureSpace ensure_space(this); 2261 emit_optional_rex_32(adr); 2262 emit(0xDD); 2263 emit_operand(3, adr); 2264 } 2265 2266 2267 void Assembler::fstp(int index) { 2268 DCHECK(is_uint3(index)); 2269 EnsureSpace ensure_space(this); 2270 emit_farith(0xDD, 0xD8, index); 2271 } 2272 2273 2274 void Assembler::fild_s(const Operand& adr) { 2275 EnsureSpace ensure_space(this); 2276 emit_optional_rex_32(adr); 2277 emit(0xDB); 2278 emit_operand(0, adr); 2279 } 2280 2281 2282 void Assembler::fild_d(const Operand& adr) { 2283 EnsureSpace ensure_space(this); 2284 emit_optional_rex_32(adr); 2285 emit(0xDF); 2286 emit_operand(5, adr); 2287 } 2288 2289 2290 void Assembler::fistp_s(const Operand& adr) { 2291 EnsureSpace ensure_space(this); 2292 emit_optional_rex_32(adr); 2293 emit(0xDB); 2294 emit_operand(3, adr); 2295 } 2296 2297 2298 void Assembler::fisttp_s(const Operand& adr) { 2299 DCHECK(IsEnabled(SSE3)); 2300 EnsureSpace ensure_space(this); 2301 emit_optional_rex_32(adr); 2302 emit(0xDB); 2303 emit_operand(1, adr); 2304 } 2305 2306 2307 void Assembler::fisttp_d(const Operand& adr) { 2308 DCHECK(IsEnabled(SSE3)); 2309 EnsureSpace ensure_space(this); 2310 emit_optional_rex_32(adr); 2311 emit(0xDD); 2312 emit_operand(1, adr); 2313 } 2314 2315 2316 void Assembler::fist_s(const Operand& adr) { 2317 EnsureSpace ensure_space(this); 2318 emit_optional_rex_32(adr); 2319 emit(0xDB); 2320 emit_operand(2, adr); 2321 } 2322 2323 2324 void Assembler::fistp_d(const Operand& adr) { 2325 EnsureSpace ensure_space(this); 2326 emit_optional_rex_32(adr); 2327 emit(0xDF); 2328 emit_operand(7, adr); 2329 } 2330 2331 2332 void Assembler::fabs() { 2333 EnsureSpace ensure_space(this); 2334 emit(0xD9); 2335 emit(0xE1); 2336 } 2337 2338 2339 void Assembler::fchs() { 2340 EnsureSpace ensure_space(this); 2341 emit(0xD9); 2342 emit(0xE0); 2343 } 2344 2345 2346 void Assembler::fcos() { 2347 EnsureSpace ensure_space(this); 2348 emit(0xD9); 2349 emit(0xFF); 2350 } 2351 2352 2353 void Assembler::fsin() { 2354 EnsureSpace ensure_space(this); 2355 emit(0xD9); 2356 emit(0xFE); 2357 } 2358 2359 2360 void Assembler::fptan() { 2361 EnsureSpace ensure_space(this); 2362 emit(0xD9); 2363 emit(0xF2); 2364 } 2365 2366 2367 void Assembler::fyl2x() { 2368 EnsureSpace ensure_space(this); 2369 emit(0xD9); 2370 emit(0xF1); 2371 } 2372 2373 2374 void Assembler::f2xm1() { 2375 EnsureSpace ensure_space(this); 2376 emit(0xD9); 2377 emit(0xF0); 2378 } 2379 2380 2381 void Assembler::fscale() { 2382 EnsureSpace ensure_space(this); 2383 emit(0xD9); 2384 emit(0xFD); 2385 } 2386 2387 2388 void Assembler::fninit() { 2389 EnsureSpace ensure_space(this); 2390 emit(0xDB); 2391 emit(0xE3); 2392 } 2393 2394 2395 void Assembler::fadd(int i) { 2396 EnsureSpace ensure_space(this); 2397 emit_farith(0xDC, 0xC0, i); 2398 } 2399 2400 2401 void Assembler::fsub(int i) { 2402 EnsureSpace ensure_space(this); 2403 emit_farith(0xDC, 0xE8, i); 2404 } 2405 2406 2407 void Assembler::fisub_s(const Operand& adr) { 2408 EnsureSpace ensure_space(this); 2409 emit_optional_rex_32(adr); 2410 emit(0xDA); 2411 emit_operand(4, adr); 2412 } 2413 2414 2415 void Assembler::fmul(int i) { 2416 EnsureSpace ensure_space(this); 2417 emit_farith(0xDC, 0xC8, i); 2418 } 2419 2420 2421 void Assembler::fdiv(int i) { 2422 EnsureSpace ensure_space(this); 2423 emit_farith(0xDC, 0xF8, i); 2424 } 2425 2426 2427 void Assembler::faddp(int i) { 2428 EnsureSpace ensure_space(this); 2429 emit_farith(0xDE, 0xC0, i); 2430 } 2431 2432 2433 void Assembler::fsubp(int i) { 2434 EnsureSpace ensure_space(this); 2435 emit_farith(0xDE, 0xE8, i); 2436 } 2437 2438 2439 void Assembler::fsubrp(int i) { 2440 EnsureSpace ensure_space(this); 2441 emit_farith(0xDE, 0xE0, i); 2442 } 2443 2444 2445 void Assembler::fmulp(int i) { 2446 EnsureSpace ensure_space(this); 2447 emit_farith(0xDE, 0xC8, i); 2448 } 2449 2450 2451 void Assembler::fdivp(int i) { 2452 EnsureSpace ensure_space(this); 2453 emit_farith(0xDE, 0xF8, i); 2454 } 2455 2456 2457 void Assembler::fprem() { 2458 EnsureSpace ensure_space(this); 2459 emit(0xD9); 2460 emit(0xF8); 2461 } 2462 2463 2464 void Assembler::fprem1() { 2465 EnsureSpace ensure_space(this); 2466 emit(0xD9); 2467 emit(0xF5); 2468 } 2469 2470 2471 void Assembler::fxch(int i) { 2472 EnsureSpace ensure_space(this); 2473 emit_farith(0xD9, 0xC8, i); 2474 } 2475 2476 2477 void Assembler::fincstp() { 2478 EnsureSpace ensure_space(this); 2479 emit(0xD9); 2480 emit(0xF7); 2481 } 2482 2483 2484 void Assembler::ffree(int i) { 2485 EnsureSpace ensure_space(this); 2486 emit_farith(0xDD, 0xC0, i); 2487 } 2488 2489 2490 void Assembler::ftst() { 2491 EnsureSpace ensure_space(this); 2492 emit(0xD9); 2493 emit(0xE4); 2494 } 2495 2496 2497 void Assembler::fucomp(int i) { 2498 EnsureSpace ensure_space(this); 2499 emit_farith(0xDD, 0xE8, i); 2500 } 2501 2502 2503 void Assembler::fucompp() { 2504 EnsureSpace ensure_space(this); 2505 emit(0xDA); 2506 emit(0xE9); 2507 } 2508 2509 2510 void Assembler::fucomi(int i) { 2511 EnsureSpace ensure_space(this); 2512 emit(0xDB); 2513 emit(0xE8 + i); 2514 } 2515 2516 2517 void Assembler::fucomip() { 2518 EnsureSpace ensure_space(this); 2519 emit(0xDF); 2520 emit(0xE9); 2521 } 2522 2523 2524 void Assembler::fcompp() { 2525 EnsureSpace ensure_space(this); 2526 emit(0xDE); 2527 emit(0xD9); 2528 } 2529 2530 2531 void Assembler::fnstsw_ax() { 2532 EnsureSpace ensure_space(this); 2533 emit(0xDF); 2534 emit(0xE0); 2535 } 2536 2537 2538 void Assembler::fwait() { 2539 EnsureSpace ensure_space(this); 2540 emit(0x9B); 2541 } 2542 2543 2544 void Assembler::frndint() { 2545 EnsureSpace ensure_space(this); 2546 emit(0xD9); 2547 emit(0xFC); 2548 } 2549 2550 2551 void Assembler::fnclex() { 2552 EnsureSpace ensure_space(this); 2553 emit(0xDB); 2554 emit(0xE2); 2555 } 2556 2557 2558 void Assembler::sahf() { 2559 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf 2560 // in 64-bit mode. Test CpuID. 2561 DCHECK(IsEnabled(SAHF)); 2562 EnsureSpace ensure_space(this); 2563 emit(0x9E); 2564 } 2565 2566 2567 void Assembler::emit_farith(int b1, int b2, int i) { 2568 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode 2569 DCHECK(is_uint3(i)); // illegal stack offset 2570 emit(b1); 2571 emit(b2 + i); 2572 } 2573 2574 2575 // SSE operations. 2576 2577 void Assembler::andps(XMMRegister dst, XMMRegister src) { 2578 EnsureSpace ensure_space(this); 2579 emit_optional_rex_32(dst, src); 2580 emit(0x0F); 2581 emit(0x54); 2582 emit_sse_operand(dst, src); 2583 } 2584 2585 2586 void Assembler::andps(XMMRegister dst, const Operand& src) { 2587 EnsureSpace ensure_space(this); 2588 emit_optional_rex_32(dst, src); 2589 emit(0x0F); 2590 emit(0x54); 2591 emit_sse_operand(dst, src); 2592 } 2593 2594 2595 void Assembler::orps(XMMRegister dst, XMMRegister src) { 2596 EnsureSpace ensure_space(this); 2597 emit_optional_rex_32(dst, src); 2598 emit(0x0F); 2599 emit(0x56); 2600 emit_sse_operand(dst, src); 2601 } 2602 2603 2604 void Assembler::orps(XMMRegister dst, const Operand& src) { 2605 EnsureSpace ensure_space(this); 2606 emit_optional_rex_32(dst, src); 2607 emit(0x0F); 2608 emit(0x56); 2609 emit_sse_operand(dst, src); 2610 } 2611 2612 2613 void Assembler::xorps(XMMRegister dst, XMMRegister src) { 2614 DCHECK(!IsEnabled(AVX)); 2615 EnsureSpace ensure_space(this); 2616 emit_optional_rex_32(dst, src); 2617 emit(0x0F); 2618 emit(0x57); 2619 emit_sse_operand(dst, src); 2620 } 2621 2622 2623 void Assembler::xorps(XMMRegister dst, const Operand& src) { 2624 DCHECK(!IsEnabled(AVX)); 2625 EnsureSpace ensure_space(this); 2626 emit_optional_rex_32(dst, src); 2627 emit(0x0F); 2628 emit(0x57); 2629 emit_sse_operand(dst, src); 2630 } 2631 2632 2633 void Assembler::addps(XMMRegister dst, XMMRegister src) { 2634 EnsureSpace ensure_space(this); 2635 emit_optional_rex_32(dst, src); 2636 emit(0x0F); 2637 emit(0x58); 2638 emit_sse_operand(dst, src); 2639 } 2640 2641 2642 void Assembler::addps(XMMRegister dst, const Operand& src) { 2643 EnsureSpace ensure_space(this); 2644 emit_optional_rex_32(dst, src); 2645 emit(0x0F); 2646 emit(0x58); 2647 emit_sse_operand(dst, src); 2648 } 2649 2650 2651 void Assembler::subps(XMMRegister dst, XMMRegister src) { 2652 EnsureSpace ensure_space(this); 2653 emit_optional_rex_32(dst, src); 2654 emit(0x0F); 2655 emit(0x5C); 2656 emit_sse_operand(dst, src); 2657 } 2658 2659 2660 void Assembler::subps(XMMRegister dst, const Operand& src) { 2661 EnsureSpace ensure_space(this); 2662 emit_optional_rex_32(dst, src); 2663 emit(0x0F); 2664 emit(0x5C); 2665 emit_sse_operand(dst, src); 2666 } 2667 2668 2669 void Assembler::mulps(XMMRegister dst, XMMRegister src) { 2670 EnsureSpace ensure_space(this); 2671 emit_optional_rex_32(dst, src); 2672 emit(0x0F); 2673 emit(0x59); 2674 emit_sse_operand(dst, src); 2675 } 2676 2677 2678 void Assembler::mulps(XMMRegister dst, const Operand& src) { 2679 EnsureSpace ensure_space(this); 2680 emit_optional_rex_32(dst, src); 2681 emit(0x0F); 2682 emit(0x59); 2683 emit_sse_operand(dst, src); 2684 } 2685 2686 2687 void Assembler::divps(XMMRegister dst, XMMRegister src) { 2688 EnsureSpace ensure_space(this); 2689 emit_optional_rex_32(dst, src); 2690 emit(0x0F); 2691 emit(0x5E); 2692 emit_sse_operand(dst, src); 2693 } 2694 2695 2696 void Assembler::divps(XMMRegister dst, const Operand& src) { 2697 EnsureSpace ensure_space(this); 2698 emit_optional_rex_32(dst, src); 2699 emit(0x0F); 2700 emit(0x5E); 2701 emit_sse_operand(dst, src); 2702 } 2703 2704 2705 // SSE 2 operations. 2706 2707 void Assembler::movd(XMMRegister dst, Register src) { 2708 DCHECK(!IsEnabled(AVX)); 2709 EnsureSpace ensure_space(this); 2710 emit(0x66); 2711 emit_optional_rex_32(dst, src); 2712 emit(0x0F); 2713 emit(0x6E); 2714 emit_sse_operand(dst, src); 2715 } 2716 2717 2718 void Assembler::movd(XMMRegister dst, const Operand& src) { 2719 DCHECK(!IsEnabled(AVX)); 2720 EnsureSpace ensure_space(this); 2721 emit(0x66); 2722 emit_optional_rex_32(dst, src); 2723 emit(0x0F); 2724 emit(0x6E); 2725 emit_sse_operand(dst, src); 2726 } 2727 2728 2729 void Assembler::movd(Register dst, XMMRegister src) { 2730 DCHECK(!IsEnabled(AVX)); 2731 EnsureSpace ensure_space(this); 2732 emit(0x66); 2733 emit_optional_rex_32(src, dst); 2734 emit(0x0F); 2735 emit(0x7E); 2736 emit_sse_operand(src, dst); 2737 } 2738 2739 2740 void Assembler::movq(XMMRegister dst, Register src) { 2741 DCHECK(!IsEnabled(AVX)); 2742 EnsureSpace ensure_space(this); 2743 emit(0x66); 2744 emit_rex_64(dst, src); 2745 emit(0x0F); 2746 emit(0x6E); 2747 emit_sse_operand(dst, src); 2748 } 2749 2750 2751 void Assembler::movq(Register dst, XMMRegister src) { 2752 DCHECK(!IsEnabled(AVX)); 2753 EnsureSpace ensure_space(this); 2754 emit(0x66); 2755 emit_rex_64(src, dst); 2756 emit(0x0F); 2757 emit(0x7E); 2758 emit_sse_operand(src, dst); 2759 } 2760 2761 2762 void Assembler::movq(XMMRegister dst, XMMRegister src) { 2763 DCHECK(!IsEnabled(AVX)); 2764 EnsureSpace ensure_space(this); 2765 if (dst.low_bits() == 4) { 2766 // Avoid unnecessary SIB byte. 2767 emit(0xf3); 2768 emit_optional_rex_32(dst, src); 2769 emit(0x0F); 2770 emit(0x7e); 2771 emit_sse_operand(dst, src); 2772 } else { 2773 emit(0x66); 2774 emit_optional_rex_32(src, dst); 2775 emit(0x0F); 2776 emit(0xD6); 2777 emit_sse_operand(src, dst); 2778 } 2779 } 2780 2781 2782 void Assembler::movdqa(const Operand& dst, XMMRegister src) { 2783 EnsureSpace ensure_space(this); 2784 emit(0x66); 2785 emit_rex_64(src, dst); 2786 emit(0x0F); 2787 emit(0x7F); 2788 emit_sse_operand(src, dst); 2789 } 2790 2791 2792 void Assembler::movdqa(XMMRegister dst, const Operand& src) { 2793 EnsureSpace ensure_space(this); 2794 emit(0x66); 2795 emit_rex_64(dst, src); 2796 emit(0x0F); 2797 emit(0x6F); 2798 emit_sse_operand(dst, src); 2799 } 2800 2801 2802 void Assembler::movdqu(const Operand& dst, XMMRegister src) { 2803 EnsureSpace ensure_space(this); 2804 emit(0xF3); 2805 emit_rex_64(src, dst); 2806 emit(0x0F); 2807 emit(0x7F); 2808 emit_sse_operand(src, dst); 2809 } 2810 2811 2812 void Assembler::movdqu(XMMRegister dst, const Operand& src) { 2813 EnsureSpace ensure_space(this); 2814 emit(0xF3); 2815 emit_rex_64(dst, src); 2816 emit(0x0F); 2817 emit(0x6F); 2818 emit_sse_operand(dst, src); 2819 } 2820 2821 2822 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) { 2823 DCHECK(IsEnabled(SSE4_1)); 2824 DCHECK(is_uint8(imm8)); 2825 EnsureSpace ensure_space(this); 2826 emit(0x66); 2827 emit_optional_rex_32(src, dst); 2828 emit(0x0F); 2829 emit(0x3A); 2830 emit(0x17); 2831 emit_sse_operand(src, dst); 2832 emit(imm8); 2833 } 2834 2835 2836 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) { 2837 DCHECK(IsEnabled(SSE4_1)); 2838 EnsureSpace ensure_space(this); 2839 emit(0x66); 2840 emit_optional_rex_32(src, dst); 2841 emit(0x0F); 2842 emit(0x3A); 2843 emit(0x16); 2844 emit_sse_operand(src, dst); 2845 emit(imm8); 2846 } 2847 2848 2849 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) { 2850 DCHECK(IsEnabled(SSE4_1)); 2851 EnsureSpace ensure_space(this); 2852 emit(0x66); 2853 emit_optional_rex_32(dst, src); 2854 emit(0x0F); 2855 emit(0x3A); 2856 emit(0x22); 2857 emit_sse_operand(dst, src); 2858 emit(imm8); 2859 } 2860 2861 2862 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) { 2863 DCHECK(IsEnabled(SSE4_1)); 2864 EnsureSpace ensure_space(this); 2865 emit(0x66); 2866 emit_optional_rex_32(dst, src); 2867 emit(0x0F); 2868 emit(0x3A); 2869 emit(0x22); 2870 emit_sse_operand(dst, src); 2871 emit(imm8); 2872 } 2873 2874 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) { 2875 DCHECK(CpuFeatures::IsSupported(SSE4_1)); 2876 DCHECK(is_uint8(imm8)); 2877 EnsureSpace ensure_space(this); 2878 emit(0x66); 2879 emit_optional_rex_32(dst, src); 2880 emit(0x0F); 2881 emit(0x3A); 2882 emit(0x21); 2883 emit_sse_operand(dst, src); 2884 emit(imm8); 2885 } 2886 2887 void Assembler::movsd(const Operand& dst, XMMRegister src) { 2888 DCHECK(!IsEnabled(AVX)); 2889 EnsureSpace ensure_space(this); 2890 emit(0xF2); // double 2891 emit_optional_rex_32(src, dst); 2892 emit(0x0F); 2893 emit(0x11); // store 2894 emit_sse_operand(src, dst); 2895 } 2896 2897 2898 void Assembler::movsd(XMMRegister dst, XMMRegister src) { 2899 DCHECK(!IsEnabled(AVX)); 2900 EnsureSpace ensure_space(this); 2901 emit(0xF2); // double 2902 emit_optional_rex_32(dst, src); 2903 emit(0x0F); 2904 emit(0x10); // load 2905 emit_sse_operand(dst, src); 2906 } 2907 2908 2909 void Assembler::movsd(XMMRegister dst, const Operand& src) { 2910 DCHECK(!IsEnabled(AVX)); 2911 EnsureSpace ensure_space(this); 2912 emit(0xF2); // double 2913 emit_optional_rex_32(dst, src); 2914 emit(0x0F); 2915 emit(0x10); // load 2916 emit_sse_operand(dst, src); 2917 } 2918 2919 2920 void Assembler::movaps(XMMRegister dst, XMMRegister src) { 2921 DCHECK(!IsEnabled(AVX)); 2922 EnsureSpace ensure_space(this); 2923 if (src.low_bits() == 4) { 2924 // Try to avoid an unnecessary SIB byte. 2925 emit_optional_rex_32(src, dst); 2926 emit(0x0F); 2927 emit(0x29); 2928 emit_sse_operand(src, dst); 2929 } else { 2930 emit_optional_rex_32(dst, src); 2931 emit(0x0F); 2932 emit(0x28); 2933 emit_sse_operand(dst, src); 2934 } 2935 } 2936 2937 2938 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) { 2939 DCHECK(is_uint8(imm8)); 2940 EnsureSpace ensure_space(this); 2941 emit_optional_rex_32(src, dst); 2942 emit(0x0F); 2943 emit(0xC6); 2944 emit_sse_operand(dst, src); 2945 emit(imm8); 2946 } 2947 2948 2949 void Assembler::movapd(XMMRegister dst, XMMRegister src) { 2950 DCHECK(!IsEnabled(AVX)); 2951 EnsureSpace ensure_space(this); 2952 if (src.low_bits() == 4) { 2953 // Try to avoid an unnecessary SIB byte. 2954 emit(0x66); 2955 emit_optional_rex_32(src, dst); 2956 emit(0x0F); 2957 emit(0x29); 2958 emit_sse_operand(src, dst); 2959 } else { 2960 emit(0x66); 2961 emit_optional_rex_32(dst, src); 2962 emit(0x0F); 2963 emit(0x28); 2964 emit_sse_operand(dst, src); 2965 } 2966 } 2967 2968 2969 void Assembler::addss(XMMRegister dst, XMMRegister src) { 2970 EnsureSpace ensure_space(this); 2971 emit(0xF3); 2972 emit_optional_rex_32(dst, src); 2973 emit(0x0F); 2974 emit(0x58); 2975 emit_sse_operand(dst, src); 2976 } 2977 2978 2979 void Assembler::addss(XMMRegister dst, const Operand& src) { 2980 EnsureSpace ensure_space(this); 2981 emit(0xF3); 2982 emit_optional_rex_32(dst, src); 2983 emit(0x0F); 2984 emit(0x58); 2985 emit_sse_operand(dst, src); 2986 } 2987 2988 2989 void Assembler::subss(XMMRegister dst, XMMRegister src) { 2990 EnsureSpace ensure_space(this); 2991 emit(0xF3); 2992 emit_optional_rex_32(dst, src); 2993 emit(0x0F); 2994 emit(0x5C); 2995 emit_sse_operand(dst, src); 2996 } 2997 2998 2999 void Assembler::subss(XMMRegister dst, const Operand& src) { 3000 EnsureSpace ensure_space(this); 3001 emit(0xF3); 3002 emit_optional_rex_32(dst, src); 3003 emit(0x0F); 3004 emit(0x5C); 3005 emit_sse_operand(dst, src); 3006 } 3007 3008 3009 void Assembler::mulss(XMMRegister dst, XMMRegister src) { 3010 EnsureSpace ensure_space(this); 3011 emit(0xF3); 3012 emit_optional_rex_32(dst, src); 3013 emit(0x0F); 3014 emit(0x59); 3015 emit_sse_operand(dst, src); 3016 } 3017 3018 3019 void Assembler::mulss(XMMRegister dst, const Operand& src) { 3020 EnsureSpace ensure_space(this); 3021 emit(0xF3); 3022 emit_optional_rex_32(dst, src); 3023 emit(0x0F); 3024 emit(0x59); 3025 emit_sse_operand(dst, src); 3026 } 3027 3028 3029 void Assembler::divss(XMMRegister dst, XMMRegister src) { 3030 EnsureSpace ensure_space(this); 3031 emit(0xF3); 3032 emit_optional_rex_32(dst, src); 3033 emit(0x0F); 3034 emit(0x5E); 3035 emit_sse_operand(dst, src); 3036 } 3037 3038 3039 void Assembler::divss(XMMRegister dst, const Operand& src) { 3040 EnsureSpace ensure_space(this); 3041 emit(0xF3); 3042 emit_optional_rex_32(dst, src); 3043 emit(0x0F); 3044 emit(0x5E); 3045 emit_sse_operand(dst, src); 3046 } 3047 3048 3049 void Assembler::maxss(XMMRegister dst, XMMRegister src) { 3050 EnsureSpace ensure_space(this); 3051 emit(0xF3); 3052 emit_optional_rex_32(dst, src); 3053 emit(0x0F); 3054 emit(0x5F); 3055 emit_sse_operand(dst, src); 3056 } 3057 3058 3059 void Assembler::maxss(XMMRegister dst, const Operand& src) { 3060 EnsureSpace ensure_space(this); 3061 emit(0xF3); 3062 emit_optional_rex_32(dst, src); 3063 emit(0x0F); 3064 emit(0x5F); 3065 emit_sse_operand(dst, src); 3066 } 3067 3068 3069 void Assembler::minss(XMMRegister dst, XMMRegister src) { 3070 EnsureSpace ensure_space(this); 3071 emit(0xF3); 3072 emit_optional_rex_32(dst, src); 3073 emit(0x0F); 3074 emit(0x5D); 3075 emit_sse_operand(dst, src); 3076 } 3077 3078 3079 void Assembler::minss(XMMRegister dst, const Operand& src) { 3080 EnsureSpace ensure_space(this); 3081 emit(0xF3); 3082 emit_optional_rex_32(dst, src); 3083 emit(0x0F); 3084 emit(0x5D); 3085 emit_sse_operand(dst, src); 3086 } 3087 3088 3089 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) { 3090 EnsureSpace ensure_space(this); 3091 emit(0xF3); 3092 emit_optional_rex_32(dst, src); 3093 emit(0x0F); 3094 emit(0x51); 3095 emit_sse_operand(dst, src); 3096 } 3097 3098 3099 void Assembler::sqrtss(XMMRegister dst, const Operand& src) { 3100 EnsureSpace ensure_space(this); 3101 emit(0xF3); 3102 emit_optional_rex_32(dst, src); 3103 emit(0x0F); 3104 emit(0x51); 3105 emit_sse_operand(dst, src); 3106 } 3107 3108 3109 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) { 3110 DCHECK(!IsEnabled(AVX)); 3111 EnsureSpace ensure_space(this); 3112 emit_optional_rex_32(dst, src); 3113 emit(0x0f); 3114 emit(0x2e); 3115 emit_sse_operand(dst, src); 3116 } 3117 3118 3119 void Assembler::ucomiss(XMMRegister dst, const Operand& src) { 3120 DCHECK(!IsEnabled(AVX)); 3121 EnsureSpace ensure_space(this); 3122 emit_optional_rex_32(dst, src); 3123 emit(0x0f); 3124 emit(0x2e); 3125 emit_sse_operand(dst, src); 3126 } 3127 3128 3129 void Assembler::movss(XMMRegister dst, XMMRegister src) { 3130 DCHECK(!IsEnabled(AVX)); 3131 EnsureSpace ensure_space(this); 3132 emit(0xF3); // single 3133 emit_optional_rex_32(dst, src); 3134 emit(0x0F); 3135 emit(0x10); // load 3136 emit_sse_operand(dst, src); 3137 } 3138 3139 3140 void Assembler::movss(XMMRegister dst, const Operand& src) { 3141 DCHECK(!IsEnabled(AVX)); 3142 EnsureSpace ensure_space(this); 3143 emit(0xF3); // single 3144 emit_optional_rex_32(dst, src); 3145 emit(0x0F); 3146 emit(0x10); // load 3147 emit_sse_operand(dst, src); 3148 } 3149 3150 3151 void Assembler::movss(const Operand& src, XMMRegister dst) { 3152 DCHECK(!IsEnabled(AVX)); 3153 EnsureSpace ensure_space(this); 3154 emit(0xF3); // single 3155 emit_optional_rex_32(dst, src); 3156 emit(0x0F); 3157 emit(0x11); // store 3158 emit_sse_operand(dst, src); 3159 } 3160 3161 3162 void Assembler::psllq(XMMRegister reg, byte imm8) { 3163 DCHECK(!IsEnabled(AVX)); 3164 EnsureSpace ensure_space(this); 3165 emit(0x66); 3166 emit_optional_rex_32(reg); 3167 emit(0x0F); 3168 emit(0x73); 3169 emit_sse_operand(rsi, reg); // rsi == 6 3170 emit(imm8); 3171 } 3172 3173 3174 void Assembler::psrlq(XMMRegister reg, byte imm8) { 3175 DCHECK(!IsEnabled(AVX)); 3176 EnsureSpace ensure_space(this); 3177 emit(0x66); 3178 emit_optional_rex_32(reg); 3179 emit(0x0F); 3180 emit(0x73); 3181 emit_sse_operand(rdx, reg); // rdx == 2 3182 emit(imm8); 3183 } 3184 3185 3186 void Assembler::pslld(XMMRegister reg, byte imm8) { 3187 EnsureSpace ensure_space(this); 3188 emit(0x66); 3189 emit_optional_rex_32(reg); 3190 emit(0x0F); 3191 emit(0x72); 3192 emit_sse_operand(rsi, reg); // rsi == 6 3193 emit(imm8); 3194 } 3195 3196 3197 void Assembler::psrld(XMMRegister reg, byte imm8) { 3198 EnsureSpace ensure_space(this); 3199 emit(0x66); 3200 emit_optional_rex_32(reg); 3201 emit(0x0F); 3202 emit(0x72); 3203 emit_sse_operand(rdx, reg); // rdx == 2 3204 emit(imm8); 3205 } 3206 3207 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) { 3208 EnsureSpace ensure_space(this); 3209 emit_optional_rex_32(dst, src); 3210 emit(0x0F); 3211 emit(0xC2); 3212 emit_sse_operand(dst, src); 3213 emit(cmp); 3214 } 3215 3216 void Assembler::cmpeqps(XMMRegister dst, XMMRegister src) { 3217 cmpps(dst, src, 0x0); 3218 } 3219 3220 void Assembler::cmpltps(XMMRegister dst, XMMRegister src) { 3221 cmpps(dst, src, 0x1); 3222 } 3223 3224 void Assembler::cmpleps(XMMRegister dst, XMMRegister src) { 3225 cmpps(dst, src, 0x2); 3226 } 3227 3228 void Assembler::cmpneqps(XMMRegister dst, XMMRegister src) { 3229 cmpps(dst, src, 0x4); 3230 } 3231 3232 void Assembler::cmpnltps(XMMRegister dst, XMMRegister src) { 3233 cmpps(dst, src, 0x5); 3234 } 3235 3236 void Assembler::cmpnleps(XMMRegister dst, XMMRegister src) { 3237 cmpps(dst, src, 0x6); 3238 } 3239 3240 void Assembler::cvttss2si(Register dst, const Operand& src) { 3241 DCHECK(!IsEnabled(AVX)); 3242 EnsureSpace ensure_space(this); 3243 emit(0xF3); 3244 emit_optional_rex_32(dst, src); 3245 emit(0x0F); 3246 emit(0x2C); 3247 emit_operand(dst, src); 3248 } 3249 3250 3251 void Assembler::cvttss2si(Register dst, XMMRegister src) { 3252 DCHECK(!IsEnabled(AVX)); 3253 EnsureSpace ensure_space(this); 3254 emit(0xF3); 3255 emit_optional_rex_32(dst, src); 3256 emit(0x0F); 3257 emit(0x2C); 3258 emit_sse_operand(dst, src); 3259 } 3260 3261 3262 void Assembler::cvttsd2si(Register dst, const Operand& src) { 3263 DCHECK(!IsEnabled(AVX)); 3264 EnsureSpace ensure_space(this); 3265 emit(0xF2); 3266 emit_optional_rex_32(dst, src); 3267 emit(0x0F); 3268 emit(0x2C); 3269 emit_operand(dst, src); 3270 } 3271 3272 3273 void Assembler::cvttsd2si(Register dst, XMMRegister src) { 3274 DCHECK(!IsEnabled(AVX)); 3275 EnsureSpace ensure_space(this); 3276 emit(0xF2); 3277 emit_optional_rex_32(dst, src); 3278 emit(0x0F); 3279 emit(0x2C); 3280 emit_sse_operand(dst, src); 3281 } 3282 3283 3284 void Assembler::cvttss2siq(Register dst, XMMRegister src) { 3285 DCHECK(!IsEnabled(AVX)); 3286 EnsureSpace ensure_space(this); 3287 emit(0xF3); 3288 emit_rex_64(dst, src); 3289 emit(0x0F); 3290 emit(0x2C); 3291 emit_sse_operand(dst, src); 3292 } 3293 3294 3295 void Assembler::cvttss2siq(Register dst, const Operand& src) { 3296 DCHECK(!IsEnabled(AVX)); 3297 EnsureSpace ensure_space(this); 3298 emit(0xF3); 3299 emit_rex_64(dst, src); 3300 emit(0x0F); 3301 emit(0x2C); 3302 emit_sse_operand(dst, src); 3303 } 3304 3305 3306 void Assembler::cvttsd2siq(Register dst, XMMRegister src) { 3307 DCHECK(!IsEnabled(AVX)); 3308 EnsureSpace ensure_space(this); 3309 emit(0xF2); 3310 emit_rex_64(dst, src); 3311 emit(0x0F); 3312 emit(0x2C); 3313 emit_sse_operand(dst, src); 3314 } 3315 3316 3317 void Assembler::cvttsd2siq(Register dst, const Operand& src) { 3318 DCHECK(!IsEnabled(AVX)); 3319 EnsureSpace ensure_space(this); 3320 emit(0xF2); 3321 emit_rex_64(dst, src); 3322 emit(0x0F); 3323 emit(0x2C); 3324 emit_sse_operand(dst, src); 3325 } 3326 3327 3328 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) { 3329 DCHECK(!IsEnabled(AVX)); 3330 EnsureSpace ensure_space(this); 3331 emit(0xF2); 3332 emit_optional_rex_32(dst, src); 3333 emit(0x0F); 3334 emit(0x2A); 3335 emit_sse_operand(dst, src); 3336 } 3337 3338 3339 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) { 3340 DCHECK(!IsEnabled(AVX)); 3341 EnsureSpace ensure_space(this); 3342 emit(0xF2); 3343 emit_optional_rex_32(dst, src); 3344 emit(0x0F); 3345 emit(0x2A); 3346 emit_sse_operand(dst, src); 3347 } 3348 3349 3350 void Assembler::cvtlsi2ss(XMMRegister dst, const Operand& src) { 3351 DCHECK(!IsEnabled(AVX)); 3352 EnsureSpace ensure_space(this); 3353 emit(0xF3); 3354 emit_optional_rex_32(dst, src); 3355 emit(0x0F); 3356 emit(0x2A); 3357 emit_sse_operand(dst, src); 3358 } 3359 3360 3361 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) { 3362 EnsureSpace ensure_space(this); 3363 emit(0xF3); 3364 emit_optional_rex_32(dst, src); 3365 emit(0x0F); 3366 emit(0x2A); 3367 emit_sse_operand(dst, src); 3368 } 3369 3370 3371 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) { 3372 DCHECK(!IsEnabled(AVX)); 3373 EnsureSpace ensure_space(this); 3374 emit(0xF3); 3375 emit_rex_64(dst, src); 3376 emit(0x0F); 3377 emit(0x2A); 3378 emit_sse_operand(dst, src); 3379 } 3380 3381 3382 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) { 3383 DCHECK(!IsEnabled(AVX)); 3384 EnsureSpace ensure_space(this); 3385 emit(0xF3); 3386 emit_rex_64(dst, src); 3387 emit(0x0F); 3388 emit(0x2A); 3389 emit_sse_operand(dst, src); 3390 } 3391 3392 3393 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) { 3394 DCHECK(!IsEnabled(AVX)); 3395 EnsureSpace ensure_space(this); 3396 emit(0xF2); 3397 emit_rex_64(dst, src); 3398 emit(0x0F); 3399 emit(0x2A); 3400 emit_sse_operand(dst, src); 3401 } 3402 3403 3404 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) { 3405 DCHECK(!IsEnabled(AVX)); 3406 EnsureSpace ensure_space(this); 3407 emit(0xF2); 3408 emit_rex_64(dst, src); 3409 emit(0x0F); 3410 emit(0x2A); 3411 emit_sse_operand(dst, src); 3412 } 3413 3414 3415 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) { 3416 DCHECK(!IsEnabled(AVX)); 3417 EnsureSpace ensure_space(this); 3418 emit(0xF3); 3419 emit_optional_rex_32(dst, src); 3420 emit(0x0F); 3421 emit(0x5A); 3422 emit_sse_operand(dst, src); 3423 } 3424 3425 3426 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) { 3427 DCHECK(!IsEnabled(AVX)); 3428 EnsureSpace ensure_space(this); 3429 emit(0xF3); 3430 emit_optional_rex_32(dst, src); 3431 emit(0x0F); 3432 emit(0x5A); 3433 emit_sse_operand(dst, src); 3434 } 3435 3436 3437 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) { 3438 DCHECK(!IsEnabled(AVX)); 3439 EnsureSpace ensure_space(this); 3440 emit(0xF2); 3441 emit_optional_rex_32(dst, src); 3442 emit(0x0F); 3443 emit(0x5A); 3444 emit_sse_operand(dst, src); 3445 } 3446 3447 3448 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) { 3449 DCHECK(!IsEnabled(AVX)); 3450 EnsureSpace ensure_space(this); 3451 emit(0xF2); 3452 emit_optional_rex_32(dst, src); 3453 emit(0x0F); 3454 emit(0x5A); 3455 emit_sse_operand(dst, src); 3456 } 3457 3458 3459 void Assembler::cvtsd2si(Register dst, XMMRegister src) { 3460 DCHECK(!IsEnabled(AVX)); 3461 EnsureSpace ensure_space(this); 3462 emit(0xF2); 3463 emit_optional_rex_32(dst, src); 3464 emit(0x0F); 3465 emit(0x2D); 3466 emit_sse_operand(dst, src); 3467 } 3468 3469 3470 void Assembler::cvtsd2siq(Register dst, XMMRegister src) { 3471 DCHECK(!IsEnabled(AVX)); 3472 EnsureSpace ensure_space(this); 3473 emit(0xF2); 3474 emit_rex_64(dst, src); 3475 emit(0x0F); 3476 emit(0x2D); 3477 emit_sse_operand(dst, src); 3478 } 3479 3480 3481 void Assembler::addsd(XMMRegister dst, XMMRegister src) { 3482 EnsureSpace ensure_space(this); 3483 emit(0xF2); 3484 emit_optional_rex_32(dst, src); 3485 emit(0x0F); 3486 emit(0x58); 3487 emit_sse_operand(dst, src); 3488 } 3489 3490 3491 void Assembler::addsd(XMMRegister dst, const Operand& src) { 3492 EnsureSpace ensure_space(this); 3493 emit(0xF2); 3494 emit_optional_rex_32(dst, src); 3495 emit(0x0F); 3496 emit(0x58); 3497 emit_sse_operand(dst, src); 3498 } 3499 3500 3501 void Assembler::mulsd(XMMRegister dst, XMMRegister src) { 3502 EnsureSpace ensure_space(this); 3503 emit(0xF2); 3504 emit_optional_rex_32(dst, src); 3505 emit(0x0F); 3506 emit(0x59); 3507 emit_sse_operand(dst, src); 3508 } 3509 3510 3511 void Assembler::mulsd(XMMRegister dst, const Operand& src) { 3512 EnsureSpace ensure_space(this); 3513 emit(0xF2); 3514 emit_optional_rex_32(dst, src); 3515 emit(0x0F); 3516 emit(0x59); 3517 emit_sse_operand(dst, src); 3518 } 3519 3520 3521 void Assembler::subsd(XMMRegister dst, XMMRegister src) { 3522 EnsureSpace ensure_space(this); 3523 emit(0xF2); 3524 emit_optional_rex_32(dst, src); 3525 emit(0x0F); 3526 emit(0x5C); 3527 emit_sse_operand(dst, src); 3528 } 3529 3530 3531 void Assembler::subsd(XMMRegister dst, const Operand& src) { 3532 EnsureSpace ensure_space(this); 3533 emit(0xF2); 3534 emit_optional_rex_32(dst, src); 3535 emit(0x0F); 3536 emit(0x5C); 3537 emit_sse_operand(dst, src); 3538 } 3539 3540 3541 void Assembler::divsd(XMMRegister dst, XMMRegister src) { 3542 EnsureSpace ensure_space(this); 3543 emit(0xF2); 3544 emit_optional_rex_32(dst, src); 3545 emit(0x0F); 3546 emit(0x5E); 3547 emit_sse_operand(dst, src); 3548 } 3549 3550 3551 void Assembler::divsd(XMMRegister dst, const Operand& src) { 3552 EnsureSpace ensure_space(this); 3553 emit(0xF2); 3554 emit_optional_rex_32(dst, src); 3555 emit(0x0F); 3556 emit(0x5E); 3557 emit_sse_operand(dst, src); 3558 } 3559 3560 3561 void Assembler::maxsd(XMMRegister dst, XMMRegister src) { 3562 EnsureSpace ensure_space(this); 3563 emit(0xF2); 3564 emit_optional_rex_32(dst, src); 3565 emit(0x0F); 3566 emit(0x5F); 3567 emit_sse_operand(dst, src); 3568 } 3569 3570 3571 void Assembler::maxsd(XMMRegister dst, const Operand& src) { 3572 EnsureSpace ensure_space(this); 3573 emit(0xF2); 3574 emit_optional_rex_32(dst, src); 3575 emit(0x0F); 3576 emit(0x5F); 3577 emit_sse_operand(dst, src); 3578 } 3579 3580 3581 void Assembler::minsd(XMMRegister dst, XMMRegister src) { 3582 EnsureSpace ensure_space(this); 3583 emit(0xF2); 3584 emit_optional_rex_32(dst, src); 3585 emit(0x0F); 3586 emit(0x5D); 3587 emit_sse_operand(dst, src); 3588 } 3589 3590 3591 void Assembler::minsd(XMMRegister dst, const Operand& src) { 3592 EnsureSpace ensure_space(this); 3593 emit(0xF2); 3594 emit_optional_rex_32(dst, src); 3595 emit(0x0F); 3596 emit(0x5D); 3597 emit_sse_operand(dst, src); 3598 } 3599 3600 3601 void Assembler::andpd(XMMRegister dst, XMMRegister src) { 3602 EnsureSpace ensure_space(this); 3603 emit(0x66); 3604 emit_optional_rex_32(dst, src); 3605 emit(0x0F); 3606 emit(0x54); 3607 emit_sse_operand(dst, src); 3608 } 3609 3610 3611 void Assembler::orpd(XMMRegister dst, XMMRegister src) { 3612 EnsureSpace ensure_space(this); 3613 emit(0x66); 3614 emit_optional_rex_32(dst, src); 3615 emit(0x0F); 3616 emit(0x56); 3617 emit_sse_operand(dst, src); 3618 } 3619 3620 3621 void Assembler::xorpd(XMMRegister dst, XMMRegister src) { 3622 DCHECK(!IsEnabled(AVX)); 3623 EnsureSpace ensure_space(this); 3624 emit(0x66); 3625 emit_optional_rex_32(dst, src); 3626 emit(0x0F); 3627 emit(0x57); 3628 emit_sse_operand(dst, src); 3629 } 3630 3631 3632 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) { 3633 DCHECK(!IsEnabled(AVX)); 3634 EnsureSpace ensure_space(this); 3635 emit(0xF2); 3636 emit_optional_rex_32(dst, src); 3637 emit(0x0F); 3638 emit(0x51); 3639 emit_sse_operand(dst, src); 3640 } 3641 3642 3643 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) { 3644 DCHECK(!IsEnabled(AVX)); 3645 EnsureSpace ensure_space(this); 3646 emit(0xF2); 3647 emit_optional_rex_32(dst, src); 3648 emit(0x0F); 3649 emit(0x51); 3650 emit_sse_operand(dst, src); 3651 } 3652 3653 3654 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) { 3655 DCHECK(!IsEnabled(AVX)); 3656 EnsureSpace ensure_space(this); 3657 emit(0x66); 3658 emit_optional_rex_32(dst, src); 3659 emit(0x0f); 3660 emit(0x2e); 3661 emit_sse_operand(dst, src); 3662 } 3663 3664 3665 void Assembler::ucomisd(XMMRegister dst, const Operand& src) { 3666 DCHECK(!IsEnabled(AVX)); 3667 EnsureSpace ensure_space(this); 3668 emit(0x66); 3669 emit_optional_rex_32(dst, src); 3670 emit(0x0f); 3671 emit(0x2e); 3672 emit_sse_operand(dst, src); 3673 } 3674 3675 3676 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) { 3677 EnsureSpace ensure_space(this); 3678 emit(0xF2); 3679 emit_optional_rex_32(dst, src); 3680 emit(0x0F); 3681 emit(0xC2); 3682 emit_sse_operand(dst, src); 3683 emit(0x01); // LT == 1 3684 } 3685 3686 3687 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3688 DCHECK(!IsEnabled(AVX)); 3689 DCHECK(IsEnabled(SSE4_1)); 3690 EnsureSpace ensure_space(this); 3691 emit(0x66); 3692 emit_optional_rex_32(dst, src); 3693 emit(0x0f); 3694 emit(0x3a); 3695 emit(0x0a); 3696 emit_sse_operand(dst, src); 3697 // Mask precision exception. 3698 emit(static_cast<byte>(mode) | 0x8); 3699 } 3700 3701 3702 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) { 3703 DCHECK(!IsEnabled(AVX)); 3704 DCHECK(IsEnabled(SSE4_1)); 3705 EnsureSpace ensure_space(this); 3706 emit(0x66); 3707 emit_optional_rex_32(dst, src); 3708 emit(0x0f); 3709 emit(0x3a); 3710 emit(0x0b); 3711 emit_sse_operand(dst, src); 3712 // Mask precision exception. 3713 emit(static_cast<byte>(mode) | 0x8); 3714 } 3715 3716 3717 void Assembler::movmskpd(Register dst, XMMRegister src) { 3718 EnsureSpace ensure_space(this); 3719 emit(0x66); 3720 emit_optional_rex_32(dst, src); 3721 emit(0x0f); 3722 emit(0x50); 3723 emit_sse_operand(dst, src); 3724 } 3725 3726 3727 void Assembler::movmskps(Register dst, XMMRegister src) { 3728 EnsureSpace ensure_space(this); 3729 emit_optional_rex_32(dst, src); 3730 emit(0x0f); 3731 emit(0x50); 3732 emit_sse_operand(dst, src); 3733 } 3734 3735 3736 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) { 3737 DCHECK(!IsEnabled(AVX)); 3738 EnsureSpace ensure_space(this); 3739 emit(0x66); 3740 emit_optional_rex_32(dst, src); 3741 emit(0x0F); 3742 emit(0x76); 3743 emit_sse_operand(dst, src); 3744 } 3745 3746 3747 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) { 3748 EnsureSpace ensure_space(this); 3749 emit(0x66); 3750 emit_optional_rex_32(dst, src); 3751 emit(0x0F); 3752 emit(0x62); 3753 emit_sse_operand(dst, src); 3754 } 3755 3756 void Assembler::punpckldq(XMMRegister dst, const Operand& src) { 3757 EnsureSpace ensure_space(this); 3758 emit(0x66); 3759 emit_optional_rex_32(dst, src); 3760 emit(0x0F); 3761 emit(0x62); 3762 emit_sse_operand(dst, src); 3763 } 3764 3765 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) { 3766 EnsureSpace ensure_space(this); 3767 emit(0x66); 3768 emit_optional_rex_32(dst, src); 3769 emit(0x0F); 3770 emit(0x6A); 3771 emit_sse_operand(dst, src); 3772 } 3773 3774 3775 // AVX instructions 3776 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3777 XMMRegister src2) { 3778 DCHECK(IsEnabled(FMA3)); 3779 EnsureSpace ensure_space(this); 3780 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3781 emit(op); 3782 emit_sse_operand(dst, src2); 3783 } 3784 3785 3786 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1, 3787 const Operand& src2) { 3788 DCHECK(IsEnabled(FMA3)); 3789 EnsureSpace ensure_space(this); 3790 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1); 3791 emit(op); 3792 emit_sse_operand(dst, src2); 3793 } 3794 3795 3796 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3797 XMMRegister src2) { 3798 DCHECK(IsEnabled(FMA3)); 3799 EnsureSpace ensure_space(this); 3800 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3801 emit(op); 3802 emit_sse_operand(dst, src2); 3803 } 3804 3805 3806 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1, 3807 const Operand& src2) { 3808 DCHECK(IsEnabled(FMA3)); 3809 EnsureSpace ensure_space(this); 3810 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0); 3811 emit(op); 3812 emit_sse_operand(dst, src2); 3813 } 3814 3815 3816 void Assembler::vmovd(XMMRegister dst, Register src) { 3817 DCHECK(IsEnabled(AVX)); 3818 EnsureSpace ensure_space(this); 3819 XMMRegister isrc = {src.code()}; 3820 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0); 3821 emit(0x6e); 3822 emit_sse_operand(dst, src); 3823 } 3824 3825 3826 void Assembler::vmovd(XMMRegister dst, const Operand& src) { 3827 DCHECK(IsEnabled(AVX)); 3828 EnsureSpace ensure_space(this); 3829 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0); 3830 emit(0x6e); 3831 emit_sse_operand(dst, src); 3832 } 3833 3834 3835 void Assembler::vmovd(Register dst, XMMRegister src) { 3836 DCHECK(IsEnabled(AVX)); 3837 EnsureSpace ensure_space(this); 3838 XMMRegister idst = {dst.code()}; 3839 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0); 3840 emit(0x7e); 3841 emit_sse_operand(src, dst); 3842 } 3843 3844 3845 void Assembler::vmovq(XMMRegister dst, Register src) { 3846 DCHECK(IsEnabled(AVX)); 3847 EnsureSpace ensure_space(this); 3848 XMMRegister isrc = {src.code()}; 3849 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1); 3850 emit(0x6e); 3851 emit_sse_operand(dst, src); 3852 } 3853 3854 3855 void Assembler::vmovq(XMMRegister dst, const Operand& src) { 3856 DCHECK(IsEnabled(AVX)); 3857 EnsureSpace ensure_space(this); 3858 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1); 3859 emit(0x6e); 3860 emit_sse_operand(dst, src); 3861 } 3862 3863 3864 void Assembler::vmovq(Register dst, XMMRegister src) { 3865 DCHECK(IsEnabled(AVX)); 3866 EnsureSpace ensure_space(this); 3867 XMMRegister idst = {dst.code()}; 3868 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1); 3869 emit(0x7e); 3870 emit_sse_operand(src, dst); 3871 } 3872 3873 3874 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, 3875 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, VexW w) { 3876 DCHECK(IsEnabled(AVX)); 3877 EnsureSpace ensure_space(this); 3878 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 3879 emit(op); 3880 emit_sse_operand(dst, src2); 3881 } 3882 3883 3884 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, 3885 const Operand& src2, SIMDPrefix pp, LeadingOpcode m, 3886 VexW w) { 3887 DCHECK(IsEnabled(AVX)); 3888 EnsureSpace ensure_space(this); 3889 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w); 3890 emit(op); 3891 emit_sse_operand(dst, src2); 3892 } 3893 3894 3895 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 3896 XMMRegister src2) { 3897 DCHECK(IsEnabled(AVX)); 3898 EnsureSpace ensure_space(this); 3899 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 3900 emit(op); 3901 emit_sse_operand(dst, src2); 3902 } 3903 3904 3905 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, 3906 const Operand& src2) { 3907 DCHECK(IsEnabled(AVX)); 3908 EnsureSpace ensure_space(this); 3909 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG); 3910 emit(op); 3911 emit_sse_operand(dst, src2); 3912 } 3913 3914 3915 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 3916 XMMRegister src2) { 3917 DCHECK(IsEnabled(AVX)); 3918 EnsureSpace ensure_space(this); 3919 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 3920 emit(op); 3921 emit_sse_operand(dst, src2); 3922 } 3923 3924 3925 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, 3926 const Operand& src2) { 3927 DCHECK(IsEnabled(AVX)); 3928 EnsureSpace ensure_space(this); 3929 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG); 3930 emit(op); 3931 emit_sse_operand(dst, src2); 3932 } 3933 3934 3935 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) { 3936 DCHECK(IsEnabled(AVX)); 3937 EnsureSpace ensure_space(this); 3938 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 3939 emit(0x2e); 3940 emit_sse_operand(dst, src); 3941 } 3942 3943 3944 void Assembler::vucomiss(XMMRegister dst, const Operand& src) { 3945 DCHECK(IsEnabled(AVX)); 3946 EnsureSpace ensure_space(this); 3947 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG); 3948 emit(0x2e); 3949 emit_sse_operand(dst, src); 3950 } 3951 3952 3953 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 3954 XMMRegister src2) { 3955 DCHECK(IsEnabled(AVX)); 3956 EnsureSpace ensure_space(this); 3957 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 3958 emit(op); 3959 emit_sse_operand(dst, src2); 3960 } 3961 3962 3963 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, 3964 const Operand& src2) { 3965 DCHECK(IsEnabled(AVX)); 3966 EnsureSpace ensure_space(this); 3967 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG); 3968 emit(op); 3969 emit_sse_operand(dst, src2); 3970 } 3971 3972 3973 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) { 3974 DCHECK(IsEnabled(BMI1)); 3975 EnsureSpace ensure_space(this); 3976 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 3977 emit(op); 3978 emit_modrm(reg, rm); 3979 } 3980 3981 3982 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) { 3983 DCHECK(IsEnabled(BMI1)); 3984 EnsureSpace ensure_space(this); 3985 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1); 3986 emit(op); 3987 emit_operand(reg, rm); 3988 } 3989 3990 3991 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) { 3992 DCHECK(IsEnabled(BMI1)); 3993 EnsureSpace ensure_space(this); 3994 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 3995 emit(op); 3996 emit_modrm(reg, rm); 3997 } 3998 3999 4000 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) { 4001 DCHECK(IsEnabled(BMI1)); 4002 EnsureSpace ensure_space(this); 4003 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0); 4004 emit(op); 4005 emit_operand(reg, rm); 4006 } 4007 4008 4009 void Assembler::tzcntq(Register dst, Register src) { 4010 DCHECK(IsEnabled(BMI1)); 4011 EnsureSpace ensure_space(this); 4012 emit(0xF3); 4013 emit_rex_64(dst, src); 4014 emit(0x0F); 4015 emit(0xBC); 4016 emit_modrm(dst, src); 4017 } 4018 4019 4020 void Assembler::tzcntq(Register dst, const Operand& src) { 4021 DCHECK(IsEnabled(BMI1)); 4022 EnsureSpace ensure_space(this); 4023 emit(0xF3); 4024 emit_rex_64(dst, src); 4025 emit(0x0F); 4026 emit(0xBC); 4027 emit_operand(dst, src); 4028 } 4029 4030 4031 void Assembler::tzcntl(Register dst, Register src) { 4032 DCHECK(IsEnabled(BMI1)); 4033 EnsureSpace ensure_space(this); 4034 emit(0xF3); 4035 emit_optional_rex_32(dst, src); 4036 emit(0x0F); 4037 emit(0xBC); 4038 emit_modrm(dst, src); 4039 } 4040 4041 4042 void Assembler::tzcntl(Register dst, const Operand& src) { 4043 DCHECK(IsEnabled(BMI1)); 4044 EnsureSpace ensure_space(this); 4045 emit(0xF3); 4046 emit_optional_rex_32(dst, src); 4047 emit(0x0F); 4048 emit(0xBC); 4049 emit_operand(dst, src); 4050 } 4051 4052 4053 void Assembler::lzcntq(Register dst, Register src) { 4054 DCHECK(IsEnabled(LZCNT)); 4055 EnsureSpace ensure_space(this); 4056 emit(0xF3); 4057 emit_rex_64(dst, src); 4058 emit(0x0F); 4059 emit(0xBD); 4060 emit_modrm(dst, src); 4061 } 4062 4063 4064 void Assembler::lzcntq(Register dst, const Operand& src) { 4065 DCHECK(IsEnabled(LZCNT)); 4066 EnsureSpace ensure_space(this); 4067 emit(0xF3); 4068 emit_rex_64(dst, src); 4069 emit(0x0F); 4070 emit(0xBD); 4071 emit_operand(dst, src); 4072 } 4073 4074 4075 void Assembler::lzcntl(Register dst, Register src) { 4076 DCHECK(IsEnabled(LZCNT)); 4077 EnsureSpace ensure_space(this); 4078 emit(0xF3); 4079 emit_optional_rex_32(dst, src); 4080 emit(0x0F); 4081 emit(0xBD); 4082 emit_modrm(dst, src); 4083 } 4084 4085 4086 void Assembler::lzcntl(Register dst, const Operand& src) { 4087 DCHECK(IsEnabled(LZCNT)); 4088 EnsureSpace ensure_space(this); 4089 emit(0xF3); 4090 emit_optional_rex_32(dst, src); 4091 emit(0x0F); 4092 emit(0xBD); 4093 emit_operand(dst, src); 4094 } 4095 4096 4097 void Assembler::popcntq(Register dst, Register src) { 4098 DCHECK(IsEnabled(POPCNT)); 4099 EnsureSpace ensure_space(this); 4100 emit(0xF3); 4101 emit_rex_64(dst, src); 4102 emit(0x0F); 4103 emit(0xB8); 4104 emit_modrm(dst, src); 4105 } 4106 4107 4108 void Assembler::popcntq(Register dst, const Operand& src) { 4109 DCHECK(IsEnabled(POPCNT)); 4110 EnsureSpace ensure_space(this); 4111 emit(0xF3); 4112 emit_rex_64(dst, src); 4113 emit(0x0F); 4114 emit(0xB8); 4115 emit_operand(dst, src); 4116 } 4117 4118 4119 void Assembler::popcntl(Register dst, Register src) { 4120 DCHECK(IsEnabled(POPCNT)); 4121 EnsureSpace ensure_space(this); 4122 emit(0xF3); 4123 emit_optional_rex_32(dst, src); 4124 emit(0x0F); 4125 emit(0xB8); 4126 emit_modrm(dst, src); 4127 } 4128 4129 4130 void Assembler::popcntl(Register dst, const Operand& src) { 4131 DCHECK(IsEnabled(POPCNT)); 4132 EnsureSpace ensure_space(this); 4133 emit(0xF3); 4134 emit_optional_rex_32(dst, src); 4135 emit(0x0F); 4136 emit(0xB8); 4137 emit_operand(dst, src); 4138 } 4139 4140 4141 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4142 Register rm) { 4143 DCHECK(IsEnabled(BMI2)); 4144 EnsureSpace ensure_space(this); 4145 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4146 emit(op); 4147 emit_modrm(reg, rm); 4148 } 4149 4150 4151 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, 4152 const Operand& rm) { 4153 DCHECK(IsEnabled(BMI2)); 4154 EnsureSpace ensure_space(this); 4155 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1); 4156 emit(op); 4157 emit_operand(reg, rm); 4158 } 4159 4160 4161 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4162 Register rm) { 4163 DCHECK(IsEnabled(BMI2)); 4164 EnsureSpace ensure_space(this); 4165 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4166 emit(op); 4167 emit_modrm(reg, rm); 4168 } 4169 4170 4171 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, 4172 const Operand& rm) { 4173 DCHECK(IsEnabled(BMI2)); 4174 EnsureSpace ensure_space(this); 4175 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0); 4176 emit(op); 4177 emit_operand(reg, rm); 4178 } 4179 4180 4181 void Assembler::rorxq(Register dst, Register src, byte imm8) { 4182 DCHECK(IsEnabled(BMI2)); 4183 DCHECK(is_uint8(imm8)); 4184 Register vreg = {0}; // VEX.vvvv unused 4185 EnsureSpace ensure_space(this); 4186 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4187 emit(0xF0); 4188 emit_modrm(dst, src); 4189 emit(imm8); 4190 } 4191 4192 4193 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) { 4194 DCHECK(IsEnabled(BMI2)); 4195 DCHECK(is_uint8(imm8)); 4196 Register vreg = {0}; // VEX.vvvv unused 4197 EnsureSpace ensure_space(this); 4198 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1); 4199 emit(0xF0); 4200 emit_operand(dst, src); 4201 emit(imm8); 4202 } 4203 4204 4205 void Assembler::rorxl(Register dst, Register src, byte imm8) { 4206 DCHECK(IsEnabled(BMI2)); 4207 DCHECK(is_uint8(imm8)); 4208 Register vreg = {0}; // VEX.vvvv unused 4209 EnsureSpace ensure_space(this); 4210 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4211 emit(0xF0); 4212 emit_modrm(dst, src); 4213 emit(imm8); 4214 } 4215 4216 4217 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) { 4218 DCHECK(IsEnabled(BMI2)); 4219 DCHECK(is_uint8(imm8)); 4220 Register vreg = {0}; // VEX.vvvv unused 4221 EnsureSpace ensure_space(this); 4222 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0); 4223 emit(0xF0); 4224 emit_operand(dst, src); 4225 emit(imm8); 4226 } 4227 4228 void Assembler::minps(XMMRegister dst, XMMRegister src) { 4229 EnsureSpace ensure_space(this); 4230 emit_optional_rex_32(dst, src); 4231 emit(0x0F); 4232 emit(0x5D); 4233 emit_sse_operand(dst, src); 4234 } 4235 4236 void Assembler::minps(XMMRegister dst, const Operand& src) { 4237 EnsureSpace ensure_space(this); 4238 emit_optional_rex_32(dst, src); 4239 emit(0x0F); 4240 emit(0x5D); 4241 emit_sse_operand(dst, src); 4242 } 4243 4244 void Assembler::maxps(XMMRegister dst, XMMRegister src) { 4245 EnsureSpace ensure_space(this); 4246 emit_optional_rex_32(dst, src); 4247 emit(0x0F); 4248 emit(0x5F); 4249 emit_sse_operand(dst, src); 4250 } 4251 4252 void Assembler::maxps(XMMRegister dst, const Operand& src) { 4253 EnsureSpace ensure_space(this); 4254 emit_optional_rex_32(dst, src); 4255 emit(0x0F); 4256 emit(0x5F); 4257 emit_sse_operand(dst, src); 4258 } 4259 4260 void Assembler::rcpps(XMMRegister dst, XMMRegister src) { 4261 EnsureSpace ensure_space(this); 4262 emit_optional_rex_32(dst, src); 4263 emit(0x0F); 4264 emit(0x53); 4265 emit_sse_operand(dst, src); 4266 } 4267 4268 void Assembler::rcpps(XMMRegister dst, const Operand& src) { 4269 EnsureSpace ensure_space(this); 4270 emit_optional_rex_32(dst, src); 4271 emit(0x0F); 4272 emit(0x53); 4273 emit_sse_operand(dst, src); 4274 } 4275 4276 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) { 4277 EnsureSpace ensure_space(this); 4278 emit_optional_rex_32(dst, src); 4279 emit(0x0F); 4280 emit(0x52); 4281 emit_sse_operand(dst, src); 4282 } 4283 4284 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) { 4285 EnsureSpace ensure_space(this); 4286 emit_optional_rex_32(dst, src); 4287 emit(0x0F); 4288 emit(0x52); 4289 emit_sse_operand(dst, src); 4290 } 4291 4292 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) { 4293 EnsureSpace ensure_space(this); 4294 emit_optional_rex_32(dst, src); 4295 emit(0x0F); 4296 emit(0x51); 4297 emit_sse_operand(dst, src); 4298 } 4299 4300 void Assembler::sqrtps(XMMRegister dst, const Operand& src) { 4301 EnsureSpace ensure_space(this); 4302 emit_optional_rex_32(dst, src); 4303 emit(0x0F); 4304 emit(0x51); 4305 emit_sse_operand(dst, src); 4306 } 4307 4308 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) { 4309 EnsureSpace ensure_space(this); 4310 emit_optional_rex_32(dst, src); 4311 emit(0x0F); 4312 emit(0x5B); 4313 emit_sse_operand(dst, src); 4314 } 4315 4316 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) { 4317 EnsureSpace ensure_space(this); 4318 emit_optional_rex_32(dst, src); 4319 emit(0x0F); 4320 emit(0x5B); 4321 emit_sse_operand(dst, src); 4322 } 4323 4324 void Assembler::movups(XMMRegister dst, XMMRegister src) { 4325 EnsureSpace ensure_space(this); 4326 if (src.low_bits() == 4) { 4327 // Try to avoid an unnecessary SIB byte. 4328 emit_optional_rex_32(src, dst); 4329 emit(0x0F); 4330 emit(0x11); 4331 emit_sse_operand(src, dst); 4332 } else { 4333 emit_optional_rex_32(dst, src); 4334 emit(0x0F); 4335 emit(0x10); 4336 emit_sse_operand(dst, src); 4337 } 4338 } 4339 4340 void Assembler::movups(XMMRegister dst, const Operand& src) { 4341 EnsureSpace ensure_space(this); 4342 emit_optional_rex_32(dst, src); 4343 emit(0x0F); 4344 emit(0x10); 4345 emit_sse_operand(dst, src); 4346 } 4347 4348 void Assembler::movups(const Operand& dst, XMMRegister src) { 4349 EnsureSpace ensure_space(this); 4350 emit_optional_rex_32(src, dst); 4351 emit(0x0F); 4352 emit(0x11); 4353 emit_sse_operand(src, dst); 4354 } 4355 4356 void Assembler::paddd(XMMRegister dst, XMMRegister src) { 4357 EnsureSpace ensure_space(this); 4358 emit(0x66); 4359 emit_optional_rex_32(dst, src); 4360 emit(0x0F); 4361 emit(0xFE); 4362 emit_sse_operand(dst, src); 4363 } 4364 4365 void Assembler::paddd(XMMRegister dst, const Operand& src) { 4366 EnsureSpace ensure_space(this); 4367 emit(0x66); 4368 emit_optional_rex_32(dst, src); 4369 emit(0x0F); 4370 emit(0xFE); 4371 emit_sse_operand(dst, src); 4372 } 4373 4374 void Assembler::psubd(XMMRegister dst, XMMRegister src) { 4375 EnsureSpace ensure_space(this); 4376 emit(0x66); 4377 emit_optional_rex_32(dst, src); 4378 emit(0x0F); 4379 emit(0xFA); 4380 emit_sse_operand(dst, src); 4381 } 4382 4383 void Assembler::psubd(XMMRegister dst, const Operand& src) { 4384 EnsureSpace ensure_space(this); 4385 emit(0x66); 4386 emit_optional_rex_32(dst, src); 4387 emit(0x0F); 4388 emit(0xFA); 4389 emit_sse_operand(dst, src); 4390 } 4391 4392 void Assembler::pmulld(XMMRegister dst, XMMRegister src) { 4393 DCHECK(IsEnabled(SSE4_1)); 4394 EnsureSpace ensure_space(this); 4395 emit(0x66); 4396 emit_optional_rex_32(dst, src); 4397 emit(0x0F); 4398 emit(0x38); 4399 emit(0x40); 4400 emit_sse_operand(dst, src); 4401 } 4402 4403 void Assembler::pmulld(XMMRegister dst, const Operand& src) { 4404 EnsureSpace ensure_space(this); 4405 emit(0x66); 4406 emit_optional_rex_32(dst, src); 4407 emit(0x0F); 4408 emit(0x38); 4409 emit(0x40); 4410 emit_sse_operand(dst, src); 4411 } 4412 4413 void Assembler::pmuludq(XMMRegister dst, XMMRegister src) { 4414 EnsureSpace ensure_space(this); 4415 emit(0x66); 4416 emit_optional_rex_32(dst, src); 4417 emit(0x0F); 4418 emit(0xF4); 4419 emit_sse_operand(dst, src); 4420 } 4421 4422 void Assembler::pmuludq(XMMRegister dst, const Operand& src) { 4423 EnsureSpace ensure_space(this); 4424 emit(0x66); 4425 emit_optional_rex_32(dst, src); 4426 emit(0x0F); 4427 emit(0xF4); 4428 emit_sse_operand(dst, src); 4429 } 4430 4431 void Assembler::psrldq(XMMRegister dst, uint8_t shift) { 4432 EnsureSpace ensure_space(this); 4433 emit(0x66); 4434 emit_optional_rex_32(dst); 4435 emit(0x0F); 4436 emit(0x73); 4437 emit_sse_operand(dst); 4438 emit(shift); 4439 } 4440 4441 void Assembler::cvtps2dq(XMMRegister dst, XMMRegister src) { 4442 EnsureSpace ensure_space(this); 4443 emit(0x66); 4444 emit_optional_rex_32(dst, src); 4445 emit(0x0F); 4446 emit(0x5B); 4447 emit_sse_operand(dst, src); 4448 } 4449 4450 void Assembler::cvtps2dq(XMMRegister dst, const Operand& src) { 4451 EnsureSpace ensure_space(this); 4452 emit(0x66); 4453 emit_optional_rex_32(dst, src); 4454 emit(0x0F); 4455 emit(0x5B); 4456 emit_sse_operand(dst, src); 4457 } 4458 4459 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) { 4460 EnsureSpace ensure_space(this); 4461 emit(0x66); 4462 emit_optional_rex_32(dst, src); 4463 emit(0x0F); 4464 emit(0x70); 4465 emit_sse_operand(dst, src); 4466 emit(shuffle); 4467 } 4468 4469 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) { 4470 Register ireg = { reg.code() }; 4471 emit_operand(ireg, adr); 4472 } 4473 4474 4475 void Assembler::emit_sse_operand(Register reg, const Operand& adr) { 4476 Register ireg = {reg.code()}; 4477 emit_operand(ireg, adr); 4478 } 4479 4480 4481 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) { 4482 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4483 } 4484 4485 4486 void Assembler::emit_sse_operand(XMMRegister dst, Register src) { 4487 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4488 } 4489 4490 4491 void Assembler::emit_sse_operand(Register dst, XMMRegister src) { 4492 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits()); 4493 } 4494 4495 void Assembler::emit_sse_operand(XMMRegister dst) { 4496 emit(0xD8 | dst.low_bits()); 4497 } 4498 4499 4500 void Assembler::db(uint8_t data) { 4501 EnsureSpace ensure_space(this); 4502 emit(data); 4503 } 4504 4505 4506 void Assembler::dd(uint32_t data) { 4507 EnsureSpace ensure_space(this); 4508 emitl(data); 4509 } 4510 4511 4512 void Assembler::dq(uint64_t data) { 4513 EnsureSpace ensure_space(this); 4514 emitq(data); 4515 } 4516 4517 4518 void Assembler::dq(Label* label) { 4519 EnsureSpace ensure_space(this); 4520 if (label->is_bound()) { 4521 internal_reference_positions_.push_back(pc_offset()); 4522 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE); 4523 } else { 4524 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); 4525 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address. 4526 if (label->is_linked()) { 4527 emitl(label->pos()); 4528 label->link_to(pc_offset() - sizeof(int32_t)); 4529 } else { 4530 DCHECK(label->is_unused()); 4531 int32_t current = pc_offset(); 4532 emitl(current); 4533 label->link_to(current); 4534 } 4535 } 4536 } 4537 4538 4539 // Relocation information implementations. 4540 4541 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { 4542 DCHECK(!RelocInfo::IsNone(rmode)); 4543 // Don't record external references unless the heap will be serialized. 4544 if (rmode == RelocInfo::EXTERNAL_REFERENCE && 4545 !serializer_enabled() && !emit_debug_code()) { 4546 return; 4547 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) { 4548 // Don't record psuedo relocation info for code age sequence mode. 4549 return; 4550 } 4551 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL); 4552 reloc_info_writer.Write(&rinfo); 4553 } 4554 4555 4556 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask | 4557 1 << RelocInfo::RUNTIME_ENTRY | 4558 1 << RelocInfo::INTERNAL_REFERENCE | 4559 1 << RelocInfo::CODE_AGE_SEQUENCE; 4560 4561 4562 bool RelocInfo::IsCodedSpecially() { 4563 // The deserializer needs to know whether a pointer is specially coded. Being 4564 // specially coded on x64 means that it is a relative 32 bit address, as used 4565 // by branch instructions. 4566 return (1 << rmode_) & kApplyMask; 4567 } 4568 4569 4570 bool RelocInfo::IsInConstantPool() { 4571 return false; 4572 } 4573 4574 4575 } // namespace internal 4576 } // namespace v8 4577 4578 #endif // V8_TARGET_ARCH_X64 4579
