Home | History | Annotate | Download | only in x86_64 
      1 /*
      2  * Copyright (C) 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "assembler_x86_64.h"
     18 
     19 #include "base/casts.h"
     20 #include "entrypoints/quick/quick_entrypoints.h"
     21 #include "memory_region.h"
     22 #include "thread.h"
     23 
     24 namespace art {
     25 namespace x86_64 {
     26 
     27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) {
     28   return os << reg.AsRegister();
     29 }
     30 
     31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) {
     32   return os << reg.AsFloatRegister();
     33 }
     34 
     35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) {
     36   return os << "ST" << static_cast<int>(reg);
     37 }
     38 
     39 void X86_64Assembler::call(CpuRegister reg) {
     40   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     41   EmitOptionalRex32(reg);
     42   EmitUint8(0xFF);
     43   EmitRegisterOperand(2, reg.LowBits());
     44 }
     45 
     46 
     47 void X86_64Assembler::call(const Address& address) {
     48   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     49   EmitOptionalRex32(address);
     50   EmitUint8(0xFF);
     51   EmitOperand(2, address);
     52 }
     53 
     54 
     55 void X86_64Assembler::call(Label* label) {
     56   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     57   EmitUint8(0xE8);
     58   static const int kSize = 5;
     59   // Offset by one because we already have emitted the opcode.
     60   EmitLabel(label, kSize - 1);
     61 }
     62 
     63 void X86_64Assembler::pushq(CpuRegister reg) {
     64   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     65   EmitOptionalRex32(reg);
     66   EmitUint8(0x50 + reg.LowBits());
     67 }
     68 
     69 
     70 void X86_64Assembler::pushq(const Address& address) {
     71   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     72   EmitOptionalRex32(address);
     73   EmitUint8(0xFF);
     74   EmitOperand(6, address);
     75 }
     76 
     77 
     78 void X86_64Assembler::pushq(const Immediate& imm) {
     79   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     80   CHECK(imm.is_int32());  // pushq only supports 32b immediate.
     81   if (imm.is_int8()) {
     82     EmitUint8(0x6A);
     83     EmitUint8(imm.value() & 0xFF);
     84   } else {
     85     EmitUint8(0x68);
     86     EmitImmediate(imm);
     87   }
     88 }
     89 
     90 
     91 void X86_64Assembler::popq(CpuRegister reg) {
     92   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     93   EmitOptionalRex32(reg);
     94   EmitUint8(0x58 + reg.LowBits());
     95 }
     96 
     97 
     98 void X86_64Assembler::popq(const Address& address) {
     99   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    100   EmitOptionalRex32(address);
    101   EmitUint8(0x8F);
    102   EmitOperand(0, address);
    103 }
    104 
    105 
    106 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) {
    107   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    108   if (imm.is_int32()) {
    109     // 32 bit. Note: sign-extends.
    110     EmitRex64(dst);
    111     EmitUint8(0xC7);
    112     EmitRegisterOperand(0, dst.LowBits());
    113     EmitInt32(static_cast<int32_t>(imm.value()));
    114   } else {
    115     EmitRex64(dst);
    116     EmitUint8(0xB8 + dst.LowBits());
    117     EmitInt64(imm.value());
    118   }
    119 }
    120 
    121 
    122 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) {
    123   CHECK(imm.is_int32());
    124   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    125   EmitOptionalRex32(dst);
    126   EmitUint8(0xB8 + dst.LowBits());
    127   EmitImmediate(imm);
    128 }
    129 
    130 
    131 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) {
    132   CHECK(imm.is_int32());
    133   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    134   EmitRex64(dst);
    135   EmitUint8(0xC7);
    136   EmitOperand(0, dst);
    137   EmitImmediate(imm);
    138 }
    139 
    140 
    141 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) {
    142   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    143   // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
    144   EmitRex64(src, dst);
    145   EmitUint8(0x89);
    146   EmitRegisterOperand(src.LowBits(), dst.LowBits());
    147 }
    148 
    149 
    150 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) {
    151   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    152   EmitOptionalRex32(dst, src);
    153   EmitUint8(0x8B);
    154   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    155 }
    156 
    157 
    158 void X86_64Assembler::movq(CpuRegister dst, const Address& src) {
    159   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    160   EmitRex64(dst, src);
    161   EmitUint8(0x8B);
    162   EmitOperand(dst.LowBits(), src);
    163 }
    164 
    165 
    166 void X86_64Assembler::movl(CpuRegister dst, const Address& src) {
    167   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    168   EmitOptionalRex32(dst, src);
    169   EmitUint8(0x8B);
    170   EmitOperand(dst.LowBits(), src);
    171 }
    172 
    173 
    174 void X86_64Assembler::movq(const Address& dst, CpuRegister src) {
    175   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    176   EmitRex64(src, dst);
    177   EmitUint8(0x89);
    178   EmitOperand(src.LowBits(), dst);
    179 }
    180 
    181 
    182 void X86_64Assembler::movl(const Address& dst, CpuRegister src) {
    183   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    184   EmitOptionalRex32(src, dst);
    185   EmitUint8(0x89);
    186   EmitOperand(src.LowBits(), dst);
    187 }
    188 
    189 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) {
    190   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    191   EmitOptionalRex32(dst);
    192   EmitUint8(0xC7);
    193   EmitOperand(0, dst);
    194   EmitImmediate(imm);
    195 }
    196 
    197 void X86_64Assembler::movntl(const Address& dst, CpuRegister src) {
    198   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    199   EmitOptionalRex32(src, dst);
    200   EmitUint8(0x0F);
    201   EmitUint8(0xC3);
    202   EmitOperand(src.LowBits(), dst);
    203 }
    204 
    205 void X86_64Assembler::movntq(const Address& dst, CpuRegister src) {
    206   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    207   EmitRex64(src, dst);
    208   EmitUint8(0x0F);
    209   EmitUint8(0xC3);
    210   EmitOperand(src.LowBits(), dst);
    211 }
    212 
    213 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) {
    214   cmov(c, dst, src, true);
    215 }
    216 
    217 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) {
    218   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    219   EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
    220   EmitUint8(0x0F);
    221   EmitUint8(0x40 + c);
    222   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    223 }
    224 
    225 
    226 void X86_64Assembler::cmov(Condition c, CpuRegister dst, const Address& src, bool is64bit) {
    227   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    228   if (is64bit) {
    229     EmitRex64(dst, src);
    230   } else {
    231     EmitOptionalRex32(dst, src);
    232   }
    233   EmitUint8(0x0F);
    234   EmitUint8(0x40 + c);
    235   EmitOperand(dst.LowBits(), src);
    236 }
    237 
    238 
    239 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) {
    240   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    241   EmitOptionalByteRegNormalizingRex32(dst, src);
    242   EmitUint8(0x0F);
    243   EmitUint8(0xB6);
    244   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    245 }
    246 
    247 
    248 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) {
    249   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    250   // Byte register is only in the source register form, so we don't use
    251   // EmitOptionalByteRegNormalizingRex32(dst, src);
    252   EmitOptionalRex32(dst, src);
    253   EmitUint8(0x0F);
    254   EmitUint8(0xB6);
    255   EmitOperand(dst.LowBits(), src);
    256 }
    257 
    258 
    259 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) {
    260   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    261   EmitOptionalByteRegNormalizingRex32(dst, src);
    262   EmitUint8(0x0F);
    263   EmitUint8(0xBE);
    264   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    265 }
    266 
    267 
    268 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) {
    269   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    270   // Byte register is only in the source register form, so we don't use
    271   // EmitOptionalByteRegNormalizingRex32(dst, src);
    272   EmitOptionalRex32(dst, src);
    273   EmitUint8(0x0F);
    274   EmitUint8(0xBE);
    275   EmitOperand(dst.LowBits(), src);
    276 }
    277 
    278 
    279 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) {
    280   LOG(FATAL) << "Use movzxb or movsxb instead.";
    281 }
    282 
    283 
    284 void X86_64Assembler::movb(const Address& dst, CpuRegister src) {
    285   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    286   EmitOptionalByteRegNormalizingRex32(src, dst);
    287   EmitUint8(0x88);
    288   EmitOperand(src.LowBits(), dst);
    289 }
    290 
    291 
    292 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) {
    293   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    294   EmitOptionalRex32(dst);
    295   EmitUint8(0xC6);
    296   EmitOperand(Register::RAX, dst);
    297   CHECK(imm.is_int8());
    298   EmitUint8(imm.value() & 0xFF);
    299 }
    300 
    301 
    302 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) {
    303   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    304   EmitOptionalRex32(dst, src);
    305   EmitUint8(0x0F);
    306   EmitUint8(0xB7);
    307   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    308 }
    309 
    310 
    311 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) {
    312   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    313   EmitOptionalRex32(dst, src);
    314   EmitUint8(0x0F);
    315   EmitUint8(0xB7);
    316   EmitOperand(dst.LowBits(), src);
    317 }
    318 
    319 
    320 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) {
    321   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    322   EmitOptionalRex32(dst, src);
    323   EmitUint8(0x0F);
    324   EmitUint8(0xBF);
    325   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    326 }
    327 
    328 
    329 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) {
    330   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    331   EmitOptionalRex32(dst, src);
    332   EmitUint8(0x0F);
    333   EmitUint8(0xBF);
    334   EmitOperand(dst.LowBits(), src);
    335 }
    336 
    337 
    338 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) {
    339   LOG(FATAL) << "Use movzxw or movsxw instead.";
    340 }
    341 
    342 
    343 void X86_64Assembler::movw(const Address& dst, CpuRegister src) {
    344   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    345   EmitOperandSizeOverride();
    346   EmitOptionalRex32(src, dst);
    347   EmitUint8(0x89);
    348   EmitOperand(src.LowBits(), dst);
    349 }
    350 
    351 
    352 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) {
    353   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    354   EmitOperandSizeOverride();
    355   EmitOptionalRex32(dst);
    356   EmitUint8(0xC7);
    357   EmitOperand(Register::RAX, dst);
    358   CHECK(imm.is_uint16() || imm.is_int16());
    359   EmitUint8(imm.value() & 0xFF);
    360   EmitUint8(imm.value() >> 8);
    361 }
    362 
    363 
    364 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) {
    365   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    366   EmitRex64(dst, src);
    367   EmitUint8(0x8D);
    368   EmitOperand(dst.LowBits(), src);
    369 }
    370 
    371 
    372 void X86_64Assembler::leal(CpuRegister dst, const Address& src) {
    373   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    374   EmitOptionalRex32(dst, src);
    375   EmitUint8(0x8D);
    376   EmitOperand(dst.LowBits(), src);
    377 }
    378 
    379 
    380 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) {
    381   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    382   EmitOptionalRex32(dst, src);
    383   EmitUint8(0x0F);
    384   EmitUint8(0x28);
    385   EmitXmmRegisterOperand(dst.LowBits(), src);
    386 }
    387 
    388 
    389 void X86_64Assembler::movaps(XmmRegister dst, const Address& src) {
    390   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    391   EmitOptionalRex32(dst, src);
    392   EmitUint8(0x0F);
    393   EmitUint8(0x28);
    394   EmitOperand(dst.LowBits(), src);
    395 }
    396 
    397 
    398 void X86_64Assembler::movups(XmmRegister dst, const Address& src) {
    399   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    400   EmitOptionalRex32(dst, src);
    401   EmitUint8(0x0F);
    402   EmitUint8(0x10);
    403   EmitOperand(dst.LowBits(), src);
    404 }
    405 
    406 
    407 void X86_64Assembler::movaps(const Address& dst, XmmRegister src) {
    408   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    409   EmitOptionalRex32(src, dst);
    410   EmitUint8(0x0F);
    411   EmitUint8(0x29);
    412   EmitOperand(src.LowBits(), dst);
    413 }
    414 
    415 
    416 void X86_64Assembler::movups(const Address& dst, XmmRegister src) {
    417   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    418   EmitOptionalRex32(src, dst);
    419   EmitUint8(0x0F);
    420   EmitUint8(0x11);
    421   EmitOperand(src.LowBits(), dst);
    422 }
    423 
    424 
    425 void X86_64Assembler::movss(XmmRegister dst, const Address& src) {
    426   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    427   EmitUint8(0xF3);
    428   EmitOptionalRex32(dst, src);
    429   EmitUint8(0x0F);
    430   EmitUint8(0x10);
    431   EmitOperand(dst.LowBits(), src);
    432 }
    433 
    434 
    435 void X86_64Assembler::movss(const Address& dst, XmmRegister src) {
    436   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    437   EmitUint8(0xF3);
    438   EmitOptionalRex32(src, dst);
    439   EmitUint8(0x0F);
    440   EmitUint8(0x11);
    441   EmitOperand(src.LowBits(), dst);
    442 }
    443 
    444 
    445 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) {
    446   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    447   EmitUint8(0xF3);
    448   EmitOptionalRex32(src, dst);  // Movss is MR encoding instead of the usual RM.
    449   EmitUint8(0x0F);
    450   EmitUint8(0x11);
    451   EmitXmmRegisterOperand(src.LowBits(), dst);
    452 }
    453 
    454 
    455 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) {
    456   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    457   EmitRex64(dst, src);
    458   EmitUint8(0x63);
    459   EmitRegisterOperand(dst.LowBits(), src.LowBits());
    460 }
    461 
    462 
    463 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) {
    464   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    465   EmitRex64(dst, src);
    466   EmitUint8(0x63);
    467   EmitOperand(dst.LowBits(), src);
    468 }
    469 
    470 
    471 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) {
    472   movd(dst, src, true);
    473 }
    474 
    475 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) {
    476   movd(dst, src, true);
    477 }
    478 
    479 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) {
    480   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    481   EmitUint8(0x66);
    482   EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
    483   EmitUint8(0x0F);
    484   EmitUint8(0x6E);
    485   EmitOperand(dst.LowBits(), Operand(src));
    486 }
    487 
    488 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) {
    489   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    490   EmitUint8(0x66);
    491   EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex());
    492   EmitUint8(0x0F);
    493   EmitUint8(0x7E);
    494   EmitOperand(src.LowBits(), Operand(dst));
    495 }
    496 
    497 
    498 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) {
    499   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    500   EmitUint8(0xF3);
    501   EmitOptionalRex32(dst, src);
    502   EmitUint8(0x0F);
    503   EmitUint8(0x58);
    504   EmitXmmRegisterOperand(dst.LowBits(), src);
    505 }
    506 
    507 
    508 void X86_64Assembler::addss(XmmRegister dst, const Address& src) {
    509   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    510   EmitUint8(0xF3);
    511   EmitOptionalRex32(dst, src);
    512   EmitUint8(0x0F);
    513   EmitUint8(0x58);
    514   EmitOperand(dst.LowBits(), src);
    515 }
    516 
    517 
    518 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) {
    519   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    520   EmitUint8(0xF3);
    521   EmitOptionalRex32(dst, src);
    522   EmitUint8(0x0F);
    523   EmitUint8(0x5C);
    524   EmitXmmRegisterOperand(dst.LowBits(), src);
    525 }
    526 
    527 
    528 void X86_64Assembler::subss(XmmRegister dst, const Address& src) {
    529   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    530   EmitUint8(0xF3);
    531   EmitOptionalRex32(dst, src);
    532   EmitUint8(0x0F);
    533   EmitUint8(0x5C);
    534   EmitOperand(dst.LowBits(), src);
    535 }
    536 
    537 
    538 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) {
    539   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    540   EmitUint8(0xF3);
    541   EmitOptionalRex32(dst, src);
    542   EmitUint8(0x0F);
    543   EmitUint8(0x59);
    544   EmitXmmRegisterOperand(dst.LowBits(), src);
    545 }
    546 
    547 
    548 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) {
    549   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    550   EmitUint8(0xF3);
    551   EmitOptionalRex32(dst, src);
    552   EmitUint8(0x0F);
    553   EmitUint8(0x59);
    554   EmitOperand(dst.LowBits(), src);
    555 }
    556 
    557 
    558 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) {
    559   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    560   EmitUint8(0xF3);
    561   EmitOptionalRex32(dst, src);
    562   EmitUint8(0x0F);
    563   EmitUint8(0x5E);
    564   EmitXmmRegisterOperand(dst.LowBits(), src);
    565 }
    566 
    567 
    568 void X86_64Assembler::divss(XmmRegister dst, const Address& src) {
    569   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    570   EmitUint8(0xF3);
    571   EmitOptionalRex32(dst, src);
    572   EmitUint8(0x0F);
    573   EmitUint8(0x5E);
    574   EmitOperand(dst.LowBits(), src);
    575 }
    576 
    577 
    578 void X86_64Assembler::addps(XmmRegister dst, XmmRegister src) {
    579   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    580   EmitOptionalRex32(dst, src);
    581   EmitUint8(0x0F);
    582   EmitUint8(0x58);
    583   EmitXmmRegisterOperand(dst.LowBits(), src);
    584 }
    585 
    586 
    587 void X86_64Assembler::subps(XmmRegister dst, XmmRegister src) {
    588   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    589   EmitOptionalRex32(dst, src);
    590   EmitUint8(0x0F);
    591   EmitUint8(0x5C);
    592   EmitXmmRegisterOperand(dst.LowBits(), src);
    593 }
    594 
    595 
    596 void X86_64Assembler::mulps(XmmRegister dst, XmmRegister src) {
    597   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    598   EmitOptionalRex32(dst, src);
    599   EmitUint8(0x0F);
    600   EmitUint8(0x59);
    601   EmitXmmRegisterOperand(dst.LowBits(), src);
    602 }
    603 
    604 
    605 void X86_64Assembler::divps(XmmRegister dst, XmmRegister src) {
    606   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    607   EmitOptionalRex32(dst, src);
    608   EmitUint8(0x0F);
    609   EmitUint8(0x5E);
    610   EmitXmmRegisterOperand(dst.LowBits(), src);
    611 }
    612 
    613 
    614 void X86_64Assembler::flds(const Address& src) {
    615   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    616   EmitUint8(0xD9);
    617   EmitOperand(0, src);
    618 }
    619 
    620 
    621 void X86_64Assembler::fsts(const Address& dst) {
    622   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    623   EmitUint8(0xD9);
    624   EmitOperand(2, dst);
    625 }
    626 
    627 
    628 void X86_64Assembler::fstps(const Address& dst) {
    629   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    630   EmitUint8(0xD9);
    631   EmitOperand(3, dst);
    632 }
    633 
    634 
    635 void X86_64Assembler::movapd(XmmRegister dst, XmmRegister src) {
    636   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    637   EmitUint8(0x66);
    638   EmitOptionalRex32(dst, src);
    639   EmitUint8(0x0F);
    640   EmitUint8(0x28);
    641   EmitXmmRegisterOperand(dst.LowBits(), src);
    642 }
    643 
    644 
    645 void X86_64Assembler::movapd(XmmRegister dst, const Address& src) {
    646   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    647   EmitUint8(0x66);
    648   EmitOptionalRex32(dst, src);
    649   EmitUint8(0x0F);
    650   EmitUint8(0x28);
    651   EmitOperand(dst.LowBits(), src);
    652 }
    653 
    654 
    655 void X86_64Assembler::movupd(XmmRegister dst, const Address& src) {
    656   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    657   EmitUint8(0x66);
    658   EmitOptionalRex32(dst, src);
    659   EmitUint8(0x0F);
    660   EmitUint8(0x10);
    661   EmitOperand(dst.LowBits(), src);
    662 }
    663 
    664 
    665 void X86_64Assembler::movapd(const Address& dst, XmmRegister src) {
    666   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    667   EmitUint8(0x66);
    668   EmitOptionalRex32(src, dst);
    669   EmitUint8(0x0F);
    670   EmitUint8(0x29);
    671   EmitOperand(src.LowBits(), dst);
    672 }
    673 
    674 
    675 void X86_64Assembler::movupd(const Address& dst, XmmRegister src) {
    676   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    677   EmitUint8(0x66);
    678   EmitOptionalRex32(src, dst);
    679   EmitUint8(0x0F);
    680   EmitUint8(0x11);
    681   EmitOperand(src.LowBits(), dst);
    682 }
    683 
    684 
    685 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) {
    686   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    687   EmitUint8(0xF2);
    688   EmitOptionalRex32(dst, src);
    689   EmitUint8(0x0F);
    690   EmitUint8(0x10);
    691   EmitOperand(dst.LowBits(), src);
    692 }
    693 
    694 
    695 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) {
    696   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    697   EmitUint8(0xF2);
    698   EmitOptionalRex32(src, dst);
    699   EmitUint8(0x0F);
    700   EmitUint8(0x11);
    701   EmitOperand(src.LowBits(), dst);
    702 }
    703 
    704 
    705 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) {
    706   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    707   EmitUint8(0xF2);
    708   EmitOptionalRex32(src, dst);  // Movsd is MR encoding instead of the usual RM.
    709   EmitUint8(0x0F);
    710   EmitUint8(0x11);
    711   EmitXmmRegisterOperand(src.LowBits(), dst);
    712 }
    713 
    714 
    715 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) {
    716   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    717   EmitUint8(0xF2);
    718   EmitOptionalRex32(dst, src);
    719   EmitUint8(0x0F);
    720   EmitUint8(0x58);
    721   EmitXmmRegisterOperand(dst.LowBits(), src);
    722 }
    723 
    724 
    725 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) {
    726   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    727   EmitUint8(0xF2);
    728   EmitOptionalRex32(dst, src);
    729   EmitUint8(0x0F);
    730   EmitUint8(0x58);
    731   EmitOperand(dst.LowBits(), src);
    732 }
    733 
    734 
    735 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) {
    736   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    737   EmitUint8(0xF2);
    738   EmitOptionalRex32(dst, src);
    739   EmitUint8(0x0F);
    740   EmitUint8(0x5C);
    741   EmitXmmRegisterOperand(dst.LowBits(), src);
    742 }
    743 
    744 
    745 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) {
    746   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    747   EmitUint8(0xF2);
    748   EmitOptionalRex32(dst, src);
    749   EmitUint8(0x0F);
    750   EmitUint8(0x5C);
    751   EmitOperand(dst.LowBits(), src);
    752 }
    753 
    754 
    755 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) {
    756   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    757   EmitUint8(0xF2);
    758   EmitOptionalRex32(dst, src);
    759   EmitUint8(0x0F);
    760   EmitUint8(0x59);
    761   EmitXmmRegisterOperand(dst.LowBits(), src);
    762 }
    763 
    764 
    765 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) {
    766   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    767   EmitUint8(0xF2);
    768   EmitOptionalRex32(dst, src);
    769   EmitUint8(0x0F);
    770   EmitUint8(0x59);
    771   EmitOperand(dst.LowBits(), src);
    772 }
    773 
    774 
    775 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) {
    776   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    777   EmitUint8(0xF2);
    778   EmitOptionalRex32(dst, src);
    779   EmitUint8(0x0F);
    780   EmitUint8(0x5E);
    781   EmitXmmRegisterOperand(dst.LowBits(), src);
    782 }
    783 
    784 
    785 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) {
    786   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    787   EmitUint8(0xF2);
    788   EmitOptionalRex32(dst, src);
    789   EmitUint8(0x0F);
    790   EmitUint8(0x5E);
    791   EmitOperand(dst.LowBits(), src);
    792 }
    793 
    794 
    795 void X86_64Assembler::addpd(XmmRegister dst, XmmRegister src) {
    796   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    797   EmitUint8(0x66);
    798   EmitOptionalRex32(dst, src);
    799   EmitUint8(0x0F);
    800   EmitUint8(0x58);
    801   EmitXmmRegisterOperand(dst.LowBits(), src);
    802 }
    803 
    804 
    805 void X86_64Assembler::subpd(XmmRegister dst, XmmRegister src) {
    806   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    807   EmitUint8(0x66);
    808   EmitOptionalRex32(dst, src);
    809   EmitUint8(0x0F);
    810   EmitUint8(0x5C);
    811   EmitXmmRegisterOperand(dst.LowBits(), src);
    812 }
    813 
    814 
    815 void X86_64Assembler::mulpd(XmmRegister dst, XmmRegister src) {
    816   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    817   EmitUint8(0x66);
    818   EmitOptionalRex32(dst, src);
    819   EmitUint8(0x0F);
    820   EmitUint8(0x59);
    821   EmitXmmRegisterOperand(dst.LowBits(), src);
    822 }
    823 
    824 
    825 void X86_64Assembler::divpd(XmmRegister dst, XmmRegister src) {
    826   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    827   EmitUint8(0x66);
    828   EmitOptionalRex32(dst, src);
    829   EmitUint8(0x0F);
    830   EmitUint8(0x5E);
    831   EmitXmmRegisterOperand(dst.LowBits(), src);
    832 }
    833 
    834 
    835 void X86_64Assembler::movdqa(XmmRegister dst, XmmRegister src) {
    836   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    837   EmitUint8(0x66);
    838   EmitOptionalRex32(dst, src);
    839   EmitUint8(0x0F);
    840   EmitUint8(0x6F);
    841   EmitXmmRegisterOperand(dst.LowBits(), src);
    842 }
    843 
    844 
    845 void X86_64Assembler::movdqa(XmmRegister dst, const Address& src) {
    846   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    847   EmitUint8(0x66);
    848   EmitOptionalRex32(dst, src);
    849   EmitUint8(0x0F);
    850   EmitUint8(0x6F);
    851   EmitOperand(dst.LowBits(), src);
    852 }
    853 
    854 
    855 void X86_64Assembler::movdqu(XmmRegister dst, const Address& src) {
    856   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    857   EmitUint8(0xF3);
    858   EmitOptionalRex32(dst, src);
    859   EmitUint8(0x0F);
    860   EmitUint8(0x6F);
    861   EmitOperand(dst.LowBits(), src);
    862 }
    863 
    864 
    865 void X86_64Assembler::movdqa(const Address& dst, XmmRegister src) {
    866   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    867   EmitUint8(0x66);
    868   EmitOptionalRex32(src, dst);
    869   EmitUint8(0x0F);
    870   EmitUint8(0x7F);
    871   EmitOperand(src.LowBits(), dst);
    872 }
    873 
    874 
    875 void X86_64Assembler::movdqu(const Address& dst, XmmRegister src) {
    876   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    877   EmitUint8(0xF3);
    878   EmitOptionalRex32(src, dst);
    879   EmitUint8(0x0F);
    880   EmitUint8(0x7F);
    881   EmitOperand(src.LowBits(), dst);
    882 }
    883 
    884 
    885 void X86_64Assembler::paddb(XmmRegister dst, XmmRegister src) {
    886   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    887   EmitUint8(0x66);
    888   EmitOptionalRex32(dst, src);
    889   EmitUint8(0x0F);
    890   EmitUint8(0xFC);
    891   EmitXmmRegisterOperand(dst.LowBits(), src);
    892 }
    893 
    894 
    895 void X86_64Assembler::psubb(XmmRegister dst, XmmRegister src) {
    896   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    897   EmitUint8(0x66);
    898   EmitOptionalRex32(dst, src);
    899   EmitUint8(0x0F);
    900   EmitUint8(0xF8);
    901   EmitXmmRegisterOperand(dst.LowBits(), src);
    902 }
    903 
    904 
    905 void X86_64Assembler::paddw(XmmRegister dst, XmmRegister src) {
    906   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    907   EmitUint8(0x66);
    908   EmitOptionalRex32(dst, src);
    909   EmitUint8(0x0F);
    910   EmitUint8(0xFD);
    911   EmitXmmRegisterOperand(dst.LowBits(), src);
    912 }
    913 
    914 
    915 void X86_64Assembler::psubw(XmmRegister dst, XmmRegister src) {
    916   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    917   EmitUint8(0x66);
    918   EmitOptionalRex32(dst, src);
    919   EmitUint8(0x0F);
    920   EmitUint8(0xF9);
    921   EmitXmmRegisterOperand(dst.LowBits(), src);
    922 }
    923 
    924 
    925 void X86_64Assembler::pmullw(XmmRegister dst, XmmRegister src) {
    926   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    927   EmitUint8(0x66);
    928   EmitOptionalRex32(dst, src);
    929   EmitUint8(0x0F);
    930   EmitUint8(0xD5);
    931   EmitXmmRegisterOperand(dst.LowBits(), src);
    932 }
    933 
    934 
    935 void X86_64Assembler::paddd(XmmRegister dst, XmmRegister src) {
    936   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    937   EmitUint8(0x66);
    938   EmitOptionalRex32(dst, src);
    939   EmitUint8(0x0F);
    940   EmitUint8(0xFE);
    941   EmitXmmRegisterOperand(dst.LowBits(), src);
    942 }
    943 
    944 
    945 void X86_64Assembler::psubd(XmmRegister dst, XmmRegister src) {
    946   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    947   EmitUint8(0x66);
    948   EmitOptionalRex32(dst, src);
    949   EmitUint8(0x0F);
    950   EmitUint8(0xFA);
    951   EmitXmmRegisterOperand(dst.LowBits(), src);
    952 }
    953 
    954 
    955 void X86_64Assembler::pmulld(XmmRegister dst, XmmRegister src) {
    956   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    957   EmitUint8(0x66);
    958   EmitOptionalRex32(dst, src);
    959   EmitUint8(0x0F);
    960   EmitUint8(0x38);
    961   EmitUint8(0x40);
    962   EmitXmmRegisterOperand(dst.LowBits(), src);
    963 }
    964 
    965 
    966 void X86_64Assembler::paddq(XmmRegister dst, XmmRegister src) {
    967   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    968   EmitUint8(0x66);
    969   EmitOptionalRex32(dst, src);
    970   EmitUint8(0x0F);
    971   EmitUint8(0xD4);
    972   EmitXmmRegisterOperand(dst.LowBits(), src);
    973 }
    974 
    975 
    976 void X86_64Assembler::psubq(XmmRegister dst, XmmRegister src) {
    977   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    978   EmitUint8(0x66);
    979   EmitOptionalRex32(dst, src);
    980   EmitUint8(0x0F);
    981   EmitUint8(0xFB);
    982   EmitXmmRegisterOperand(dst.LowBits(), src);
    983 }
    984 
    985 
    986 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) {
    987   cvtsi2ss(dst, src, false);
    988 }
    989 
    990 
    991 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) {
    992   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
    993   EmitUint8(0xF3);
    994   if (is64bit) {
    995     // Emit a REX.W prefix if the operand size is 64 bits.
    996     EmitRex64(dst, src);
    997   } else {
    998     EmitOptionalRex32(dst, src);
    999   }
   1000   EmitUint8(0x0F);
   1001   EmitUint8(0x2A);
   1002   EmitOperand(dst.LowBits(), Operand(src));
   1003 }
   1004 
   1005 
   1006 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) {
   1007   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1008   EmitUint8(0xF3);
   1009   if (is64bit) {
   1010     // Emit a REX.W prefix if the operand size is 64 bits.
   1011     EmitRex64(dst, src);
   1012   } else {
   1013     EmitOptionalRex32(dst, src);
   1014   }
   1015   EmitUint8(0x0F);
   1016   EmitUint8(0x2A);
   1017   EmitOperand(dst.LowBits(), src);
   1018 }
   1019 
   1020 
   1021 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) {
   1022   cvtsi2sd(dst, src, false);
   1023 }
   1024 
   1025 
   1026 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) {
   1027   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1028   EmitUint8(0xF2);
   1029   if (is64bit) {
   1030     // Emit a REX.W prefix if the operand size is 64 bits.
   1031     EmitRex64(dst, src);
   1032   } else {
   1033     EmitOptionalRex32(dst, src);
   1034   }
   1035   EmitUint8(0x0F);
   1036   EmitUint8(0x2A);
   1037   EmitOperand(dst.LowBits(), Operand(src));
   1038 }
   1039 
   1040 
   1041 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) {
   1042   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1043   EmitUint8(0xF2);
   1044   if (is64bit) {
   1045     // Emit a REX.W prefix if the operand size is 64 bits.
   1046     EmitRex64(dst, src);
   1047   } else {
   1048     EmitOptionalRex32(dst, src);
   1049   }
   1050   EmitUint8(0x0F);
   1051   EmitUint8(0x2A);
   1052   EmitOperand(dst.LowBits(), src);
   1053 }
   1054 
   1055 
   1056 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) {
   1057   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1058   EmitUint8(0xF3);
   1059   EmitOptionalRex32(dst, src);
   1060   EmitUint8(0x0F);
   1061   EmitUint8(0x2D);
   1062   EmitXmmRegisterOperand(dst.LowBits(), src);
   1063 }
   1064 
   1065 
   1066 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) {
   1067   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1068   EmitUint8(0xF3);
   1069   EmitOptionalRex32(dst, src);
   1070   EmitUint8(0x0F);
   1071   EmitUint8(0x5A);
   1072   EmitXmmRegisterOperand(dst.LowBits(), src);
   1073 }
   1074 
   1075 
   1076 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) {
   1077   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1078   EmitUint8(0xF3);
   1079   EmitOptionalRex32(dst, src);
   1080   EmitUint8(0x0F);
   1081   EmitUint8(0x5A);
   1082   EmitOperand(dst.LowBits(), src);
   1083 }
   1084 
   1085 
   1086 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) {
   1087   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1088   EmitUint8(0xF2);
   1089   EmitOptionalRex32(dst, src);
   1090   EmitUint8(0x0F);
   1091   EmitUint8(0x2D);
   1092   EmitXmmRegisterOperand(dst.LowBits(), src);
   1093 }
   1094 
   1095 
   1096 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) {
   1097   cvttss2si(dst, src, false);
   1098 }
   1099 
   1100 
   1101 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) {
   1102   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1103   EmitUint8(0xF3);
   1104   if (is64bit) {
   1105     // Emit a REX.W prefix if the operand size is 64 bits.
   1106     EmitRex64(dst, src);
   1107   } else {
   1108     EmitOptionalRex32(dst, src);
   1109   }
   1110   EmitUint8(0x0F);
   1111   EmitUint8(0x2C);
   1112   EmitXmmRegisterOperand(dst.LowBits(), src);
   1113 }
   1114 
   1115 
   1116 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) {
   1117   cvttsd2si(dst, src, false);
   1118 }
   1119 
   1120 
   1121 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) {
   1122   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1123   EmitUint8(0xF2);
   1124   if (is64bit) {
   1125     // Emit a REX.W prefix if the operand size is 64 bits.
   1126     EmitRex64(dst, src);
   1127   } else {
   1128     EmitOptionalRex32(dst, src);
   1129   }
   1130   EmitUint8(0x0F);
   1131   EmitUint8(0x2C);
   1132   EmitXmmRegisterOperand(dst.LowBits(), src);
   1133 }
   1134 
   1135 
   1136 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) {
   1137   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1138   EmitUint8(0xF2);
   1139   EmitOptionalRex32(dst, src);
   1140   EmitUint8(0x0F);
   1141   EmitUint8(0x5A);
   1142   EmitXmmRegisterOperand(dst.LowBits(), src);
   1143 }
   1144 
   1145 
   1146 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) {
   1147   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1148   EmitUint8(0xF2);
   1149   EmitOptionalRex32(dst, src);
   1150   EmitUint8(0x0F);
   1151   EmitUint8(0x5A);
   1152   EmitOperand(dst.LowBits(), src);
   1153 }
   1154 
   1155 
   1156 void X86_64Assembler::cvtdq2ps(XmmRegister dst, XmmRegister src) {
   1157   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1158   EmitOptionalRex32(dst, src);
   1159   EmitUint8(0x0F);
   1160   EmitUint8(0x5B);
   1161   EmitXmmRegisterOperand(dst.LowBits(), src);
   1162 }
   1163 
   1164 
   1165 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) {
   1166   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1167   EmitUint8(0xF3);
   1168   EmitOptionalRex32(dst, src);
   1169   EmitUint8(0x0F);
   1170   EmitUint8(0xE6);
   1171   EmitXmmRegisterOperand(dst.LowBits(), src);
   1172 }
   1173 
   1174 
   1175 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) {
   1176   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1177   EmitOptionalRex32(a, b);
   1178   EmitUint8(0x0F);
   1179   EmitUint8(0x2F);
   1180   EmitXmmRegisterOperand(a.LowBits(), b);
   1181 }
   1182 
   1183 
   1184 void X86_64Assembler::comiss(XmmRegister a, const Address& b) {
   1185   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1186   EmitOptionalRex32(a, b);
   1187   EmitUint8(0x0F);
   1188   EmitUint8(0x2F);
   1189   EmitOperand(a.LowBits(), b);
   1190 }
   1191 
   1192 
   1193 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) {
   1194   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1195   EmitUint8(0x66);
   1196   EmitOptionalRex32(a, b);
   1197   EmitUint8(0x0F);
   1198   EmitUint8(0x2F);
   1199   EmitXmmRegisterOperand(a.LowBits(), b);
   1200 }
   1201 
   1202 
   1203 void X86_64Assembler::comisd(XmmRegister a, const Address& b) {
   1204   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1205   EmitUint8(0x66);
   1206   EmitOptionalRex32(a, b);
   1207   EmitUint8(0x0F);
   1208   EmitUint8(0x2F);
   1209   EmitOperand(a.LowBits(), b);
   1210 }
   1211 
   1212 
   1213 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) {
   1214   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1215   EmitOptionalRex32(a, b);
   1216   EmitUint8(0x0F);
   1217   EmitUint8(0x2E);
   1218   EmitXmmRegisterOperand(a.LowBits(), b);
   1219 }
   1220 
   1221 
   1222 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) {
   1223   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1224   EmitOptionalRex32(a, b);
   1225   EmitUint8(0x0F);
   1226   EmitUint8(0x2E);
   1227   EmitOperand(a.LowBits(), b);
   1228 }
   1229 
   1230 
   1231 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) {
   1232   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1233   EmitUint8(0x66);
   1234   EmitOptionalRex32(a, b);
   1235   EmitUint8(0x0F);
   1236   EmitUint8(0x2E);
   1237   EmitXmmRegisterOperand(a.LowBits(), b);
   1238 }
   1239 
   1240 
   1241 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) {
   1242   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1243   EmitUint8(0x66);
   1244   EmitOptionalRex32(a, b);
   1245   EmitUint8(0x0F);
   1246   EmitUint8(0x2E);
   1247   EmitOperand(a.LowBits(), b);
   1248 }
   1249 
   1250 
   1251 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
   1252   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1253   EmitUint8(0x66);
   1254   EmitOptionalRex32(dst, src);
   1255   EmitUint8(0x0F);
   1256   EmitUint8(0x3A);
   1257   EmitUint8(0x0B);
   1258   EmitXmmRegisterOperand(dst.LowBits(), src);
   1259   EmitUint8(imm.value());
   1260 }
   1261 
   1262 
   1263 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) {
   1264   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1265   EmitUint8(0x66);
   1266   EmitOptionalRex32(dst, src);
   1267   EmitUint8(0x0F);
   1268   EmitUint8(0x3A);
   1269   EmitUint8(0x0A);
   1270   EmitXmmRegisterOperand(dst.LowBits(), src);
   1271   EmitUint8(imm.value());
   1272 }
   1273 
   1274 
   1275 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) {
   1276   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1277   EmitUint8(0xF2);
   1278   EmitOptionalRex32(dst, src);
   1279   EmitUint8(0x0F);
   1280   EmitUint8(0x51);
   1281   EmitXmmRegisterOperand(dst.LowBits(), src);
   1282 }
   1283 
   1284 
   1285 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) {
   1286   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1287   EmitUint8(0xF3);
   1288   EmitOptionalRex32(dst, src);
   1289   EmitUint8(0x0F);
   1290   EmitUint8(0x51);
   1291   EmitXmmRegisterOperand(dst.LowBits(), src);
   1292 }
   1293 
   1294 
   1295 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) {
   1296   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1297   EmitUint8(0x66);
   1298   EmitOptionalRex32(dst, src);
   1299   EmitUint8(0x0F);
   1300   EmitUint8(0x57);
   1301   EmitOperand(dst.LowBits(), src);
   1302 }
   1303 
   1304 
   1305 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) {
   1306   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1307   EmitUint8(0x66);
   1308   EmitOptionalRex32(dst, src);
   1309   EmitUint8(0x0F);
   1310   EmitUint8(0x57);
   1311   EmitXmmRegisterOperand(dst.LowBits(), src);
   1312 }
   1313 
   1314 
   1315 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) {
   1316   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1317   EmitOptionalRex32(dst, src);
   1318   EmitUint8(0x0F);
   1319   EmitUint8(0x57);
   1320   EmitOperand(dst.LowBits(), src);
   1321 }
   1322 
   1323 
   1324 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) {
   1325   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1326   EmitOptionalRex32(dst, src);
   1327   EmitUint8(0x0F);
   1328   EmitUint8(0x57);
   1329   EmitXmmRegisterOperand(dst.LowBits(), src);
   1330 }
   1331 
   1332 
   1333 void X86_64Assembler::pxor(XmmRegister dst, XmmRegister src) {
   1334   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1335   EmitUint8(0x66);
   1336   EmitOptionalRex32(dst, src);
   1337   EmitUint8(0x0F);
   1338   EmitUint8(0xEF);
   1339   EmitXmmRegisterOperand(dst.LowBits(), src);
   1340 }
   1341 
   1342 
   1343 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) {
   1344   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1345   EmitUint8(0x66);
   1346   EmitOptionalRex32(dst, src);
   1347   EmitUint8(0x0F);
   1348   EmitUint8(0x54);
   1349   EmitOperand(dst.LowBits(), src);
   1350 }
   1351 
   1352 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) {
   1353   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1354   EmitUint8(0x66);
   1355   EmitOptionalRex32(dst, src);
   1356   EmitUint8(0x0F);
   1357   EmitUint8(0x54);
   1358   EmitXmmRegisterOperand(dst.LowBits(), src);
   1359 }
   1360 
   1361 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) {
   1362   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1363   EmitOptionalRex32(dst, src);
   1364   EmitUint8(0x0F);
   1365   EmitUint8(0x54);
   1366   EmitXmmRegisterOperand(dst.LowBits(), src);
   1367 }
   1368 
   1369 void X86_64Assembler::pand(XmmRegister dst, XmmRegister src) {
   1370   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1371   EmitUint8(0x66);
   1372   EmitOptionalRex32(dst, src);
   1373   EmitUint8(0x0F);
   1374   EmitUint8(0xDB);
   1375   EmitXmmRegisterOperand(dst.LowBits(), src);
   1376 }
   1377 
   1378 void X86_64Assembler::andnpd(XmmRegister dst, XmmRegister src) {
   1379   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1380   EmitUint8(0x66);
   1381   EmitOptionalRex32(dst, src);
   1382   EmitUint8(0x0F);
   1383   EmitUint8(0x55);
   1384   EmitXmmRegisterOperand(dst.LowBits(), src);
   1385 }
   1386 
   1387 void X86_64Assembler::andnps(XmmRegister dst, XmmRegister src) {
   1388   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1389   EmitOptionalRex32(dst, src);
   1390   EmitUint8(0x0F);
   1391   EmitUint8(0x55);
   1392   EmitXmmRegisterOperand(dst.LowBits(), src);
   1393 }
   1394 
   1395 void X86_64Assembler::pandn(XmmRegister dst, XmmRegister src) {
   1396   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1397   EmitUint8(0x66);
   1398   EmitOptionalRex32(dst, src);
   1399   EmitUint8(0x0F);
   1400   EmitUint8(0xDF);
   1401   EmitXmmRegisterOperand(dst.LowBits(), src);
   1402 }
   1403 
   1404 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) {
   1405   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1406   EmitUint8(0x66);
   1407   EmitOptionalRex32(dst, src);
   1408   EmitUint8(0x0F);
   1409   EmitUint8(0x56);
   1410   EmitXmmRegisterOperand(dst.LowBits(), src);
   1411 }
   1412 
   1413 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) {
   1414   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1415   EmitOptionalRex32(dst, src);
   1416   EmitUint8(0x0F);
   1417   EmitUint8(0x56);
   1418   EmitXmmRegisterOperand(dst.LowBits(), src);
   1419 }
   1420 
   1421 void X86_64Assembler::por(XmmRegister dst, XmmRegister src) {
   1422   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1423   EmitUint8(0x66);
   1424   EmitOptionalRex32(dst, src);
   1425   EmitUint8(0x0F);
   1426   EmitUint8(0xEB);
   1427   EmitXmmRegisterOperand(dst.LowBits(), src);
   1428 }
   1429 
   1430 void X86_64Assembler::pavgb(XmmRegister dst, XmmRegister src) {
   1431   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1432   EmitUint8(0x66);
   1433   EmitOptionalRex32(dst, src);
   1434   EmitUint8(0x0F);
   1435   EmitUint8(0xE0);
   1436   EmitXmmRegisterOperand(dst.LowBits(), src);
   1437 }
   1438 
   1439 void X86_64Assembler::pavgw(XmmRegister dst, XmmRegister src) {
   1440   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1441   EmitUint8(0x66);
   1442   EmitOptionalRex32(dst, src);
   1443   EmitUint8(0x0F);
   1444   EmitUint8(0xE3);
   1445   EmitXmmRegisterOperand(dst.LowBits(), src);
   1446 }
   1447 
   1448 void X86_64Assembler::pminsb(XmmRegister dst, XmmRegister src) {
   1449   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1450   EmitUint8(0x66);
   1451   EmitOptionalRex32(dst, src);
   1452   EmitUint8(0x0F);
   1453   EmitUint8(0x38);
   1454   EmitUint8(0x38);
   1455   EmitXmmRegisterOperand(dst.LowBits(), src);
   1456 }
   1457 
   1458 void X86_64Assembler::pmaxsb(XmmRegister dst, XmmRegister src) {
   1459   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1460   EmitUint8(0x66);
   1461   EmitOptionalRex32(dst, src);
   1462   EmitUint8(0x0F);
   1463   EmitUint8(0x38);
   1464   EmitUint8(0x3C);
   1465   EmitXmmRegisterOperand(dst.LowBits(), src);
   1466 }
   1467 
   1468 void X86_64Assembler::pminsw(XmmRegister dst, XmmRegister src) {
   1469   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1470   EmitUint8(0x66);
   1471   EmitOptionalRex32(dst, src);
   1472   EmitUint8(0x0F);
   1473   EmitUint8(0xEA);
   1474   EmitXmmRegisterOperand(dst.LowBits(), src);
   1475 }
   1476 
   1477 void X86_64Assembler::pmaxsw(XmmRegister dst, XmmRegister src) {
   1478   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1479   EmitUint8(0x66);
   1480   EmitOptionalRex32(dst, src);
   1481   EmitUint8(0x0F);
   1482   EmitUint8(0xEE);
   1483   EmitXmmRegisterOperand(dst.LowBits(), src);
   1484 }
   1485 
   1486 void X86_64Assembler::pminsd(XmmRegister dst, XmmRegister src) {
   1487   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1488   EmitUint8(0x66);
   1489   EmitOptionalRex32(dst, src);
   1490   EmitUint8(0x0F);
   1491   EmitUint8(0x38);
   1492   EmitUint8(0x39);
   1493   EmitXmmRegisterOperand(dst.LowBits(), src);
   1494 }
   1495 
   1496 void X86_64Assembler::pmaxsd(XmmRegister dst, XmmRegister src) {
   1497   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1498   EmitUint8(0x66);
   1499   EmitOptionalRex32(dst, src);
   1500   EmitUint8(0x0F);
   1501   EmitUint8(0x38);
   1502   EmitUint8(0x3D);
   1503   EmitXmmRegisterOperand(dst.LowBits(), src);
   1504 }
   1505 
   1506 void X86_64Assembler::pminub(XmmRegister dst, XmmRegister src) {
   1507   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1508   EmitUint8(0x66);
   1509   EmitOptionalRex32(dst, src);
   1510   EmitUint8(0x0F);
   1511   EmitUint8(0xDA);
   1512   EmitXmmRegisterOperand(dst.LowBits(), src);
   1513 }
   1514 
   1515 void X86_64Assembler::pmaxub(XmmRegister dst, XmmRegister src) {
   1516   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1517   EmitUint8(0x66);
   1518   EmitOptionalRex32(dst, src);
   1519   EmitUint8(0x0F);
   1520   EmitUint8(0xDE);
   1521   EmitXmmRegisterOperand(dst.LowBits(), src);
   1522 }
   1523 
   1524 void X86_64Assembler::pminuw(XmmRegister dst, XmmRegister src) {
   1525   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1526   EmitUint8(0x66);
   1527   EmitOptionalRex32(dst, src);
   1528   EmitUint8(0x0F);
   1529   EmitUint8(0x38);
   1530   EmitUint8(0x3A);
   1531   EmitXmmRegisterOperand(dst.LowBits(), src);
   1532 }
   1533 
   1534 void X86_64Assembler::pmaxuw(XmmRegister dst, XmmRegister src) {
   1535   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1536   EmitUint8(0x66);
   1537   EmitOptionalRex32(dst, src);
   1538   EmitUint8(0x0F);
   1539   EmitUint8(0x38);
   1540   EmitUint8(0x3E);
   1541   EmitXmmRegisterOperand(dst.LowBits(), src);
   1542 }
   1543 
   1544 void X86_64Assembler::pminud(XmmRegister dst, XmmRegister src) {
   1545   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1546   EmitUint8(0x66);
   1547   EmitOptionalRex32(dst, src);
   1548   EmitUint8(0x0F);
   1549   EmitUint8(0x38);
   1550   EmitUint8(0x3B);
   1551   EmitXmmRegisterOperand(dst.LowBits(), src);
   1552 }
   1553 
   1554 void X86_64Assembler::pmaxud(XmmRegister dst, XmmRegister src) {
   1555   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1556   EmitUint8(0x66);
   1557   EmitOptionalRex32(dst, src);
   1558   EmitUint8(0x0F);
   1559   EmitUint8(0x38);
   1560   EmitUint8(0x3F);
   1561   EmitXmmRegisterOperand(dst.LowBits(), src);
   1562 }
   1563 
   1564 void X86_64Assembler::minps(XmmRegister dst, XmmRegister src) {
   1565   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1566   EmitOptionalRex32(dst, src);
   1567   EmitUint8(0x0F);
   1568   EmitUint8(0x5D);
   1569   EmitXmmRegisterOperand(dst.LowBits(), src);
   1570 }
   1571 
   1572 void X86_64Assembler::maxps(XmmRegister dst, XmmRegister src) {
   1573   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1574   EmitOptionalRex32(dst, src);
   1575   EmitUint8(0x0F);
   1576   EmitUint8(0x5F);
   1577   EmitXmmRegisterOperand(dst.LowBits(), src);
   1578 }
   1579 
   1580 void X86_64Assembler::minpd(XmmRegister dst, XmmRegister src) {
   1581   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1582   EmitUint8(0x66);
   1583   EmitOptionalRex32(dst, src);
   1584   EmitUint8(0x0F);
   1585   EmitUint8(0x5D);
   1586   EmitXmmRegisterOperand(dst.LowBits(), src);
   1587 }
   1588 
   1589 void X86_64Assembler::maxpd(XmmRegister dst, XmmRegister src) {
   1590   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1591   EmitUint8(0x66);
   1592   EmitOptionalRex32(dst, src);
   1593   EmitUint8(0x0F);
   1594   EmitUint8(0x5F);
   1595   EmitXmmRegisterOperand(dst.LowBits(), src);
   1596 }
   1597 
   1598 void X86_64Assembler::pcmpeqb(XmmRegister dst, XmmRegister src) {
   1599   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1600   EmitUint8(0x66);
   1601   EmitOptionalRex32(dst, src);
   1602   EmitUint8(0x0F);
   1603   EmitUint8(0x74);
   1604   EmitXmmRegisterOperand(dst.LowBits(), src);
   1605 }
   1606 
   1607 void X86_64Assembler::pcmpeqw(XmmRegister dst, XmmRegister src) {
   1608   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1609   EmitUint8(0x66);
   1610   EmitOptionalRex32(dst, src);
   1611   EmitUint8(0x0F);
   1612   EmitUint8(0x75);
   1613   EmitXmmRegisterOperand(dst.LowBits(), src);
   1614 }
   1615 
   1616 void X86_64Assembler::pcmpeqd(XmmRegister dst, XmmRegister src) {
   1617   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1618   EmitUint8(0x66);
   1619   EmitOptionalRex32(dst, src);
   1620   EmitUint8(0x0F);
   1621   EmitUint8(0x76);
   1622   EmitXmmRegisterOperand(dst.LowBits(), src);
   1623 }
   1624 
   1625 void X86_64Assembler::pcmpeqq(XmmRegister dst, XmmRegister src) {
   1626   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1627   EmitUint8(0x66);
   1628   EmitOptionalRex32(dst, src);
   1629   EmitUint8(0x0F);
   1630   EmitUint8(0x38);
   1631   EmitUint8(0x29);
   1632   EmitXmmRegisterOperand(dst.LowBits(), src);
   1633 }
   1634 
   1635 void X86_64Assembler::pcmpgtb(XmmRegister dst, XmmRegister src) {
   1636   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1637   EmitUint8(0x66);
   1638   EmitOptionalRex32(dst, src);
   1639   EmitUint8(0x0F);
   1640   EmitUint8(0x64);
   1641   EmitXmmRegisterOperand(dst.LowBits(), src);
   1642 }
   1643 
   1644 void X86_64Assembler::pcmpgtw(XmmRegister dst, XmmRegister src) {
   1645   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1646   EmitUint8(0x66);
   1647   EmitOptionalRex32(dst, src);
   1648   EmitUint8(0x0F);
   1649   EmitUint8(0x65);
   1650   EmitXmmRegisterOperand(dst.LowBits(), src);
   1651 }
   1652 
   1653 void X86_64Assembler::pcmpgtd(XmmRegister dst, XmmRegister src) {
   1654   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1655   EmitUint8(0x66);
   1656   EmitOptionalRex32(dst, src);
   1657   EmitUint8(0x0F);
   1658   EmitUint8(0x66);
   1659   EmitXmmRegisterOperand(dst.LowBits(), src);
   1660 }
   1661 
   1662 void X86_64Assembler::pcmpgtq(XmmRegister dst, XmmRegister src) {
   1663   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1664   EmitUint8(0x66);
   1665   EmitOptionalRex32(dst, src);
   1666   EmitUint8(0x0F);
   1667   EmitUint8(0x38);
   1668   EmitUint8(0x37);
   1669   EmitXmmRegisterOperand(dst.LowBits(), src);
   1670 }
   1671 
   1672 void X86_64Assembler::shufpd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
   1673   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1674   EmitUint8(0x66);
   1675   EmitOptionalRex32(dst, src);
   1676   EmitUint8(0x0F);
   1677   EmitUint8(0xC6);
   1678   EmitXmmRegisterOperand(dst.LowBits(), src);
   1679   EmitUint8(imm.value());
   1680 }
   1681 
   1682 
   1683 void X86_64Assembler::shufps(XmmRegister dst, XmmRegister src, const Immediate& imm) {
   1684   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1685   EmitOptionalRex32(dst, src);
   1686   EmitUint8(0x0F);
   1687   EmitUint8(0xC6);
   1688   EmitXmmRegisterOperand(dst.LowBits(), src);
   1689   EmitUint8(imm.value());
   1690 }
   1691 
   1692 
   1693 void X86_64Assembler::pshufd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
   1694   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1695   EmitUint8(0x66);
   1696   EmitOptionalRex32(dst, src);
   1697   EmitUint8(0x0F);
   1698   EmitUint8(0x70);
   1699   EmitXmmRegisterOperand(dst.LowBits(), src);
   1700   EmitUint8(imm.value());
   1701 }
   1702 
   1703 
   1704 void X86_64Assembler::punpcklbw(XmmRegister dst, XmmRegister src) {
   1705   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1706   EmitUint8(0x66);
   1707   EmitOptionalRex32(dst, src);
   1708   EmitUint8(0x0F);
   1709   EmitUint8(0x60);
   1710   EmitXmmRegisterOperand(dst.LowBits(), src);
   1711 }
   1712 
   1713 
   1714 void X86_64Assembler::punpcklwd(XmmRegister dst, XmmRegister src) {
   1715   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1716   EmitUint8(0x66);
   1717   EmitOptionalRex32(dst, src);
   1718   EmitUint8(0x0F);
   1719   EmitUint8(0x61);
   1720   EmitXmmRegisterOperand(dst.LowBits(), src);
   1721 }
   1722 
   1723 
   1724 void X86_64Assembler::punpckldq(XmmRegister dst, XmmRegister src) {
   1725   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1726   EmitUint8(0x66);
   1727   EmitOptionalRex32(dst, src);
   1728   EmitUint8(0x0F);
   1729   EmitUint8(0x62);
   1730   EmitXmmRegisterOperand(dst.LowBits(), src);
   1731 }
   1732 
   1733 
   1734 void X86_64Assembler::punpcklqdq(XmmRegister dst, XmmRegister src) {
   1735   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1736   EmitUint8(0x66);
   1737   EmitOptionalRex32(dst, src);
   1738   EmitUint8(0x0F);
   1739   EmitUint8(0x6C);
   1740   EmitXmmRegisterOperand(dst.LowBits(), src);
   1741 }
   1742 
   1743 
   1744 void X86_64Assembler::psllw(XmmRegister reg, const Immediate& shift_count) {
   1745   DCHECK(shift_count.is_uint8());
   1746   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1747   EmitUint8(0x66);
   1748   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1749   EmitUint8(0x0F);
   1750   EmitUint8(0x71);
   1751   EmitXmmRegisterOperand(6, reg);
   1752   EmitUint8(shift_count.value());
   1753 }
   1754 
   1755 
   1756 void X86_64Assembler::pslld(XmmRegister reg, const Immediate& shift_count) {
   1757   DCHECK(shift_count.is_uint8());
   1758   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1759   EmitUint8(0x66);
   1760   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1761   EmitUint8(0x0F);
   1762   EmitUint8(0x72);
   1763   EmitXmmRegisterOperand(6, reg);
   1764   EmitUint8(shift_count.value());
   1765 }
   1766 
   1767 
   1768 void X86_64Assembler::psllq(XmmRegister reg, const Immediate& shift_count) {
   1769   DCHECK(shift_count.is_uint8());
   1770   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1771   EmitUint8(0x66);
   1772   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1773   EmitUint8(0x0F);
   1774   EmitUint8(0x73);
   1775   EmitXmmRegisterOperand(6, reg);
   1776   EmitUint8(shift_count.value());
   1777 }
   1778 
   1779 
   1780 void X86_64Assembler::psraw(XmmRegister reg, const Immediate& shift_count) {
   1781   DCHECK(shift_count.is_uint8());
   1782   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1783   EmitUint8(0x66);
   1784   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1785   EmitUint8(0x0F);
   1786   EmitUint8(0x71);
   1787   EmitXmmRegisterOperand(4, reg);
   1788   EmitUint8(shift_count.value());
   1789 }
   1790 
   1791 
   1792 void X86_64Assembler::psrad(XmmRegister reg, const Immediate& shift_count) {
   1793   DCHECK(shift_count.is_uint8());
   1794   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1795   EmitUint8(0x66);
   1796   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1797   EmitUint8(0x0F);
   1798   EmitUint8(0x72);
   1799   EmitXmmRegisterOperand(4, reg);
   1800   EmitUint8(shift_count.value());
   1801 }
   1802 
   1803 
   1804 void X86_64Assembler::psrlw(XmmRegister reg, const Immediate& shift_count) {
   1805   DCHECK(shift_count.is_uint8());
   1806   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1807   EmitUint8(0x66);
   1808   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1809   EmitUint8(0x0F);
   1810   EmitUint8(0x71);
   1811   EmitXmmRegisterOperand(2, reg);
   1812   EmitUint8(shift_count.value());
   1813 }
   1814 
   1815 
   1816 void X86_64Assembler::psrld(XmmRegister reg, const Immediate& shift_count) {
   1817   DCHECK(shift_count.is_uint8());
   1818   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1819   EmitUint8(0x66);
   1820   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1821   EmitUint8(0x0F);
   1822   EmitUint8(0x72);
   1823   EmitXmmRegisterOperand(2, reg);
   1824   EmitUint8(shift_count.value());
   1825 }
   1826 
   1827 
   1828 void X86_64Assembler::psrlq(XmmRegister reg, const Immediate& shift_count) {
   1829   DCHECK(shift_count.is_uint8());
   1830   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1831   EmitUint8(0x66);
   1832   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   1833   EmitUint8(0x0F);
   1834   EmitUint8(0x73);
   1835   EmitXmmRegisterOperand(2, reg);
   1836   EmitUint8(shift_count.value());
   1837 }
   1838 
   1839 
   1840 void X86_64Assembler::fldl(const Address& src) {
   1841   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1842   EmitUint8(0xDD);
   1843   EmitOperand(0, src);
   1844 }
   1845 
   1846 
   1847 void X86_64Assembler::fstl(const Address& dst) {
   1848   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1849   EmitUint8(0xDD);
   1850   EmitOperand(2, dst);
   1851 }
   1852 
   1853 
   1854 void X86_64Assembler::fstpl(const Address& dst) {
   1855   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1856   EmitUint8(0xDD);
   1857   EmitOperand(3, dst);
   1858 }
   1859 
   1860 
   1861 void X86_64Assembler::fstsw() {
   1862   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1863   EmitUint8(0x9B);
   1864   EmitUint8(0xDF);
   1865   EmitUint8(0xE0);
   1866 }
   1867 
   1868 
   1869 void X86_64Assembler::fnstcw(const Address& dst) {
   1870   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1871   EmitUint8(0xD9);
   1872   EmitOperand(7, dst);
   1873 }
   1874 
   1875 
   1876 void X86_64Assembler::fldcw(const Address& src) {
   1877   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1878   EmitUint8(0xD9);
   1879   EmitOperand(5, src);
   1880 }
   1881 
   1882 
   1883 void X86_64Assembler::fistpl(const Address& dst) {
   1884   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1885   EmitUint8(0xDF);
   1886   EmitOperand(7, dst);
   1887 }
   1888 
   1889 
   1890 void X86_64Assembler::fistps(const Address& dst) {
   1891   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1892   EmitUint8(0xDB);
   1893   EmitOperand(3, dst);
   1894 }
   1895 
   1896 
   1897 void X86_64Assembler::fildl(const Address& src) {
   1898   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1899   EmitUint8(0xDF);
   1900   EmitOperand(5, src);
   1901 }
   1902 
   1903 
   1904 void X86_64Assembler::filds(const Address& src) {
   1905   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1906   EmitUint8(0xDB);
   1907   EmitOperand(0, src);
   1908 }
   1909 
   1910 
   1911 void X86_64Assembler::fincstp() {
   1912   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1913   EmitUint8(0xD9);
   1914   EmitUint8(0xF7);
   1915 }
   1916 
   1917 
   1918 void X86_64Assembler::ffree(const Immediate& index) {
   1919   CHECK_LT(index.value(), 7);
   1920   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1921   EmitUint8(0xDD);
   1922   EmitUint8(0xC0 + index.value());
   1923 }
   1924 
   1925 
   1926 void X86_64Assembler::fsin() {
   1927   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1928   EmitUint8(0xD9);
   1929   EmitUint8(0xFE);
   1930 }
   1931 
   1932 
   1933 void X86_64Assembler::fcos() {
   1934   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1935   EmitUint8(0xD9);
   1936   EmitUint8(0xFF);
   1937 }
   1938 
   1939 
   1940 void X86_64Assembler::fptan() {
   1941   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1942   EmitUint8(0xD9);
   1943   EmitUint8(0xF2);
   1944 }
   1945 
   1946 void X86_64Assembler::fucompp() {
   1947   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1948   EmitUint8(0xDA);
   1949   EmitUint8(0xE9);
   1950 }
   1951 
   1952 
   1953 void X86_64Assembler::fprem() {
   1954   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1955   EmitUint8(0xD9);
   1956   EmitUint8(0xF8);
   1957 }
   1958 
   1959 
   1960 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) {
   1961   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1962   // There is a short version for rax.
   1963   // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
   1964   // work.
   1965   const bool src_rax = src.AsRegister() == RAX;
   1966   const bool dst_rax = dst.AsRegister() == RAX;
   1967   if (src_rax || dst_rax) {
   1968     EmitOptionalRex32(src_rax ? dst : src);
   1969     EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
   1970     return;
   1971   }
   1972 
   1973   // General case.
   1974   EmitOptionalRex32(src, dst);
   1975   EmitUint8(0x87);
   1976   EmitRegisterOperand(src.LowBits(), dst.LowBits());
   1977 }
   1978 
   1979 
   1980 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) {
   1981   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   1982   // There is a short version for rax.
   1983   // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
   1984   // work.
   1985   const bool src_rax = src.AsRegister() == RAX;
   1986   const bool dst_rax = dst.AsRegister() == RAX;
   1987   if (src_rax || dst_rax) {
   1988     // If src == target, emit a nop instead.
   1989     if (src_rax && dst_rax) {
   1990       EmitUint8(0x90);
   1991     } else {
   1992       EmitRex64(src_rax ? dst : src);
   1993       EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
   1994     }
   1995     return;
   1996   }
   1997 
   1998   // General case.
   1999   EmitRex64(src, dst);
   2000   EmitUint8(0x87);
   2001   EmitRegisterOperand(src.LowBits(), dst.LowBits());
   2002 }
   2003 
   2004 
   2005 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) {
   2006   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2007   EmitOptionalRex32(reg, address);
   2008   EmitUint8(0x87);
   2009   EmitOperand(reg.LowBits(), address);
   2010 }
   2011 
   2012 
   2013 void X86_64Assembler::cmpb(const Address& address, const Immediate& imm) {
   2014   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2015   CHECK(imm.is_int32());
   2016   EmitOptionalRex32(address);
   2017   EmitUint8(0x80);
   2018   EmitOperand(7, address);
   2019   EmitUint8(imm.value() & 0xFF);
   2020 }
   2021 
   2022 
   2023 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) {
   2024   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2025   CHECK(imm.is_int32());
   2026   EmitOperandSizeOverride();
   2027   EmitOptionalRex32(address);
   2028   EmitComplex(7, address, imm);
   2029 }
   2030 
   2031 
   2032 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) {
   2033   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2034   CHECK(imm.is_int32());
   2035   EmitOptionalRex32(reg);
   2036   EmitComplex(7, Operand(reg), imm);
   2037 }
   2038 
   2039 
   2040 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) {
   2041   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2042   EmitOptionalRex32(reg0, reg1);
   2043   EmitUint8(0x3B);
   2044   EmitOperand(reg0.LowBits(), Operand(reg1));
   2045 }
   2046 
   2047 
   2048 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) {
   2049   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2050   EmitOptionalRex32(reg, address);
   2051   EmitUint8(0x3B);
   2052   EmitOperand(reg.LowBits(), address);
   2053 }
   2054 
   2055 
   2056 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) {
   2057   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2058   EmitOptionalRex32(reg, address);
   2059   EmitUint8(0x39);
   2060   EmitOperand(reg.LowBits(), address);
   2061 }
   2062 
   2063 
   2064 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) {
   2065   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2066   CHECK(imm.is_int32());
   2067   EmitOptionalRex32(address);
   2068   EmitComplex(7, address, imm);
   2069 }
   2070 
   2071 
   2072 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) {
   2073   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2074   EmitRex64(reg0, reg1);
   2075   EmitUint8(0x3B);
   2076   EmitOperand(reg0.LowBits(), Operand(reg1));
   2077 }
   2078 
   2079 
   2080 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) {
   2081   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2082   CHECK(imm.is_int32());  // cmpq only supports 32b immediate.
   2083   EmitRex64(reg);
   2084   EmitComplex(7, Operand(reg), imm);
   2085 }
   2086 
   2087 
   2088 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) {
   2089   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2090   EmitRex64(reg, address);
   2091   EmitUint8(0x3B);
   2092   EmitOperand(reg.LowBits(), address);
   2093 }
   2094 
   2095 
   2096 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) {
   2097   CHECK(imm.is_int32());  // cmpq only supports 32b immediate.
   2098   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2099   EmitRex64(address);
   2100   EmitComplex(7, address, imm);
   2101 }
   2102 
   2103 
   2104 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) {
   2105   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2106   EmitOptionalRex32(dst, src);
   2107   EmitUint8(0x03);
   2108   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   2109 }
   2110 
   2111 
   2112 void X86_64Assembler::addl(CpuRegister reg, const Address& address) {
   2113   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2114   EmitOptionalRex32(reg, address);
   2115   EmitUint8(0x03);
   2116   EmitOperand(reg.LowBits(), address);
   2117 }
   2118 
   2119 
   2120 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) {
   2121   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2122   EmitOptionalRex32(reg1, reg2);
   2123   EmitUint8(0x85);
   2124   EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
   2125 }
   2126 
   2127 
   2128 void X86_64Assembler::testl(CpuRegister reg, const Address& address) {
   2129   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2130   EmitOptionalRex32(reg, address);
   2131   EmitUint8(0x85);
   2132   EmitOperand(reg.LowBits(), address);
   2133 }
   2134 
   2135 
   2136 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) {
   2137   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2138   // For registers that have a byte variant (RAX, RBX, RCX, and RDX)
   2139   // we only test the byte CpuRegister to keep the encoding short.
   2140   if (immediate.is_uint8() && reg.AsRegister() < 4) {
   2141     // Use zero-extended 8-bit immediate.
   2142     if (reg.AsRegister() == RAX) {
   2143       EmitUint8(0xA8);
   2144     } else {
   2145       EmitUint8(0xF6);
   2146       EmitUint8(0xC0 + reg.AsRegister());
   2147     }
   2148     EmitUint8(immediate.value() & 0xFF);
   2149   } else if (reg.AsRegister() == RAX) {
   2150     // Use short form if the destination is RAX.
   2151     EmitUint8(0xA9);
   2152     EmitImmediate(immediate);
   2153   } else {
   2154     EmitOptionalRex32(reg);
   2155     EmitUint8(0xF7);
   2156     EmitOperand(0, Operand(reg));
   2157     EmitImmediate(immediate);
   2158   }
   2159 }
   2160 
   2161 
   2162 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) {
   2163   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2164   EmitRex64(reg1, reg2);
   2165   EmitUint8(0x85);
   2166   EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
   2167 }
   2168 
   2169 
   2170 void X86_64Assembler::testq(CpuRegister reg, const Address& address) {
   2171   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2172   EmitRex64(reg, address);
   2173   EmitUint8(0x85);
   2174   EmitOperand(reg.LowBits(), address);
   2175 }
   2176 
   2177 
   2178 void X86_64Assembler::testb(const Address& dst, const Immediate& imm) {
   2179   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2180   EmitOptionalRex32(dst);
   2181   EmitUint8(0xF6);
   2182   EmitOperand(Register::RAX, dst);
   2183   CHECK(imm.is_int8());
   2184   EmitUint8(imm.value() & 0xFF);
   2185 }
   2186 
   2187 
   2188 void X86_64Assembler::testl(const Address& dst, const Immediate& imm) {
   2189   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2190   EmitOptionalRex32(dst);
   2191   EmitUint8(0xF7);
   2192   EmitOperand(0, dst);
   2193   EmitImmediate(imm);
   2194 }
   2195 
   2196 
   2197 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) {
   2198   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2199   EmitOptionalRex32(dst, src);
   2200   EmitUint8(0x23);
   2201   EmitOperand(dst.LowBits(), Operand(src));
   2202 }
   2203 
   2204 
   2205 void X86_64Assembler::andl(CpuRegister reg, const Address& address) {
   2206   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2207   EmitOptionalRex32(reg, address);
   2208   EmitUint8(0x23);
   2209   EmitOperand(reg.LowBits(), address);
   2210 }
   2211 
   2212 
   2213 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) {
   2214   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2215   EmitOptionalRex32(dst);
   2216   EmitComplex(4, Operand(dst), imm);
   2217 }
   2218 
   2219 
   2220 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) {
   2221   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2222   CHECK(imm.is_int32());  // andq only supports 32b immediate.
   2223   EmitRex64(reg);
   2224   EmitComplex(4, Operand(reg), imm);
   2225 }
   2226 
   2227 
   2228 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) {
   2229   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2230   EmitRex64(dst, src);
   2231   EmitUint8(0x23);
   2232   EmitOperand(dst.LowBits(), Operand(src));
   2233 }
   2234 
   2235 
   2236 void X86_64Assembler::andq(CpuRegister dst, const Address& src) {
   2237   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2238   EmitRex64(dst, src);
   2239   EmitUint8(0x23);
   2240   EmitOperand(dst.LowBits(), src);
   2241 }
   2242 
   2243 
   2244 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) {
   2245   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2246   EmitOptionalRex32(dst, src);
   2247   EmitUint8(0x0B);
   2248   EmitOperand(dst.LowBits(), Operand(src));
   2249 }
   2250 
   2251 
   2252 void X86_64Assembler::orl(CpuRegister reg, const Address& address) {
   2253   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2254   EmitOptionalRex32(reg, address);
   2255   EmitUint8(0x0B);
   2256   EmitOperand(reg.LowBits(), address);
   2257 }
   2258 
   2259 
   2260 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) {
   2261   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2262   EmitOptionalRex32(dst);
   2263   EmitComplex(1, Operand(dst), imm);
   2264 }
   2265 
   2266 
   2267 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) {
   2268   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2269   CHECK(imm.is_int32());  // orq only supports 32b immediate.
   2270   EmitRex64(dst);
   2271   EmitComplex(1, Operand(dst), imm);
   2272 }
   2273 
   2274 
   2275 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) {
   2276   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2277   EmitRex64(dst, src);
   2278   EmitUint8(0x0B);
   2279   EmitOperand(dst.LowBits(), Operand(src));
   2280 }
   2281 
   2282 
   2283 void X86_64Assembler::orq(CpuRegister dst, const Address& src) {
   2284   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2285   EmitRex64(dst, src);
   2286   EmitUint8(0x0B);
   2287   EmitOperand(dst.LowBits(), src);
   2288 }
   2289 
   2290 
   2291 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) {
   2292   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2293   EmitOptionalRex32(dst, src);
   2294   EmitUint8(0x33);
   2295   EmitOperand(dst.LowBits(), Operand(src));
   2296 }
   2297 
   2298 
   2299 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) {
   2300   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2301   EmitOptionalRex32(reg, address);
   2302   EmitUint8(0x33);
   2303   EmitOperand(reg.LowBits(), address);
   2304 }
   2305 
   2306 
   2307 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) {
   2308   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2309   EmitOptionalRex32(dst);
   2310   EmitComplex(6, Operand(dst), imm);
   2311 }
   2312 
   2313 
   2314 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) {
   2315   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2316   EmitRex64(dst, src);
   2317   EmitUint8(0x33);
   2318   EmitOperand(dst.LowBits(), Operand(src));
   2319 }
   2320 
   2321 
   2322 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) {
   2323   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2324   CHECK(imm.is_int32());  // xorq only supports 32b immediate.
   2325   EmitRex64(dst);
   2326   EmitComplex(6, Operand(dst), imm);
   2327 }
   2328 
   2329 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) {
   2330   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2331   EmitRex64(dst, src);
   2332   EmitUint8(0x33);
   2333   EmitOperand(dst.LowBits(), src);
   2334 }
   2335 
   2336 
   2337 #if 0
   2338 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) {
   2339   // REX.WRXB
   2340   // W - 64-bit operand
   2341   // R - MODRM.reg
   2342   // X - SIB.index
   2343   // B - MODRM.rm/SIB.base
   2344   uint8_t rex = force ? 0x40 : 0;
   2345   if (w) {
   2346     rex |= 0x48;  // REX.W000
   2347   }
   2348   if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) {
   2349     rex |= 0x44;  // REX.0R00
   2350     *r = static_cast<Register>(*r - 8);
   2351   }
   2352   if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) {
   2353     rex |= 0x42;  // REX.00X0
   2354     *x = static_cast<Register>(*x - 8);
   2355   }
   2356   if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) {
   2357     rex |= 0x41;  // REX.000B
   2358     *b = static_cast<Register>(*b - 8);
   2359   }
   2360   if (rex != 0) {
   2361     EmitUint8(rex);
   2362   }
   2363 }
   2364 
   2365 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) {
   2366   // REX.WRXB
   2367   // W - 64-bit operand
   2368   // R - MODRM.reg
   2369   // X - SIB.index
   2370   // B - MODRM.rm/SIB.base
   2371   uint8_t rex = mem->rex();
   2372   if (force) {
   2373     rex |= 0x40;  // REX.0000
   2374   }
   2375   if (w) {
   2376     rex |= 0x48;  // REX.W000
   2377   }
   2378   if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) {
   2379     rex |= 0x44;  // REX.0R00
   2380     *dst = static_cast<Register>(*dst - 8);
   2381   }
   2382   if (rex != 0) {
   2383     EmitUint8(rex);
   2384   }
   2385 }
   2386 
   2387 void rex_mem_reg(bool force, bool w, Address* mem, Register* src);
   2388 #endif
   2389 
   2390 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) {
   2391   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2392   EmitOptionalRex32(reg);
   2393   EmitComplex(0, Operand(reg), imm);
   2394 }
   2395 
   2396 
   2397 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) {
   2398   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2399   CHECK(imm.is_int32());  // addq only supports 32b immediate.
   2400   EmitRex64(reg);
   2401   EmitComplex(0, Operand(reg), imm);
   2402 }
   2403 
   2404 
   2405 void X86_64Assembler::addq(CpuRegister dst, const Address& address) {
   2406   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2407   EmitRex64(dst, address);
   2408   EmitUint8(0x03);
   2409   EmitOperand(dst.LowBits(), address);
   2410 }
   2411 
   2412 
   2413 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) {
   2414   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2415   // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
   2416   EmitRex64(src, dst);
   2417   EmitUint8(0x01);
   2418   EmitRegisterOperand(src.LowBits(), dst.LowBits());
   2419 }
   2420 
   2421 
   2422 void X86_64Assembler::addl(const Address& address, CpuRegister reg) {
   2423   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2424   EmitOptionalRex32(reg, address);
   2425   EmitUint8(0x01);
   2426   EmitOperand(reg.LowBits(), address);
   2427 }
   2428 
   2429 
   2430 void X86_64Assembler::addl(const Address& address, const Immediate& imm) {
   2431   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2432   EmitOptionalRex32(address);
   2433   EmitComplex(0, address, imm);
   2434 }
   2435 
   2436 
   2437 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) {
   2438   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2439   EmitOptionalRex32(dst, src);
   2440   EmitUint8(0x2B);
   2441   EmitOperand(dst.LowBits(), Operand(src));
   2442 }
   2443 
   2444 
   2445 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) {
   2446   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2447   EmitOptionalRex32(reg);
   2448   EmitComplex(5, Operand(reg), imm);
   2449 }
   2450 
   2451 
   2452 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) {
   2453   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2454   CHECK(imm.is_int32());  // subq only supports 32b immediate.
   2455   EmitRex64(reg);
   2456   EmitComplex(5, Operand(reg), imm);
   2457 }
   2458 
   2459 
   2460 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) {
   2461   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2462   EmitRex64(dst, src);
   2463   EmitUint8(0x2B);
   2464   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   2465 }
   2466 
   2467 
   2468 void X86_64Assembler::subq(CpuRegister reg, const Address& address) {
   2469   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2470   EmitRex64(reg, address);
   2471   EmitUint8(0x2B);
   2472   EmitOperand(reg.LowBits() & 7, address);
   2473 }
   2474 
   2475 
   2476 void X86_64Assembler::subl(CpuRegister reg, const Address& address) {
   2477   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2478   EmitOptionalRex32(reg, address);
   2479   EmitUint8(0x2B);
   2480   EmitOperand(reg.LowBits(), address);
   2481 }
   2482 
   2483 
   2484 void X86_64Assembler::cdq() {
   2485   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2486   EmitUint8(0x99);
   2487 }
   2488 
   2489 
   2490 void X86_64Assembler::cqo() {
   2491   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2492   EmitRex64();
   2493   EmitUint8(0x99);
   2494 }
   2495 
   2496 
   2497 void X86_64Assembler::idivl(CpuRegister reg) {
   2498   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2499   EmitOptionalRex32(reg);
   2500   EmitUint8(0xF7);
   2501   EmitUint8(0xF8 | reg.LowBits());
   2502 }
   2503 
   2504 
   2505 void X86_64Assembler::idivq(CpuRegister reg) {
   2506   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2507   EmitRex64(reg);
   2508   EmitUint8(0xF7);
   2509   EmitUint8(0xF8 | reg.LowBits());
   2510 }
   2511 
   2512 
   2513 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) {
   2514   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2515   EmitOptionalRex32(dst, src);
   2516   EmitUint8(0x0F);
   2517   EmitUint8(0xAF);
   2518   EmitOperand(dst.LowBits(), Operand(src));
   2519 }
   2520 
   2521 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src, const Immediate& imm) {
   2522   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2523   CHECK(imm.is_int32());  // imull only supports 32b immediate.
   2524 
   2525   EmitOptionalRex32(dst, src);
   2526 
   2527   // See whether imm can be represented as a sign-extended 8bit value.
   2528   int32_t v32 = static_cast<int32_t>(imm.value());
   2529   if (IsInt<8>(v32)) {
   2530     // Sign-extension works.
   2531     EmitUint8(0x6B);
   2532     EmitOperand(dst.LowBits(), Operand(src));
   2533     EmitUint8(static_cast<uint8_t>(v32 & 0xFF));
   2534   } else {
   2535     // Not representable, use full immediate.
   2536     EmitUint8(0x69);
   2537     EmitOperand(dst.LowBits(), Operand(src));
   2538     EmitImmediate(imm);
   2539   }
   2540 }
   2541 
   2542 
   2543 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) {
   2544   imull(reg, reg, imm);
   2545 }
   2546 
   2547 
   2548 void X86_64Assembler::imull(CpuRegister reg, const Address& address) {
   2549   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2550   EmitOptionalRex32(reg, address);
   2551   EmitUint8(0x0F);
   2552   EmitUint8(0xAF);
   2553   EmitOperand(reg.LowBits(), address);
   2554 }
   2555 
   2556 
   2557 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) {
   2558   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2559   EmitRex64(dst, src);
   2560   EmitUint8(0x0F);
   2561   EmitUint8(0xAF);
   2562   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   2563 }
   2564 
   2565 
   2566 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) {
   2567   imulq(reg, reg, imm);
   2568 }
   2569 
   2570 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) {
   2571   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2572   CHECK(imm.is_int32());  // imulq only supports 32b immediate.
   2573 
   2574   EmitRex64(dst, reg);
   2575 
   2576   // See whether imm can be represented as a sign-extended 8bit value.
   2577   int64_t v64 = imm.value();
   2578   if (IsInt<8>(v64)) {
   2579     // Sign-extension works.
   2580     EmitUint8(0x6B);
   2581     EmitOperand(dst.LowBits(), Operand(reg));
   2582     EmitUint8(static_cast<uint8_t>(v64 & 0xFF));
   2583   } else {
   2584     // Not representable, use full immediate.
   2585     EmitUint8(0x69);
   2586     EmitOperand(dst.LowBits(), Operand(reg));
   2587     EmitImmediate(imm);
   2588   }
   2589 }
   2590 
   2591 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) {
   2592   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2593   EmitRex64(reg, address);
   2594   EmitUint8(0x0F);
   2595   EmitUint8(0xAF);
   2596   EmitOperand(reg.LowBits(), address);
   2597 }
   2598 
   2599 
   2600 void X86_64Assembler::imull(CpuRegister reg) {
   2601   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2602   EmitOptionalRex32(reg);
   2603   EmitUint8(0xF7);
   2604   EmitOperand(5, Operand(reg));
   2605 }
   2606 
   2607 
   2608 void X86_64Assembler::imulq(CpuRegister reg) {
   2609   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2610   EmitRex64(reg);
   2611   EmitUint8(0xF7);
   2612   EmitOperand(5, Operand(reg));
   2613 }
   2614 
   2615 
   2616 void X86_64Assembler::imull(const Address& address) {
   2617   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2618   EmitOptionalRex32(address);
   2619   EmitUint8(0xF7);
   2620   EmitOperand(5, address);
   2621 }
   2622 
   2623 
   2624 void X86_64Assembler::mull(CpuRegister reg) {
   2625   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2626   EmitOptionalRex32(reg);
   2627   EmitUint8(0xF7);
   2628   EmitOperand(4, Operand(reg));
   2629 }
   2630 
   2631 
   2632 void X86_64Assembler::mull(const Address& address) {
   2633   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2634   EmitOptionalRex32(address);
   2635   EmitUint8(0xF7);
   2636   EmitOperand(4, address);
   2637 }
   2638 
   2639 
   2640 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) {
   2641   EmitGenericShift(false, 4, reg, imm);
   2642 }
   2643 
   2644 
   2645 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) {
   2646   EmitGenericShift(true, 4, reg, imm);
   2647 }
   2648 
   2649 
   2650 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) {
   2651   EmitGenericShift(false, 4, operand, shifter);
   2652 }
   2653 
   2654 
   2655 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) {
   2656   EmitGenericShift(true, 4, operand, shifter);
   2657 }
   2658 
   2659 
   2660 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) {
   2661   EmitGenericShift(false, 5, reg, imm);
   2662 }
   2663 
   2664 
   2665 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) {
   2666   EmitGenericShift(true, 5, reg, imm);
   2667 }
   2668 
   2669 
   2670 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) {
   2671   EmitGenericShift(false, 5, operand, shifter);
   2672 }
   2673 
   2674 
   2675 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) {
   2676   EmitGenericShift(true, 5, operand, shifter);
   2677 }
   2678 
   2679 
   2680 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) {
   2681   EmitGenericShift(false, 7, reg, imm);
   2682 }
   2683 
   2684 
   2685 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) {
   2686   EmitGenericShift(false, 7, operand, shifter);
   2687 }
   2688 
   2689 
   2690 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) {
   2691   EmitGenericShift(true, 7, reg, imm);
   2692 }
   2693 
   2694 
   2695 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) {
   2696   EmitGenericShift(true, 7, operand, shifter);
   2697 }
   2698 
   2699 
   2700 void X86_64Assembler::roll(CpuRegister reg, const Immediate& imm) {
   2701   EmitGenericShift(false, 0, reg, imm);
   2702 }
   2703 
   2704 
   2705 void X86_64Assembler::roll(CpuRegister operand, CpuRegister shifter) {
   2706   EmitGenericShift(false, 0, operand, shifter);
   2707 }
   2708 
   2709 
   2710 void X86_64Assembler::rorl(CpuRegister reg, const Immediate& imm) {
   2711   EmitGenericShift(false, 1, reg, imm);
   2712 }
   2713 
   2714 
   2715 void X86_64Assembler::rorl(CpuRegister operand, CpuRegister shifter) {
   2716   EmitGenericShift(false, 1, operand, shifter);
   2717 }
   2718 
   2719 
   2720 void X86_64Assembler::rolq(CpuRegister reg, const Immediate& imm) {
   2721   EmitGenericShift(true, 0, reg, imm);
   2722 }
   2723 
   2724 
   2725 void X86_64Assembler::rolq(CpuRegister operand, CpuRegister shifter) {
   2726   EmitGenericShift(true, 0, operand, shifter);
   2727 }
   2728 
   2729 
   2730 void X86_64Assembler::rorq(CpuRegister reg, const Immediate& imm) {
   2731   EmitGenericShift(true, 1, reg, imm);
   2732 }
   2733 
   2734 
   2735 void X86_64Assembler::rorq(CpuRegister operand, CpuRegister shifter) {
   2736   EmitGenericShift(true, 1, operand, shifter);
   2737 }
   2738 
   2739 
   2740 void X86_64Assembler::negl(CpuRegister reg) {
   2741   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2742   EmitOptionalRex32(reg);
   2743   EmitUint8(0xF7);
   2744   EmitOperand(3, Operand(reg));
   2745 }
   2746 
   2747 
   2748 void X86_64Assembler::negq(CpuRegister reg) {
   2749   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2750   EmitRex64(reg);
   2751   EmitUint8(0xF7);
   2752   EmitOperand(3, Operand(reg));
   2753 }
   2754 
   2755 
   2756 void X86_64Assembler::notl(CpuRegister reg) {
   2757   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2758   EmitOptionalRex32(reg);
   2759   EmitUint8(0xF7);
   2760   EmitUint8(0xD0 | reg.LowBits());
   2761 }
   2762 
   2763 
   2764 void X86_64Assembler::notq(CpuRegister reg) {
   2765   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2766   EmitRex64(reg);
   2767   EmitUint8(0xF7);
   2768   EmitOperand(2, Operand(reg));
   2769 }
   2770 
   2771 
   2772 void X86_64Assembler::enter(const Immediate& imm) {
   2773   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2774   EmitUint8(0xC8);
   2775   CHECK(imm.is_uint16()) << imm.value();
   2776   EmitUint8(imm.value() & 0xFF);
   2777   EmitUint8((imm.value() >> 8) & 0xFF);
   2778   EmitUint8(0x00);
   2779 }
   2780 
   2781 
   2782 void X86_64Assembler::leave() {
   2783   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2784   EmitUint8(0xC9);
   2785 }
   2786 
   2787 
   2788 void X86_64Assembler::ret() {
   2789   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2790   EmitUint8(0xC3);
   2791 }
   2792 
   2793 
   2794 void X86_64Assembler::ret(const Immediate& imm) {
   2795   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2796   EmitUint8(0xC2);
   2797   CHECK(imm.is_uint16());
   2798   EmitUint8(imm.value() & 0xFF);
   2799   EmitUint8((imm.value() >> 8) & 0xFF);
   2800 }
   2801 
   2802 
   2803 
   2804 void X86_64Assembler::nop() {
   2805   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2806   EmitUint8(0x90);
   2807 }
   2808 
   2809 
   2810 void X86_64Assembler::int3() {
   2811   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2812   EmitUint8(0xCC);
   2813 }
   2814 
   2815 
   2816 void X86_64Assembler::hlt() {
   2817   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2818   EmitUint8(0xF4);
   2819 }
   2820 
   2821 
   2822 void X86_64Assembler::j(Condition condition, Label* label) {
   2823   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2824   if (label->IsBound()) {
   2825     static const int kShortSize = 2;
   2826     static const int kLongSize = 6;
   2827     int offset = label->Position() - buffer_.Size();
   2828     CHECK_LE(offset, 0);
   2829     if (IsInt<8>(offset - kShortSize)) {
   2830       EmitUint8(0x70 + condition);
   2831       EmitUint8((offset - kShortSize) & 0xFF);
   2832     } else {
   2833       EmitUint8(0x0F);
   2834       EmitUint8(0x80 + condition);
   2835       EmitInt32(offset - kLongSize);
   2836     }
   2837   } else {
   2838     EmitUint8(0x0F);
   2839     EmitUint8(0x80 + condition);
   2840     EmitLabelLink(label);
   2841   }
   2842 }
   2843 
   2844 
   2845 void X86_64Assembler::j(Condition condition, NearLabel* label) {
   2846   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2847   if (label->IsBound()) {
   2848     static const int kShortSize = 2;
   2849     int offset = label->Position() - buffer_.Size();
   2850     CHECK_LE(offset, 0);
   2851     CHECK(IsInt<8>(offset - kShortSize));
   2852     EmitUint8(0x70 + condition);
   2853     EmitUint8((offset - kShortSize) & 0xFF);
   2854   } else {
   2855     EmitUint8(0x70 + condition);
   2856     EmitLabelLink(label);
   2857   }
   2858 }
   2859 
   2860 
   2861 void X86_64Assembler::jrcxz(NearLabel* label) {
   2862   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2863   if (label->IsBound()) {
   2864     static const int kShortSize = 2;
   2865     int offset = label->Position() - buffer_.Size();
   2866     CHECK_LE(offset, 0);
   2867     CHECK(IsInt<8>(offset - kShortSize));
   2868     EmitUint8(0xE3);
   2869     EmitUint8((offset - kShortSize) & 0xFF);
   2870   } else {
   2871     EmitUint8(0xE3);
   2872     EmitLabelLink(label);
   2873   }
   2874 }
   2875 
   2876 
   2877 void X86_64Assembler::jmp(CpuRegister reg) {
   2878   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2879   EmitOptionalRex32(reg);
   2880   EmitUint8(0xFF);
   2881   EmitRegisterOperand(4, reg.LowBits());
   2882 }
   2883 
   2884 void X86_64Assembler::jmp(const Address& address) {
   2885   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2886   EmitOptionalRex32(address);
   2887   EmitUint8(0xFF);
   2888   EmitOperand(4, address);
   2889 }
   2890 
   2891 void X86_64Assembler::jmp(Label* label) {
   2892   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2893   if (label->IsBound()) {
   2894     static const int kShortSize = 2;
   2895     static const int kLongSize = 5;
   2896     int offset = label->Position() - buffer_.Size();
   2897     CHECK_LE(offset, 0);
   2898     if (IsInt<8>(offset - kShortSize)) {
   2899       EmitUint8(0xEB);
   2900       EmitUint8((offset - kShortSize) & 0xFF);
   2901     } else {
   2902       EmitUint8(0xE9);
   2903       EmitInt32(offset - kLongSize);
   2904     }
   2905   } else {
   2906     EmitUint8(0xE9);
   2907     EmitLabelLink(label);
   2908   }
   2909 }
   2910 
   2911 
   2912 void X86_64Assembler::jmp(NearLabel* label) {
   2913   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2914   if (label->IsBound()) {
   2915     static const int kShortSize = 2;
   2916     int offset = label->Position() - buffer_.Size();
   2917     CHECK_LE(offset, 0);
   2918     CHECK(IsInt<8>(offset - kShortSize));
   2919     EmitUint8(0xEB);
   2920     EmitUint8((offset - kShortSize) & 0xFF);
   2921   } else {
   2922     EmitUint8(0xEB);
   2923     EmitLabelLink(label);
   2924   }
   2925 }
   2926 
   2927 
   2928 void X86_64Assembler::rep_movsw() {
   2929   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2930   EmitUint8(0x66);
   2931   EmitUint8(0xF3);
   2932   EmitUint8(0xA5);
   2933 }
   2934 
   2935 
   2936 X86_64Assembler* X86_64Assembler::lock() {
   2937   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2938   EmitUint8(0xF0);
   2939   return this;
   2940 }
   2941 
   2942 
   2943 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) {
   2944   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2945   EmitOptionalRex32(reg, address);
   2946   EmitUint8(0x0F);
   2947   EmitUint8(0xB1);
   2948   EmitOperand(reg.LowBits(), address);
   2949 }
   2950 
   2951 
   2952 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) {
   2953   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2954   EmitRex64(reg, address);
   2955   EmitUint8(0x0F);
   2956   EmitUint8(0xB1);
   2957   EmitOperand(reg.LowBits(), address);
   2958 }
   2959 
   2960 
   2961 void X86_64Assembler::mfence() {
   2962   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2963   EmitUint8(0x0F);
   2964   EmitUint8(0xAE);
   2965   EmitUint8(0xF0);
   2966 }
   2967 
   2968 
   2969 X86_64Assembler* X86_64Assembler::gs() {
   2970   // TODO: gs is a prefix and not an instruction
   2971   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2972   EmitUint8(0x65);
   2973   return this;
   2974 }
   2975 
   2976 
   2977 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) {
   2978   int value = imm.value();
   2979   if (value != 0) {
   2980     if (value > 0) {
   2981       addl(reg, imm);
   2982     } else {
   2983       subl(reg, Immediate(value));
   2984     }
   2985   }
   2986 }
   2987 
   2988 
   2989 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) {
   2990   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   2991   // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh).
   2992   if (dst.NeedsRex() || dst.AsRegister() > 3) {
   2993     EmitOptionalRex(true, false, false, false, dst.NeedsRex());
   2994   }
   2995   EmitUint8(0x0F);
   2996   EmitUint8(0x90 + condition);
   2997   EmitUint8(0xC0 + dst.LowBits());
   2998 }
   2999 
   3000 void X86_64Assembler::bswapl(CpuRegister dst) {
   3001   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3002   EmitOptionalRex(false, false, false, false, dst.NeedsRex());
   3003   EmitUint8(0x0F);
   3004   EmitUint8(0xC8 + dst.LowBits());
   3005 }
   3006 
   3007 void X86_64Assembler::bswapq(CpuRegister dst) {
   3008   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3009   EmitOptionalRex(false, true, false, false, dst.NeedsRex());
   3010   EmitUint8(0x0F);
   3011   EmitUint8(0xC8 + dst.LowBits());
   3012 }
   3013 
   3014 void X86_64Assembler::bsfl(CpuRegister dst, CpuRegister src) {
   3015   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3016   EmitOptionalRex32(dst, src);
   3017   EmitUint8(0x0F);
   3018   EmitUint8(0xBC);
   3019   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3020 }
   3021 
   3022 void X86_64Assembler::bsfl(CpuRegister dst, const Address& src) {
   3023   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3024   EmitOptionalRex32(dst, src);
   3025   EmitUint8(0x0F);
   3026   EmitUint8(0xBC);
   3027   EmitOperand(dst.LowBits(), src);
   3028 }
   3029 
   3030 void X86_64Assembler::bsfq(CpuRegister dst, CpuRegister src) {
   3031   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3032   EmitRex64(dst, src);
   3033   EmitUint8(0x0F);
   3034   EmitUint8(0xBC);
   3035   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3036 }
   3037 
   3038 void X86_64Assembler::bsfq(CpuRegister dst, const Address& src) {
   3039   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3040   EmitRex64(dst, src);
   3041   EmitUint8(0x0F);
   3042   EmitUint8(0xBC);
   3043   EmitOperand(dst.LowBits(), src);
   3044 }
   3045 
   3046 void X86_64Assembler::bsrl(CpuRegister dst, CpuRegister src) {
   3047   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3048   EmitOptionalRex32(dst, src);
   3049   EmitUint8(0x0F);
   3050   EmitUint8(0xBD);
   3051   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3052 }
   3053 
   3054 void X86_64Assembler::bsrl(CpuRegister dst, const Address& src) {
   3055   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3056   EmitOptionalRex32(dst, src);
   3057   EmitUint8(0x0F);
   3058   EmitUint8(0xBD);
   3059   EmitOperand(dst.LowBits(), src);
   3060 }
   3061 
   3062 void X86_64Assembler::bsrq(CpuRegister dst, CpuRegister src) {
   3063   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3064   EmitRex64(dst, src);
   3065   EmitUint8(0x0F);
   3066   EmitUint8(0xBD);
   3067   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3068 }
   3069 
   3070 void X86_64Assembler::bsrq(CpuRegister dst, const Address& src) {
   3071   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3072   EmitRex64(dst, src);
   3073   EmitUint8(0x0F);
   3074   EmitUint8(0xBD);
   3075   EmitOperand(dst.LowBits(), src);
   3076 }
   3077 
   3078 void X86_64Assembler::popcntl(CpuRegister dst, CpuRegister src) {
   3079   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3080   EmitUint8(0xF3);
   3081   EmitOptionalRex32(dst, src);
   3082   EmitUint8(0x0F);
   3083   EmitUint8(0xB8);
   3084   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3085 }
   3086 
   3087 void X86_64Assembler::popcntl(CpuRegister dst, const Address& src) {
   3088   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3089   EmitUint8(0xF3);
   3090   EmitOptionalRex32(dst, src);
   3091   EmitUint8(0x0F);
   3092   EmitUint8(0xB8);
   3093   EmitOperand(dst.LowBits(), src);
   3094 }
   3095 
   3096 void X86_64Assembler::popcntq(CpuRegister dst, CpuRegister src) {
   3097   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3098   EmitUint8(0xF3);
   3099   EmitRex64(dst, src);
   3100   EmitUint8(0x0F);
   3101   EmitUint8(0xB8);
   3102   EmitRegisterOperand(dst.LowBits(), src.LowBits());
   3103 }
   3104 
   3105 void X86_64Assembler::popcntq(CpuRegister dst, const Address& src) {
   3106   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3107   EmitUint8(0xF3);
   3108   EmitRex64(dst, src);
   3109   EmitUint8(0x0F);
   3110   EmitUint8(0xB8);
   3111   EmitOperand(dst.LowBits(), src);
   3112 }
   3113 
   3114 void X86_64Assembler::repne_scasb() {
   3115   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3116   EmitUint8(0xF2);
   3117   EmitUint8(0xAE);
   3118 }
   3119 
   3120 void X86_64Assembler::repne_scasw() {
   3121   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3122   EmitUint8(0x66);
   3123   EmitUint8(0xF2);
   3124   EmitUint8(0xAF);
   3125 }
   3126 
   3127 void X86_64Assembler::repe_cmpsw() {
   3128   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3129   EmitUint8(0x66);
   3130   EmitUint8(0xF3);
   3131   EmitUint8(0xA7);
   3132 }
   3133 
   3134 
   3135 void X86_64Assembler::repe_cmpsl() {
   3136   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3137   EmitUint8(0xF3);
   3138   EmitUint8(0xA7);
   3139 }
   3140 
   3141 
   3142 void X86_64Assembler::repe_cmpsq() {
   3143   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3144   EmitUint8(0xF3);
   3145   EmitRex64();
   3146   EmitUint8(0xA7);
   3147 }
   3148 
   3149 
   3150 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) {
   3151   // TODO: Need to have a code constants table.
   3152   int64_t constant = bit_cast<int64_t, double>(value);
   3153   pushq(Immediate(High32Bits(constant)));
   3154   pushq(Immediate(Low32Bits(constant)));
   3155   movsd(dst, Address(CpuRegister(RSP), 0));
   3156   addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t)));
   3157 }
   3158 
   3159 
   3160 void X86_64Assembler::Align(int alignment, int offset) {
   3161   CHECK(IsPowerOfTwo(alignment));
   3162   // Emit nop instruction until the real position is aligned.
   3163   while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) {
   3164     nop();
   3165   }
   3166 }
   3167 
   3168 
   3169 void X86_64Assembler::Bind(Label* label) {
   3170   int bound = buffer_.Size();
   3171   CHECK(!label->IsBound());  // Labels can only be bound once.
   3172   while (label->IsLinked()) {
   3173     int position = label->LinkPosition();
   3174     int next = buffer_.Load<int32_t>(position);
   3175     buffer_.Store<int32_t>(position, bound - (position + 4));
   3176     label->position_ = next;
   3177   }
   3178   label->BindTo(bound);
   3179 }
   3180 
   3181 
   3182 void X86_64Assembler::Bind(NearLabel* label) {
   3183   int bound = buffer_.Size();
   3184   CHECK(!label->IsBound());  // Labels can only be bound once.
   3185   while (label->IsLinked()) {
   3186     int position = label->LinkPosition();
   3187     uint8_t delta = buffer_.Load<uint8_t>(position);
   3188     int offset = bound - (position + 1);
   3189     CHECK(IsInt<8>(offset));
   3190     buffer_.Store<int8_t>(position, offset);
   3191     label->position_ = delta != 0u ? label->position_ - delta : 0;
   3192   }
   3193   label->BindTo(bound);
   3194 }
   3195 
   3196 
   3197 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) {
   3198   CHECK_GE(reg_or_opcode, 0);
   3199   CHECK_LT(reg_or_opcode, 8);
   3200   const int length = operand.length_;
   3201   CHECK_GT(length, 0);
   3202   // Emit the ModRM byte updated with the given reg value.
   3203   CHECK_EQ(operand.encoding_[0] & 0x38, 0);
   3204   EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3));
   3205   // Emit the rest of the encoded operand.
   3206   for (int i = 1; i < length; i++) {
   3207     EmitUint8(operand.encoding_[i]);
   3208   }
   3209   AssemblerFixup* fixup = operand.GetFixup();
   3210   if (fixup != nullptr) {
   3211     EmitFixup(fixup);
   3212   }
   3213 }
   3214 
   3215 
   3216 void X86_64Assembler::EmitImmediate(const Immediate& imm) {
   3217   if (imm.is_int32()) {
   3218     EmitInt32(static_cast<int32_t>(imm.value()));
   3219   } else {
   3220     EmitInt64(imm.value());
   3221   }
   3222 }
   3223 
   3224 
   3225 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode,
   3226                                   const Operand& operand,
   3227                                   const Immediate& immediate) {
   3228   CHECK_GE(reg_or_opcode, 0);
   3229   CHECK_LT(reg_or_opcode, 8);
   3230   if (immediate.is_int8()) {
   3231     // Use sign-extended 8-bit immediate.
   3232     EmitUint8(0x83);
   3233     EmitOperand(reg_or_opcode, operand);
   3234     EmitUint8(immediate.value() & 0xFF);
   3235   } else if (operand.IsRegister(CpuRegister(RAX))) {
   3236     // Use short form if the destination is eax.
   3237     EmitUint8(0x05 + (reg_or_opcode << 3));
   3238     EmitImmediate(immediate);
   3239   } else {
   3240     EmitUint8(0x81);
   3241     EmitOperand(reg_or_opcode, operand);
   3242     EmitImmediate(immediate);
   3243   }
   3244 }
   3245 
   3246 
   3247 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) {
   3248   if (label->IsBound()) {
   3249     int offset = label->Position() - buffer_.Size();
   3250     CHECK_LE(offset, 0);
   3251     EmitInt32(offset - instruction_size);
   3252   } else {
   3253     EmitLabelLink(label);
   3254   }
   3255 }
   3256 
   3257 
   3258 void X86_64Assembler::EmitLabelLink(Label* label) {
   3259   CHECK(!label->IsBound());
   3260   int position = buffer_.Size();
   3261   EmitInt32(label->position_);
   3262   label->LinkTo(position);
   3263 }
   3264 
   3265 
   3266 void X86_64Assembler::EmitLabelLink(NearLabel* label) {
   3267   CHECK(!label->IsBound());
   3268   int position = buffer_.Size();
   3269   if (label->IsLinked()) {
   3270     // Save the delta in the byte that we have to play with.
   3271     uint32_t delta = position - label->LinkPosition();
   3272     CHECK(IsUint<8>(delta));
   3273     EmitUint8(delta & 0xFF);
   3274   } else {
   3275     EmitUint8(0);
   3276   }
   3277   label->LinkTo(position);
   3278 }
   3279 
   3280 
   3281 void X86_64Assembler::EmitGenericShift(bool wide,
   3282                                        int reg_or_opcode,
   3283                                        CpuRegister reg,
   3284                                        const Immediate& imm) {
   3285   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3286   CHECK(imm.is_int8());
   3287   if (wide) {
   3288     EmitRex64(reg);
   3289   } else {
   3290     EmitOptionalRex32(reg);
   3291   }
   3292   if (imm.value() == 1) {
   3293     EmitUint8(0xD1);
   3294     EmitOperand(reg_or_opcode, Operand(reg));
   3295   } else {
   3296     EmitUint8(0xC1);
   3297     EmitOperand(reg_or_opcode, Operand(reg));
   3298     EmitUint8(imm.value() & 0xFF);
   3299   }
   3300 }
   3301 
   3302 
   3303 void X86_64Assembler::EmitGenericShift(bool wide,
   3304                                        int reg_or_opcode,
   3305                                        CpuRegister operand,
   3306                                        CpuRegister shifter) {
   3307   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3308   CHECK_EQ(shifter.AsRegister(), RCX);
   3309   if (wide) {
   3310     EmitRex64(operand);
   3311   } else {
   3312     EmitOptionalRex32(operand);
   3313   }
   3314   EmitUint8(0xD3);
   3315   EmitOperand(reg_or_opcode, Operand(operand));
   3316 }
   3317 
   3318 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) {
   3319   // REX.WRXB
   3320   // W - 64-bit operand
   3321   // R - MODRM.reg
   3322   // X - SIB.index
   3323   // B - MODRM.rm/SIB.base
   3324   uint8_t rex = force ? 0x40 : 0;
   3325   if (w) {
   3326     rex |= 0x48;  // REX.W000
   3327   }
   3328   if (r) {
   3329     rex |= 0x44;  // REX.0R00
   3330   }
   3331   if (x) {
   3332     rex |= 0x42;  // REX.00X0
   3333   }
   3334   if (b) {
   3335     rex |= 0x41;  // REX.000B
   3336   }
   3337   if (rex != 0) {
   3338     EmitUint8(rex);
   3339   }
   3340 }
   3341 
   3342 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) {
   3343   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
   3344 }
   3345 
   3346 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) {
   3347   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
   3348 }
   3349 
   3350 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) {
   3351   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
   3352 }
   3353 
   3354 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) {
   3355   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
   3356 }
   3357 
   3358 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) {
   3359   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
   3360 }
   3361 
   3362 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) {
   3363   uint8_t rex = operand.rex();
   3364   if (rex != 0) {
   3365     EmitUint8(rex);
   3366   }
   3367 }
   3368 
   3369 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) {
   3370   uint8_t rex = operand.rex();
   3371   if (dst.NeedsRex()) {
   3372     rex |= 0x44;  // REX.0R00
   3373   }
   3374   if (rex != 0) {
   3375     EmitUint8(rex);
   3376   }
   3377 }
   3378 
   3379 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) {
   3380   uint8_t rex = operand.rex();
   3381   if (dst.NeedsRex()) {
   3382     rex |= 0x44;  // REX.0R00
   3383   }
   3384   if (rex != 0) {
   3385     EmitUint8(rex);
   3386   }
   3387 }
   3388 
   3389 void X86_64Assembler::EmitRex64() {
   3390   EmitOptionalRex(false, true, false, false, false);
   3391 }
   3392 
   3393 void X86_64Assembler::EmitRex64(CpuRegister reg) {
   3394   EmitOptionalRex(false, true, false, false, reg.NeedsRex());
   3395 }
   3396 
   3397 void X86_64Assembler::EmitRex64(const Operand& operand) {
   3398   uint8_t rex = operand.rex();
   3399   rex |= 0x48;  // REX.W000
   3400   EmitUint8(rex);
   3401 }
   3402 
   3403 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) {
   3404   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
   3405 }
   3406 
   3407 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) {
   3408   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
   3409 }
   3410 
   3411 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) {
   3412   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
   3413 }
   3414 
   3415 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) {
   3416   uint8_t rex = 0x48 | operand.rex();  // REX.W000
   3417   if (dst.NeedsRex()) {
   3418     rex |= 0x44;  // REX.0R00
   3419   }
   3420   EmitUint8(rex);
   3421 }
   3422 
   3423 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) {
   3424   uint8_t rex = 0x48 | operand.rex();  // REX.W000
   3425   if (dst.NeedsRex()) {
   3426     rex |= 0x44;  // REX.0R00
   3427   }
   3428   EmitUint8(rex);
   3429 }
   3430 
   3431 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) {
   3432   // For src, SPL, BPL, SIL, DIL need the rex prefix.
   3433   bool force = src.AsRegister() > 3;
   3434   EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex());
   3435 }
   3436 
   3437 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) {
   3438   uint8_t rex = operand.rex();
   3439   // For dst, SPL, BPL, SIL, DIL need the rex prefix.
   3440   bool force = dst.AsRegister() > 3;
   3441   if (force) {
   3442     rex |= 0x40;  // REX.0000
   3443   }
   3444   if (dst.NeedsRex()) {
   3445     rex |= 0x44;  // REX.0R00
   3446   }
   3447   if (rex != 0) {
   3448     EmitUint8(rex);
   3449   }
   3450 }
   3451 
   3452 void X86_64Assembler::AddConstantArea() {
   3453   ArrayRef<const int32_t> area = constant_area_.GetBuffer();
   3454   for (size_t i = 0, e = area.size(); i < e; i++) {
   3455     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   3456     EmitInt32(area[i]);
   3457   }
   3458 }
   3459 
   3460 size_t ConstantArea::AppendInt32(int32_t v) {
   3461   size_t result = buffer_.size() * elem_size_;
   3462   buffer_.push_back(v);
   3463   return result;
   3464 }
   3465 
   3466 size_t ConstantArea::AddInt32(int32_t v) {
   3467   // Look for an existing match.
   3468   for (size_t i = 0, e = buffer_.size(); i < e; i++) {
   3469     if (v == buffer_[i]) {
   3470       return i * elem_size_;
   3471     }
   3472   }
   3473 
   3474   // Didn't match anything.
   3475   return AppendInt32(v);
   3476 }
   3477 
   3478 size_t ConstantArea::AddInt64(int64_t v) {
   3479   int32_t v_low = v;
   3480   int32_t v_high = v >> 32;
   3481   if (buffer_.size() > 1) {
   3482     // Ensure we don't pass the end of the buffer.
   3483     for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) {
   3484       if (v_low == buffer_[i] && v_high == buffer_[i + 1]) {
   3485         return i * elem_size_;
   3486       }
   3487     }
   3488   }
   3489 
   3490   // Didn't match anything.
   3491   size_t result = buffer_.size() * elem_size_;
   3492   buffer_.push_back(v_low);
   3493   buffer_.push_back(v_high);
   3494   return result;
   3495 }
   3496 
   3497 size_t ConstantArea::AddDouble(double v) {
   3498   // Treat the value as a 64-bit integer value.
   3499   return AddInt64(bit_cast<int64_t, double>(v));
   3500 }
   3501 
   3502 size_t ConstantArea::AddFloat(float v) {
   3503   // Treat the value as a 32-bit integer value.
   3504   return AddInt32(bit_cast<int32_t, float>(v));
   3505 }
   3506 
   3507 }  // namespace x86_64
   3508 }  // namespace art
   3509