Home | History | Annotate | Download | only in priv
      1 /* -*- mode: C; c-basic-offset: 3; -*- */
      2 
      3 /*---------------------------------------------------------------*/
      4 /*--- begin                                  host_s390_defs.c ---*/
      5 /*---------------------------------------------------------------*/
      6 
      7 /*
      8    This file is part of Valgrind, a dynamic binary instrumentation
      9    framework.
     10 
     11    Copyright IBM Corp. 2010-2015
     12    Copyright (C) 2012-2015  Florian Krohm   (britzel (at) acm.org)
     13 
     14    This program is free software; you can redistribute it and/or
     15    modify it under the terms of the GNU General Public License as
     16    published by the Free Software Foundation; either version 2 of the
     17    License, or (at your option) any later version.
     18 
     19    This program is distributed in the hope that it will be useful, but
     20    WITHOUT ANY WARRANTY; without even the implied warranty of
     21    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     22    General Public License for more details.
     23 
     24    You should have received a copy of the GNU General Public License
     25    along with this program; if not, write to the Free Software
     26    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     27    02110-1301, USA.
     28 
     29    The GNU General Public License is contained in the file COPYING.
     30 */
     31 
     32 /* Contributed by Florian Krohm */
     33 
     34 #include "libvex_basictypes.h"
     35 #include "libvex.h"
     36 #include "libvex_trc_values.h"
     37 #include "libvex_s390x_common.h"
     38 
     39 #include "main_util.h"
     40 #include "main_globals.h"
     41 #include "host_generic_regs.h"
     42 #include "host_s390_defs.h"
     43 #include "s390_disasm.h"
     44 #include "guest_s390_defs.h"    /* S390X_GUEST_OFFSET */
     45 #include <stdarg.h>
     46 
     47 /*------------------------------------------------------------*/
     48 /*--- Forward declarations                                 ---*/
     49 /*------------------------------------------------------------*/
     50 
     51 static Bool s390_insn_is_reg_reg_move(const s390_insn *, HReg *src, HReg *dst);
     52 static void s390_insn_map_regs(HRegRemap *, s390_insn *);
     53 static void s390_insn_get_reg_usage(HRegUsage *u, const s390_insn *);
     54 static UInt s390_tchain_load64_len(void);
     55 
     56 
     57 /*------------------------------------------------------------*/
     58 /*--- Registers                                            ---*/
     59 /*------------------------------------------------------------*/
     60 
     61 /* A mapping from register number to register index */
     62 static Int gpr_index[16];  // GPR regno -> register index
     63 static Int fpr_index[16];  // FPR regno -> register index
     64 
     65 HReg
     66 s390_hreg_gpr(UInt regno)
     67 {
     68    Int ix = gpr_index[regno];
     69    vassert(ix >= 0);
     70    return mkHReg(/*virtual*/False, HRcInt64, regno, ix);
     71 }
     72 
     73 HReg
     74 s390_hreg_fpr(UInt regno)
     75 {
     76    Int ix = fpr_index[regno];
     77    vassert(ix >= 0);
     78    return mkHReg(/*virtual*/False, HRcFlt64, regno, ix);
     79 }
     80 
     81 static __inline__ UInt
     82 hregNumber(HReg reg)
     83 {
     84    return hregEncoding(reg);
     85 }
     86 
     87 /* Decompile the given register into a static buffer and return it */
     88 const HChar *
     89 s390_hreg_as_string(HReg reg)
     90 {
     91    static HChar buf[10];
     92 
     93    static const HChar ireg_names[16][5] = {
     94       "%r0",  "%r1",  "%r2",  "%r3",  "%r4",  "%r5",  "%r6",  "%r7",
     95       "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15"
     96    };
     97 
     98    static const HChar freg_names[16][5] = {
     99       "%f0",  "%f1",  "%f2",  "%f3",  "%f4",  "%f5",  "%f6",  "%f7",
    100       "%f8",  "%f9",  "%f10", "%f11", "%f12", "%f13", "%f14", "%f15"
    101    };
    102 
    103    UInt r;  /* hregNumber() returns an UInt */
    104 
    105    r = hregNumber(reg);
    106 
    107    /* Be generic for all virtual regs. */
    108    if (hregIsVirtual(reg)) {
    109       buf[0] = '\0';
    110       switch (hregClass(reg)) {
    111       case HRcInt64: vex_sprintf(buf, "%%vR%u", r); break;
    112       case HRcFlt64: vex_sprintf(buf, "%%vF%u", r); break;
    113       default:       goto fail;
    114       }
    115       return buf;
    116    }
    117 
    118    /* But specific for real regs. */
    119    vassert(r < 16);
    120 
    121    switch (hregClass(reg)) {
    122    case HRcInt64: return ireg_names[r];
    123    case HRcFlt64: return freg_names[r];
    124    default:       goto fail;
    125    }
    126 
    127  fail: vpanic("s390_hreg_as_string");
    128 }
    129 
    130 
    131 /* Return the real register that holds the guest state pointer */
    132 HReg
    133 s390_hreg_guest_state_pointer(void)
    134 {
    135    return s390_hreg_gpr(S390_REGNO_GUEST_STATE_POINTER);
    136 }
    137 
    138 
    139 /* Is VALUE within the domain of a 20-bit signed integer. */
    140 static __inline__ Bool
    141 fits_signed_20bit(Int value)
    142 {
    143    UInt uval = value;
    144    return ((Int)(uval << 12) >> 12) == value;
    145 }
    146 
    147 
    148 /* Is VALUE within the domain of a 12-bit unsigned integer. */
    149 static __inline__ Bool
    150 fits_unsigned_12bit(Int value)
    151 {
    152    return (value & 0xFFF) == value;
    153 }
    154 
    155 /*------------------------------------------------------------*/
    156 /*--- Addressing modes (amodes)                            ---*/
    157 /*------------------------------------------------------------*/
    158 
    159 /* Construct a b12 amode. */
    160 s390_amode *
    161 s390_amode_b12(Int d, HReg b)
    162 {
    163    s390_amode *am = LibVEX_Alloc_inline(sizeof(s390_amode));
    164 
    165    vassert(fits_unsigned_12bit(d));
    166 
    167    am->tag = S390_AMODE_B12;
    168    am->d = d;
    169    am->b = b;
    170    am->x = s390_hreg_gpr(0);  /* hregNumber(am->x) == 0 */
    171 
    172    return am;
    173 }
    174 
    175 
    176 /* Construct a b20 amode. */
    177 s390_amode *
    178 s390_amode_b20(Int d, HReg b)
    179 {
    180    s390_amode *am = LibVEX_Alloc_inline(sizeof(s390_amode));
    181 
    182    vassert(fits_signed_20bit(d));
    183 
    184    am->tag = S390_AMODE_B20;
    185    am->d = d;
    186    am->b = b;
    187    am->x = s390_hreg_gpr(0);  /* hregNumber(am->x) == 0 */
    188 
    189    return am;
    190 }
    191 
    192 
    193 /* Construct a bx12 amode. */
    194 s390_amode *
    195 s390_amode_bx12(Int d, HReg b, HReg x)
    196 {
    197    s390_amode *am = LibVEX_Alloc_inline(sizeof(s390_amode));
    198 
    199    vassert(fits_unsigned_12bit(d));
    200    vassert(hregNumber(b) != 0);
    201    vassert(hregNumber(x) != 0);
    202 
    203    am->tag = S390_AMODE_BX12;
    204    am->d = d;
    205    am->b = b;
    206    am->x = x;
    207 
    208    return am;
    209 }
    210 
    211 
    212 /* Construct a bx20 amode. */
    213 s390_amode *
    214 s390_amode_bx20(Int d, HReg b, HReg x)
    215 {
    216    s390_amode *am = LibVEX_Alloc_inline(sizeof(s390_amode));
    217 
    218    vassert(fits_signed_20bit(d));
    219    vassert(hregNumber(b) != 0);
    220    vassert(hregNumber(x) != 0);
    221 
    222    am->tag = S390_AMODE_BX20;
    223    am->d = d;
    224    am->b = b;
    225    am->x = x;
    226 
    227    return am;
    228 }
    229 
    230 
    231 /* Construct an AMODE for accessing the guest state at OFFSET.
    232    OFFSET can be at most 3 * sizeof(VexGuestS390XState) + LibVEX_N_SPILL_BYTES
    233    which may be too large for a B12 addressing mode.
    234    Use a B20 amode as a fallback which will be safe for any offset.
    235 */
    236 s390_amode *
    237 s390_amode_for_guest_state(Int offset)
    238 {
    239    if (fits_unsigned_12bit(offset))
    240       return s390_amode_b12(offset, s390_hreg_guest_state_pointer());
    241 
    242    if (fits_signed_20bit(offset))
    243       return s390_amode_b20(offset, s390_hreg_guest_state_pointer());
    244 
    245    vpanic("invalid guest state offset");
    246 }
    247 
    248 
    249 /* Decompile the given amode into a static buffer and return it. */
    250 const HChar *
    251 s390_amode_as_string(const s390_amode *am)
    252 {
    253    static HChar buf[30];
    254    HChar *p;
    255 
    256    buf[0] = '\0';
    257    p = buf;
    258 
    259    switch (am->tag) {
    260    case S390_AMODE_B12:
    261    case S390_AMODE_B20:
    262       vex_sprintf(p, "%d(%s)", am->d, s390_hreg_as_string(am->b));
    263       break;
    264 
    265    case S390_AMODE_BX12:
    266    case S390_AMODE_BX20:
    267       /* s390_hreg_as_string returns pointer to local buffer. Need to
    268          split this into two printfs */
    269       p += vex_sprintf(p, "%d(%s,", am->d, s390_hreg_as_string(am->x));
    270       vex_sprintf(p, "%s)", s390_hreg_as_string(am->b));
    271       break;
    272 
    273    default:
    274       vpanic("s390_amode_as_string");
    275    }
    276 
    277    return buf;
    278 }
    279 
    280 
    281 /* Helper function for s390_amode_is_sane */
    282 static __inline__ Bool
    283 is_virtual_gpr(HReg reg)
    284 {
    285    return hregIsVirtual(reg) && hregClass(reg) == HRcInt64;
    286 }
    287 
    288 
    289 /* Sanity check for an amode */
    290 Bool
    291 s390_amode_is_sane(const s390_amode *am)
    292 {
    293    switch (am->tag) {
    294    case S390_AMODE_B12:
    295       return is_virtual_gpr(am->b) && fits_unsigned_12bit(am->d);
    296 
    297    case S390_AMODE_B20:
    298       return is_virtual_gpr(am->b) && fits_signed_20bit(am->d);
    299 
    300    case S390_AMODE_BX12:
    301       return is_virtual_gpr(am->b) && is_virtual_gpr(am->x) &&
    302              fits_unsigned_12bit(am->d);
    303 
    304    case S390_AMODE_BX20:
    305       return is_virtual_gpr(am->b) && is_virtual_gpr(am->x) &&
    306              fits_signed_20bit(am->d);
    307 
    308    default:
    309       vpanic("s390_amode_is_sane");
    310    }
    311 }
    312 
    313 
    314 /* Record the register use of an amode */
    315 static void
    316 s390_amode_get_reg_usage(HRegUsage *u, const s390_amode *am)
    317 {
    318    switch (am->tag) {
    319    case S390_AMODE_B12:
    320    case S390_AMODE_B20:
    321       addHRegUse(u, HRmRead, am->b);
    322       return;
    323 
    324    case S390_AMODE_BX12:
    325    case S390_AMODE_BX20:
    326       addHRegUse(u, HRmRead, am->b);
    327       addHRegUse(u, HRmRead, am->x);
    328       return;
    329 
    330    default:
    331       vpanic("s390_amode_get_reg_usage");
    332    }
    333 }
    334 
    335 
    336 static void
    337 s390_amode_map_regs(HRegRemap *m, s390_amode *am)
    338 {
    339    switch (am->tag) {
    340    case S390_AMODE_B12:
    341    case S390_AMODE_B20:
    342       am->b = lookupHRegRemap(m, am->b);
    343       return;
    344 
    345    case S390_AMODE_BX12:
    346    case S390_AMODE_BX20:
    347       am->b = lookupHRegRemap(m, am->b);
    348       am->x = lookupHRegRemap(m, am->x);
    349       return;
    350 
    351    default:
    352       vpanic("s390_amode_map_regs");
    353    }
    354 }
    355 
    356 
    357 void
    358 ppS390AMode(const s390_amode *am)
    359 {
    360    vex_printf("%s", s390_amode_as_string(am));
    361 }
    362 
    363 void
    364 ppS390Instr(const s390_insn *insn, Bool mode64)
    365 {
    366    vex_printf("%s", s390_insn_as_string(insn));
    367 }
    368 
    369 void
    370 ppHRegS390(HReg reg)
    371 {
    372    vex_printf("%s", s390_hreg_as_string(reg));
    373 }
    374 
    375 /*------------------------------------------------------------*/
    376 /*--- Helpers for register allocation                      ---*/
    377 /*------------------------------------------------------------*/
    378 
    379 /* Initialise and return the "register universe", i.e. a list of
    380    all hardware registers. Called once. */
    381 const RRegUniverse *
    382 getRRegUniverse_S390(void)
    383 {
    384    static RRegUniverse all_regs;
    385    static Bool initialised = False;
    386    RRegUniverse *ru = &all_regs;
    387 
    388    if (LIKELY(initialised))
    389       return ru;
    390 
    391    RRegUniverse__init(ru);
    392 
    393    /* Assign invalid values to the gpr/fpr_index */
    394    for (UInt i = 0; i < sizeof gpr_index / sizeof gpr_index[0]; ++i)
    395       gpr_index[i] = -1;
    396    for (UInt i = 0; i < sizeof fpr_index / sizeof fpr_index[0]; ++i)
    397       fpr_index[i] = -1;
    398 
    399    /* Add the registers that are available to the register allocator.
    400       GPRs:  registers 1..11 are available
    401       FPRs:  registers 0..15 are available
    402              FPR12 - FPR15 are also used as register pairs for 128-bit
    403              floating point operations
    404    */
    405    UInt regno;
    406    for (regno = 1; regno <= 11; ++regno) {
    407       gpr_index[regno] = ru->size;
    408       ru->regs[ru->size++] = s390_hreg_gpr(regno);
    409    }
    410    for (regno = 0; regno <= 15; ++regno) {
    411       fpr_index[regno] = ru->size;
    412       ru->regs[ru->size++] = s390_hreg_fpr(regno);
    413    }
    414    ru->allocable = ru->size;
    415 
    416    /* Add the registers that are not available for allocation.
    417       r0  -- cannot be used as a base or index register
    418       r12 -- scratch register for translation chaining support
    419       r13 -- guest state pointer
    420       r14 -- link register
    421       r15 -- stack pointer
    422    */
    423    UInt other[] = { 0, 12, 13, 14, 15 };
    424    for (UInt i = 0; i < sizeof other / sizeof other[0]; ++i) {
    425       gpr_index[other[i]] = ru->size;
    426       ru->regs[ru->size++] = s390_hreg_gpr(other[i]);
    427    }
    428 
    429    /* Sanity checking */
    430    for (UInt i = 0; i < sizeof gpr_index / sizeof gpr_index[0]; ++i)
    431       vassert(gpr_index[i] >= 0);
    432    for (UInt i = 0; i < sizeof fpr_index / sizeof fpr_index[0]; ++i)
    433       vassert(fpr_index[i] >= 0);
    434 
    435    initialised = True;
    436    return ru;
    437 }
    438 
    439 /* Tell the register allocator how the given instruction uses the registers
    440    it refers to. */
    441 void
    442 getRegUsage_S390Instr(HRegUsage *u, const s390_insn *insn, Bool mode64)
    443 {
    444    s390_insn_get_reg_usage(u, insn);
    445 }
    446 
    447 
    448 /* Map the registers of the given instruction */
    449 void
    450 mapRegs_S390Instr(HRegRemap *m, s390_insn *insn, Bool mode64)
    451 {
    452    s390_insn_map_regs(m, insn);
    453 }
    454 
    455 
    456 /* Figure out if the given insn represents a reg-reg move, and if so
    457    assign the source and destination to *src and *dst.  If in doubt say No.
    458    Used by the register allocator to do move coalescing. */
    459 Bool
    460 isMove_S390Instr(const s390_insn *insn, HReg *src, HReg *dst)
    461 {
    462    return s390_insn_is_reg_reg_move(insn, src, dst);
    463 }
    464 
    465 
    466 /* Generate s390 spill/reload instructions under the direction of the
    467    register allocator.  Note it's critical these don't write the
    468    condition codes. This is like an Ist_Put */
    469 void
    470 genSpill_S390(HInstr **i1, HInstr **i2, HReg rreg, Int offsetB, Bool mode64)
    471 {
    472    s390_amode *am;
    473 
    474    vassert(offsetB >= 0);
    475    vassert(!hregIsVirtual(rreg));
    476 
    477    *i1 = *i2 = NULL;
    478 
    479    am = s390_amode_for_guest_state(offsetB);
    480 
    481    switch (hregClass(rreg)) {
    482    case HRcInt64:
    483    case HRcFlt64:
    484       *i1 = s390_insn_store(8, am, rreg);
    485       return;
    486 
    487    default:
    488       ppHRegClass(hregClass(rreg));
    489       vpanic("genSpill_S390: unimplemented regclass");
    490    }
    491 }
    492 
    493 
    494 /* This is like an Iex_Get */
    495 void
    496 genReload_S390(HInstr **i1, HInstr **i2, HReg rreg, Int offsetB, Bool mode64)
    497 {
    498    s390_amode *am;
    499 
    500    vassert(offsetB >= 0);
    501    vassert(!hregIsVirtual(rreg));
    502 
    503    *i1 = *i2 = NULL;
    504 
    505    am = s390_amode_for_guest_state(offsetB);
    506 
    507    switch (hregClass(rreg)) {
    508    case HRcInt64:
    509    case HRcFlt64:
    510       *i1 = s390_insn_load(8, rreg, am);
    511       return;
    512 
    513    default:
    514       ppHRegClass(hregClass(rreg));
    515       vpanic("genReload_S390: unimplemented regclass");
    516    }
    517 }
    518 
    519 /* Helper function for s390_insn_get_reg_usage */
    520 static void
    521 s390_opnd_RMI_get_reg_usage(HRegUsage *u, s390_opnd_RMI op)
    522 {
    523    switch (op.tag) {
    524    case S390_OPND_REG:
    525       addHRegUse(u, HRmRead, op.variant.reg);
    526       break;
    527 
    528    case S390_OPND_AMODE:
    529       s390_amode_get_reg_usage(u, op.variant.am);
    530       break;
    531 
    532    case S390_OPND_IMMEDIATE:
    533       break;
    534 
    535    default:
    536       vpanic("s390_opnd_RMI_get_reg_usage");
    537    }
    538 }
    539 
    540 
    541 /* Tell the register allocator how the given insn uses the registers */
    542 static void
    543 s390_insn_get_reg_usage(HRegUsage *u, const s390_insn *insn)
    544 {
    545    initHRegUsage(u);
    546 
    547    switch (insn->tag) {
    548    case S390_INSN_LOAD:
    549       addHRegUse(u, HRmWrite, insn->variant.load.dst);
    550       s390_amode_get_reg_usage(u, insn->variant.load.src);
    551       break;
    552 
    553    case S390_INSN_LOAD_IMMEDIATE:
    554       addHRegUse(u, HRmWrite, insn->variant.load_immediate.dst);
    555       break;
    556 
    557    case S390_INSN_STORE:
    558       addHRegUse(u, HRmRead, insn->variant.store.src);
    559       s390_amode_get_reg_usage(u, insn->variant.store.dst);
    560       break;
    561 
    562    case S390_INSN_MOVE:
    563       addHRegUse(u, HRmRead,  insn->variant.move.src);
    564       addHRegUse(u, HRmWrite, insn->variant.move.dst);
    565       break;
    566 
    567    case S390_INSN_MEMCPY:
    568       s390_amode_get_reg_usage(u, insn->variant.memcpy.src);
    569       s390_amode_get_reg_usage(u, insn->variant.memcpy.dst);
    570       break;
    571 
    572    case S390_INSN_COND_MOVE:
    573       s390_opnd_RMI_get_reg_usage(u, insn->variant.cond_move.src);
    574       addHRegUse(u, HRmWrite, insn->variant.cond_move.dst);
    575       break;
    576 
    577    case S390_INSN_ALU:
    578       addHRegUse(u, HRmWrite, insn->variant.alu.dst);
    579       addHRegUse(u, HRmRead,  insn->variant.alu.dst);  /* op1 */
    580       s390_opnd_RMI_get_reg_usage(u, insn->variant.alu.op2);
    581       break;
    582 
    583    case S390_INSN_SMUL:
    584    case S390_INSN_UMUL:
    585       addHRegUse(u, HRmRead,  insn->variant.mul.dst_lo);  /* op1 */
    586       addHRegUse(u, HRmWrite, insn->variant.mul.dst_lo);
    587       addHRegUse(u, HRmWrite, insn->variant.mul.dst_hi);
    588       s390_opnd_RMI_get_reg_usage(u, insn->variant.mul.op2);
    589       break;
    590 
    591    case S390_INSN_SDIV:
    592    case S390_INSN_UDIV:
    593       addHRegUse(u, HRmRead,  insn->variant.div.op1_lo);
    594       addHRegUse(u, HRmRead,  insn->variant.div.op1_hi);
    595       addHRegUse(u, HRmWrite, insn->variant.div.op1_lo);
    596       addHRegUse(u, HRmWrite, insn->variant.div.op1_hi);
    597       s390_opnd_RMI_get_reg_usage(u, insn->variant.div.op2);
    598       break;
    599 
    600    case S390_INSN_DIVS:
    601       addHRegUse(u, HRmRead,  insn->variant.divs.op1);
    602       addHRegUse(u, HRmWrite, insn->variant.divs.op1); /* quotient */
    603       addHRegUse(u, HRmWrite, insn->variant.divs.rem); /* remainder */
    604       s390_opnd_RMI_get_reg_usage(u, insn->variant.divs.op2);
    605       break;
    606 
    607    case S390_INSN_CLZ:
    608       addHRegUse(u, HRmWrite, insn->variant.clz.num_bits);
    609       addHRegUse(u, HRmWrite, insn->variant.clz.clobber);
    610       s390_opnd_RMI_get_reg_usage(u, insn->variant.clz.src);
    611       break;
    612 
    613    case S390_INSN_UNOP:
    614       addHRegUse(u, HRmWrite, insn->variant.unop.dst);
    615       s390_opnd_RMI_get_reg_usage(u, insn->variant.unop.src);
    616       break;
    617 
    618    case S390_INSN_TEST:
    619       s390_opnd_RMI_get_reg_usage(u, insn->variant.test.src);
    620       break;
    621 
    622    case S390_INSN_CC2BOOL:
    623       addHRegUse(u, HRmWrite, insn->variant.cc2bool.dst);
    624       break;
    625 
    626    case S390_INSN_CAS:
    627       addHRegUse(u, HRmRead,  insn->variant.cas.op1);
    628       s390_amode_get_reg_usage(u, insn->variant.cas.op2);
    629       addHRegUse(u, HRmRead,  insn->variant.cas.op3);
    630       addHRegUse(u, HRmWrite,  insn->variant.cas.old_mem);
    631       break;
    632 
    633    case S390_INSN_CDAS: {
    634       s390_cdas *cdas = insn->variant.cdas.details;
    635 
    636       addHRegUse(u, HRmRead,  cdas->op1_high);
    637       addHRegUse(u, HRmRead,  cdas->op1_low);
    638       s390_amode_get_reg_usage(u, cdas->op2);
    639       addHRegUse(u, HRmRead,  cdas->op3_high);
    640       addHRegUse(u, HRmRead,  cdas->op3_low);
    641       addHRegUse(u, HRmWrite, cdas->old_mem_high);
    642       addHRegUse(u, HRmWrite, cdas->old_mem_low);
    643       addHRegUse(u, HRmWrite, cdas->scratch);
    644       break;
    645    }
    646 
    647    case S390_INSN_COMPARE:
    648       addHRegUse(u, HRmRead, insn->variant.compare.src1);
    649       s390_opnd_RMI_get_reg_usage(u, insn->variant.compare.src2);
    650       break;
    651 
    652    case S390_INSN_HELPER_CALL: {
    653       UInt i;
    654 
    655       /* Assume that all volatile registers are clobbered. ABI says,
    656          volatile registers are: r0 - r5. Valgrind's register allocator
    657          does not know about r0, so we can leave that out */
    658       for (i = 1; i <= 5; ++i) {
    659          addHRegUse(u, HRmWrite, s390_hreg_gpr(i));
    660       }
    661 
    662       /* Ditto for floating point registers. f0 - f7 are volatile */
    663       for (i = 0; i <= 7; ++i) {
    664          addHRegUse(u, HRmWrite, s390_hreg_fpr(i));
    665       }
    666 
    667       /* The registers that are used for passing arguments will be read.
    668          Not all of them may, but in general we need to assume that. */
    669       for (i = 0; i < insn->variant.helper_call.details->num_args; ++i) {
    670          addHRegUse(u, HRmRead, s390_hreg_gpr(s390_gprno_from_arg_index(i)));
    671       }
    672 
    673       /* s390_insn_helper_call_emit also reads / writes the link register
    674          and stack pointer. But those registers are not visible to the
    675          register allocator. So we don't need to do anything for them. */
    676       break;
    677    }
    678 
    679    case S390_INSN_BFP_TRIOP:
    680       addHRegUse(u, HRmWrite, insn->variant.bfp_triop.dst);
    681       addHRegUse(u, HRmRead,  insn->variant.bfp_triop.dst);  /* first */
    682       addHRegUse(u, HRmRead,  insn->variant.bfp_triop.op2);  /* second */
    683       addHRegUse(u, HRmRead,  insn->variant.bfp_triop.op3);  /* third */
    684       break;
    685 
    686    case S390_INSN_BFP_BINOP:
    687       addHRegUse(u, HRmWrite, insn->variant.bfp_binop.dst_hi);
    688       addHRegUse(u, HRmRead,  insn->variant.bfp_binop.dst_hi);  /* left */
    689       addHRegUse(u, HRmRead,  insn->variant.bfp_binop.op2_hi);  /* right */
    690       if (insn->size == 16) {
    691          addHRegUse(u, HRmWrite, insn->variant.bfp_binop.dst_lo);
    692          addHRegUse(u, HRmRead,  insn->variant.bfp_binop.dst_lo);  /* left */
    693          addHRegUse(u, HRmRead,  insn->variant.bfp_binop.op2_lo);  /* right */
    694       }
    695       break;
    696 
    697    case S390_INSN_BFP_UNOP:
    698       addHRegUse(u, HRmWrite, insn->variant.bfp_unop.dst_hi);
    699       addHRegUse(u, HRmRead,  insn->variant.bfp_unop.op_hi);  /* operand */
    700       if (insn->size == 16) {
    701          addHRegUse(u, HRmWrite, insn->variant.bfp_unop.dst_lo);
    702          addHRegUse(u, HRmRead,  insn->variant.bfp_unop.op_lo);  /* operand */
    703       }
    704       break;
    705 
    706    case S390_INSN_BFP_COMPARE:
    707       addHRegUse(u, HRmWrite, insn->variant.bfp_compare.dst);
    708       addHRegUse(u, HRmRead,  insn->variant.bfp_compare.op1_hi);  /* left */
    709       addHRegUse(u, HRmRead,  insn->variant.bfp_compare.op2_hi);  /* right */
    710       if (insn->size == 16) {
    711          addHRegUse(u, HRmRead,  insn->variant.bfp_compare.op1_lo);  /* left */
    712          addHRegUse(u, HRmRead,  insn->variant.bfp_compare.op2_lo);  /* right */
    713       }
    714       break;
    715 
    716    case S390_INSN_BFP_CONVERT:
    717       addHRegUse(u, HRmWrite, insn->variant.bfp_convert.dst_hi);
    718       if (! hregIsInvalid(insn->variant.bfp_convert.dst_lo))
    719          addHRegUse(u, HRmWrite, insn->variant.bfp_convert.dst_lo);
    720       addHRegUse(u, HRmRead,  insn->variant.bfp_convert.op_hi);
    721       if (! hregIsInvalid(insn->variant.bfp_convert.op_lo))
    722          addHRegUse(u, HRmRead, insn->variant.bfp_convert.op_lo);
    723       break;
    724 
    725    case S390_INSN_DFP_BINOP: {
    726       s390_dfp_binop *dfp_binop = insn->variant.dfp_binop.details;
    727 
    728       addHRegUse(u, HRmWrite, dfp_binop->dst_hi);
    729       addHRegUse(u, HRmRead,  dfp_binop->op2_hi);  /* left */
    730       addHRegUse(u, HRmRead,  dfp_binop->op3_hi);  /* right */
    731       if (insn->size == 16) {
    732          addHRegUse(u, HRmWrite, dfp_binop->dst_lo);
    733          addHRegUse(u, HRmRead,  dfp_binop->op2_lo);  /* left */
    734          addHRegUse(u, HRmRead,  dfp_binop->op3_lo);  /* right */
    735       }
    736       break;
    737    }
    738 
    739    case S390_INSN_DFP_UNOP:
    740       addHRegUse(u, HRmWrite, insn->variant.dfp_unop.dst_hi);
    741       addHRegUse(u, HRmRead,  insn->variant.dfp_unop.op_hi);  /* operand */
    742       if (insn->size == 16) {
    743          addHRegUse(u, HRmWrite, insn->variant.dfp_unop.dst_lo);
    744          addHRegUse(u, HRmRead,  insn->variant.dfp_unop.op_lo);  /* operand */
    745       }
    746       break;
    747 
    748    case S390_INSN_DFP_INTOP:
    749       addHRegUse(u, HRmWrite, insn->variant.dfp_intop.dst_hi);
    750       addHRegUse(u, HRmRead,  insn->variant.dfp_intop.op2);
    751       addHRegUse(u, HRmRead,  insn->variant.dfp_intop.op3_hi);
    752       if (insn->size == 16) {
    753          addHRegUse(u, HRmWrite, insn->variant.dfp_intop.dst_lo);
    754          addHRegUse(u, HRmRead,  insn->variant.dfp_intop.op3_lo);
    755       }
    756       break;
    757 
    758    case S390_INSN_DFP_COMPARE:
    759       addHRegUse(u, HRmWrite, insn->variant.dfp_compare.dst);
    760       addHRegUse(u, HRmRead,  insn->variant.dfp_compare.op1_hi);  /* left */
    761       addHRegUse(u, HRmRead,  insn->variant.dfp_compare.op2_hi);  /* right */
    762       if (insn->size == 16) {
    763          addHRegUse(u, HRmRead,  insn->variant.dfp_compare.op1_lo);  /* left */
    764          addHRegUse(u, HRmRead,  insn->variant.dfp_compare.op2_lo);  /* right */
    765       }
    766       break;
    767 
    768    case S390_INSN_DFP_CONVERT:
    769       addHRegUse(u, HRmWrite, insn->variant.dfp_convert.dst_hi);
    770       if (! hregIsInvalid(insn->variant.dfp_convert.dst_lo))
    771          addHRegUse(u, HRmWrite, insn->variant.dfp_convert.dst_lo);
    772       addHRegUse(u, HRmRead,  insn->variant.dfp_convert.op_hi);  /* operand */
    773       if (! hregIsInvalid(insn->variant.dfp_convert.op_lo))
    774          addHRegUse(u, HRmRead, insn->variant.dfp_convert.op_lo); /* operand */
    775       break;
    776 
    777    case S390_INSN_DFP_REROUND:
    778       addHRegUse(u, HRmWrite, insn->variant.dfp_reround.dst_hi);
    779       addHRegUse(u, HRmRead,  insn->variant.dfp_reround.op2);     /* left */
    780       addHRegUse(u, HRmRead,  insn->variant.dfp_reround.op3_hi);  /* right */
    781       if (insn->size == 16) {
    782          addHRegUse(u, HRmWrite, insn->variant.dfp_reround.dst_lo);
    783          addHRegUse(u, HRmRead,  insn->variant.dfp_reround.op3_lo); /* right */
    784       }
    785       break;
    786 
    787    case S390_INSN_FP_CONVERT: {
    788       s390_fp_convert *fp_convert = insn->variant.fp_convert.details;
    789 
    790       addHRegUse(u, HRmWrite, fp_convert->dst_hi);
    791       if (! hregIsInvalid(fp_convert->dst_lo))
    792          addHRegUse(u, HRmWrite, fp_convert->dst_lo);
    793       addHRegUse(u, HRmRead,  fp_convert->op_hi);
    794       if (! hregIsInvalid(fp_convert->op_lo))
    795          addHRegUse(u, HRmRead, fp_convert->op_lo);
    796       addHRegUse(u, HRmWrite, fp_convert->r1);
    797       break;
    798    }
    799 
    800    case S390_INSN_MIMM:
    801       s390_amode_get_reg_usage(u, insn->variant.mimm.dst);
    802       break;
    803 
    804    case S390_INSN_MADD:
    805       s390_amode_get_reg_usage(u, insn->variant.madd.dst);
    806       break;
    807 
    808    case S390_INSN_MFENCE:
    809       break;
    810 
    811    case S390_INSN_SET_FPC_BFPRM:
    812       addHRegUse(u, HRmRead,  insn->variant.set_fpc_bfprm.mode);
    813       break;
    814 
    815    case S390_INSN_SET_FPC_DFPRM:
    816       addHRegUse(u, HRmRead,  insn->variant.set_fpc_dfprm.mode);
    817       break;
    818 
    819    case S390_INSN_EVCHECK:
    820       s390_amode_get_reg_usage(u, insn->variant.evcheck.counter);
    821       s390_amode_get_reg_usage(u, insn->variant.evcheck.fail_addr);
    822       break;
    823 
    824    case S390_INSN_PROFINC:
    825       /* Does not use any register visible to the register allocator */
    826       break;
    827 
    828    case S390_INSN_XDIRECT:
    829       s390_amode_get_reg_usage(u, insn->variant.xdirect.guest_IA);
    830       break;
    831 
    832    case S390_INSN_XINDIR:
    833       addHRegUse(u, HRmRead, insn->variant.xindir.dst);
    834       s390_amode_get_reg_usage(u, insn->variant.xindir.guest_IA);
    835       break;
    836 
    837    case S390_INSN_XASSISTED:
    838       addHRegUse(u, HRmRead, insn->variant.xassisted.dst);
    839       s390_amode_get_reg_usage(u, insn->variant.xassisted.guest_IA);
    840       break;
    841 
    842    default:
    843       vpanic("s390_insn_get_reg_usage");
    844    }
    845 }
    846 
    847 
    848 /* Helper function for s390_insn_map_regs */
    849 static void
    850 s390_opnd_RMI_map_regs(HRegRemap *m, s390_opnd_RMI *op)
    851 {
    852    switch (op->tag) {
    853    case S390_OPND_REG:
    854       op->variant.reg = lookupHRegRemap(m, op->variant.reg);
    855       break;
    856 
    857    case S390_OPND_IMMEDIATE:
    858       break;
    859 
    860    case S390_OPND_AMODE:
    861       s390_amode_map_regs(m, op->variant.am);
    862       break;
    863 
    864    default:
    865       vpanic("s390_opnd_RMI_map_regs");
    866    }
    867 }
    868 
    869 
    870 static void
    871 s390_insn_map_regs(HRegRemap *m, s390_insn *insn)
    872 {
    873    switch (insn->tag) {
    874    case S390_INSN_LOAD:
    875       insn->variant.load.dst = lookupHRegRemap(m, insn->variant.load.dst);
    876       s390_amode_map_regs(m, insn->variant.load.src);
    877       break;
    878 
    879    case S390_INSN_STORE:
    880       s390_amode_map_regs(m, insn->variant.store.dst);
    881       insn->variant.store.src = lookupHRegRemap(m, insn->variant.store.src);
    882       break;
    883 
    884    case S390_INSN_MOVE:
    885       insn->variant.move.dst = lookupHRegRemap(m, insn->variant.move.dst);
    886       insn->variant.move.src = lookupHRegRemap(m, insn->variant.move.src);
    887       break;
    888 
    889    case S390_INSN_MEMCPY:
    890       s390_amode_map_regs(m, insn->variant.memcpy.dst);
    891       s390_amode_map_regs(m, insn->variant.memcpy.src);
    892       break;
    893 
    894    case S390_INSN_COND_MOVE:
    895       insn->variant.cond_move.dst = lookupHRegRemap(m, insn->variant.cond_move.dst);
    896       s390_opnd_RMI_map_regs(m, &insn->variant.cond_move.src);
    897       break;
    898 
    899    case S390_INSN_LOAD_IMMEDIATE:
    900       insn->variant.load_immediate.dst =
    901          lookupHRegRemap(m, insn->variant.load_immediate.dst);
    902       break;
    903 
    904    case S390_INSN_ALU:
    905       insn->variant.alu.dst = lookupHRegRemap(m, insn->variant.alu.dst);
    906       s390_opnd_RMI_map_regs(m, &insn->variant.alu.op2);
    907       break;
    908 
    909    case S390_INSN_SMUL:
    910    case S390_INSN_UMUL:
    911       insn->variant.mul.dst_hi = lookupHRegRemap(m, insn->variant.mul.dst_hi);
    912       insn->variant.mul.dst_lo = lookupHRegRemap(m, insn->variant.mul.dst_lo);
    913       s390_opnd_RMI_map_regs(m, &insn->variant.mul.op2);
    914       break;
    915 
    916    case S390_INSN_SDIV:
    917    case S390_INSN_UDIV:
    918       insn->variant.div.op1_hi = lookupHRegRemap(m, insn->variant.div.op1_hi);
    919       insn->variant.div.op1_lo = lookupHRegRemap(m, insn->variant.div.op1_lo);
    920       s390_opnd_RMI_map_regs(m, &insn->variant.div.op2);
    921       break;
    922 
    923    case S390_INSN_DIVS:
    924       insn->variant.divs.op1 = lookupHRegRemap(m, insn->variant.divs.op1);
    925       insn->variant.divs.rem = lookupHRegRemap(m, insn->variant.divs.rem);
    926       s390_opnd_RMI_map_regs(m, &insn->variant.divs.op2);
    927       break;
    928 
    929    case S390_INSN_CLZ:
    930       insn->variant.clz.num_bits = lookupHRegRemap(m, insn->variant.clz.num_bits);
    931       insn->variant.clz.clobber  = lookupHRegRemap(m, insn->variant.clz.clobber);
    932       s390_opnd_RMI_map_regs(m, &insn->variant.clz.src);
    933       break;
    934 
    935    case S390_INSN_UNOP:
    936       insn->variant.unop.dst = lookupHRegRemap(m, insn->variant.unop.dst);
    937       s390_opnd_RMI_map_regs(m, &insn->variant.unop.src);
    938       break;
    939 
    940    case S390_INSN_TEST:
    941       s390_opnd_RMI_map_regs(m, &insn->variant.test.src);
    942       break;
    943 
    944    case S390_INSN_CC2BOOL:
    945       insn->variant.cc2bool.dst = lookupHRegRemap(m, insn->variant.cc2bool.dst);
    946       break;
    947 
    948    case S390_INSN_CAS:
    949       insn->variant.cas.op1 = lookupHRegRemap(m, insn->variant.cas.op1);
    950       s390_amode_map_regs(m, insn->variant.cas.op2);
    951       insn->variant.cas.op3 = lookupHRegRemap(m, insn->variant.cas.op3);
    952       insn->variant.cas.old_mem = lookupHRegRemap(m, insn->variant.cas.old_mem);
    953       break;
    954 
    955    case S390_INSN_CDAS: {
    956       s390_cdas *cdas = insn->variant.cdas.details;
    957 
    958       cdas->op1_high = lookupHRegRemap(m, cdas->op1_high);
    959       cdas->op1_low  = lookupHRegRemap(m, cdas->op1_low);
    960       s390_amode_map_regs(m, cdas->op2);
    961       cdas->op3_high = lookupHRegRemap(m, cdas->op3_high);
    962       cdas->op3_low  = lookupHRegRemap(m, cdas->op3_low);
    963       cdas->old_mem_high = lookupHRegRemap(m, cdas->old_mem_high);
    964       cdas->old_mem_low  = lookupHRegRemap(m, cdas->old_mem_low);
    965       cdas->scratch  = lookupHRegRemap(m, cdas->scratch);
    966       break;
    967    }
    968 
    969    case S390_INSN_COMPARE:
    970       insn->variant.compare.src1 = lookupHRegRemap(m, insn->variant.compare.src1);
    971       s390_opnd_RMI_map_regs(m, &insn->variant.compare.src2);
    972       break;
    973 
    974    case S390_INSN_HELPER_CALL:
    975       /* s390_insn_helper_call_emit also reads / writes the link register
    976          and stack pointer. But those registers are not visible to the
    977          register allocator. So we don't need to do anything for them.
    978          As for the arguments of the helper call -- they will be loaded into
    979          non-virtual registers. Again, we don't need to do anything for those
    980          here. */
    981       break;
    982 
    983    case S390_INSN_BFP_TRIOP:
    984       insn->variant.bfp_triop.dst =
    985          lookupHRegRemap(m, insn->variant.bfp_triop.dst);
    986       insn->variant.bfp_triop.op2 =
    987          lookupHRegRemap(m, insn->variant.bfp_triop.op2);
    988       insn->variant.bfp_triop.op3 =
    989          lookupHRegRemap(m, insn->variant.bfp_triop.op3);
    990       break;
    991 
    992    case S390_INSN_BFP_BINOP:
    993       insn->variant.bfp_binop.dst_hi =
    994          lookupHRegRemap(m, insn->variant.bfp_binop.dst_hi);
    995       insn->variant.bfp_binop.op2_hi =
    996          lookupHRegRemap(m, insn->variant.bfp_binop.op2_hi);
    997       if (insn->size == 16) {
    998          insn->variant.bfp_binop.dst_lo =
    999             lookupHRegRemap(m, insn->variant.bfp_binop.dst_lo);
   1000          insn->variant.bfp_binop.op2_lo  =
   1001             lookupHRegRemap(m, insn->variant.bfp_binop.op2_lo);
   1002       }
   1003       break;
   1004 
   1005    case S390_INSN_BFP_UNOP:
   1006       insn->variant.bfp_unop.dst_hi =
   1007          lookupHRegRemap(m, insn->variant.bfp_unop.dst_hi);
   1008       insn->variant.bfp_unop.op_hi  =
   1009          lookupHRegRemap(m, insn->variant.bfp_unop.op_hi);
   1010       if (insn->size == 16) {
   1011          insn->variant.bfp_unop.dst_lo =
   1012             lookupHRegRemap(m, insn->variant.bfp_unop.dst_lo);
   1013          insn->variant.bfp_unop.op_lo  =
   1014             lookupHRegRemap(m, insn->variant.bfp_unop.op_lo);
   1015       }
   1016       break;
   1017 
   1018    case S390_INSN_BFP_COMPARE:
   1019       insn->variant.bfp_compare.dst =
   1020          lookupHRegRemap(m, insn->variant.bfp_compare.dst);
   1021       insn->variant.bfp_compare.op1_hi =
   1022          lookupHRegRemap(m, insn->variant.bfp_compare.op1_hi);
   1023       insn->variant.bfp_compare.op2_hi =
   1024          lookupHRegRemap(m, insn->variant.bfp_compare.op2_hi);
   1025       if (insn->size == 16) {
   1026          insn->variant.bfp_compare.op1_lo =
   1027             lookupHRegRemap(m, insn->variant.bfp_compare.op1_lo);
   1028          insn->variant.bfp_compare.op2_lo =
   1029             lookupHRegRemap(m, insn->variant.bfp_compare.op2_lo);
   1030       }
   1031       break;
   1032 
   1033    case S390_INSN_BFP_CONVERT:
   1034       insn->variant.bfp_convert.dst_hi =
   1035          lookupHRegRemap(m, insn->variant.bfp_convert.dst_hi);
   1036       if (! hregIsInvalid(insn->variant.bfp_convert.dst_lo))
   1037          insn->variant.bfp_convert.dst_lo =
   1038             lookupHRegRemap(m, insn->variant.bfp_convert.dst_lo);
   1039       insn->variant.bfp_convert.op_hi =
   1040          lookupHRegRemap(m, insn->variant.bfp_convert.op_hi);
   1041       if (! hregIsInvalid(insn->variant.bfp_convert.op_lo))
   1042          insn->variant.bfp_convert.op_lo =
   1043             lookupHRegRemap(m, insn->variant.bfp_convert.op_lo);
   1044       break;
   1045 
   1046    case S390_INSN_DFP_BINOP: {
   1047       s390_dfp_binop *dfp_binop = insn->variant.dfp_binop.details;
   1048 
   1049       dfp_binop->dst_hi = lookupHRegRemap(m, dfp_binop->dst_hi);
   1050       dfp_binop->op2_hi = lookupHRegRemap(m, dfp_binop->op2_hi);
   1051       dfp_binop->op3_hi = lookupHRegRemap(m, dfp_binop->op3_hi);
   1052       if (insn->size == 16) {
   1053          dfp_binop->dst_lo = lookupHRegRemap(m, dfp_binop->dst_lo);
   1054          dfp_binop->op2_lo = lookupHRegRemap(m, dfp_binop->op2_lo);
   1055          dfp_binop->op3_lo = lookupHRegRemap(m, dfp_binop->op3_lo);
   1056       }
   1057       break;
   1058    }
   1059 
   1060    case S390_INSN_DFP_UNOP:
   1061       insn->variant.dfp_unop.dst_hi =
   1062          lookupHRegRemap(m, insn->variant.dfp_unop.dst_hi);
   1063       insn->variant.dfp_unop.op_hi  =
   1064          lookupHRegRemap(m, insn->variant.dfp_unop.op_hi);
   1065       if (insn->size == 16) {
   1066          insn->variant.dfp_unop.dst_lo =
   1067             lookupHRegRemap(m, insn->variant.dfp_unop.dst_lo);
   1068          insn->variant.dfp_unop.op_lo  =
   1069             lookupHRegRemap(m, insn->variant.dfp_unop.op_lo);
   1070       }
   1071       break;
   1072 
   1073    case S390_INSN_DFP_INTOP:
   1074       insn->variant.dfp_intop.dst_hi =
   1075          lookupHRegRemap(m, insn->variant.dfp_intop.dst_hi);
   1076       insn->variant.dfp_intop.op2    =
   1077          lookupHRegRemap(m, insn->variant.dfp_intop.op2);
   1078       insn->variant.dfp_intop.op3_hi =
   1079          lookupHRegRemap(m, insn->variant.dfp_intop.op3_hi);
   1080       if (insn->size == 16) {
   1081          insn->variant.dfp_intop.dst_lo =
   1082             lookupHRegRemap(m, insn->variant.dfp_intop.dst_lo);
   1083          insn->variant.dfp_intop.op3_lo =
   1084             lookupHRegRemap(m, insn->variant.dfp_intop.op3_lo);
   1085       }
   1086       break;
   1087 
   1088    case S390_INSN_DFP_COMPARE:
   1089       insn->variant.dfp_compare.dst =
   1090          lookupHRegRemap(m, insn->variant.dfp_compare.dst);
   1091       insn->variant.dfp_compare.op1_hi =
   1092          lookupHRegRemap(m, insn->variant.dfp_compare.op1_hi);
   1093       insn->variant.dfp_compare.op2_hi =
   1094          lookupHRegRemap(m, insn->variant.dfp_compare.op2_hi);
   1095       if (insn->size == 16) {
   1096          insn->variant.dfp_compare.op1_lo =
   1097             lookupHRegRemap(m, insn->variant.dfp_compare.op1_lo);
   1098          insn->variant.dfp_compare.op2_lo =
   1099             lookupHRegRemap(m, insn->variant.dfp_compare.op2_lo);
   1100       }
   1101       break;
   1102 
   1103    case S390_INSN_DFP_CONVERT:
   1104       insn->variant.dfp_convert.dst_hi =
   1105          lookupHRegRemap(m, insn->variant.dfp_convert.dst_hi);
   1106       if (! hregIsInvalid(insn->variant.dfp_convert.dst_lo))
   1107          insn->variant.dfp_convert.dst_lo =
   1108             lookupHRegRemap(m, insn->variant.dfp_convert.dst_lo);
   1109       insn->variant.dfp_convert.op_hi =
   1110          lookupHRegRemap(m, insn->variant.dfp_convert.op_hi);
   1111       if (! hregIsInvalid(insn->variant.dfp_convert.op_lo))
   1112          insn->variant.dfp_convert.op_lo =
   1113             lookupHRegRemap(m, insn->variant.dfp_convert.op_lo);
   1114       break;
   1115 
   1116    case S390_INSN_DFP_REROUND:
   1117       insn->variant.dfp_reround.dst_hi =
   1118          lookupHRegRemap(m, insn->variant.dfp_reround.dst_hi);
   1119       insn->variant.dfp_reround.op2    =
   1120          lookupHRegRemap(m, insn->variant.dfp_reround.op2);
   1121       insn->variant.dfp_reround.op3_hi =
   1122          lookupHRegRemap(m, insn->variant.dfp_reround.op3_hi);
   1123       if (insn->size == 16) {
   1124          insn->variant.dfp_reround.dst_lo =
   1125             lookupHRegRemap(m, insn->variant.dfp_reround.dst_lo);
   1126          insn->variant.dfp_reround.op3_lo =
   1127             lookupHRegRemap(m, insn->variant.dfp_reround.op3_lo);
   1128       }
   1129       break;
   1130 
   1131    case S390_INSN_FP_CONVERT: {
   1132       s390_fp_convert *fp_convert = insn->variant.fp_convert.details;
   1133 
   1134       fp_convert->dst_hi = lookupHRegRemap(m, fp_convert->dst_hi);
   1135       if (! hregIsInvalid(fp_convert->dst_lo))
   1136          fp_convert->dst_lo = lookupHRegRemap(m, fp_convert->dst_lo);
   1137       fp_convert->op_hi = lookupHRegRemap(m, fp_convert->op_hi);
   1138       if (! hregIsInvalid(fp_convert->op_lo))
   1139          fp_convert->op_lo = lookupHRegRemap(m, fp_convert->op_lo);
   1140       fp_convert->r1 = lookupHRegRemap(m, fp_convert->r1);
   1141       break;
   1142    }
   1143 
   1144    case S390_INSN_MIMM:
   1145       s390_amode_map_regs(m, insn->variant.mimm.dst);
   1146       break;
   1147 
   1148    case S390_INSN_MADD:
   1149       s390_amode_map_regs(m, insn->variant.madd.dst);
   1150       break;
   1151 
   1152    case S390_INSN_MFENCE:
   1153       break;
   1154 
   1155    case S390_INSN_SET_FPC_BFPRM:
   1156       insn->variant.set_fpc_bfprm.mode =
   1157          lookupHRegRemap(m, insn->variant.set_fpc_bfprm.mode);
   1158       break;
   1159 
   1160    case S390_INSN_SET_FPC_DFPRM:
   1161       insn->variant.set_fpc_dfprm.mode =
   1162          lookupHRegRemap(m, insn->variant.set_fpc_dfprm.mode);
   1163       break;
   1164 
   1165    case S390_INSN_EVCHECK:
   1166       s390_amode_map_regs(m, insn->variant.evcheck.counter);
   1167       s390_amode_map_regs(m, insn->variant.evcheck.fail_addr);
   1168       break;
   1169 
   1170    case S390_INSN_PROFINC:
   1171       /* Does not use any register visible to the register allocator */
   1172       break;
   1173 
   1174    case S390_INSN_XDIRECT:
   1175       s390_amode_map_regs(m, insn->variant.xdirect.guest_IA);
   1176       break;
   1177 
   1178    case S390_INSN_XINDIR:
   1179       s390_amode_map_regs(m, insn->variant.xindir.guest_IA);
   1180       insn->variant.xindir.dst =
   1181          lookupHRegRemap(m, insn->variant.xindir.dst);
   1182       break;
   1183 
   1184    case S390_INSN_XASSISTED:
   1185       s390_amode_map_regs(m, insn->variant.xassisted.guest_IA);
   1186       insn->variant.xassisted.dst =
   1187          lookupHRegRemap(m, insn->variant.xassisted.dst);
   1188       break;
   1189 
   1190    default:
   1191       vpanic("s390_insn_map_regs");
   1192    }
   1193 }
   1194 
   1195 
   1196 /* Return True, if INSN is a move between two registers of the same class.
   1197    In that case assign the source and destination registers to SRC and DST,
   1198    respectively. */
   1199 static Bool
   1200 s390_insn_is_reg_reg_move(const s390_insn *insn, HReg *src, HReg *dst)
   1201 {
   1202    if (insn->tag == S390_INSN_MOVE &&
   1203        hregClass(insn->variant.move.src) == hregClass(insn->variant.move.dst)) {
   1204       *src = insn->variant.move.src;
   1205       *dst = insn->variant.move.dst;
   1206       return True;
   1207    }
   1208 
   1209    return False;
   1210 }
   1211 
   1212 
   1213 /*------------------------------------------------------------*/
   1214 /*--- Functions to emit a sequence of bytes                ---*/
   1215 /*------------------------------------------------------------*/
   1216 
   1217 static __inline__ UChar *
   1218 emit_2bytes(UChar *p, ULong val)
   1219 {
   1220    return (UChar *)__builtin_memcpy(p, ((UChar *)&val) + 6, 2) + 2;
   1221 }
   1222 
   1223 
   1224 static __inline__ UChar *
   1225 emit_4bytes(UChar *p, ULong val)
   1226 {
   1227    return (UChar *)__builtin_memcpy(p, ((UChar *)&val) + 4, 4) + 4;
   1228 }
   1229 
   1230 
   1231 static __inline__ UChar *
   1232 emit_6bytes(UChar *p, ULong val)
   1233 {
   1234    return (UChar *)__builtin_memcpy(p, ((UChar *)&val) + 2, 6) + 6;
   1235 }
   1236 
   1237 
   1238 /*------------------------------------------------------------*/
   1239 /*--- Functions to emit various instruction formats        ---*/
   1240 /*------------------------------------------------------------*/
   1241 
   1242 static UChar *
   1243 emit_RI(UChar *p, UInt op, UChar r1, UShort i2)
   1244 {
   1245    ULong the_insn = op;
   1246 
   1247    the_insn |= ((ULong)r1) << 20;
   1248    the_insn |= ((ULong)i2) << 0;
   1249 
   1250    return emit_4bytes(p, the_insn);
   1251 }
   1252 
   1253 
   1254 static UChar *
   1255 emit_RIL(UChar *p, ULong op, UChar r1, UInt i2)
   1256 {
   1257    ULong the_insn = op;
   1258 
   1259    the_insn |= ((ULong)r1) << 36;
   1260    the_insn |= ((ULong)i2) << 0;
   1261 
   1262    return emit_6bytes(p, the_insn);
   1263 }
   1264 
   1265 
   1266 static UChar *
   1267 emit_RR(UChar *p, UInt op, UChar r1, UChar r2)
   1268 {
   1269    ULong the_insn = op;
   1270 
   1271    the_insn |= ((ULong)r1) << 4;
   1272    the_insn |= ((ULong)r2) << 0;
   1273 
   1274    return emit_2bytes(p, the_insn);
   1275 }
   1276 
   1277 
   1278 static UChar *
   1279 emit_RRE(UChar *p, UInt op, UChar r1, UChar r2)
   1280 {
   1281    ULong the_insn = op;
   1282 
   1283    the_insn |= ((ULong)r1) << 4;
   1284    the_insn |= ((ULong)r2) << 0;
   1285 
   1286    return emit_4bytes(p, the_insn);
   1287 }
   1288 
   1289 
   1290 static UChar *
   1291 emit_RRF(UChar *p, UInt op, UChar r1, UChar r3, UChar r2)
   1292 {
   1293    ULong the_insn = op;
   1294 
   1295    the_insn |= ((ULong)r1) << 12;
   1296    the_insn |= ((ULong)r3) << 4;
   1297    the_insn |= ((ULong)r2) << 0;
   1298 
   1299    return emit_4bytes(p, the_insn);
   1300 }
   1301 
   1302 
   1303 static UChar *
   1304 emit_RRF2(UChar *p, UInt op, UChar m3, UChar m4, UChar r1, UChar r2)
   1305 {
   1306    ULong the_insn = op;
   1307 
   1308    the_insn |= ((ULong)m3) << 12;
   1309    the_insn |= ((ULong)m4) << 8;
   1310    the_insn |= ((ULong)r1) << 4;
   1311    the_insn |= ((ULong)r2) << 0;
   1312 
   1313    return emit_4bytes(p, the_insn);
   1314 }
   1315 
   1316 
   1317 static UChar *
   1318 emit_RRF3(UChar *p, UInt op, UChar r3, UChar r1, UChar r2)
   1319 {
   1320    ULong the_insn = op;
   1321 
   1322    the_insn |= ((ULong)r3) << 12;
   1323    the_insn |= ((ULong)r1) << 4;
   1324    the_insn |= ((ULong)r2) << 0;
   1325 
   1326    return emit_4bytes(p, the_insn);
   1327 }
   1328 
   1329 
   1330 static UChar *
   1331 emit_RRF4(UChar *p, UInt op, UChar r3, UChar m4, UChar r1, UChar r2)
   1332 {
   1333    ULong the_insn = op;
   1334 
   1335    the_insn |= ((ULong)r3) << 12;
   1336    the_insn |= ((ULong)m4) << 8;
   1337    the_insn |= ((ULong)r1) << 4;
   1338    the_insn |= ((ULong)r2) << 0;
   1339 
   1340    return emit_4bytes(p, the_insn);
   1341 }
   1342 
   1343 
   1344 static UChar *
   1345 emit_RRF5(UChar *p, UInt op, UChar m4, UChar r1, UChar r2)
   1346 {
   1347    ULong the_insn = op;
   1348 
   1349    the_insn |= ((ULong)m4) << 8;
   1350    the_insn |= ((ULong)r1) << 4;
   1351    the_insn |= ((ULong)r2) << 0;
   1352 
   1353    return emit_4bytes(p, the_insn);
   1354 }
   1355 
   1356 
   1357 static UChar *
   1358 emit_RS(UChar *p, UInt op, UChar r1, UChar r3, UChar b2, UShort d2)
   1359 {
   1360    ULong the_insn = op;
   1361 
   1362    the_insn |= ((ULong)r1) << 20;
   1363    the_insn |= ((ULong)r3) << 16;
   1364    the_insn |= ((ULong)b2) << 12;
   1365    the_insn |= ((ULong)d2) << 0;
   1366 
   1367    return emit_4bytes(p, the_insn);
   1368 }
   1369 
   1370 
   1371 static UChar *
   1372 emit_RSY(UChar *p, ULong op, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   1373 {
   1374    ULong the_insn = op;
   1375 
   1376    the_insn |= ((ULong)r1) << 36;
   1377    the_insn |= ((ULong)r3) << 32;
   1378    the_insn |= ((ULong)b2) << 28;
   1379    the_insn |= ((ULong)dl2) << 16;
   1380    the_insn |= ((ULong)dh2) << 8;
   1381 
   1382    return emit_6bytes(p, the_insn);
   1383 }
   1384 
   1385 
   1386 static UChar *
   1387 emit_RX(UChar *p, UInt op, UChar r1, UChar x2, UChar b2, UShort d2)
   1388 {
   1389    ULong the_insn = op;
   1390 
   1391    the_insn |= ((ULong)r1) << 20;
   1392    the_insn |= ((ULong)x2) << 16;
   1393    the_insn |= ((ULong)b2) << 12;
   1394    the_insn |= ((ULong)d2) << 0;
   1395 
   1396    return emit_4bytes(p, the_insn);
   1397 }
   1398 
   1399 
   1400 static UChar *
   1401 emit_RXF(UChar *p, ULong op, UChar r3, UChar x2, UChar b2, UShort d2, UChar r1)
   1402 {
   1403    ULong the_insn = op;
   1404 
   1405    the_insn |= ((ULong)r3) << 36;
   1406    the_insn |= ((ULong)x2) << 32;
   1407    the_insn |= ((ULong)b2) << 28;
   1408    the_insn |= ((ULong)d2) << 16;
   1409    the_insn |= ((ULong)r1) << 12;
   1410 
   1411    return emit_6bytes(p, the_insn);
   1412 }
   1413 
   1414 
   1415 static UChar *
   1416 emit_RXY(UChar *p, ULong op, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1417 {
   1418    ULong the_insn = op;
   1419 
   1420    the_insn |= ((ULong)r1) << 36;
   1421    the_insn |= ((ULong)x2) << 32;
   1422    the_insn |= ((ULong)b2) << 28;
   1423    the_insn |= ((ULong)dl2) << 16;
   1424    the_insn |= ((ULong)dh2) << 8;
   1425 
   1426    return emit_6bytes(p, the_insn);
   1427 }
   1428 
   1429 
   1430 static UChar *
   1431 emit_S(UChar *p, UInt op, UChar b2, UShort d2)
   1432 {
   1433    ULong the_insn = op;
   1434 
   1435    the_insn |= ((ULong)b2) << 12;
   1436    the_insn |= ((ULong)d2) << 0;
   1437 
   1438    return emit_4bytes(p, the_insn);
   1439 }
   1440 
   1441 
   1442 static UChar *
   1443 emit_SI(UChar *p, UInt op, UChar i2, UChar b1, UShort d1)
   1444 {
   1445    ULong the_insn = op;
   1446 
   1447    the_insn |= ((ULong)i2) << 16;
   1448    the_insn |= ((ULong)b1) << 12;
   1449    the_insn |= ((ULong)d1) << 0;
   1450 
   1451    return emit_4bytes(p, the_insn);
   1452 }
   1453 
   1454 
   1455 static UChar *
   1456 emit_SIL(UChar *p, ULong op, UChar b1, UShort d1, UShort i2)
   1457 {
   1458    ULong the_insn = op;
   1459 
   1460    the_insn |= ((ULong)b1) << 28;
   1461    the_insn |= ((ULong)d1) << 16;
   1462    the_insn |= ((ULong)i2) << 0;
   1463 
   1464    return emit_6bytes(p, the_insn);
   1465 }
   1466 
   1467 
   1468 static UChar *
   1469 emit_SIY(UChar *p, ULong op, UChar i2, UChar b1, UShort dl1, UChar dh1)
   1470 {
   1471    ULong the_insn = op;
   1472 
   1473    the_insn |= ((ULong)i2) << 32;
   1474    the_insn |= ((ULong)b1) << 28;
   1475    the_insn |= ((ULong)dl1) << 16;
   1476    the_insn |= ((ULong)dh1) << 8;
   1477 
   1478    return emit_6bytes(p, the_insn);
   1479 }
   1480 
   1481 
   1482 static UChar *
   1483 emit_SSa(UChar *p, ULong op, UChar l, UChar b1, UShort d1, UChar b2, UShort d2)
   1484 {
   1485    ULong the_insn = op;
   1486 
   1487    the_insn |= ((ULong)l)  << 32;
   1488    the_insn |= ((ULong)b1) << 28;
   1489    the_insn |= ((ULong)d1) << 16;
   1490    the_insn |= ((ULong)b2) << 12;
   1491    the_insn |= ((ULong)d2) << 0;
   1492 
   1493    return emit_6bytes(p, the_insn);
   1494 }
   1495 
   1496 
   1497 /*------------------------------------------------------------*/
   1498 /*--- Functions to emit particular instructions            ---*/
   1499 /*------------------------------------------------------------*/
   1500 
   1501 static UChar *
   1502 s390_emit_AR(UChar *p, UChar r1, UChar r2)
   1503 {
   1504    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1505       s390_disasm(ENC3(MNM, GPR, GPR), "ar", r1, r2);
   1506 
   1507    return emit_RR(p, 0x1a00, r1, r2);
   1508 }
   1509 
   1510 
   1511 static UChar *
   1512 s390_emit_AGR(UChar *p, UChar r1, UChar r2)
   1513 {
   1514    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1515       s390_disasm(ENC3(MNM, GPR, GPR), "agr", r1, r2);
   1516 
   1517    return emit_RRE(p, 0xb9080000, r1, r2);
   1518 }
   1519 
   1520 
   1521 static UChar *
   1522 s390_emit_A(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1523 {
   1524    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1525       s390_disasm(ENC3(MNM, GPR, UDXB), "a", r1, d2, x2, b2);
   1526 
   1527    return emit_RX(p, 0x5a000000, r1, x2, b2, d2);
   1528 }
   1529 
   1530 
   1531 static UChar *
   1532 s390_emit_AY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1533 {
   1534    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1535       s390_disasm(ENC3(MNM, GPR, SDXB), "ay", r1, dh2, dl2, x2, b2);
   1536 
   1537    return emit_RXY(p, 0xe3000000005aULL, r1, x2, b2, dl2, dh2);
   1538 }
   1539 
   1540 
   1541 static UChar *
   1542 s390_emit_AG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1543 {
   1544    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1545       s390_disasm(ENC3(MNM, GPR, SDXB), "ag", r1, dh2, dl2, x2, b2);
   1546 
   1547    return emit_RXY(p, 0xe30000000008ULL, r1, x2, b2, dl2, dh2);
   1548 }
   1549 
   1550 
   1551 static UChar *
   1552 s390_emit_AFI(UChar *p, UChar r1, UInt i2)
   1553 {
   1554    vassert(s390_host_has_eimm);
   1555 
   1556    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1557       s390_disasm(ENC3(MNM, GPR, INT), "afi", r1, i2);
   1558 
   1559    return emit_RIL(p, 0xc20900000000ULL, r1, i2);
   1560 }
   1561 
   1562 
   1563 static UChar *
   1564 s390_emit_AGFI(UChar *p, UChar r1, UInt i2)
   1565 {
   1566    vassert(s390_host_has_eimm);
   1567 
   1568    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1569       s390_disasm(ENC3(MNM, GPR, INT), "agfi", r1, i2);
   1570 
   1571    return emit_RIL(p, 0xc20800000000ULL, r1, i2);
   1572 }
   1573 
   1574 
   1575 static UChar *
   1576 s390_emit_AH(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1577 {
   1578    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1579       s390_disasm(ENC3(MNM, GPR, UDXB), "ah", r1, d2, x2, b2);
   1580 
   1581    return emit_RX(p, 0x4a000000, r1, x2, b2, d2);
   1582 }
   1583 
   1584 
   1585 static UChar *
   1586 s390_emit_AHY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1587 {
   1588    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1589       s390_disasm(ENC3(MNM, GPR, SDXB), "ahy", r1, dh2, dl2, x2, b2);
   1590 
   1591    return emit_RXY(p, 0xe3000000007aULL, r1, x2, b2, dl2, dh2);
   1592 }
   1593 
   1594 
   1595 static UChar *
   1596 s390_emit_AHI(UChar *p, UChar r1, UShort i2)
   1597 {
   1598    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1599       s390_disasm(ENC3(MNM, GPR, INT), "ahi", r1, (Int)(Short)i2);
   1600 
   1601    return emit_RI(p, 0xa70a0000, r1, i2);
   1602 }
   1603 
   1604 
   1605 static UChar *
   1606 s390_emit_AGHI(UChar *p, UChar r1, UShort i2)
   1607 {
   1608    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1609       s390_disasm(ENC3(MNM, GPR, INT), "aghi", r1, (Int)(Short)i2);
   1610 
   1611    return emit_RI(p, 0xa70b0000, r1, i2);
   1612 }
   1613 
   1614 
   1615 static UChar *
   1616 s390_emit_AGSI(UChar *p, UChar i2, UChar b1, UShort dl1, UChar dh1)
   1617 {
   1618    vassert(s390_host_has_gie);
   1619 
   1620    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1621       s390_disasm(ENC3(MNM, SDXB, INT), "agsi", dh1, dl1, 0, b1, (Int)(Char)i2);
   1622 
   1623    return emit_SIY(p, 0xeb000000007aULL, i2, b1, dl1, dh1);
   1624 }
   1625 
   1626 
   1627 static UChar *
   1628 s390_emit_ASI(UChar *p, UChar i2, UChar b1, UShort dl1, UChar dh1)
   1629 {
   1630    vassert(s390_host_has_gie);
   1631 
   1632    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1633       s390_disasm(ENC3(MNM, SDXB, INT), "asi", dh1, dl1, 0, b1, (Int)(Char)i2);
   1634 
   1635    return emit_SIY(p, 0xeb000000006aULL, i2, b1, dl1, dh1);
   1636 }
   1637 
   1638 
   1639 static UChar *
   1640 s390_emit_NR(UChar *p, UChar r1, UChar r2)
   1641 {
   1642    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1643       s390_disasm(ENC3(MNM, GPR, GPR), "nr", r1, r2);
   1644 
   1645    return emit_RR(p, 0x1400, r1, r2);
   1646 }
   1647 
   1648 
   1649 static UChar *
   1650 s390_emit_NGR(UChar *p, UChar r1, UChar r2)
   1651 {
   1652    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1653       s390_disasm(ENC3(MNM, GPR, GPR), "ngr", r1, r2);
   1654 
   1655    return emit_RRE(p, 0xb9800000, r1, r2);
   1656 }
   1657 
   1658 
   1659 static UChar *
   1660 s390_emit_N(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1661 {
   1662    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1663       s390_disasm(ENC3(MNM, GPR, UDXB), "n", r1, d2, x2, b2);
   1664 
   1665    return emit_RX(p, 0x54000000, r1, x2, b2, d2);
   1666 }
   1667 
   1668 
   1669 static UChar *
   1670 s390_emit_NY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1671 {
   1672    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1673       s390_disasm(ENC3(MNM, GPR, SDXB), "ny", r1, dh2, dl2, x2, b2);
   1674 
   1675    return emit_RXY(p, 0xe30000000054ULL, r1, x2, b2, dl2, dh2);
   1676 }
   1677 
   1678 
   1679 static UChar *
   1680 s390_emit_NG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1681 {
   1682    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1683       s390_disasm(ENC3(MNM, GPR, SDXB), "ng", r1, dh2, dl2, x2, b2);
   1684 
   1685    return emit_RXY(p, 0xe30000000080ULL, r1, x2, b2, dl2, dh2);
   1686 }
   1687 
   1688 
   1689 static UChar *
   1690 s390_emit_NIHF(UChar *p, UChar r1, UInt i2)
   1691 {
   1692    vassert(s390_host_has_eimm);
   1693 
   1694    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1695       s390_disasm(ENC3(MNM, GPR, UINT), "nihf", r1, i2);
   1696 
   1697    return emit_RIL(p, 0xc00a00000000ULL, r1, i2);
   1698 }
   1699 
   1700 
   1701 static UChar *
   1702 s390_emit_NILF(UChar *p, UChar r1, UInt i2)
   1703 {
   1704    vassert(s390_host_has_eimm);
   1705 
   1706    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1707       s390_disasm(ENC3(MNM, GPR, UINT), "nilf", r1, i2);
   1708 
   1709    return emit_RIL(p, 0xc00b00000000ULL, r1, i2);
   1710 }
   1711 
   1712 
   1713 static UChar *
   1714 s390_emit_NILL(UChar *p, UChar r1, UShort i2)
   1715 {
   1716    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1717       s390_disasm(ENC3(MNM, GPR, UINT), "nill", r1, i2);
   1718 
   1719    return emit_RI(p, 0xa5070000, r1, i2);
   1720 }
   1721 
   1722 
   1723 static UChar *
   1724 s390_emit_BASR(UChar *p, UChar r1, UChar r2)
   1725 {
   1726    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1727       s390_disasm(ENC3(MNM, GPR, GPR), "basr", r1, r2);
   1728 
   1729    return emit_RR(p, 0x0d00, r1, r2);
   1730 }
   1731 
   1732 
   1733 static UChar *
   1734 s390_emit_BCR(UChar *p, UChar r1, UChar r2)
   1735 {
   1736    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1737       s390_disasm(ENC2(XMNM, GPR), S390_XMNM_BCR, r1, r2);
   1738 
   1739    return emit_RR(p, 0x0700, r1, r2);
   1740 }
   1741 
   1742 
   1743 static UChar *
   1744 s390_emit_BRC(UChar *p, UChar r1, UShort i2)
   1745 {
   1746    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1747       s390_disasm(ENC2(XMNM, PCREL), S390_XMNM_BRC, r1, (Int)(Short)i2);
   1748 
   1749    return emit_RI(p, 0xa7040000, r1, i2);
   1750 }
   1751 
   1752 
   1753 static UChar *
   1754 s390_emit_BRCL(UChar *p, UChar r1, ULong i2)
   1755 {
   1756    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1757       s390_disasm(ENC2(XMNM, PCREL), S390_XMNM_BRCL, r1, i2);
   1758 
   1759    return emit_RIL(p, 0xc00400000000ULL, r1, i2);
   1760 }
   1761 
   1762 
   1763 static UChar *
   1764 s390_emit_CR(UChar *p, UChar r1, UChar r2)
   1765 {
   1766    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1767       s390_disasm(ENC3(MNM, GPR, GPR), "cr", r1, r2);
   1768 
   1769    return emit_RR(p, 0x1900, r1, r2);
   1770 }
   1771 
   1772 
   1773 static UChar *
   1774 s390_emit_CGR(UChar *p, UChar r1, UChar r2)
   1775 {
   1776    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1777       s390_disasm(ENC3(MNM, GPR, GPR), "cgr", r1, r2);
   1778 
   1779    return emit_RRE(p, 0xb9200000, r1, r2);
   1780 }
   1781 
   1782 
   1783 static UChar *
   1784 s390_emit_C(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1785 {
   1786    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1787       s390_disasm(ENC3(MNM, GPR, UDXB), "c", r1, d2, x2, b2);
   1788 
   1789    return emit_RX(p, 0x59000000, r1, x2, b2, d2);
   1790 }
   1791 
   1792 
   1793 static UChar *
   1794 s390_emit_CY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1795 {
   1796    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1797       s390_disasm(ENC3(MNM, GPR, SDXB), "cy", r1, dh2, dl2, x2, b2);
   1798 
   1799    return emit_RXY(p, 0xe30000000059ULL, r1, x2, b2, dl2, dh2);
   1800 }
   1801 
   1802 
   1803 static UChar *
   1804 s390_emit_CG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1805 {
   1806    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1807       s390_disasm(ENC3(MNM, GPR, SDXB), "cg", r1, dh2, dl2, x2, b2);
   1808 
   1809    return emit_RXY(p, 0xe30000000020ULL, r1, x2, b2, dl2, dh2);
   1810 }
   1811 
   1812 
   1813 static UChar *
   1814 s390_emit_CFI(UChar *p, UChar r1, UInt i2)
   1815 {
   1816    vassert(s390_host_has_eimm);
   1817 
   1818    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1819       s390_disasm(ENC3(MNM, GPR, INT), "cfi", r1, i2);
   1820 
   1821    return emit_RIL(p, 0xc20d00000000ULL, r1, i2);
   1822 }
   1823 
   1824 
   1825 static UChar *
   1826 s390_emit_CGFI(UChar *p, UChar r1, UInt i2)
   1827 {
   1828    vassert(s390_host_has_eimm);
   1829 
   1830    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1831       s390_disasm(ENC3(MNM, GPR, INT), "cgfi", r1, i2);
   1832 
   1833    return emit_RIL(p, 0xc20c00000000ULL, r1, i2);
   1834 }
   1835 
   1836 
   1837 static UChar *
   1838 s390_emit_CS(UChar *p, UChar r1, UChar r3, UChar b2, UShort d2)
   1839 {
   1840    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1841       s390_disasm(ENC4(MNM, GPR, GPR, UDXB), "cs", r1, r3, d2, 0, b2);
   1842 
   1843    return emit_RS(p, 0xba000000, r1, r3, b2, d2);
   1844 }
   1845 
   1846 
   1847 static UChar *
   1848 s390_emit_CSY(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   1849 {
   1850    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1851       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "csy", r1, r3, dh2, dl2, 0, b2);
   1852 
   1853    return emit_RSY(p, 0xeb0000000014ULL, r1, r3, b2, dl2, dh2);
   1854 }
   1855 
   1856 
   1857 static UChar *
   1858 s390_emit_CSG(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   1859 {
   1860    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1861       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "csg", r1, r3, dh2, dl2, 0, b2);
   1862 
   1863    return emit_RSY(p, 0xeb0000000030ULL, r1, r3, b2, dl2, dh2);
   1864 }
   1865 
   1866 
   1867 static UChar *
   1868 s390_emit_CDS(UChar *p, UChar r1, UChar r3, UChar b2, UShort d2)
   1869 {
   1870    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1871       s390_disasm(ENC4(MNM, GPR, GPR, UDXB), "cds", r1, r3, d2, 0, b2);
   1872 
   1873    return emit_RS(p, 0xbb000000, r1, r3, b2, d2);
   1874 }
   1875 
   1876 
   1877 static UChar *
   1878 s390_emit_CDSY(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   1879 {
   1880    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1881       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "cdsy", r1, r3, dh2, dl2, 0, b2);
   1882 
   1883    return emit_RSY(p, 0xeb0000000031ULL, r1, r3, b2, dl2, dh2);
   1884 }
   1885 
   1886 
   1887 static UChar *
   1888 s390_emit_CDSG(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   1889 {
   1890    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1891       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "cdsg", r1, r3, dh2, dl2, 0, b2);
   1892 
   1893    return emit_RSY(p, 0xeb000000003eULL, r1, r3, b2, dl2, dh2);
   1894 }
   1895 
   1896 
   1897 static UChar *
   1898 s390_emit_CLR(UChar *p, UChar r1, UChar r2)
   1899 {
   1900    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1901       s390_disasm(ENC3(MNM, GPR, GPR), "clr", r1, r2);
   1902 
   1903    return emit_RR(p, 0x1500, r1, r2);
   1904 }
   1905 
   1906 
   1907 static UChar *
   1908 s390_emit_CLGR(UChar *p, UChar r1, UChar r2)
   1909 {
   1910    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1911       s390_disasm(ENC3(MNM, GPR, GPR), "clgr", r1, r2);
   1912 
   1913    return emit_RRE(p, 0xb9210000, r1, r2);
   1914 }
   1915 
   1916 
   1917 static UChar *
   1918 s390_emit_CL(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1919 {
   1920    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1921       s390_disasm(ENC3(MNM, GPR, UDXB), "cl", r1, d2, x2, b2);
   1922 
   1923    return emit_RX(p, 0x55000000, r1, x2, b2, d2);
   1924 }
   1925 
   1926 
   1927 static UChar *
   1928 s390_emit_CLY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1929 {
   1930    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1931       s390_disasm(ENC3(MNM, GPR, SDXB), "cly", r1, dh2, dl2, x2, b2);
   1932 
   1933    return emit_RXY(p, 0xe30000000055ULL, r1, x2, b2, dl2, dh2);
   1934 }
   1935 
   1936 
   1937 static UChar *
   1938 s390_emit_CLG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   1939 {
   1940    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1941       s390_disasm(ENC3(MNM, GPR, SDXB), "clg", r1, dh2, dl2, x2, b2);
   1942 
   1943    return emit_RXY(p, 0xe30000000021ULL, r1, x2, b2, dl2, dh2);
   1944 }
   1945 
   1946 
   1947 static UChar *
   1948 s390_emit_CLFI(UChar *p, UChar r1, UInt i2)
   1949 {
   1950    vassert(s390_host_has_eimm);
   1951 
   1952    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1953       s390_disasm(ENC3(MNM, GPR, UINT), "clfi", r1, i2);
   1954 
   1955    return emit_RIL(p, 0xc20f00000000ULL, r1, i2);
   1956 }
   1957 
   1958 
   1959 static UChar *
   1960 s390_emit_CLGFI(UChar *p, UChar r1, UInt i2)
   1961 {
   1962    vassert(s390_host_has_eimm);
   1963 
   1964    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1965       s390_disasm(ENC3(MNM, GPR, UINT), "clgfi", r1, i2);
   1966 
   1967    return emit_RIL(p, 0xc20e00000000ULL, r1, i2);
   1968 }
   1969 
   1970 
   1971 static UChar *
   1972 s390_emit_DR(UChar *p, UChar r1, UChar r2)
   1973 {
   1974    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1975       s390_disasm(ENC3(MNM, GPR, GPR), "dr", r1, r2);
   1976 
   1977    return emit_RR(p, 0x1d00, r1, r2);
   1978 }
   1979 
   1980 
   1981 static UChar *
   1982 s390_emit_D(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   1983 {
   1984    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1985       s390_disasm(ENC3(MNM, GPR, UDXB), "d", r1, d2, x2, b2);
   1986 
   1987    return emit_RX(p, 0x5d000000, r1, x2, b2, d2);
   1988 }
   1989 
   1990 
   1991 static UChar *
   1992 s390_emit_DLR(UChar *p, UChar r1, UChar r2)
   1993 {
   1994    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   1995       s390_disasm(ENC3(MNM, GPR, GPR), "dlr", r1, r2);
   1996 
   1997    return emit_RRE(p, 0xb9970000, r1, r2);
   1998 }
   1999 
   2000 
   2001 static UChar *
   2002 s390_emit_DLGR(UChar *p, UChar r1, UChar r2)
   2003 {
   2004    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2005       s390_disasm(ENC3(MNM, GPR, GPR), "dlgr", r1, r2);
   2006 
   2007    return emit_RRE(p, 0xb9870000, r1, r2);
   2008 }
   2009 
   2010 
   2011 static UChar *
   2012 s390_emit_DL(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2013 {
   2014    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2015       s390_disasm(ENC3(MNM, GPR, SDXB), "dl", r1, dh2, dl2, x2, b2);
   2016 
   2017    return emit_RXY(p, 0xe30000000097ULL, r1, x2, b2, dl2, dh2);
   2018 }
   2019 
   2020 
   2021 static UChar *
   2022 s390_emit_DLG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2023 {
   2024    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2025       s390_disasm(ENC3(MNM, GPR, SDXB), "dlg", r1, dh2, dl2, x2, b2);
   2026 
   2027    return emit_RXY(p, 0xe30000000087ULL, r1, x2, b2, dl2, dh2);
   2028 }
   2029 
   2030 
   2031 static UChar *
   2032 s390_emit_DSGR(UChar *p, UChar r1, UChar r2)
   2033 {
   2034    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2035       s390_disasm(ENC3(MNM, GPR, GPR), "dsgr", r1, r2);
   2036 
   2037    return emit_RRE(p, 0xb90d0000, r1, r2);
   2038 }
   2039 
   2040 
   2041 static UChar *
   2042 s390_emit_DSG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2043 {
   2044    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2045       s390_disasm(ENC3(MNM, GPR, SDXB), "dsg", r1, dh2, dl2, x2, b2);
   2046 
   2047    return emit_RXY(p, 0xe3000000000dULL, r1, x2, b2, dl2, dh2);
   2048 }
   2049 
   2050 
   2051 static UChar *
   2052 s390_emit_XR(UChar *p, UChar r1, UChar r2)
   2053 {
   2054    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2055       s390_disasm(ENC3(MNM, GPR, GPR), "xr", r1, r2);
   2056 
   2057    return emit_RR(p, 0x1700, r1, r2);
   2058 }
   2059 
   2060 
   2061 static UChar *
   2062 s390_emit_XGR(UChar *p, UChar r1, UChar r2)
   2063 {
   2064    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2065       s390_disasm(ENC3(MNM, GPR, GPR), "xgr", r1, r2);
   2066 
   2067    return emit_RRE(p, 0xb9820000, r1, r2);
   2068 }
   2069 
   2070 
   2071 static UChar *
   2072 s390_emit_X(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2073 {
   2074    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2075       s390_disasm(ENC3(MNM, GPR, UDXB), "x", r1, d2, x2, b2);
   2076 
   2077    return emit_RX(p, 0x57000000, r1, x2, b2, d2);
   2078 }
   2079 
   2080 
   2081 static UChar *
   2082 s390_emit_XY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2083 {
   2084    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2085       s390_disasm(ENC3(MNM, GPR, SDXB), "xy", r1, dh2, dl2, x2, b2);
   2086 
   2087    return emit_RXY(p, 0xe30000000057ULL, r1, x2, b2, dl2, dh2);
   2088 }
   2089 
   2090 
   2091 static UChar *
   2092 s390_emit_XG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2093 {
   2094    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2095       s390_disasm(ENC3(MNM, GPR, SDXB), "xg", r1, dh2, dl2, x2, b2);
   2096 
   2097    return emit_RXY(p, 0xe30000000082ULL, r1, x2, b2, dl2, dh2);
   2098 }
   2099 
   2100 
   2101 static UChar *
   2102 s390_emit_XIHF(UChar *p, UChar r1, UInt i2)
   2103 {
   2104    vassert(s390_host_has_eimm);
   2105 
   2106    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2107       s390_disasm(ENC3(MNM, GPR, UINT), "xihf", r1, i2);
   2108 
   2109    return emit_RIL(p, 0xc00600000000ULL, r1, i2);
   2110 }
   2111 
   2112 
   2113 static UChar *
   2114 s390_emit_XILF(UChar *p, UChar r1, UInt i2)
   2115 {
   2116    vassert(s390_host_has_eimm);
   2117 
   2118    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2119       s390_disasm(ENC3(MNM, GPR, UINT), "xilf", r1, i2);
   2120 
   2121    return emit_RIL(p, 0xc00700000000ULL, r1, i2);
   2122 }
   2123 
   2124 
   2125 static UChar *
   2126 s390_emit_XC(UChar *p, UInt l, UChar b1, UShort d1, UChar b2, UShort d2)
   2127 {
   2128    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2129       s390_disasm(ENC3(MNM, UDLB, UDXB), "xc", d1, l, b1, d2, 0, b2);
   2130 
   2131    return emit_SSa(p, 0xd70000000000ULL, l, b1, d1, b2, d2);
   2132 }
   2133 
   2134 
   2135 static UChar *
   2136 s390_emit_FLOGR(UChar *p, UChar r1, UChar r2)
   2137 {
   2138    vassert(s390_host_has_eimm);
   2139 
   2140    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2141       s390_disasm(ENC3(MNM, GPR, GPR), "flogr", r1, r2);
   2142 
   2143    return emit_RRE(p, 0xb9830000, r1, r2);
   2144 }
   2145 
   2146 
   2147 static UChar *
   2148 s390_emit_IC(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2149 {
   2150    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2151       s390_disasm(ENC3(MNM, GPR, UDXB), "ic", r1, d2, x2, b2);
   2152 
   2153    return emit_RX(p, 0x43000000, r1, x2, b2, d2);
   2154 }
   2155 
   2156 
   2157 static UChar *
   2158 s390_emit_ICY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2159 {
   2160    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2161       s390_disasm(ENC3(MNM, GPR, SDXB), "icy", r1, dh2, dl2, x2, b2);
   2162 
   2163    return emit_RXY(p, 0xe30000000073ULL, r1, x2, b2, dl2, dh2);
   2164 }
   2165 
   2166 
   2167 static UChar *
   2168 s390_emit_IIHF(UChar *p, UChar r1, UInt i2)
   2169 {
   2170    vassert(s390_host_has_eimm);
   2171 
   2172    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2173       s390_disasm(ENC3(MNM, GPR, UINT), "iihf", r1, i2);
   2174 
   2175    return emit_RIL(p, 0xc00800000000ULL, r1, i2);
   2176 }
   2177 
   2178 
   2179 static UChar *
   2180 s390_emit_IIHH(UChar *p, UChar r1, UShort i2)
   2181 {
   2182    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2183       s390_disasm(ENC3(MNM, GPR, UINT), "iihh", r1, i2);
   2184 
   2185    return emit_RI(p, 0xa5000000, r1, i2);
   2186 }
   2187 
   2188 
   2189 static UChar *
   2190 s390_emit_IIHL(UChar *p, UChar r1, UShort i2)
   2191 {
   2192    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2193       s390_disasm(ENC3(MNM, GPR, UINT), "iihl", r1, i2);
   2194 
   2195    return emit_RI(p, 0xa5010000, r1, i2);
   2196 }
   2197 
   2198 
   2199 static UChar *
   2200 s390_emit_IILF(UChar *p, UChar r1, UInt i2)
   2201 {
   2202    vassert(s390_host_has_eimm);
   2203 
   2204    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2205       s390_disasm(ENC3(MNM, GPR, UINT), "iilf", r1, i2);
   2206 
   2207    return emit_RIL(p, 0xc00900000000ULL, r1, i2);
   2208 }
   2209 
   2210 
   2211 static UChar *
   2212 s390_emit_IILH(UChar *p, UChar r1, UShort i2)
   2213 {
   2214    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2215       s390_disasm(ENC3(MNM, GPR, UINT), "iilh", r1, i2);
   2216 
   2217    return emit_RI(p, 0xa5020000, r1, i2);
   2218 }
   2219 
   2220 
   2221 static UChar *
   2222 s390_emit_IILL(UChar *p, UChar r1, UShort i2)
   2223 {
   2224    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2225       s390_disasm(ENC3(MNM, GPR, UINT), "iill", r1, i2);
   2226 
   2227    return emit_RI(p, 0xa5030000, r1, i2);
   2228 }
   2229 
   2230 
   2231 static UChar *
   2232 s390_emit_IPM(UChar *p, UChar r1, UChar r2)
   2233 {
   2234    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2235       s390_disasm(ENC2(MNM, GPR), "ipm", r1);
   2236 
   2237    return emit_RRE(p, 0xb2220000, r1, r2);
   2238 }
   2239 
   2240 
   2241 static UChar *
   2242 s390_emit_LR(UChar *p, UChar r1, UChar r2)
   2243 {
   2244    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2245       s390_disasm(ENC3(MNM, GPR, GPR), "lr", r1, r2);
   2246 
   2247    return emit_RR(p, 0x1800, r1, r2);
   2248 }
   2249 
   2250 
   2251 static UChar *
   2252 s390_emit_LGR(UChar *p, UChar r1, UChar r2)
   2253 {
   2254    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2255       s390_disasm(ENC3(MNM, GPR, GPR), "lgr", r1, r2);
   2256 
   2257    return emit_RRE(p, 0xb9040000, r1, r2);
   2258 }
   2259 
   2260 
   2261 static UChar *
   2262 s390_emit_LGFR(UChar *p, UChar r1, UChar r2)
   2263 {
   2264    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2265       s390_disasm(ENC3(MNM, GPR, GPR), "lgfr", r1, r2);
   2266 
   2267    return emit_RRE(p, 0xb9140000, r1, r2);
   2268 }
   2269 
   2270 
   2271 static UChar *
   2272 s390_emit_L(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2273 {
   2274    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2275       s390_disasm(ENC3(MNM, GPR, UDXB), "l", r1, d2, x2, b2);
   2276 
   2277    return emit_RX(p, 0x58000000, r1, x2, b2, d2);
   2278 }
   2279 
   2280 
   2281 static UChar *
   2282 s390_emit_LY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2283 {
   2284    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2285       s390_disasm(ENC3(MNM, GPR, SDXB), "ly", r1, dh2, dl2, x2, b2);
   2286 
   2287    return emit_RXY(p, 0xe30000000058ULL, r1, x2, b2, dl2, dh2);
   2288 }
   2289 
   2290 
   2291 static UChar *
   2292 s390_emit_LG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2293 {
   2294    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2295       s390_disasm(ENC3(MNM, GPR, SDXB), "lg", r1, dh2, dl2, x2, b2);
   2296 
   2297    return emit_RXY(p, 0xe30000000004ULL, r1, x2, b2, dl2, dh2);
   2298 }
   2299 
   2300 
   2301 static UChar *
   2302 s390_emit_LGF(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2303 {
   2304    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2305       s390_disasm(ENC3(MNM, GPR, SDXB), "lgf", r1, dh2, dl2, x2, b2);
   2306 
   2307    return emit_RXY(p, 0xe30000000014ULL, r1, x2, b2, dl2, dh2);
   2308 }
   2309 
   2310 
   2311 static UChar *
   2312 s390_emit_LGFI(UChar *p, UChar r1, UInt i2)
   2313 {
   2314    vassert(s390_host_has_eimm);
   2315 
   2316    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2317       s390_disasm(ENC3(MNM, GPR, INT), "lgfi", r1, i2);
   2318 
   2319    return emit_RIL(p, 0xc00100000000ULL, r1, i2);
   2320 }
   2321 
   2322 
   2323 static UChar *
   2324 s390_emit_LTR(UChar *p, UChar r1, UChar r2)
   2325 {
   2326    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2327       s390_disasm(ENC3(MNM, GPR, GPR), "ltr", r1, r2);
   2328 
   2329    return emit_RR(p, 0x1200, r1, r2);
   2330 }
   2331 
   2332 
   2333 static UChar *
   2334 s390_emit_LTGR(UChar *p, UChar r1, UChar r2)
   2335 {
   2336    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2337       s390_disasm(ENC3(MNM, GPR, GPR), "ltgr", r1, r2);
   2338 
   2339    return emit_RRE(p, 0xb9020000, r1, r2);
   2340 }
   2341 
   2342 
   2343 static UChar *
   2344 s390_emit_LT(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2345 {
   2346    vassert(s390_host_has_eimm);
   2347 
   2348    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2349       s390_disasm(ENC3(MNM, GPR, SDXB), "lt", r1, dh2, dl2, x2, b2);
   2350 
   2351    return emit_RXY(p, 0xe30000000012ULL, r1, x2, b2, dl2, dh2);
   2352 }
   2353 
   2354 
   2355 static UChar *
   2356 s390_emit_LTG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2357 {
   2358    vassert(s390_host_has_eimm);
   2359 
   2360    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2361       s390_disasm(ENC3(MNM, GPR, SDXB), "ltg", r1, dh2, dl2, x2, b2);
   2362 
   2363    return emit_RXY(p, 0xe30000000002ULL, r1, x2, b2, dl2, dh2);
   2364 }
   2365 
   2366 
   2367 static UChar *
   2368 s390_emit_LBR(UChar *p, UChar r1, UChar r2)
   2369 {
   2370    vassert(s390_host_has_eimm);
   2371 
   2372    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2373       s390_disasm(ENC3(MNM, GPR, GPR), "lbr", r1, r2);
   2374 
   2375    return emit_RRE(p, 0xb9260000, r1, r2);
   2376 }
   2377 
   2378 
   2379 static UChar *
   2380 s390_emit_LGBR(UChar *p, UChar r1, UChar r2)
   2381 {
   2382    vassert(s390_host_has_eimm);
   2383 
   2384    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2385       s390_disasm(ENC3(MNM, GPR, GPR), "lgbr", r1, r2);
   2386 
   2387    return emit_RRE(p, 0xb9060000, r1, r2);
   2388 }
   2389 
   2390 
   2391 static UChar *
   2392 s390_emit_LB(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2393 {
   2394    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2395       s390_disasm(ENC3(MNM, GPR, SDXB), "lb", r1, dh2, dl2, x2, b2);
   2396 
   2397    return emit_RXY(p, 0xe30000000076ULL, r1, x2, b2, dl2, dh2);
   2398 }
   2399 
   2400 
   2401 static UChar *
   2402 s390_emit_LGB(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2403 {
   2404    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2405       s390_disasm(ENC3(MNM, GPR, SDXB), "lgb", r1, dh2, dl2, x2, b2);
   2406 
   2407    return emit_RXY(p, 0xe30000000077ULL, r1, x2, b2, dl2, dh2);
   2408 }
   2409 
   2410 
   2411 static UChar *
   2412 s390_emit_LCR(UChar *p, UChar r1, UChar r2)
   2413 {
   2414    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2415       s390_disasm(ENC3(MNM, GPR, GPR), "lcr", r1, r2);
   2416 
   2417    return emit_RR(p, 0x1300, r1, r2);
   2418 }
   2419 
   2420 
   2421 static UChar *
   2422 s390_emit_LCGR(UChar *p, UChar r1, UChar r2)
   2423 {
   2424    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2425       s390_disasm(ENC3(MNM, GPR, GPR), "lcgr", r1, r2);
   2426 
   2427    return emit_RRE(p, 0xb9030000, r1, r2);
   2428 }
   2429 
   2430 
   2431 static UChar *
   2432 s390_emit_LHR(UChar *p, UChar r1, UChar r2)
   2433 {
   2434    vassert(s390_host_has_eimm);
   2435 
   2436    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2437       s390_disasm(ENC3(MNM, GPR, GPR), "lhr", r1, r2);
   2438 
   2439    return emit_RRE(p, 0xb9270000, r1, r2);
   2440 }
   2441 
   2442 
   2443 static UChar *
   2444 s390_emit_LGHR(UChar *p, UChar r1, UChar r2)
   2445 {
   2446    vassert(s390_host_has_eimm);
   2447 
   2448    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2449       s390_disasm(ENC3(MNM, GPR, GPR), "lghr", r1, r2);
   2450 
   2451    return emit_RRE(p, 0xb9070000, r1, r2);
   2452 }
   2453 
   2454 
   2455 static UChar *
   2456 s390_emit_LH(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2457 {
   2458    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2459       s390_disasm(ENC3(MNM, GPR, UDXB), "lh", r1, d2, x2, b2);
   2460 
   2461    return emit_RX(p, 0x48000000, r1, x2, b2, d2);
   2462 }
   2463 
   2464 
   2465 static UChar *
   2466 s390_emit_LHY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2467 {
   2468    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2469       s390_disasm(ENC3(MNM, GPR, SDXB), "lhy", r1, dh2, dl2, x2, b2);
   2470 
   2471    return emit_RXY(p, 0xe30000000078ULL, r1, x2, b2, dl2, dh2);
   2472 }
   2473 
   2474 
   2475 static UChar *
   2476 s390_emit_LGH(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2477 {
   2478    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2479       s390_disasm(ENC3(MNM, GPR, SDXB), "lgh", r1, dh2, dl2, x2, b2);
   2480 
   2481    return emit_RXY(p, 0xe30000000015ULL, r1, x2, b2, dl2, dh2);
   2482 }
   2483 
   2484 
   2485 static UChar *
   2486 s390_emit_LHI(UChar *p, UChar r1, UShort i2)
   2487 {
   2488    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2489       s390_disasm(ENC3(MNM, GPR, INT), "lhi", r1, (Int)(Short)i2);
   2490 
   2491    return emit_RI(p, 0xa7080000, r1, i2);
   2492 }
   2493 
   2494 
   2495 static UChar *
   2496 s390_emit_LGHI(UChar *p, UChar r1, UShort i2)
   2497 {
   2498    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2499       s390_disasm(ENC3(MNM, GPR, INT), "lghi", r1, (Int)(Short)i2);
   2500 
   2501    return emit_RI(p, 0xa7090000, r1, i2);
   2502 }
   2503 
   2504 
   2505 static UChar *
   2506 s390_emit_LLGFR(UChar *p, UChar r1, UChar r2)
   2507 {
   2508    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2509       s390_disasm(ENC3(MNM, GPR, GPR), "llgfr", r1, r2);
   2510 
   2511    return emit_RRE(p, 0xb9160000, r1, r2);
   2512 }
   2513 
   2514 
   2515 static UChar *
   2516 s390_emit_LLGF(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2517 {
   2518    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2519       s390_disasm(ENC3(MNM, GPR, SDXB), "llgf", r1, dh2, dl2, x2, b2);
   2520 
   2521    return emit_RXY(p, 0xe30000000016ULL, r1, x2, b2, dl2, dh2);
   2522 }
   2523 
   2524 
   2525 static UChar *
   2526 s390_emit_LLCR(UChar *p, UChar r1, UChar r2)
   2527 {
   2528    vassert(s390_host_has_eimm);
   2529 
   2530    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2531       s390_disasm(ENC3(MNM, GPR, GPR), "llcr", r1, r2);
   2532 
   2533    return emit_RRE(p, 0xb9940000, r1, r2);
   2534 }
   2535 
   2536 
   2537 static UChar *
   2538 s390_emit_LLGCR(UChar *p, UChar r1, UChar r2)
   2539 {
   2540    vassert(s390_host_has_eimm);
   2541 
   2542    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2543       s390_disasm(ENC3(MNM, GPR, GPR), "llgcr", r1, r2);
   2544 
   2545    return emit_RRE(p, 0xb9840000, r1, r2);
   2546 }
   2547 
   2548 
   2549 static UChar *
   2550 s390_emit_LLC(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2551 {
   2552    vassert(s390_host_has_eimm);
   2553 
   2554    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2555       s390_disasm(ENC3(MNM, GPR, SDXB), "llc", r1, dh2, dl2, x2, b2);
   2556 
   2557    return emit_RXY(p, 0xe30000000094ULL, r1, x2, b2, dl2, dh2);
   2558 }
   2559 
   2560 
   2561 static UChar *
   2562 s390_emit_LLGC(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2563 {
   2564    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2565       s390_disasm(ENC3(MNM, GPR, SDXB), "llgc", r1, dh2, dl2, x2, b2);
   2566 
   2567    return emit_RXY(p, 0xe30000000090ULL, r1, x2, b2, dl2, dh2);
   2568 }
   2569 
   2570 
   2571 static UChar *
   2572 s390_emit_LLHR(UChar *p, UChar r1, UChar r2)
   2573 {
   2574    vassert(s390_host_has_eimm);
   2575 
   2576    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2577       s390_disasm(ENC3(MNM, GPR, GPR), "llhr", r1, r2);
   2578 
   2579    return emit_RRE(p, 0xb9950000, r1, r2);
   2580 }
   2581 
   2582 
   2583 static UChar *
   2584 s390_emit_LLGHR(UChar *p, UChar r1, UChar r2)
   2585 {
   2586    vassert(s390_host_has_eimm);
   2587 
   2588    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2589       s390_disasm(ENC3(MNM, GPR, GPR), "llghr", r1, r2);
   2590 
   2591    return emit_RRE(p, 0xb9850000, r1, r2);
   2592 }
   2593 
   2594 
   2595 static UChar *
   2596 s390_emit_LLH(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2597 {
   2598    vassert(s390_host_has_eimm);
   2599 
   2600    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2601       s390_disasm(ENC3(MNM, GPR, SDXB), "llh", r1, dh2, dl2, x2, b2);
   2602 
   2603    return emit_RXY(p, 0xe30000000095ULL, r1, x2, b2, dl2, dh2);
   2604 }
   2605 
   2606 
   2607 static UChar *
   2608 s390_emit_LLGH(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2609 {
   2610    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2611       s390_disasm(ENC3(MNM, GPR, SDXB), "llgh", r1, dh2, dl2, x2, b2);
   2612 
   2613    return emit_RXY(p, 0xe30000000091ULL, r1, x2, b2, dl2, dh2);
   2614 }
   2615 
   2616 
   2617 static UChar *
   2618 s390_emit_LLILF(UChar *p, UChar r1, UInt i2)
   2619 {
   2620    vassert(s390_host_has_eimm);
   2621 
   2622    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2623       s390_disasm(ENC3(MNM, GPR, UINT), "llilf", r1, i2);
   2624 
   2625    return emit_RIL(p, 0xc00f00000000ULL, r1, i2);
   2626 }
   2627 
   2628 
   2629 static UChar *
   2630 s390_emit_LLILH(UChar *p, UChar r1, UShort i2)
   2631 {
   2632    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2633       s390_disasm(ENC3(MNM, GPR, UINT), "llilh", r1, i2);
   2634 
   2635    return emit_RI(p, 0xa50e0000, r1, i2);
   2636 }
   2637 
   2638 
   2639 static UChar *
   2640 s390_emit_LLILL(UChar *p, UChar r1, UShort i2)
   2641 {
   2642    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2643       s390_disasm(ENC3(MNM, GPR, UINT), "llill", r1, i2);
   2644 
   2645    return emit_RI(p, 0xa50f0000, r1, i2);
   2646 }
   2647 
   2648 
   2649 static UChar *
   2650 s390_emit_MR(UChar *p, UChar r1, UChar r2)
   2651 {
   2652    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2653       s390_disasm(ENC3(MNM, GPR, GPR), "mr", r1, r2);
   2654 
   2655    return emit_RR(p, 0x1c00, r1, r2);
   2656 }
   2657 
   2658 
   2659 static UChar *
   2660 s390_emit_M(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2661 {
   2662    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2663       s390_disasm(ENC3(MNM, GPR, UDXB), "m", r1, d2, x2, b2);
   2664 
   2665    return emit_RX(p, 0x5c000000, r1, x2, b2, d2);
   2666 }
   2667 
   2668 
   2669 static UChar *
   2670 s390_emit_MFY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2671 {
   2672    vassert(s390_host_has_gie);
   2673 
   2674    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2675       s390_disasm(ENC3(MNM, GPR, SDXB), "mfy", r1, dh2, dl2, x2, b2);
   2676 
   2677    return emit_RXY(p, 0xe3000000005cULL, r1, x2, b2, dl2, dh2);
   2678 }
   2679 
   2680 
   2681 static UChar *
   2682 s390_emit_MH(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2683 {
   2684    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2685       s390_disasm(ENC3(MNM, GPR, UDXB), "mh", r1, d2, x2, b2);
   2686 
   2687    return emit_RX(p, 0x4c000000, r1, x2, b2, d2);
   2688 }
   2689 
   2690 
   2691 static UChar *
   2692 s390_emit_MHY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2693 {
   2694    vassert(s390_host_has_gie);
   2695 
   2696    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2697       s390_disasm(ENC3(MNM, GPR, SDXB), "mhy", r1, dh2, dl2, x2, b2);
   2698 
   2699    return emit_RXY(p, 0xe3000000007cULL, r1, x2, b2, dl2, dh2);
   2700 }
   2701 
   2702 
   2703 static UChar *
   2704 s390_emit_MHI(UChar *p, UChar r1, UShort i2)
   2705 {
   2706    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2707       s390_disasm(ENC3(MNM, GPR, INT), "mhi", r1, (Int)(Short)i2);
   2708 
   2709    return emit_RI(p, 0xa70c0000, r1, i2);
   2710 }
   2711 
   2712 
   2713 static UChar *
   2714 s390_emit_MLR(UChar *p, UChar r1, UChar r2)
   2715 {
   2716    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2717       s390_disasm(ENC3(MNM, GPR, GPR), "mlr", r1, r2);
   2718 
   2719    return emit_RRE(p, 0xb9960000, r1, r2);
   2720 }
   2721 
   2722 
   2723 static UChar *
   2724 s390_emit_MLGR(UChar *p, UChar r1, UChar r2)
   2725 {
   2726    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2727       s390_disasm(ENC3(MNM, GPR, GPR), "mlgr", r1, r2);
   2728 
   2729    return emit_RRE(p, 0xb9860000, r1, r2);
   2730 }
   2731 
   2732 
   2733 static UChar *
   2734 s390_emit_ML(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2735 {
   2736    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2737       s390_disasm(ENC3(MNM, GPR, SDXB), "ml", r1, dh2, dl2, x2, b2);
   2738 
   2739    return emit_RXY(p, 0xe30000000096ULL, r1, x2, b2, dl2, dh2);
   2740 }
   2741 
   2742 
   2743 static UChar *
   2744 s390_emit_MLG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2745 {
   2746    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2747       s390_disasm(ENC3(MNM, GPR, SDXB), "mlg", r1, dh2, dl2, x2, b2);
   2748 
   2749    return emit_RXY(p, 0xe30000000086ULL, r1, x2, b2, dl2, dh2);
   2750 }
   2751 
   2752 
   2753 static UChar *
   2754 s390_emit_MSR(UChar *p, UChar r1, UChar r2)
   2755 {
   2756    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2757       s390_disasm(ENC3(MNM, GPR, GPR), "msr", r1, r2);
   2758 
   2759    return emit_RRE(p, 0xb2520000, r1, r2);
   2760 }
   2761 
   2762 
   2763 static UChar *
   2764 s390_emit_MSGR(UChar *p, UChar r1, UChar r2)
   2765 {
   2766    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2767       s390_disasm(ENC3(MNM, GPR, GPR), "msgr", r1, r2);
   2768 
   2769    return emit_RRE(p, 0xb90c0000, r1, r2);
   2770 }
   2771 
   2772 
   2773 static UChar *
   2774 s390_emit_MS(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2775 {
   2776    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2777       s390_disasm(ENC3(MNM, GPR, UDXB), "ms", r1, d2, x2, b2);
   2778 
   2779    return emit_RX(p, 0x71000000, r1, x2, b2, d2);
   2780 }
   2781 
   2782 
   2783 static UChar *
   2784 s390_emit_MSY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2785 {
   2786    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2787       s390_disasm(ENC3(MNM, GPR, SDXB), "msy", r1, dh2, dl2, x2, b2);
   2788 
   2789    return emit_RXY(p, 0xe30000000051ULL, r1, x2, b2, dl2, dh2);
   2790 }
   2791 
   2792 
   2793 static UChar *
   2794 s390_emit_MSG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2795 {
   2796    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2797       s390_disasm(ENC3(MNM, GPR, SDXB), "msg", r1, dh2, dl2, x2, b2);
   2798 
   2799    return emit_RXY(p, 0xe3000000000cULL, r1, x2, b2, dl2, dh2);
   2800 }
   2801 
   2802 
   2803 static UChar *
   2804 s390_emit_MSFI(UChar *p, UChar r1, UInt i2)
   2805 {
   2806    vassert(s390_host_has_gie);
   2807 
   2808    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2809       s390_disasm(ENC3(MNM, GPR, INT), "msfi", r1, i2);
   2810 
   2811    return emit_RIL(p, 0xc20100000000ULL, r1, i2);
   2812 }
   2813 
   2814 
   2815 static UChar *
   2816 s390_emit_MSGFI(UChar *p, UChar r1, UInt i2)
   2817 {
   2818    vassert(s390_host_has_gie);
   2819 
   2820    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2821       s390_disasm(ENC3(MNM, GPR, INT), "msgfi", r1, i2);
   2822 
   2823    return emit_RIL(p, 0xc20000000000ULL, r1, i2);
   2824 }
   2825 
   2826 
   2827 static UChar *
   2828 s390_emit_MVC(UChar *p, UInt l, UChar b1, UShort d1, UChar b2, UShort d2)
   2829 {
   2830    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2831       s390_disasm(ENC3(MNM, UDLB, UDXB), "mvc", d1, l, b1, d2, 0, b2);
   2832 
   2833    return emit_SSa(p, 0xd20000000000ULL, l, b1, d1, b2, d2);
   2834 }
   2835 
   2836 
   2837 static UChar *
   2838 s390_emit_MVI(UChar *p, UChar i2, UChar b1, UShort d1)
   2839 {
   2840    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2841       s390_disasm(ENC3(MNM, UDXB, INT), "mvi", d1, 0, b1, i2);
   2842 
   2843    return emit_SI(p, 0x92000000, i2, b1, d1);
   2844 }
   2845 
   2846 
   2847 static UChar *
   2848 s390_emit_MVHHI(UChar *p, UChar b1, UShort d1, UShort i2)
   2849 {
   2850    vassert(s390_host_has_gie);
   2851 
   2852    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2853       s390_disasm(ENC3(MNM, UDXB, INT), "mvhhi", d1, 0, b1, i2);
   2854 
   2855    return emit_SIL(p, 0xe54400000000ULL, b1, d1, i2);
   2856 }
   2857 
   2858 
   2859 static UChar *
   2860 s390_emit_MVHI(UChar *p, UChar b1, UShort d1, UShort i2)
   2861 {
   2862    vassert(s390_host_has_gie);
   2863 
   2864    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2865       s390_disasm(ENC3(MNM, UDXB, INT), "mvhi", d1, 0, b1, i2);
   2866 
   2867    return emit_SIL(p, 0xe54c00000000ULL, b1, d1, i2);
   2868 }
   2869 
   2870 
   2871 static UChar *
   2872 s390_emit_MVGHI(UChar *p, UChar b1, UShort d1, UShort i2)
   2873 {
   2874    vassert(s390_host_has_gie);
   2875 
   2876    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2877       s390_disasm(ENC3(MNM, UDXB, INT), "mvghi", d1, 0, b1, i2);
   2878 
   2879    return emit_SIL(p, 0xe54800000000ULL, b1, d1, i2);
   2880 }
   2881 
   2882 
   2883 static UChar *
   2884 s390_emit_OR(UChar *p, UChar r1, UChar r2)
   2885 {
   2886    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2887       s390_disasm(ENC3(MNM, GPR, GPR), "or", r1, r2);
   2888 
   2889    return emit_RR(p, 0x1600, r1, r2);
   2890 }
   2891 
   2892 
   2893 static UChar *
   2894 s390_emit_OGR(UChar *p, UChar r1, UChar r2)
   2895 {
   2896    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2897       s390_disasm(ENC3(MNM, GPR, GPR), "ogr", r1, r2);
   2898 
   2899    return emit_RRE(p, 0xb9810000, r1, r2);
   2900 }
   2901 
   2902 
   2903 static UChar *
   2904 s390_emit_O(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   2905 {
   2906    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2907       s390_disasm(ENC3(MNM, GPR, UDXB), "o", r1, d2, x2, b2);
   2908 
   2909    return emit_RX(p, 0x56000000, r1, x2, b2, d2);
   2910 }
   2911 
   2912 
   2913 static UChar *
   2914 s390_emit_OY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2915 {
   2916    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2917       s390_disasm(ENC3(MNM, GPR, SDXB), "oy", r1, dh2, dl2, x2, b2);
   2918 
   2919    return emit_RXY(p, 0xe30000000056ULL, r1, x2, b2, dl2, dh2);
   2920 }
   2921 
   2922 
   2923 static UChar *
   2924 s390_emit_OG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   2925 {
   2926    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2927       s390_disasm(ENC3(MNM, GPR, SDXB), "og", r1, dh2, dl2, x2, b2);
   2928 
   2929    return emit_RXY(p, 0xe30000000081ULL, r1, x2, b2, dl2, dh2);
   2930 }
   2931 
   2932 
   2933 static UChar *
   2934 s390_emit_OIHF(UChar *p, UChar r1, UInt i2)
   2935 {
   2936    vassert(s390_host_has_eimm);
   2937 
   2938    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2939       s390_disasm(ENC3(MNM, GPR, UINT), "oihf", r1, i2);
   2940 
   2941    return emit_RIL(p, 0xc00c00000000ULL, r1, i2);
   2942 }
   2943 
   2944 
   2945 static UChar *
   2946 s390_emit_OILF(UChar *p, UChar r1, UInt i2)
   2947 {
   2948    vassert(s390_host_has_eimm);
   2949 
   2950    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2951       s390_disasm(ENC3(MNM, GPR, UINT), "oilf", r1, i2);
   2952 
   2953    return emit_RIL(p, 0xc00d00000000ULL, r1, i2);
   2954 }
   2955 
   2956 
   2957 static UChar *
   2958 s390_emit_OILL(UChar *p, UChar r1, UShort i2)
   2959 {
   2960    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2961       s390_disasm(ENC3(MNM, GPR, UINT), "oill", r1, i2);
   2962 
   2963    return emit_RI(p, 0xa50b0000, r1, i2);
   2964 }
   2965 
   2966 
   2967 static UChar *
   2968 s390_emit_SLL(UChar *p, UChar r1, UChar b2, UShort d2)
   2969 {
   2970    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2971       s390_disasm(ENC3(MNM, GPR, UDXB), "sll", r1, d2, 0, b2);
   2972 
   2973    return emit_RS(p, 0x89000000, r1, 0, b2, d2);
   2974 }
   2975 
   2976 
   2977 static UChar *
   2978 s390_emit_SLLG(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   2979 {
   2980    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2981       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "sllg", r1, r3, dh2, dl2, 0, b2);
   2982 
   2983    return emit_RSY(p, 0xeb000000000dULL, r1, r3, b2, dl2, dh2);
   2984 }
   2985 
   2986 
   2987 static UChar *
   2988 s390_emit_SRA(UChar *p, UChar r1, UChar b2, UShort d2)
   2989 {
   2990    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   2991       s390_disasm(ENC3(MNM, GPR, UDXB), "sra", r1, d2, 0, b2);
   2992 
   2993    return emit_RS(p, 0x8a000000, r1, 0, b2, d2);
   2994 }
   2995 
   2996 
   2997 static UChar *
   2998 s390_emit_SRAG(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   2999 {
   3000    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3001       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "srag", r1, r3, dh2, dl2, 0, b2);
   3002 
   3003    return emit_RSY(p, 0xeb000000000aULL, r1, r3, b2, dl2, dh2);
   3004 }
   3005 
   3006 
   3007 static UChar *
   3008 s390_emit_SRL(UChar *p, UChar r1, UChar b2, UShort d2)
   3009 {
   3010    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3011       s390_disasm(ENC3(MNM, GPR, UDXB), "srl", r1, d2, 0, b2);
   3012 
   3013    return emit_RS(p, 0x88000000, r1, 0, b2, d2);
   3014 }
   3015 
   3016 
   3017 static UChar *
   3018 s390_emit_SRLG(UChar *p, UChar r1, UChar r3, UChar b2, UShort dl2, UChar dh2)
   3019 {
   3020    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3021       s390_disasm(ENC4(MNM, GPR, GPR, SDXB), "srlg", r1, r3, dh2, dl2, 0, b2);
   3022 
   3023    return emit_RSY(p, 0xeb000000000cULL, r1, r3, b2, dl2, dh2);
   3024 }
   3025 
   3026 
   3027 static UChar *
   3028 s390_emit_ST(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3029 {
   3030    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3031       s390_disasm(ENC3(MNM, GPR, UDXB), "st", r1, d2, x2, b2);
   3032 
   3033    return emit_RX(p, 0x50000000, r1, x2, b2, d2);
   3034 }
   3035 
   3036 
   3037 static UChar *
   3038 s390_emit_STY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3039 {
   3040    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3041       s390_disasm(ENC3(MNM, GPR, SDXB), "sty", r1, dh2, dl2, x2, b2);
   3042 
   3043    return emit_RXY(p, 0xe30000000050ULL, r1, x2, b2, dl2, dh2);
   3044 }
   3045 
   3046 
   3047 static UChar *
   3048 s390_emit_STG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3049 {
   3050    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3051       s390_disasm(ENC3(MNM, GPR, SDXB), "stg", r1, dh2, dl2, x2, b2);
   3052 
   3053    return emit_RXY(p, 0xe30000000024ULL, r1, x2, b2, dl2, dh2);
   3054 }
   3055 
   3056 
   3057 static UChar *
   3058 s390_emit_STC(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3059 {
   3060    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3061       s390_disasm(ENC3(MNM, GPR, UDXB), "stc", r1, d2, x2, b2);
   3062 
   3063    return emit_RX(p, 0x42000000, r1, x2, b2, d2);
   3064 }
   3065 
   3066 
   3067 static UChar *
   3068 s390_emit_STCY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3069 {
   3070    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3071       s390_disasm(ENC3(MNM, GPR, SDXB), "stcy", r1, dh2, dl2, x2, b2);
   3072 
   3073    return emit_RXY(p, 0xe30000000072ULL, r1, x2, b2, dl2, dh2);
   3074 }
   3075 
   3076 
   3077 static UChar *
   3078 s390_emit_STH(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3079 {
   3080    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3081       s390_disasm(ENC3(MNM, GPR, UDXB), "sth", r1, d2, x2, b2);
   3082 
   3083    return emit_RX(p, 0x40000000, r1, x2, b2, d2);
   3084 }
   3085 
   3086 
   3087 static UChar *
   3088 s390_emit_STHY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3089 {
   3090    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3091       s390_disasm(ENC3(MNM, GPR, SDXB), "sthy", r1, dh2, dl2, x2, b2);
   3092 
   3093    return emit_RXY(p, 0xe30000000070ULL, r1, x2, b2, dl2, dh2);
   3094 }
   3095 
   3096 
   3097 static UChar *
   3098 s390_emit_SR(UChar *p, UChar r1, UChar r2)
   3099 {
   3100    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3101       s390_disasm(ENC3(MNM, GPR, GPR), "sr", r1, r2);
   3102 
   3103    return emit_RR(p, 0x1b00, r1, r2);
   3104 }
   3105 
   3106 
   3107 static UChar *
   3108 s390_emit_SGR(UChar *p, UChar r1, UChar r2)
   3109 {
   3110    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3111       s390_disasm(ENC3(MNM, GPR, GPR), "sgr", r1, r2);
   3112 
   3113    return emit_RRE(p, 0xb9090000, r1, r2);
   3114 }
   3115 
   3116 
   3117 static UChar *
   3118 s390_emit_S(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3119 {
   3120    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3121       s390_disasm(ENC3(MNM, GPR, UDXB), "s", r1, d2, x2, b2);
   3122 
   3123    return emit_RX(p, 0x5b000000, r1, x2, b2, d2);
   3124 }
   3125 
   3126 
   3127 static UChar *
   3128 s390_emit_SY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3129 {
   3130    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3131       s390_disasm(ENC3(MNM, GPR, SDXB), "sy", r1, dh2, dl2, x2, b2);
   3132 
   3133    return emit_RXY(p, 0xe3000000005bULL, r1, x2, b2, dl2, dh2);
   3134 }
   3135 
   3136 
   3137 static UChar *
   3138 s390_emit_SG(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3139 {
   3140    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3141       s390_disasm(ENC3(MNM, GPR, SDXB), "sg", r1, dh2, dl2, x2, b2);
   3142 
   3143    return emit_RXY(p, 0xe30000000009ULL, r1, x2, b2, dl2, dh2);
   3144 }
   3145 
   3146 
   3147 static UChar *
   3148 s390_emit_SH(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3149 {
   3150    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3151       s390_disasm(ENC3(MNM, GPR, UDXB), "sh", r1, d2, x2, b2);
   3152 
   3153    return emit_RX(p, 0x4b000000, r1, x2, b2, d2);
   3154 }
   3155 
   3156 
   3157 static UChar *
   3158 s390_emit_SHY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3159 {
   3160    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3161       s390_disasm(ENC3(MNM, GPR, SDXB), "shy", r1, dh2, dl2, x2, b2);
   3162 
   3163    return emit_RXY(p, 0xe3000000007bULL, r1, x2, b2, dl2, dh2);
   3164 }
   3165 
   3166 
   3167 static UChar *
   3168 s390_emit_SLFI(UChar *p, UChar r1, UInt i2)
   3169 {
   3170    vassert(s390_host_has_eimm);
   3171 
   3172    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3173       s390_disasm(ENC3(MNM, GPR, UINT), "slfi", r1, i2);
   3174 
   3175    return emit_RIL(p, 0xc20500000000ULL, r1, i2);
   3176 }
   3177 
   3178 
   3179 static UChar *
   3180 s390_emit_SLGFI(UChar *p, UChar r1, UInt i2)
   3181 {
   3182    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3183       s390_disasm(ENC3(MNM, GPR, UINT), "slgfi", r1, i2);
   3184 
   3185    return emit_RIL(p, 0xc20400000000ULL, r1, i2);
   3186 }
   3187 
   3188 
   3189 static UChar *
   3190 s390_emit_LDR(UChar *p, UChar r1, UChar r2)
   3191 {
   3192    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3193       s390_disasm(ENC3(MNM, FPR, FPR), "ldr", r1, r2);
   3194 
   3195    return emit_RR(p, 0x2800, r1, r2);
   3196 }
   3197 
   3198 
   3199 static UChar *
   3200 s390_emit_LE(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3201 {
   3202    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3203       s390_disasm(ENC3(MNM, FPR, UDXB), "le", r1, d2, x2, b2);
   3204 
   3205    return emit_RX(p, 0x78000000, r1, x2, b2, d2);
   3206 }
   3207 
   3208 
   3209 static UChar *
   3210 s390_emit_LD(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3211 {
   3212    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3213       s390_disasm(ENC3(MNM, FPR, UDXB), "ld", r1, d2, x2, b2);
   3214 
   3215    return emit_RX(p, 0x68000000, r1, x2, b2, d2);
   3216 }
   3217 
   3218 
   3219 static UChar *
   3220 s390_emit_LEY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3221 {
   3222    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3223       s390_disasm(ENC3(MNM, FPR, SDXB), "ley", r1, dh2, dl2, x2, b2);
   3224 
   3225    return emit_RXY(p, 0xed0000000064ULL, r1, x2, b2, dl2, dh2);
   3226 }
   3227 
   3228 
   3229 static UChar *
   3230 s390_emit_LDY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3231 {
   3232    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3233       s390_disasm(ENC3(MNM, FPR, SDXB), "ldy", r1, dh2, dl2, x2, b2);
   3234 
   3235    return emit_RXY(p, 0xed0000000065ULL, r1, x2, b2, dl2, dh2);
   3236 }
   3237 
   3238 
   3239 static UChar *
   3240 s390_emit_LFPC(UChar *p, UChar b2, UShort d2)
   3241 {
   3242    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3243       s390_disasm(ENC2(MNM, UDXB), "lfpc", d2, 0, b2);
   3244 
   3245    return emit_S(p, 0xb29d0000, b2, d2);
   3246 }
   3247 
   3248 
   3249 static UChar *
   3250 s390_emit_LDGR(UChar *p, UChar r1, UChar r2)
   3251 {
   3252    vassert(s390_host_has_fgx);
   3253 
   3254    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3255       s390_disasm(ENC3(MNM, FPR, GPR), "ldgr", r1, r2);
   3256 
   3257    return emit_RRE(p, 0xb3c10000, r1, r2);
   3258 }
   3259 
   3260 
   3261 static UChar *
   3262 s390_emit_LGDR(UChar *p, UChar r1, UChar r2)
   3263 {
   3264    vassert(s390_host_has_fgx);
   3265 
   3266    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3267       s390_disasm(ENC3(MNM, GPR, FPR), "lgdr", r1, r2);
   3268 
   3269    return emit_RRE(p, 0xb3cd0000, r1, r2);
   3270 }
   3271 
   3272 
   3273 static UChar *
   3274 s390_emit_LZER(UChar *p, UChar r1, UChar r2)
   3275 {
   3276    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3277       s390_disasm(ENC2(MNM, FPR), "lzer", r1);
   3278 
   3279    return emit_RRE(p, 0xb3740000, r1, r2);
   3280 }
   3281 
   3282 
   3283 static UChar *
   3284 s390_emit_LZDR(UChar *p, UChar r1, UChar r2)
   3285 {
   3286    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3287       s390_disasm(ENC2(MNM, FPR), "lzdr", r1);
   3288 
   3289    return emit_RRE(p, 0xb3750000, r1, r2);
   3290 }
   3291 
   3292 
   3293 static UChar *
   3294 s390_emit_SFPC(UChar *p, UChar r1)
   3295 {
   3296    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3297       s390_disasm(ENC2(MNM, GPR), "sfpc", r1);
   3298 
   3299    return emit_RRE(p, 0xb3840000, r1, 0);
   3300 }
   3301 
   3302 
   3303 static UChar *
   3304 s390_emit_STE(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3305 {
   3306    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3307       s390_disasm(ENC3(MNM, FPR, UDXB), "ste", r1, d2, x2, b2);
   3308 
   3309    return emit_RX(p, 0x70000000, r1, x2, b2, d2);
   3310 }
   3311 
   3312 
   3313 static UChar *
   3314 s390_emit_STD(UChar *p, UChar r1, UChar x2, UChar b2, UShort d2)
   3315 {
   3316    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3317       s390_disasm(ENC3(MNM, FPR, UDXB), "std", r1, d2, x2, b2);
   3318 
   3319    return emit_RX(p, 0x60000000, r1, x2, b2, d2);
   3320 }
   3321 
   3322 
   3323 static UChar *
   3324 s390_emit_STEY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3325 {
   3326    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3327       s390_disasm(ENC3(MNM, FPR, SDXB), "stey", r1, dh2, dl2, x2, b2);
   3328 
   3329    return emit_RXY(p, 0xed0000000066ULL, r1, x2, b2, dl2, dh2);
   3330 }
   3331 
   3332 
   3333 static UChar *
   3334 s390_emit_STDY(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl2, UChar dh2)
   3335 {
   3336    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3337       s390_disasm(ENC3(MNM, FPR, SDXB), "stdy", r1, dh2, dl2, x2, b2);
   3338 
   3339    return emit_RXY(p, 0xed0000000067ULL, r1, x2, b2, dl2, dh2);
   3340 }
   3341 
   3342 
   3343 static UChar *
   3344 s390_emit_STFPC(UChar *p, UChar b2, UShort d2)
   3345 {
   3346    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3347       s390_disasm(ENC2(MNM, UDXB), "stfpc", d2, 0, b2);
   3348 
   3349    return emit_S(p, 0xb29c0000, b2, d2);
   3350 }
   3351 
   3352 
   3353 static UChar *
   3354 s390_emit_AEBR(UChar *p, UChar r1, UChar r2)
   3355 {
   3356    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3357       s390_disasm(ENC3(MNM, FPR, FPR), "aebr", r1, r2);
   3358 
   3359    return emit_RRE(p, 0xb30a0000, r1, r2);
   3360 }
   3361 
   3362 
   3363 static UChar *
   3364 s390_emit_ADBR(UChar *p, UChar r1, UChar r2)
   3365 {
   3366    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3367       s390_disasm(ENC3(MNM, FPR, FPR), "adbr", r1, r2);
   3368 
   3369    return emit_RRE(p, 0xb31a0000, r1, r2);
   3370 }
   3371 
   3372 
   3373 static UChar *
   3374 s390_emit_AXBR(UChar *p, UChar r1, UChar r2)
   3375 {
   3376    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3377       s390_disasm(ENC3(MNM, FPR, FPR), "axbr", r1, r2);
   3378 
   3379    return emit_RRE(p, 0xb34a0000, r1, r2);
   3380 }
   3381 
   3382 
   3383 static UChar *
   3384 s390_emit_CEBR(UChar *p, UChar r1, UChar r2)
   3385 {
   3386    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3387       s390_disasm(ENC3(MNM, FPR, FPR), "cebr", r1, r2);
   3388 
   3389    return emit_RRE(p, 0xb3090000, r1, r2);
   3390 }
   3391 
   3392 
   3393 static UChar *
   3394 s390_emit_CDBR(UChar *p, UChar r1, UChar r2)
   3395 {
   3396    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3397       s390_disasm(ENC3(MNM, FPR, FPR), "cdbr", r1, r2);
   3398 
   3399    return emit_RRE(p, 0xb3190000, r1, r2);
   3400 }
   3401 
   3402 
   3403 static UChar *
   3404 s390_emit_CXBR(UChar *p, UChar r1, UChar r2)
   3405 {
   3406    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3407       s390_disasm(ENC3(MNM, FPR, FPR), "cxbr", r1, r2);
   3408 
   3409    return emit_RRE(p, 0xb3490000, r1, r2);
   3410 }
   3411 
   3412 
   3413 static UChar *
   3414 s390_emit_CEFBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3415 {
   3416    vassert(m4 == 0);
   3417    vassert(m3 == 0 || s390_host_has_fpext);
   3418 
   3419    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3420       if (m3 == 0)
   3421          s390_disasm(ENC3(MNM, FPR, GPR), "cefbr", r1, r2);
   3422       else
   3423          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3424                      "cefbra", r1, m3, r2, m4);
   3425    }
   3426 
   3427    return emit_RRF2(p, 0xb3940000, m3, m4, r1, r2);
   3428 }
   3429 
   3430 
   3431 static UChar *
   3432 s390_emit_CDFBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3433 {
   3434    vassert(m4 == 0);
   3435    vassert(m3 == 0 || s390_host_has_fpext);
   3436 
   3437    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3438       if (m3 == 0)
   3439          s390_disasm(ENC3(MNM, FPR, GPR), "cdfbr", r1, r2);
   3440       else
   3441          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3442                      "cdfbra", r1, m3, r2, m4);
   3443    }
   3444 
   3445    return emit_RRF2(p, 0xb3950000, m3, m4, r1, r2);
   3446 }
   3447 
   3448 
   3449 static UChar *
   3450 s390_emit_CXFBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3451 {
   3452    vassert(m4 == 0);
   3453    vassert(m3 == 0 || s390_host_has_fpext);
   3454 
   3455    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3456       if (m3 == 0)
   3457          s390_disasm(ENC3(MNM, FPR, GPR), "cxfbr", r1, r2);
   3458       else
   3459          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3460                      "cxfbra", r1, m3, r2, m4);
   3461    }
   3462 
   3463    return emit_RRF2(p, 0xb3960000, m3, m4, r1, r2);
   3464 }
   3465 
   3466 
   3467 static UChar *
   3468 s390_emit_CEGBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3469 {
   3470    vassert(m4 == 0);
   3471    vassert(m3 == 0 || s390_host_has_fpext);
   3472 
   3473    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3474       if (m3 == 0)
   3475          s390_disasm(ENC3(MNM, FPR, GPR), "cegbr", r1, r2);
   3476       else
   3477          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3478                      "cegbra", r1, m3, r2, m4);
   3479    }
   3480 
   3481    return emit_RRF2(p, 0xb3a40000, m3, m4, r1, r2);
   3482 }
   3483 
   3484 
   3485 static UChar *
   3486 s390_emit_CDGBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3487 {
   3488    vassert(m4 == 0);
   3489    vassert(m3 == 0 || s390_host_has_fpext);
   3490 
   3491    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3492       if (m3 == 0)
   3493          s390_disasm(ENC3(MNM, FPR, GPR), "cdgbr", r1, r2);
   3494       else
   3495          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3496                      "cdgbra", r1, m3, r2, m4);
   3497    }
   3498 
   3499    return emit_RRF2(p, 0xb3a50000, m3, m4, r1, r2);
   3500 }
   3501 
   3502 
   3503 static UChar *
   3504 s390_emit_CXGBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3505 {
   3506    vassert(m4 == 0);
   3507    vassert(m3 == 0 || s390_host_has_fpext);
   3508 
   3509    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3510       if (m3 == 0)
   3511          s390_disasm(ENC3(MNM, FPR, GPR), "cxgbr", r1, r2);
   3512       else
   3513          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT),
   3514                      "cxgbra", r1, m3, r2, m4);
   3515    }
   3516 
   3517    return emit_RRF2(p, 0xb3a60000, m3, m4, r1, r2);
   3518 }
   3519 
   3520 
   3521 static UChar *
   3522 s390_emit_CELFBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3523 {
   3524    vassert(m4 == 0);
   3525    vassert(s390_host_has_fpext);
   3526 
   3527    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3528       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "celfbr", r1, m3, r2, m4);
   3529 
   3530    return emit_RRF2(p, 0xb3900000, m3, m4, r1, r2);
   3531 }
   3532 
   3533 
   3534 static UChar *
   3535 s390_emit_CDLFBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3536 {
   3537    vassert(m4 == 0);
   3538    vassert(s390_host_has_fpext);
   3539 
   3540    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3541       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdlfbr", r1, m3, r2, m4);
   3542 
   3543    return emit_RRF2(p, 0xb3910000, m3, m4, r1, r2);
   3544 }
   3545 
   3546 
   3547 static UChar *
   3548 s390_emit_CXLFBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3549 {
   3550    vassert(m4 == 0);
   3551    vassert(s390_host_has_fpext);
   3552 
   3553    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3554       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cxlfbr", r1, m3, r2, m4);
   3555 
   3556    return emit_RRF2(p, 0xb3920000, m3, m4, r1, r2);
   3557 }
   3558 
   3559 
   3560 static UChar *
   3561 s390_emit_CELGBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3562 {
   3563    vassert(m4 == 0);
   3564    vassert(s390_host_has_fpext);
   3565 
   3566    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3567       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "celgbr", r1, m3, r2, m4);
   3568 
   3569    return emit_RRF2(p, 0xb3a00000, m3, m4, r1, r2);
   3570 }
   3571 
   3572 
   3573 static UChar *
   3574 s390_emit_CDLGBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3575 {
   3576    vassert(m4 == 0);
   3577    vassert(s390_host_has_fpext);
   3578 
   3579    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3580       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdlgbr", r1, m3, r2, m4);
   3581 
   3582    return emit_RRF2(p, 0xb3a10000, m3, m4, r1, r2);
   3583 }
   3584 
   3585 
   3586 static UChar *
   3587 s390_emit_CXLGBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3588 {
   3589    vassert(m4 == 0);
   3590    vassert(s390_host_has_fpext);
   3591 
   3592    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3593       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cxlgbr", r1, m3, r2, m4);
   3594 
   3595    return emit_RRF2(p, 0xb3a20000, m3, m4, r1, r2);
   3596 }
   3597 
   3598 
   3599 static UChar *
   3600 s390_emit_CLFEBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3601 {
   3602    vassert(m4 == 0);
   3603    vassert(s390_host_has_fpext);
   3604 
   3605    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3606       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clfebr", r1, m3, r2, m4);
   3607 
   3608    return emit_RRF2(p, 0xb39c0000, m3, m4, r1, r2);
   3609 }
   3610 
   3611 
   3612 static UChar *
   3613 s390_emit_CLFDBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3614 {
   3615    vassert(m4 == 0);
   3616    vassert(s390_host_has_fpext);
   3617 
   3618    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3619       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clfdbr", r1, m3, r2, m4);
   3620 
   3621    return emit_RRF2(p, 0xb39d0000, m3, m4, r1, r2);
   3622 }
   3623 
   3624 
   3625 static UChar *
   3626 s390_emit_CLFXBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3627 {
   3628    vassert(m4 == 0);
   3629    vassert(s390_host_has_fpext);
   3630 
   3631    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3632       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clfxbr", r1, m3, r2, m4);
   3633 
   3634    return emit_RRF2(p, 0xb39e0000, m3, m4, r1, r2);
   3635 }
   3636 
   3637 
   3638 static UChar *
   3639 s390_emit_CLGEBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3640 {
   3641    vassert(m4 == 0);
   3642    vassert(s390_host_has_fpext);
   3643 
   3644    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3645       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clgebr", r1, m3, r2, m4);
   3646 
   3647    return emit_RRF2(p, 0xb3ac0000, m3, m4, r1, r2);
   3648 }
   3649 
   3650 
   3651 static UChar *
   3652 s390_emit_CLGDBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3653 {
   3654    vassert(m4 == 0);
   3655    vassert(s390_host_has_fpext);
   3656 
   3657    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3658       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clgdbr", r1, m3, r2, m4);
   3659 
   3660    return emit_RRF2(p, 0xb3ad0000, m3, m4, r1, r2);
   3661 }
   3662 
   3663 
   3664 static UChar *
   3665 s390_emit_CLGXBR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3666 {
   3667    vassert(m4 == 0);
   3668    vassert(s390_host_has_fpext);
   3669 
   3670    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3671       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clgxbr", r1, m3, r2, m4);
   3672 
   3673    return emit_RRF2(p, 0xb3ae0000, m3, m4, r1, r2);
   3674 }
   3675 
   3676 
   3677 static UChar *
   3678 s390_emit_CFEBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3679 {
   3680    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3681       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cfebr", r1, r3, r2);
   3682 
   3683    return emit_RRF3(p, 0xb3980000, r3, r1, r2);
   3684 }
   3685 
   3686 
   3687 static UChar *
   3688 s390_emit_CFDBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3689 {
   3690    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3691       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cfdbr", r1, r3, r2);
   3692 
   3693    return emit_RRF3(p, 0xb3990000, r3, r1, r2);
   3694 }
   3695 
   3696 
   3697 static UChar *
   3698 s390_emit_CFXBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3699 {
   3700    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3701       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cfxbr", r1, r3, r2);
   3702 
   3703    return emit_RRF3(p, 0xb39a0000, r3, r1, r2);
   3704 }
   3705 
   3706 
   3707 static UChar *
   3708 s390_emit_CGEBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3709 {
   3710    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3711       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cgebr", r1, r3, r2);
   3712 
   3713    return emit_RRF3(p, 0xb3a80000, r3, r1, r2);
   3714 }
   3715 
   3716 
   3717 static UChar *
   3718 s390_emit_CGDBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3719 {
   3720    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3721       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cgdbr", r1, r3, r2);
   3722 
   3723    return emit_RRF3(p, 0xb3a90000, r3, r1, r2);
   3724 }
   3725 
   3726 
   3727 static UChar *
   3728 s390_emit_CGXBR(UChar *p, UChar r3, UChar r1, UChar r2)
   3729 {
   3730    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3731       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cgxbr", r1, r3, r2);
   3732 
   3733    return emit_RRF3(p, 0xb3aa0000, r3, r1, r2);
   3734 }
   3735 
   3736 
   3737 static UChar *
   3738 s390_emit_DEBR(UChar *p, UChar r1, UChar r2)
   3739 {
   3740    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3741       s390_disasm(ENC3(MNM, FPR, FPR), "debr", r1, r2);
   3742 
   3743    return emit_RRE(p, 0xb30d0000, r1, r2);
   3744 }
   3745 
   3746 
   3747 static UChar *
   3748 s390_emit_DDBR(UChar *p, UChar r1, UChar r2)
   3749 {
   3750    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3751       s390_disasm(ENC3(MNM, FPR, FPR), "ddbr", r1, r2);
   3752 
   3753    return emit_RRE(p, 0xb31d0000, r1, r2);
   3754 }
   3755 
   3756 
   3757 static UChar *
   3758 s390_emit_DXBR(UChar *p, UChar r1, UChar r2)
   3759 {
   3760    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3761       s390_disasm(ENC3(MNM, FPR, FPR), "dxbr", r1, r2);
   3762 
   3763    return emit_RRE(p, 0xb34d0000, r1, r2);
   3764 }
   3765 
   3766 
   3767 static UChar *
   3768 s390_emit_LCEBR(UChar *p, UChar r1, UChar r2)
   3769 {
   3770    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3771       s390_disasm(ENC3(MNM, FPR, FPR), "lcebr", r1, r2);
   3772 
   3773    return emit_RRE(p, 0xb3030000, r1, r2);
   3774 }
   3775 
   3776 
   3777 static UChar *
   3778 s390_emit_LCDBR(UChar *p, UChar r1, UChar r2)
   3779 {
   3780    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3781       s390_disasm(ENC3(MNM, FPR, FPR), "lcdbr", r1, r2);
   3782 
   3783    return emit_RRE(p, 0xb3130000, r1, r2);
   3784 }
   3785 
   3786 
   3787 static UChar *
   3788 s390_emit_LCXBR(UChar *p, UChar r1, UChar r2)
   3789 {
   3790    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3791       s390_disasm(ENC3(MNM, FPR, FPR), "lcxbr", r1, r2);
   3792 
   3793    return emit_RRE(p, 0xb3430000, r1, r2);
   3794 }
   3795 
   3796 
   3797 static UChar *
   3798 s390_emit_LDEBR(UChar *p, UChar r1, UChar r2)
   3799 {
   3800    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3801       s390_disasm(ENC3(MNM, FPR, FPR), "ldebr", r1, r2);
   3802 
   3803    return emit_RRE(p, 0xb3040000, r1, r2);
   3804 }
   3805 
   3806 
   3807 static UChar *
   3808 s390_emit_LXDBR(UChar *p, UChar r1, UChar r2)
   3809 {
   3810    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3811       s390_disasm(ENC3(MNM, FPR, FPR), "lxdbr", r1, r2);
   3812 
   3813    return emit_RRE(p, 0xb3050000, r1, r2);
   3814 }
   3815 
   3816 
   3817 static UChar *
   3818 s390_emit_LXEBR(UChar *p, UChar r1, UChar r2)
   3819 {
   3820    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3821       s390_disasm(ENC3(MNM, FPR, FPR), "lxebr", r1, r2);
   3822 
   3823    return emit_RRE(p, 0xb3060000, r1, r2);
   3824 }
   3825 
   3826 
   3827 static UChar *
   3828 s390_emit_LNEBR(UChar *p, UChar r1, UChar r2)
   3829 {
   3830    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3831       s390_disasm(ENC3(MNM, FPR, FPR), "lnebr", r1, r2);
   3832 
   3833    return emit_RRE(p, 0xb3010000, r1, r2);
   3834 }
   3835 
   3836 
   3837 static UChar *
   3838 s390_emit_LNDBR(UChar *p, UChar r1, UChar r2)
   3839 {
   3840    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3841       s390_disasm(ENC3(MNM, FPR, FPR), "lndbr", r1, r2);
   3842 
   3843    return emit_RRE(p, 0xb3110000, r1, r2);
   3844 }
   3845 
   3846 
   3847 static UChar *
   3848 s390_emit_LNXBR(UChar *p, UChar r1, UChar r2)
   3849 {
   3850    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3851       s390_disasm(ENC3(MNM, FPR, FPR), "lnxbr", r1, r2);
   3852 
   3853    return emit_RRE(p, 0xb3410000, r1, r2);
   3854 }
   3855 
   3856 
   3857 static UChar *
   3858 s390_emit_LPEBR(UChar *p, UChar r1, UChar r2)
   3859 {
   3860    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3861       s390_disasm(ENC3(MNM, FPR, FPR), "lpebr", r1, r2);
   3862 
   3863    return emit_RRE(p, 0xb3000000, r1, r2);
   3864 }
   3865 
   3866 
   3867 static UChar *
   3868 s390_emit_LPDBR(UChar *p, UChar r1, UChar r2)
   3869 {
   3870    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3871       s390_disasm(ENC3(MNM, FPR, FPR), "lpdbr", r1, r2);
   3872 
   3873    return emit_RRE(p, 0xb3100000, r1, r2);
   3874 }
   3875 
   3876 
   3877 static UChar *
   3878 s390_emit_LPXBR(UChar *p, UChar r1, UChar r2)
   3879 {
   3880    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3881       s390_disasm(ENC3(MNM, FPR, FPR), "lpxbr", r1, r2);
   3882 
   3883    return emit_RRE(p, 0xb3400000, r1, r2);
   3884 }
   3885 
   3886 
   3887 static UChar *
   3888 s390_emit_LEDBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3889 {
   3890    vassert(m4 == 0);
   3891    vassert(m3 == 0 || s390_host_has_fpext);
   3892 
   3893    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3894       if (m3 == 0)
   3895          s390_disasm(ENC3(MNM, FPR, FPR), "ledbr", r1, r2);
   3896       else
   3897          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3898                      "ledbra", r1, m3, r2, m4);
   3899    }
   3900 
   3901    return emit_RRF2(p, 0xb3440000, m3, m4, r1, r2);
   3902 }
   3903 
   3904 
   3905 static UChar *
   3906 s390_emit_LDXBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3907 {
   3908    vassert(m4 == 0);
   3909    vassert(m3 == 0 || s390_host_has_fpext);
   3910 
   3911    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3912       if (m3 == 0)
   3913          s390_disasm(ENC3(MNM, FPR, FPR), "ldxbr", r1, r2);
   3914       else
   3915          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3916                      "ldxbra", r1, m3, r2, m4);
   3917    }
   3918 
   3919    return emit_RRF2(p, 0xb3450000, m3, m4, r1, r2);
   3920 }
   3921 
   3922 
   3923 static UChar *
   3924 s390_emit_LEXBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3925 {
   3926    vassert(m4 == 0);
   3927    vassert(m3 == 0 || s390_host_has_fpext);
   3928 
   3929    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3930       if (m3 == 0)
   3931          s390_disasm(ENC3(MNM, FPR, FPR), "lexbr", r1, r2);
   3932       else
   3933          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3934                      "lexbra", r1, m3, r2, m4);
   3935    }
   3936 
   3937    return emit_RRF2(p, 0xb3460000, m3, m4, r1, r2);
   3938 }
   3939 
   3940 
   3941 static UChar *
   3942 s390_emit_FIEBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3943 {
   3944    vassert(m3 == 0 || s390_host_has_fpext);
   3945 
   3946    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3947       if (m4 == 0)
   3948          s390_disasm(ENC4(MNM, FPR, UINT, FPR), "fiebr", r1, m3, r2);
   3949       else
   3950          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3951                      "fiebra", r1, m3, r2, m4);
   3952    }
   3953 
   3954    return emit_RRF2(p, 0xb3570000, m3, m4, r1, r2);
   3955 }
   3956 
   3957 
   3958 static UChar *
   3959 s390_emit_FIDBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3960 {
   3961    vassert(m3 == 0 || s390_host_has_fpext);
   3962 
   3963    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3964       if (m4 == 0)
   3965          s390_disasm(ENC4(MNM, FPR, UINT, FPR), "fidbr", r1, m3, r2);
   3966       else
   3967          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3968                      "fidbra", r1, m3, r2, m4);
   3969    }
   3970 
   3971    return emit_RRF2(p, 0xb35f0000, m3, m4, r1, r2);
   3972 }
   3973 
   3974 
   3975 static UChar *
   3976 s390_emit_FIXBRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   3977 {
   3978    vassert(m3 == 0 || s390_host_has_fpext);
   3979 
   3980    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   3981       if (m4 == 0)
   3982          s390_disasm(ENC4(MNM, FPR, UINT, FPR), "fixbr", r1, m3, r2);
   3983       else
   3984          s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT),
   3985                      "fixbra", r1, m3, r2, m4);
   3986    }
   3987 
   3988    return emit_RRF2(p, 0xb3470000, m3, m4, r1, r2);
   3989 }
   3990 
   3991 
   3992 static UChar *
   3993 s390_emit_MEEBR(UChar *p, UChar r1, UChar r2)
   3994 {
   3995    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   3996       s390_disasm(ENC3(MNM, FPR, FPR), "meebr", r1, r2);
   3997 
   3998    return emit_RRE(p, 0xb3170000, r1, r2);
   3999 }
   4000 
   4001 
   4002 static UChar *
   4003 s390_emit_MDBR(UChar *p, UChar r1, UChar r2)
   4004 {
   4005    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4006       s390_disasm(ENC3(MNM, FPR, FPR), "mdbr", r1, r2);
   4007 
   4008    return emit_RRE(p, 0xb31c0000, r1, r2);
   4009 }
   4010 
   4011 
   4012 static UChar *
   4013 s390_emit_MXBR(UChar *p, UChar r1, UChar r2)
   4014 {
   4015    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4016       s390_disasm(ENC3(MNM, FPR, FPR), "mxbr", r1, r2);
   4017 
   4018    return emit_RRE(p, 0xb34c0000, r1, r2);
   4019 }
   4020 
   4021 
   4022 static UChar *
   4023 s390_emit_MAEBR(UChar *p, UChar r1, UChar r3, UChar r2)
   4024 {
   4025    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4026       s390_disasm(ENC4(MNM, FPR, FPR, FPR), "maebr", r1, r3, r2);
   4027 
   4028    return emit_RRF(p, 0xb30e0000, r1, r3, r2);
   4029 }
   4030 
   4031 
   4032 static UChar *
   4033 s390_emit_MADBR(UChar *p, UChar r1, UChar r3, UChar r2)
   4034 {
   4035    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4036       s390_disasm(ENC4(MNM, FPR, FPR, FPR), "madbr", r1, r3, r2);
   4037 
   4038    return emit_RRF(p, 0xb31e0000, r1, r3, r2);
   4039 }
   4040 
   4041 
   4042 static UChar *
   4043 s390_emit_MSEBR(UChar *p, UChar r1, UChar r3, UChar r2)
   4044 {
   4045    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4046       s390_disasm(ENC4(MNM, FPR, FPR, FPR), "msebr", r1, r3, r2);
   4047 
   4048    return emit_RRF(p, 0xb30f0000, r1, r3, r2);
   4049 }
   4050 
   4051 
   4052 static UChar *
   4053 s390_emit_MSDBR(UChar *p, UChar r1, UChar r3, UChar r2)
   4054 {
   4055    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4056       s390_disasm(ENC4(MNM, FPR, FPR, FPR), "msdbr", r1, r3, r2);
   4057 
   4058    return emit_RRF(p, 0xb31f0000, r1, r3, r2);
   4059 }
   4060 
   4061 
   4062 static UChar *
   4063 s390_emit_SQEBR(UChar *p, UChar r1, UChar r2)
   4064 {
   4065    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4066       s390_disasm(ENC3(MNM, FPR, FPR), "sqebr", r1, r2);
   4067 
   4068    return emit_RRE(p, 0xb3140000, r1, r2);
   4069 }
   4070 
   4071 
   4072 static UChar *
   4073 s390_emit_SQDBR(UChar *p, UChar r1, UChar r2)
   4074 {
   4075    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4076       s390_disasm(ENC3(MNM, FPR, FPR), "sqdbr", r1, r2);
   4077 
   4078    return emit_RRE(p, 0xb3150000, r1, r2);
   4079 }
   4080 
   4081 
   4082 static UChar *
   4083 s390_emit_SQXBR(UChar *p, UChar r1, UChar r2)
   4084 {
   4085    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4086       s390_disasm(ENC3(MNM, FPR, FPR), "sqxbr", r1, r2);
   4087 
   4088    return emit_RRE(p, 0xb3160000, r1, r2);
   4089 }
   4090 
   4091 
   4092 static UChar *
   4093 s390_emit_SEBR(UChar *p, UChar r1, UChar r2)
   4094 {
   4095    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4096       s390_disasm(ENC3(MNM, FPR, FPR), "sebr", r1, r2);
   4097 
   4098    return emit_RRE(p, 0xb30b0000, r1, r2);
   4099 }
   4100 
   4101 
   4102 static UChar *
   4103 s390_emit_SDBR(UChar *p, UChar r1, UChar r2)
   4104 {
   4105    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4106       s390_disasm(ENC3(MNM, FPR, FPR), "sdbr", r1, r2);
   4107 
   4108    return emit_RRE(p, 0xb31b0000, r1, r2);
   4109 }
   4110 
   4111 
   4112 static UChar *
   4113 s390_emit_SXBR(UChar *p, UChar r1, UChar r2)
   4114 {
   4115    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4116       s390_disasm(ENC3(MNM, FPR, FPR), "sxbr", r1, r2);
   4117 
   4118    return emit_RRE(p, 0xb34b0000, r1, r2);
   4119 }
   4120 
   4121 
   4122 static UChar *
   4123 s390_emit_ADTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4124 {
   4125    vassert(s390_host_has_dfp);
   4126    vassert(m4 == 0 || s390_host_has_fpext);
   4127    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4128       if (m4 == 0)
   4129          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "adtr", r1, r2, r3);
   4130       else
   4131          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "adtra", r1, r2, r3, m4);
   4132    }
   4133 
   4134    return emit_RRF4(p, 0xb3d20000, r3, m4, r1, r2);
   4135 }
   4136 
   4137 
   4138 static UChar *
   4139 s390_emit_AXTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4140 {
   4141    vassert(s390_host_has_dfp);
   4142    vassert(m4 == 0 || s390_host_has_fpext);
   4143    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4144       if (m4 == 0)
   4145          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "axtr", r1, r2, r3);
   4146       else
   4147          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "axtra", r1, r2, r3, m4);
   4148    }
   4149 
   4150    return emit_RRF4(p, 0xb3da0000, r3, m4, r1, r2);
   4151 }
   4152 
   4153 
   4154 static UChar *
   4155 s390_emit_CDTR(UChar *p, UChar r1, UChar r2)
   4156 {
   4157    vassert(s390_host_has_dfp);
   4158    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4159       s390_disasm(ENC3(MNM, FPR, FPR), "cdtr", r1, r2);
   4160 
   4161    return emit_RRE(p, 0xb3e40000, r1, r2);
   4162 }
   4163 
   4164 
   4165 static UChar *
   4166 s390_emit_CXTR(UChar *p, UChar r1, UChar r2)
   4167 {
   4168    vassert(s390_host_has_dfp);
   4169    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4170       s390_disasm(ENC3(MNM, FPR, FPR), "cxtr", r1, r2);
   4171 
   4172    return emit_RRE(p, 0xb3ec0000, r1, r2);
   4173 }
   4174 
   4175 
   4176 static UChar *
   4177 s390_emit_CDGTRA(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4178 {
   4179    vassert(s390_host_has_dfp);
   4180    vassert(m4 == 0);
   4181    vassert(m3 == 0 || s390_host_has_fpext);
   4182 
   4183    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4184       if (m3 == 0)
   4185          s390_disasm(ENC3(MNM, FPR, GPR), "cdgtr", r1, r2);
   4186       else
   4187          s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdgtra", r1, m3, r2, m4);
   4188    }
   4189 
   4190    return emit_RRF2(p, 0xb3f10000, m3, m4, r1, r2);
   4191 }
   4192 
   4193 
   4194 static UChar *
   4195 s390_emit_CXGTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4196 {
   4197    vassert(s390_host_has_dfp);
   4198    vassert(m4 == 0);
   4199    /* rounding mode m3 is not considered, as the corresponding
   4200       IRop (Iop_I64StoD128) does not take rounding mode. */
   4201    vassert(m3 == 0);
   4202 
   4203    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4204       s390_disasm(ENC3(MNM, FPR, GPR), "cxgtr", r1, r2);
   4205 
   4206    return emit_RRF2(p, 0xb3f90000, m3, m4, r1, r2);
   4207 }
   4208 
   4209 
   4210 static UChar *
   4211 s390_emit_CDFTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4212 {
   4213    vassert(m4 == 0);
   4214    vassert(s390_host_has_dfp);
   4215    vassert(s390_host_has_fpext);
   4216 
   4217    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4218       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdftr", r1, m3, r2, m4);
   4219 
   4220    return emit_RRF2(p, 0xb9510000, m3, m4, r1, r2);
   4221 }
   4222 
   4223 
   4224 static UChar *
   4225 s390_emit_CXFTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4226 {
   4227    vassert(m4 == 0);
   4228    vassert(s390_host_has_dfp);
   4229    vassert(s390_host_has_fpext);
   4230 
   4231    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4232       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cxftr", r1, m3, r2, m4);
   4233 
   4234    return emit_RRF2(p, 0xb9590000, m3, m4, r1, r2);
   4235 }
   4236 
   4237 
   4238 static UChar *
   4239 s390_emit_CDLFTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4240 {
   4241    vassert(m4 == 0);
   4242    vassert(s390_host_has_dfp);
   4243    vassert(s390_host_has_fpext);
   4244 
   4245    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4246       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdlftr", r1, m3, r2, m4);
   4247 
   4248    return emit_RRF2(p, 0xb9530000, m3, m4, r1, r2);
   4249 }
   4250 
   4251 
   4252 static UChar *
   4253 s390_emit_CXLFTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4254 {
   4255    vassert(m4 == 0);
   4256    vassert(s390_host_has_dfp);
   4257    vassert(s390_host_has_fpext);
   4258 
   4259    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4260       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cxlftr", r1, m3, r2, m4);
   4261 
   4262    return emit_RRF2(p, 0xb95b0000, m3, m4, r1, r2);
   4263 }
   4264 
   4265 
   4266 static UChar *
   4267 s390_emit_CDLGTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4268 {
   4269    vassert(m4 == 0);
   4270    vassert(s390_host_has_dfp);
   4271    vassert(s390_host_has_fpext);
   4272 
   4273    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4274       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cdlgtr", r1, m3, r2, m4);
   4275 
   4276    return emit_RRF2(p, 0xb9520000, m3, m4, r1, r2);
   4277 }
   4278 
   4279 
   4280 static UChar *
   4281 s390_emit_CXLGTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4282 {
   4283    vassert(m4 == 0);
   4284    vassert(s390_host_has_dfp);
   4285    vassert(s390_host_has_fpext);
   4286 
   4287    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4288       s390_disasm(ENC5(MNM, FPR, UINT, GPR, UINT), "cxlgtr", r1, m3, r2, m4);
   4289 
   4290    return emit_RRF2(p, 0xb95a0000, m3, m4, r1, r2);
   4291 }
   4292 
   4293 
   4294 static UChar *
   4295 s390_emit_CEDTR(UChar *p, UChar r1, UChar r2)
   4296 {
   4297    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4298       s390_disasm(ENC3(MNM, FPR, FPR), "cedtr", r1, r2);
   4299 
   4300    return emit_RRE(p, 0xb3f40000, r1, r2);
   4301 }
   4302 
   4303 
   4304 static UChar *
   4305 s390_emit_CEXTR(UChar *p, UChar r1, UChar r2)
   4306 {
   4307    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4308       s390_disasm(ENC3(MNM, FPR, FPR), "cextr", r1, r2);
   4309 
   4310    return emit_RRE(p, 0xb3fc0000, r1, r2);
   4311 }
   4312 
   4313 
   4314 static UChar *
   4315 s390_emit_CFDTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4316 {
   4317    vassert(m4 == 0);
   4318    vassert(s390_host_has_dfp);
   4319    vassert(s390_host_has_fpext);
   4320 
   4321    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4322       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "cfdtr", r1, m3, r2, m4);
   4323 
   4324    return emit_RRF2(p, 0xb9410000, m3, m4, r1, r2);
   4325 }
   4326 
   4327 
   4328 static UChar *
   4329 s390_emit_CFXTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4330 {
   4331    vassert(m4 == 0);
   4332    vassert(s390_host_has_dfp);
   4333    vassert(s390_host_has_fpext);
   4334 
   4335    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4336       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "cfxtr", r1, m3, r2, m4);
   4337 
   4338    return emit_RRF2(p, 0xb9490000, m3, m4, r1, r2);
   4339 }
   4340 
   4341 
   4342 static UChar *
   4343 s390_emit_CGDTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4344 {
   4345    vassert(s390_host_has_dfp);
   4346    vassert(m4 == 0);
   4347    vassert(s390_host_has_fpext || m3 < 1 || m3 > 7);
   4348 
   4349    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4350       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cgdtr", r1, m3, r2);
   4351 
   4352    return emit_RRF2(p, 0xb3e10000, m3, m4, r1, r2);
   4353 }
   4354 
   4355 
   4356 static UChar *
   4357 s390_emit_CGXTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4358 {
   4359    vassert(s390_host_has_dfp);
   4360    vassert(m4 == 0);
   4361    vassert(s390_host_has_fpext || m3 < 1 || m3 > 7);
   4362 
   4363    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4364       s390_disasm(ENC4(MNM, GPR, UINT, FPR), "cgxtr", r1, m3, r2);
   4365 
   4366    return emit_RRF2(p, 0xb3e90000, m3, m4, r1, r2);
   4367 }
   4368 
   4369 
   4370 static UChar *
   4371 s390_emit_CLFDTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4372 {
   4373    vassert(m4 == 0);
   4374    vassert(s390_host_has_dfp);
   4375    vassert(s390_host_has_fpext);
   4376 
   4377    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4378       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clfdtr", r1, m3, r2, m4);
   4379 
   4380    return emit_RRF2(p, 0xb9430000, m3, m4, r1, r2);
   4381 }
   4382 
   4383 
   4384 static UChar *
   4385 s390_emit_CLFXTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4386 {
   4387    vassert(m4 == 0);
   4388    vassert(s390_host_has_dfp);
   4389    vassert(s390_host_has_fpext);
   4390 
   4391    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4392       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clfxtr", r1, m3, r2, m4);
   4393 
   4394    return emit_RRF2(p, 0xb94b0000, m3, m4, r1, r2);
   4395 }
   4396 
   4397 
   4398 static UChar *
   4399 s390_emit_CLGDTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4400 {
   4401    vassert(m4 == 0);
   4402    vassert(s390_host_has_dfp);
   4403    vassert(s390_host_has_fpext);
   4404 
   4405    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4406       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clgdtr", r1, m3, r2, m4);
   4407 
   4408    return emit_RRF2(p, 0xb9420000, m3, m4, r1, r2);
   4409 }
   4410 
   4411 
   4412 static UChar *
   4413 s390_emit_CLGXTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4414 {
   4415    vassert(m4 == 0);
   4416    vassert(s390_host_has_dfp);
   4417    vassert(s390_host_has_fpext);
   4418 
   4419    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4420       s390_disasm(ENC5(MNM, GPR, UINT, FPR, UINT), "clgxtr", r1, m3, r2, m4);
   4421 
   4422    return emit_RRF2(p, 0xb94a0000, m3, m4, r1, r2);
   4423 }
   4424 
   4425 
   4426 static UChar *
   4427 s390_emit_DDTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4428 {
   4429    vassert(s390_host_has_dfp);
   4430    vassert(m4 == 0 || s390_host_has_fpext);
   4431    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4432       if (m4 == 0)
   4433          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "ddtr", r1, r2, r3);
   4434       else
   4435          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "ddtra", r1, r2, r3, m4);
   4436    }
   4437 
   4438    return emit_RRF4(p, 0xb3d10000, r3, m4, r1, r2);
   4439 }
   4440 
   4441 
   4442 static UChar *
   4443 s390_emit_DXTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4444 {
   4445    vassert(s390_host_has_dfp);
   4446    vassert(m4 == 0 || s390_host_has_fpext);
   4447    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4448       if (m4 == 0)
   4449          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "dxtr", r1, r2, r3);
   4450       else
   4451          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "dxtra", r1, r2, r3, m4);
   4452    }
   4453 
   4454    return emit_RRF4(p, 0xb3d90000, r3, m4, r1, r2);
   4455 }
   4456 
   4457 
   4458 static UChar *
   4459 s390_emit_EEDTR(UChar *p, UChar r1, UChar r2)
   4460 {
   4461    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4462       s390_disasm(ENC3(MNM, GPR, FPR), "eedtr", r1, r2);
   4463 
   4464    return emit_RRE(p, 0xb3e50000, r1, r2);
   4465 }
   4466 
   4467 
   4468 static UChar *
   4469 s390_emit_EEXTR(UChar *p, UChar r1, UChar r2)
   4470 {
   4471    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4472       s390_disasm(ENC3(MNM, GPR, FPR), "eextr", r1, r2);
   4473 
   4474    return emit_RRE(p, 0xb3ed0000, r1, r2);
   4475 }
   4476 
   4477 
   4478 static UChar *
   4479 s390_emit_ESDTR(UChar *p, UChar r1, UChar r2)
   4480 {
   4481    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4482       s390_disasm(ENC3(MNM, GPR, FPR), "esdtr", r1, r2);
   4483 
   4484    return emit_RRE(p, 0xb3e70000, r1, r2);
   4485 }
   4486 
   4487 
   4488 static UChar *
   4489 s390_emit_ESXTR(UChar *p, UChar r1, UChar r2)
   4490 {
   4491    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4492       s390_disasm(ENC3(MNM, GPR, FPR), "esxtr", r1, r2);
   4493 
   4494    return emit_RRE(p, 0xb3ef0000, r1, r2);
   4495 }
   4496 
   4497 
   4498 static UChar *
   4499 s390_emit_IEDTR(UChar *p, UChar r3, UChar r1, UChar r2)
   4500 {
   4501    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4502       s390_disasm(ENC4(MNM, FPR, FPR, GPR), "iedtr", r1, r3, r2);
   4503 
   4504    return emit_RRF(p, 0xb3f60000, r3, r1, r2);
   4505 }
   4506 
   4507 
   4508 static UChar *
   4509 s390_emit_IEXTR(UChar *p, UChar r3, UChar r1, UChar r2)
   4510 {
   4511    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4512       s390_disasm(ENC4(MNM, FPR, FPR, GPR), "iextr", r1, r3, r2);
   4513 
   4514    return emit_RRF(p, 0xb3fe0000, r3, r1, r2);
   4515 }
   4516 
   4517 
   4518 static UChar *
   4519 s390_emit_LDETR(UChar *p, UChar m4, UChar r1, UChar r2)
   4520 {
   4521    vassert(s390_host_has_dfp);
   4522    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4523       s390_disasm(ENC4(MNM, FPR, FPR, UINT), "ldetr", r1, r2, m4);
   4524 
   4525    return emit_RRF5(p, 0xb3d40000, m4, r1, r2);
   4526 }
   4527 
   4528 
   4529 static UChar *
   4530 s390_emit_LXDTR(UChar *p, UChar m4, UChar r1, UChar r2)
   4531 {
   4532    vassert(s390_host_has_dfp);
   4533    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4534       s390_disasm(ENC4(MNM, FPR, FPR, UINT), "lxdtr", r1, r2, m4);
   4535 
   4536    return emit_RRF5(p, 0xb3dc0000, m4, r1, r2);
   4537 }
   4538 
   4539 
   4540 static UChar *
   4541 s390_emit_LEDTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4542 {
   4543    vassert(s390_host_has_dfp);
   4544    vassert(m4 == 0);
   4545    vassert(s390_host_has_fpext || m3 < 1 || m3 > 7);
   4546 
   4547    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4548       s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT), "ledtr", r1, m3, r2, m4);
   4549 
   4550    return emit_RRF2(p, 0xb3d50000, m3, m4, r1, r2);
   4551 }
   4552 
   4553 
   4554 static UChar *
   4555 s390_emit_LDXTR(UChar *p, UChar m3, UChar m4, UChar r1, UChar r2)
   4556 {
   4557    vassert(s390_host_has_dfp);
   4558    vassert(m4 == 0);
   4559    vassert(s390_host_has_fpext || m3 < 1 || m3 > 7);
   4560 
   4561    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4562       s390_disasm(ENC5(MNM, FPR, UINT, FPR, UINT), "ldxtr", r1, m3, r2, m4);
   4563 
   4564    return emit_RRF2(p, 0xb3dd0000, m3, m4, r1, r2);
   4565 }
   4566 
   4567 
   4568 static UChar *
   4569 s390_emit_MDTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4570 {
   4571    vassert(s390_host_has_dfp);
   4572    vassert(m4 == 0 || s390_host_has_fpext);
   4573    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4574       if (m4 == 0)
   4575          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "mdtr", r1, r2, r3);
   4576       else
   4577          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "mdtra", r1, r2, r3, m4);
   4578    }
   4579 
   4580    return emit_RRF4(p, 0xb3d00000, r3, m4, r1, r2);
   4581 }
   4582 
   4583 
   4584 static UChar *
   4585 s390_emit_MXTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4586 {
   4587    vassert(s390_host_has_dfp);
   4588    vassert(m4 == 0 || s390_host_has_fpext);
   4589    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4590       if (m4 == 0)
   4591          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "mxtr", r1, r2, r3);
   4592       else
   4593          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "mxtra", r1, r2, r3, m4);
   4594    }
   4595 
   4596    return emit_RRF4(p, 0xb3d80000, r3, m4, r1, r2);
   4597 }
   4598 
   4599 
   4600 static UChar *
   4601 emit_E(UChar *p, UInt op)
   4602 {
   4603    ULong the_insn = op;
   4604 
   4605    return emit_2bytes(p, the_insn);
   4606 }
   4607 
   4608 
   4609 static UChar *
   4610 s390_emit_PFPO(UChar *p)
   4611 {
   4612    vassert(s390_host_has_pfpo);
   4613    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4614       s390_disasm(ENC1(MNM), "pfpo");
   4615    }
   4616 
   4617    return emit_E(p, 0x010a);
   4618 }
   4619 
   4620 
   4621 static UChar *
   4622 s390_emit_QADTR(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4623 {
   4624    vassert(s390_host_has_dfp);
   4625    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4626       s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "qadtr", r1, r3, r2, m4);
   4627 
   4628    return emit_RRF4(p, 0xb3f50000, r3, m4, r1, r2);
   4629 }
   4630 
   4631 
   4632 static UChar *
   4633 s390_emit_QAXTR(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4634 {
   4635    vassert(s390_host_has_dfp);
   4636    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4637       s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "qaxtr", r1, r3, r2, m4);
   4638 
   4639    return emit_RRF4(p, 0xb3fd0000, r3, m4, r1, r2);
   4640 }
   4641 
   4642 
   4643 static UChar *
   4644 s390_emit_RRDTR(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4645 {
   4646    vassert(s390_host_has_dfp);
   4647    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4648       s390_disasm(ENC5(MNM, FPR, FPR, GPR, UINT), "rrdtr", r1, r3, r2, m4);
   4649 
   4650    return emit_RRF4(p, 0xb3f70000, r3, m4, r1, r2);
   4651 }
   4652 
   4653 
   4654 static UChar *
   4655 s390_emit_RRXTR(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4656 {
   4657    vassert(s390_host_has_dfp);
   4658    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4659       s390_disasm(ENC5(MNM, FPR, FPR, GPR, UINT), "rrxtr", r1, r3, r2, m4);
   4660 
   4661    return emit_RRF4(p, 0xb3ff0000, r3, m4, r1, r2);
   4662 }
   4663 
   4664 
   4665 static UChar *
   4666 s390_emit_SDTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4667 {
   4668    vassert(s390_host_has_dfp);
   4669    vassert(m4 == 0 || s390_host_has_fpext);
   4670    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4671       if (m4 == 0)
   4672          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "sdtr", r1, r2, r3);
   4673       else
   4674          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "sdtra", r1, r2, r3, m4);
   4675    }
   4676 
   4677    return emit_RRF4(p, 0xb3d30000, r3, m4, r1, r2);
   4678 }
   4679 
   4680 
   4681 static UChar *
   4682 s390_emit_SXTRA(UChar *p, UChar r3, UChar m4, UChar r1, UChar r2)
   4683 {
   4684    vassert(s390_host_has_dfp);
   4685    vassert(m4 == 0 || s390_host_has_fpext);
   4686    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM)) {
   4687       if (m4 == 0)
   4688          s390_disasm(ENC4(MNM, FPR, FPR, FPR), "sxtr", r1, r2, r3);
   4689       else
   4690          s390_disasm(ENC5(MNM, FPR, FPR, FPR, UINT), "sxtra", r1, r2, r3, m4);
   4691    }
   4692 
   4693    return emit_RRF4(p, 0xb3db0000, r3, m4, r1, r2);
   4694 }
   4695 
   4696 
   4697 static UChar *
   4698 s390_emit_SLDT(UChar *p, UChar r3, UChar r1, UChar r2)
   4699 {
   4700    vassert(s390_host_has_dfp);
   4701    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4702       s390_disasm(ENC4(MNM, FPR, FPR, UDXB), "sldt", r1, r3, 0, 0, r2);
   4703 
   4704    return emit_RXF(p, 0xED0000000040ULL, r3, 0, r2, 0, r1);
   4705 }
   4706 
   4707 
   4708 static UChar *
   4709 s390_emit_SLXT(UChar *p, UChar r3, UChar r1, UChar r2)
   4710 {
   4711    vassert(s390_host_has_dfp);
   4712    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4713       s390_disasm(ENC4(MNM, FPR, FPR, UDXB), "slxt", r1, r3, 0, 0, r2);
   4714 
   4715    return emit_RXF(p, 0xED0000000048ULL, r3, 0, r2, 0, r1);
   4716 }
   4717 
   4718 
   4719 static UChar *
   4720 s390_emit_SRDT(UChar *p, UChar r3, UChar r1, UChar r2)
   4721 {
   4722    vassert(s390_host_has_dfp);
   4723    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4724       s390_disasm(ENC4(MNM, FPR, FPR, UDXB), "srdt", r1, r3, 0, 0, r2);
   4725 
   4726    return emit_RXF(p, 0xED0000000041ULL, r3, 0, r2, 0, r1);
   4727 }
   4728 
   4729 
   4730 static UChar *
   4731 s390_emit_SRXT(UChar *p, UChar r3, UChar r1, UChar r2)
   4732 {
   4733    vassert(s390_host_has_dfp);
   4734    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4735       s390_disasm(ENC4(MNM, FPR, FPR, UDXB), "srxt", r1, r3, 0, 0, r2);
   4736 
   4737    return emit_RXF(p, 0xED0000000049ULL, r3, 0, r2, 0, r1);
   4738 }
   4739 
   4740 
   4741 static UChar *
   4742 s390_emit_LOCGR(UChar *p, UChar m3, UChar r1, UChar r2)
   4743 {
   4744    vassert(s390_host_has_lsc);
   4745    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4746       s390_disasm(ENC4(MNM, GPR, GPR, UINT), "locgr", r1, r2, m3);
   4747 
   4748    return emit_RRF3(p, 0xb9e20000, m3, r1, r2);
   4749 }
   4750 
   4751 
   4752 static UChar *
   4753 s390_emit_LOC(UChar *p, UChar r1, UChar m3, UChar b2, UShort dl2, UChar dh2)
   4754 {
   4755    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4756       s390_disasm(ENC4(MNM, GPR, UINT, SDXB), "loc", r1, m3, dh2, dl2, 0, b2);
   4757 
   4758    return emit_RSY(p, 0xeb00000000f2ULL, r1, m3, b2, dl2, dh2);
   4759 }
   4760 
   4761 
   4762 static UChar *
   4763 s390_emit_LOCG(UChar *p, UChar r1, UChar m3, UChar b2, UShort dl2, UChar dh2)
   4764 {
   4765    if (UNLIKELY(vex_traceflags & VEX_TRACE_ASM))
   4766       s390_disasm(ENC4(MNM, GPR, UINT, SDXB), "locg", r1, m3, dh2, dl2, 0, b2);
   4767 
   4768    return emit_RSY(p, 0xeb00000000e2ULL, r1, m3, b2, dl2, dh2);
   4769 }
   4770 
   4771 
   4772 /* Provide a symbolic name for register "R0" */
   4773 #define R0 0
   4774 
   4775 /* Split up a 20-bit displacement into its high and low piece
   4776    suitable for passing as function arguments */
   4777 #define DISP20(d) (((UInt)d) & 0xFFF), ((((UInt)d) >> 12) & 0xFF)
   4778 
   4779 /*---------------------------------------------------------------*/
   4780 /*--- Helper functions                                        ---*/
   4781 /*---------------------------------------------------------------*/
   4782 
   4783 static __inline__ Bool
   4784 uint_fits_signed_16bit(UInt val)
   4785 {
   4786    UInt v = val & 0xFFFFu;
   4787 
   4788    /* sign extend */
   4789    v = (Int)(v << 16) >> 16;
   4790 
   4791    return val == v;
   4792 }
   4793 
   4794 
   4795 static __inline__ Bool
   4796 ulong_fits_signed_16bit(ULong val)
   4797 {
   4798    ULong v = val & 0xFFFFu;
   4799 
   4800    /* sign extend */
   4801    v = (Long)(v << 48) >> 48;
   4802 
   4803    return val == v;
   4804 }
   4805 
   4806 
   4807 static __inline__ Bool
   4808 ulong_fits_signed_32bit(ULong val)
   4809 {
   4810    ULong v = val & 0xFFFFFFFFu;
   4811 
   4812    /* sign extend */
   4813    v = (Long)(v << 32) >> 32;
   4814 
   4815    return val == v;
   4816 }
   4817 
   4818 
   4819 static __inline__ Bool
   4820 ulong_fits_unsigned_32bit(ULong val)
   4821 {
   4822    return (val & 0xFFFFFFFFu) == val;
   4823 }
   4824 
   4825 
   4826 /* Load a 64-bit immediate VAL into register REG. */
   4827 static UChar *
   4828 s390_emit_load_64imm(UChar *p, UChar reg, ULong val)
   4829 {
   4830    if (ulong_fits_signed_16bit(val)) {
   4831       return s390_emit_LGHI(p, reg, val);
   4832    }
   4833 
   4834    if (s390_host_has_eimm) {
   4835       if (ulong_fits_unsigned_32bit(val)) {
   4836          return s390_emit_LLILF(p, reg, val);
   4837       }
   4838       if (ulong_fits_signed_32bit(val)) {
   4839          /* LGFI's sign extension will recreate the correct 64-bit value */
   4840          return s390_emit_LGFI(p, reg, val);
   4841       }
   4842       /* Do it in two steps: upper half [0:31] and lower half [32:63] */
   4843       p =  s390_emit_IIHF(p, reg, val >> 32);
   4844       return s390_emit_IILF(p, reg, val & 0xFFFFFFFF);
   4845    }
   4846 
   4847    /* Fall back */
   4848    if (ulong_fits_unsigned_32bit(val)) {
   4849       p = s390_emit_LLILH(p, reg, (val >> 16) & 0xFFFF); /* sets val[32:47]
   4850                                                             val[0:31] = 0 */
   4851       p = s390_emit_IILL(p, reg, val & 0xFFFF);          /* sets val[48:63] */
   4852       return p;
   4853    }
   4854 
   4855    p = s390_emit_IIHH(p, reg, (val >> 48) & 0xFFFF);
   4856    p = s390_emit_IIHL(p, reg, (val >> 32) & 0xFFFF);
   4857    p = s390_emit_IILH(p, reg, (val >> 16) & 0xFFFF);
   4858    p = s390_emit_IILL(p, reg, val & 0xFFFF);
   4859 
   4860    return p;
   4861 }
   4862 
   4863 /* Load a 32-bit immediate VAL into register REG. */
   4864 static UChar *
   4865 s390_emit_load_32imm(UChar *p, UChar reg, UInt val)
   4866 {
   4867    if (uint_fits_signed_16bit(val)) {
   4868       /* LHI's sign extension will recreate the correct 32-bit value */
   4869       return s390_emit_LHI(p, reg, val);
   4870    }
   4871    if (s390_host_has_eimm) {
   4872       return s390_emit_IILF(p, reg, val);
   4873    }
   4874    /* val[0:15]  --> (val >> 16) & 0xFFFF
   4875       val[16:31] --> val & 0xFFFF */
   4876    p = s390_emit_IILH(p, reg, (val >> 16) & 0xFFFF);
   4877    return s390_emit_IILL(p, reg, val & 0xFFFF);
   4878 }
   4879 
   4880 /*------------------------------------------------------------*/
   4881 /*--- Wrapper functions                                    ---*/
   4882 /*------------------------------------------------------------*/
   4883 
   4884 /* r1[32:63],r1+1[32:63] = r1+1[32:63] * memory[op2addr][0:31] */
   4885 static UChar *
   4886 s390_emit_MFYw(UChar *p, UChar r1, UChar x, UChar b,  UShort dl, UChar dh)
   4887 {
   4888    if (s390_host_has_gie) {
   4889       return s390_emit_MFY(p, r1, x, b, dl, dh);
   4890    }
   4891 
   4892    /* Load from memory into R0, then MULTIPLY with R1 */
   4893    p = s390_emit_LY(p, R0, x, b, dl, dh);
   4894    return s390_emit_MR(p, r1, R0);
   4895 }
   4896 
   4897 /* r1[32:63] = r1[32:63] * memory[op2addr][0:15] */
   4898 static UChar *
   4899 s390_emit_MHYw(UChar *p, UChar r1, UChar x, UChar b,  UShort dl, UChar dh)
   4900 {
   4901    if (s390_host_has_gie) {
   4902       return s390_emit_MHY(p, r1, x, b, dl, dh);
   4903    }
   4904 
   4905    /* Load from memory into R0, then MULTIPLY with R1 */
   4906    p = s390_emit_LHY(p, R0, x, b, dl, dh);
   4907    return s390_emit_MSR(p, r1, R0);
   4908 }
   4909 
   4910 /* r1[32:63] = r1[32:63] * i2 */
   4911 static UChar *
   4912 s390_emit_MSFIw(UChar *p, UChar r1, UInt i2)
   4913 {
   4914    if (s390_host_has_gie) {
   4915       return s390_emit_MSFI(p, r1, i2);
   4916    }
   4917 
   4918    /* Load I2 into R0; then MULTIPLY R0 with R1 */
   4919    p = s390_emit_load_32imm(p, R0, i2);
   4920    return s390_emit_MSR(p, r1, R0);
   4921 }
   4922 
   4923 
   4924 /* r1[32:63] = r1[32:63] & i2 */
   4925 static UChar *
   4926 s390_emit_NILFw(UChar *p, UChar r1, UInt i2)
   4927 {
   4928    if (s390_host_has_eimm) {
   4929       return s390_emit_NILF(p, r1, i2);
   4930    }
   4931 
   4932    /* Load I2 into R0; then AND R0 with R1 */
   4933    p = s390_emit_load_32imm(p, R0, i2);
   4934    return s390_emit_NR(p, r1, R0);
   4935 }
   4936 
   4937 
   4938 /* r1[32:63] = r1[32:63] | i2 */
   4939 static UChar *
   4940 s390_emit_OILFw(UChar *p, UChar r1, UInt i2)
   4941 {
   4942    if (s390_host_has_eimm) {
   4943       return s390_emit_OILF(p, r1, i2);
   4944    }
   4945 
   4946    /* Load I2 into R0; then AND R0 with R1 */
   4947    p = s390_emit_load_32imm(p, R0, i2);
   4948    return s390_emit_OR(p, r1, R0);
   4949 }
   4950 
   4951 
   4952 /* r1[32:63] = r1[32:63] ^ i2 */
   4953 static UChar *
   4954 s390_emit_XILFw(UChar *p, UChar r1, UInt i2)
   4955 {
   4956    if (s390_host_has_eimm) {
   4957       return s390_emit_XILF(p, r1, i2);
   4958    }
   4959 
   4960    /* Load I2 into R0; then AND R0 with R1 */
   4961    p = s390_emit_load_32imm(p, R0, i2);
   4962    return s390_emit_XR(p, r1, R0);
   4963 }
   4964 
   4965 
   4966 /*  r1[32:63] = sign_extend(r2[56:63]) */
   4967 static UChar *
   4968 s390_emit_LBRw(UChar *p, UChar r1, UChar r2)
   4969 {
   4970    if (s390_host_has_eimm) {
   4971       return s390_emit_LBR(p, r1, r2);
   4972    }
   4973 
   4974    p = s390_emit_LR(p, r1, r2);               /* r1 = r2 */
   4975    p = s390_emit_SLL(p, r1, R0, 24);          /* r1 = r1 << 24  */
   4976    return s390_emit_SRA(p, r1, R0, 24);       /* r1 = r1 >>a 24 */
   4977 }
   4978 
   4979 
   4980 /*  r1[0:63] = sign_extend(r2[56:63]) */
   4981 static UChar *
   4982 s390_emit_LGBRw(UChar *p, UChar r1, UChar r2)
   4983 {
   4984    if (s390_host_has_eimm) {
   4985       return s390_emit_LGBR(p, r1, r2);
   4986    }
   4987 
   4988    p = s390_emit_LR(p, r1, r2);                       /* r1 = r2 */
   4989    p = s390_emit_SLLG(p, r1, r1, R0, DISP20(56));     /* r1 = r1 << 56  */
   4990    return s390_emit_SRAG(p, r1, r1, R0, DISP20(56));  /* r1 = r1 >>a 56 */
   4991 }
   4992 
   4993 
   4994 /* r1[32:63] = sign_extend(r2[48:63]) */
   4995 static UChar *
   4996 s390_emit_LHRw(UChar *p, UChar r1, UChar r2)
   4997 {
   4998    if (s390_host_has_eimm) {
   4999       return s390_emit_LHR(p, r1, r2);
   5000    }
   5001 
   5002    p = s390_emit_LR(p, r1, r2);               /* r1 = r2 */
   5003    p = s390_emit_SLL(p, r1, R0, 16);          /* r1 = r1 << 16  */
   5004    return s390_emit_SRA(p, r1, R0, 16);       /* r1 = r1 >>a 16 */
   5005 }
   5006 
   5007 
   5008 /* r1[0:63] = sign_extend(r2[48:63]) */
   5009 static UChar *
   5010 s390_emit_LGHRw(UChar *p, UChar r1, UChar r2)
   5011 {
   5012    if (s390_host_has_eimm) {
   5013       return s390_emit_LGHR(p, r1, r2);
   5014    }
   5015 
   5016    p = s390_emit_LR(p, r1, r2);               /* r1 = r2 */
   5017    p = s390_emit_SLLG(p, r1, r1, R0, DISP20(48));     /* r1 = r1 << 48  */
   5018    return s390_emit_SRAG(p, r1, r1, R0, DISP20(48));  /* r1 = r1 >>a 48 */
   5019 }
   5020 
   5021 
   5022 /* r1[0:63] = sign_extend(i2) */
   5023 static UChar *
   5024 s390_emit_LGFIw(UChar *p, UChar r1, UInt i2)
   5025 {
   5026    if (s390_host_has_eimm) {
   5027       return s390_emit_LGFI(p, r1, i2);
   5028    }
   5029 
   5030    p = s390_emit_load_32imm(p, R0, i2);
   5031    return s390_emit_LGFR(p, r1, R0);
   5032 }
   5033 
   5034 
   5035 /* r1[32:63] = zero_extend($r2[56:63]) */
   5036 static UChar *
   5037 s390_emit_LLCRw(UChar *p, UChar r1, UChar r2)
   5038 {
   5039    if (s390_host_has_eimm) {
   5040       return s390_emit_LLCR(p, r1, r2);
   5041    }
   5042 
   5043    p = s390_emit_LR(p, r1, r2);
   5044    p = s390_emit_LHI(p, R0, 0xFF);
   5045    return s390_emit_NR(p, r1, R0);
   5046 }
   5047 
   5048 
   5049 /* r1[0:63] = zero_extend($r2[56:63]) */
   5050 static UChar *
   5051 s390_emit_LLGCRw(UChar *p, UChar r1, UChar r2)
   5052 {
   5053    if (s390_host_has_eimm) {
   5054       return s390_emit_LLGCR(p, r1, r2);
   5055    }
   5056 
   5057    p = s390_emit_LR(p, r1, r2);
   5058    p = s390_emit_LLILL(p, R0, 0xFF);
   5059    return s390_emit_NGR(p, r1, R0);
   5060 }
   5061 
   5062 
   5063 /* r1[32:63] = zero_extend(r2[48:63]) */
   5064 static UChar *
   5065 s390_emit_LLHRw(UChar *p, UChar r1, UChar r2)
   5066 {
   5067    if (s390_host_has_eimm) {
   5068       return s390_emit_LLHR(p, r1, r2);
   5069    }
   5070 
   5071    p = s390_emit_LR(p, r1, r2);
   5072    p = s390_emit_LLILL(p, R0, 0xFFFF);
   5073    return s390_emit_NR(p, r1, R0);
   5074 }
   5075 
   5076 
   5077 /* r1[0:63] = zero_extend(r2[48:63]) */
   5078 static UChar *
   5079 s390_emit_LLGHRw(UChar *p, UChar r1, UChar r2)
   5080 {
   5081    if (s390_host_has_eimm) {
   5082       return s390_emit_LLGHR(p, r1, r2);
   5083    }
   5084 
   5085    p = s390_emit_LR(p, r1, r2);
   5086    p = s390_emit_LLILL(p, R0, 0xFFFF);
   5087    return s390_emit_NGR(p, r1, R0);
   5088 }
   5089 
   5090 
   5091 /* r1[32:63] = zero_extend(mem[op2addr][0:7]) */
   5092 static UChar *
   5093 s390_emit_LLCw(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl, UChar dh)
   5094 {
   5095    if (s390_host_has_eimm) {
   5096       return s390_emit_LLC(p, r1, x2, b2, dl, dh);
   5097    }
   5098 
   5099    if (dh == 0) {
   5100       p = s390_emit_IC(p, r1, x2, b2, dl);
   5101    } else {
   5102       p = s390_emit_ICY(p, r1, x2, b2, dl, dh);
   5103    }
   5104    p = s390_emit_LLILL(p, R0, 0xFF);
   5105    return s390_emit_NR(p, r1, R0);
   5106 }
   5107 
   5108 
   5109 /* r1[32:63] = zero_extend(mem[op2addr][0:15]) */
   5110 static UChar *
   5111 s390_emit_LLHw(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl, UChar dh)
   5112 {
   5113    if (s390_host_has_eimm) {
   5114       return s390_emit_LLH(p, r1, x2, b2, dl, dh);
   5115    }
   5116 
   5117    p = s390_emit_LLGH(p, r1, x2, b2, dl, dh);
   5118    p = s390_emit_LLILL(p, R0, 0xFFFF);
   5119    return s390_emit_NR(p, r1, R0);
   5120 }
   5121 
   5122 
   5123 /* r1[0:63] = zero_extend(i2) */
   5124 static UChar *
   5125 s390_emit_LLILFw(UChar *p, UChar r1, UInt i2)
   5126 {
   5127    if (s390_host_has_eimm) {
   5128       return s390_emit_LLILF(p, r1, i2);
   5129    }
   5130 
   5131    p = s390_emit_LLILH(p, r1, (i2 >> 16) & 0xFFFF);  /* i2[0:15] */
   5132    return s390_emit_OILL(p, r1, i2 & 0xFFFF);
   5133 }
   5134 
   5135 
   5136 /* r1[32:63] = r1[32:63] + i2 */
   5137 static UChar *
   5138 s390_emit_AFIw(UChar *p, UChar r1, UInt i2)
   5139 {
   5140    if (s390_host_has_eimm) {
   5141       return s390_emit_AFI(p, r1, i2);
   5142    }
   5143    /* Load 32 bit immediate to R0 then add */
   5144    p = s390_emit_load_32imm(p, R0, i2);
   5145    return s390_emit_AR(p, r1, R0);
   5146 }
   5147 
   5148 
   5149 /* r1[32:63] = r1[32:63] - i2 */
   5150 static UChar *
   5151 s390_emit_SLFIw(UChar *p, UChar r1, UInt i2)
   5152 {
   5153    if (s390_host_has_eimm) {
   5154       return s390_emit_SLFI(p, r1, i2);
   5155    }
   5156 
   5157    /* Load 32 bit immediate to R0 then subtract */
   5158    p = s390_emit_load_32imm(p, R0, i2);
   5159    return s390_emit_SR(p, r1, R0);
   5160 }
   5161 
   5162 
   5163 /* r1[0:63] = r1[0:63] - zero_extend(i2) */
   5164 static UChar *
   5165 s390_emit_SLGFIw(UChar *p, UChar r1, UInt i2)
   5166 {
   5167    if (s390_host_has_eimm) {
   5168       return s390_emit_SLGFI(p, r1, i2);
   5169    }
   5170 
   5171    /* Load zero-extended 32 bit immediate to R0 then subtract */
   5172    p = s390_emit_load_64imm(p, R0, i2);
   5173    return s390_emit_SGR(p, r1, R0);
   5174 }
   5175 
   5176 
   5177 static UChar *
   5178 s390_emit_LTw(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl, UChar dh)
   5179 {
   5180    if (s390_host_has_eimm) {
   5181       return s390_emit_LT(p, r1, x2, b2, dl, dh);
   5182    }
   5183    /* Load 32 bit from memory to R0 then compare */
   5184    if (dh == 0) {
   5185       p = s390_emit_L(p, R0, x2, b2, dl);
   5186    } else {
   5187       p = s390_emit_LY(p, R0, x2, b2, dl, dh);
   5188    }
   5189    return s390_emit_LTR(p, r1, R0);
   5190 }
   5191 
   5192 
   5193 static UChar *
   5194 s390_emit_LTGw(UChar *p, UChar r1, UChar x2, UChar b2, UShort dl, UChar dh)
   5195 {
   5196    if (s390_host_has_eimm) {
   5197       return s390_emit_LTG(p, r1, x2, b2, dl, dh);
   5198    }
   5199    /* Load 64 bit from memory to R0 then compare */
   5200    p = s390_emit_LG(p, R0, x2, b2, dl, dh);
   5201    return s390_emit_LTGR(p, r1, R0);
   5202 }
   5203 
   5204 
   5205 static UChar *
   5206 s390_emit_CFIw(UChar *p, UChar r1, UInt i2)
   5207 {
   5208    if (s390_host_has_eimm) {
   5209       return s390_emit_CFI(p, r1, i2);
   5210    }
   5211    /* Load 32 bit immediate to R0 then compare */
   5212    p = s390_emit_load_32imm(p, R0, i2);
   5213    return s390_emit_CR(p, r1, R0);
   5214 }
   5215 
   5216 
   5217 static UChar *
   5218 s390_emit_CLFIw(UChar *p, UChar r1, UInt i2)
   5219 {
   5220    if (s390_host_has_eimm) {
   5221       return s390_emit_CLFI(p, r1, i2);
   5222    }
   5223    /* Load 32 bit immediate to R0 then compare */
   5224    p = s390_emit_load_32imm(p, R0, i2);
   5225    return s390_emit_CLR(p, r1, R0);
   5226 }
   5227 
   5228 
   5229 static UChar *
   5230 s390_emit_LGDRw(UChar *p, UChar r1, UChar r2)
   5231 {
   5232    if (s390_host_has_fgx) {
   5233       return s390_emit_LGDR(p, r1, r2);
   5234    }
   5235 
   5236    /* Store the FPR at memory[sp - 8]. This is safe because SP grows towards
   5237       smaller addresses and is 8-byte aligned. Then load the GPR from that
   5238       memory location/ */
   5239    p = s390_emit_STDY(p, r2, R0, S390_REGNO_STACK_POINTER, DISP20(-8));
   5240    return s390_emit_LG(p, r1, R0, S390_REGNO_STACK_POINTER, DISP20(-8));
   5241 }
   5242 
   5243 
   5244 static UChar *
   5245 s390_emit_LDGRw(UChar *p, UChar r1, UChar r2)
   5246 {
   5247    if (s390_host_has_fgx) {
   5248       return s390_emit_LDGR(p, r1, r2);
   5249    }
   5250 
   5251    /* Store the GPR at memory[sp - 8]. This is safe because SP grows towards
   5252       smaller addresses and is 8-byte aligned. Then load the FPR from that
   5253       memory location/ */
   5254    p = s390_emit_STG(p, r2, R0, S390_REGNO_STACK_POINTER, DISP20(-8));
   5255    return s390_emit_LDY(p, r1, R0, S390_REGNO_STACK_POINTER, DISP20(-8));
   5256 }
   5257 
   5258 
   5259 /*---------------------------------------------------------------*/
   5260 /*--- Constructors for the various s390_insn kinds            ---*/
   5261 /*---------------------------------------------------------------*/
   5262 
   5263 s390_insn *
   5264 s390_insn_load(UChar size, HReg dst, s390_amode *src)
   5265 {
   5266    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5267 
   5268    insn->tag  = S390_INSN_LOAD;
   5269    insn->size = size;
   5270    insn->variant.load.src  = src;
   5271    insn->variant.load.dst  = dst;
   5272 
   5273    vassert(size == 1 || size == 2 || size == 4 || size == 8);
   5274 
   5275    return insn;
   5276 }
   5277 
   5278 
   5279 s390_insn *
   5280 s390_insn_store(UChar size, s390_amode *dst, HReg src)
   5281 {
   5282    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5283 
   5284    insn->tag  = S390_INSN_STORE;
   5285    insn->size = size;
   5286    insn->variant.store.src  = src;
   5287    insn->variant.store.dst  = dst;
   5288 
   5289    vassert(size == 1 || size == 2 || size == 4 || size == 8);
   5290 
   5291    return insn;
   5292 }
   5293 
   5294 
   5295 s390_insn *
   5296 s390_insn_move(UChar size, HReg dst, HReg src)
   5297 {
   5298    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5299 
   5300    insn->tag  = S390_INSN_MOVE;
   5301    insn->size = size;
   5302    insn->variant.move.src  = src;
   5303    insn->variant.move.dst  = dst;
   5304 
   5305    vassert(size == 1 || size == 2 || size == 4 || size == 8);
   5306 
   5307    return insn;
   5308 }
   5309 
   5310 
   5311 s390_insn *
   5312 s390_insn_memcpy(UChar size, s390_amode *dst, s390_amode *src)
   5313 {
   5314    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5315 
   5316    /* This insn will be mapped to MVC which requires base register
   5317       plus 12-bit displacement */
   5318    vassert(src->tag == S390_AMODE_B12);
   5319    vassert(dst->tag == S390_AMODE_B12);
   5320 
   5321    insn->tag  = S390_INSN_MEMCPY;
   5322    insn->size = size;
   5323    insn->variant.memcpy.src = src;
   5324    insn->variant.memcpy.dst = dst;
   5325 
   5326    vassert(size == 1 || size == 2 || size == 4 || size == 8);
   5327 
   5328    return insn;
   5329 }
   5330 
   5331 
   5332 s390_insn *
   5333 s390_insn_cond_move(UChar size, s390_cc_t cond, HReg dst, s390_opnd_RMI src)
   5334 {
   5335    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5336 
   5337    insn->tag  = S390_INSN_COND_MOVE;
   5338    insn->size = size;
   5339    insn->variant.cond_move.cond = cond;
   5340    insn->variant.cond_move.src  = src;
   5341    insn->variant.cond_move.dst  = dst;
   5342 
   5343    vassert(size == 1 || size == 2 || size == 4 || size == 8);
   5344 
   5345    return insn;
   5346 }
   5347 
   5348 
   5349 s390_insn *
   5350 s390_insn_load_immediate(UChar size, HReg dst, ULong value)
   5351 {
   5352    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5353 
   5354    insn->tag  = S390_INSN_LOAD_IMMEDIATE;
   5355    insn->size = size;
   5356    insn->variant.load_immediate.dst   = dst;
   5357    insn->variant.load_immediate.value = value;
   5358 
   5359    return insn;
   5360 }
   5361 
   5362 
   5363 s390_insn *
   5364 s390_insn_alu(UChar size, s390_alu_t tag, HReg dst, s390_opnd_RMI op2)
   5365 {
   5366    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5367 
   5368    insn->tag  = S390_INSN_ALU;
   5369    insn->size = size;
   5370    insn->variant.alu.tag = tag;
   5371    insn->variant.alu.dst = dst;
   5372    insn->variant.alu.op2 = op2;
   5373 
   5374    return insn;
   5375 }
   5376 
   5377 
   5378 s390_insn *
   5379 s390_insn_mul(UChar size, HReg dst_hi, HReg dst_lo, s390_opnd_RMI op2,
   5380               Bool signed_multiply)
   5381 {
   5382    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5383 
   5384    vassert(! hregIsVirtual(dst_hi));
   5385    vassert(! hregIsVirtual(dst_lo));
   5386 
   5387    insn->tag  = signed_multiply ? S390_INSN_SMUL : S390_INSN_UMUL;
   5388    insn->size = size;
   5389    insn->variant.mul.dst_hi = dst_hi;
   5390    insn->variant.mul.dst_lo = dst_lo;
   5391    insn->variant.mul.op2 = op2;
   5392 
   5393    return insn;
   5394 }
   5395 
   5396 
   5397 s390_insn *
   5398 s390_insn_div(UChar size, HReg op1_hi, HReg op1_lo, s390_opnd_RMI op2,
   5399               Bool signed_divide)
   5400 {
   5401    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5402 
   5403    vassert(size == 4 || size == 8);
   5404    vassert(! hregIsVirtual(op1_hi));
   5405    vassert(! hregIsVirtual(op1_lo));
   5406 
   5407    insn->tag  = signed_divide ? S390_INSN_SDIV : S390_INSN_UDIV;
   5408    insn->size = size;
   5409    insn->variant.div.op1_hi = op1_hi;
   5410    insn->variant.div.op1_lo = op1_lo;
   5411    insn->variant.div.op2 = op2;
   5412 
   5413    return insn;
   5414 }
   5415 
   5416 
   5417 s390_insn *
   5418 s390_insn_divs(UChar size, HReg rem, HReg op1, s390_opnd_RMI op2)
   5419 {
   5420    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5421 
   5422    vassert(size == 8);
   5423    vassert(! hregIsVirtual(op1));
   5424    vassert(! hregIsVirtual(rem));
   5425 
   5426    insn->tag  = S390_INSN_DIVS;
   5427    insn->size = size;
   5428    insn->variant.divs.rem = rem;   /* remainder */
   5429    insn->variant.divs.op1 = op1;   /* also quotient */
   5430    insn->variant.divs.op2 = op2;
   5431 
   5432    return insn;
   5433 }
   5434 
   5435 
   5436 s390_insn *
   5437 s390_insn_clz(UChar size, HReg num_bits, HReg clobber, s390_opnd_RMI src)
   5438 {
   5439    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5440 
   5441    vassert(size == 8);
   5442    vassert(! hregIsVirtual(num_bits));
   5443    vassert(! hregIsVirtual(clobber));
   5444 
   5445    insn->tag  = S390_INSN_CLZ;
   5446    insn->size = size;
   5447    insn->variant.clz.num_bits = num_bits;
   5448    insn->variant.clz.clobber  = clobber;
   5449    insn->variant.clz.src = src;
   5450 
   5451    return insn;
   5452 }
   5453 
   5454 
   5455 s390_insn *
   5456 s390_insn_unop(UChar size, s390_unop_t tag, HReg dst, s390_opnd_RMI opnd)
   5457 {
   5458    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5459 
   5460    insn->tag  = S390_INSN_UNOP;
   5461    insn->size = size;
   5462    insn->variant.unop.tag = tag;
   5463    insn->variant.unop.dst = dst;
   5464    insn->variant.unop.src = opnd;
   5465 
   5466    return insn;
   5467 }
   5468 
   5469 
   5470 s390_insn *
   5471 s390_insn_test(UChar size, s390_opnd_RMI src)
   5472 {
   5473    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5474 
   5475    vassert(size == 4 || size == 8);
   5476 
   5477    insn->tag  = S390_INSN_TEST;
   5478    insn->size = size;
   5479    insn->variant.test.src = src;
   5480 
   5481    return insn;
   5482 }
   5483 
   5484 
   5485 s390_insn *
   5486 s390_insn_cc2bool(HReg dst, s390_cc_t cond)
   5487 {
   5488    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5489 
   5490    insn->tag  = S390_INSN_CC2BOOL;
   5491    insn->size = 0;   /* does not matter */
   5492    insn->variant.cc2bool.cond = cond;
   5493    insn->variant.cc2bool.dst  = dst;
   5494 
   5495    return insn;
   5496 }
   5497 
   5498 
   5499 s390_insn *
   5500 s390_insn_cas(UChar size, HReg op1, s390_amode *op2, HReg op3, HReg old_mem)
   5501 {
   5502    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5503 
   5504    vassert(size == 4 || size == 8);
   5505    vassert(hregNumber(op2->x) == 0);
   5506    vassert(op2->tag == S390_AMODE_B12 || op2->tag == S390_AMODE_B20);
   5507 
   5508    insn->tag  = S390_INSN_CAS;
   5509    insn->size = size;
   5510    insn->variant.cas.op1 = op1;
   5511    insn->variant.cas.op2 = op2;
   5512    insn->variant.cas.op3 = op3;
   5513    insn->variant.cas.old_mem = old_mem;
   5514 
   5515    return insn;
   5516 }
   5517 
   5518 
   5519 s390_insn *
   5520 s390_insn_cdas(UChar size, HReg op1_high, HReg op1_low, s390_amode *op2,
   5521                HReg op3_high, HReg op3_low, HReg old_mem_high, HReg old_mem_low,
   5522                HReg scratch)
   5523 {
   5524    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5525    s390_cdas *cdas = LibVEX_Alloc_inline(sizeof(s390_cdas));
   5526 
   5527    vassert(size == 4 || size == 8);
   5528    vassert(hregNumber(op2->x) == 0);
   5529    vassert(hregNumber(scratch) == 1);  /* r0,r1 used as scratch reg pair */
   5530    vassert(op2->tag == S390_AMODE_B12 || op2->tag == S390_AMODE_B20);
   5531 
   5532    insn->tag  = S390_INSN_CDAS;
   5533    insn->size = size;
   5534    insn->variant.cdas.details = cdas;
   5535 
   5536    cdas->op1_high = op1_high;
   5537    cdas->op1_low  = op1_low;
   5538    cdas->op2 = op2;
   5539    cdas->op3_high = op3_high;
   5540    cdas->op3_low  = op3_low;
   5541    cdas->old_mem_high = old_mem_high;
   5542    cdas->old_mem_low  = old_mem_low;
   5543    cdas->scratch = scratch;
   5544 
   5545    return insn;
   5546 }
   5547 
   5548 
   5549 s390_insn *
   5550 s390_insn_compare(UChar size, HReg src1, s390_opnd_RMI src2,
   5551                   Bool signed_comparison)
   5552 {
   5553    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5554 
   5555    vassert(size == 4 || size == 8);
   5556 
   5557    insn->tag  = S390_INSN_COMPARE;
   5558    insn->size = size;
   5559    insn->variant.compare.src1 = src1;
   5560    insn->variant.compare.src2 = src2;
   5561    insn->variant.compare.signed_comparison = signed_comparison;
   5562 
   5563    return insn;
   5564 }
   5565 
   5566 
   5567 s390_insn *
   5568 s390_insn_helper_call(s390_cc_t cond, Addr64 target, UInt num_args,
   5569                       const HChar *name, RetLoc rloc)
   5570 {
   5571    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5572    s390_helper_call *helper_call = LibVEX_Alloc_inline(sizeof(s390_helper_call));
   5573 
   5574    insn->tag  = S390_INSN_HELPER_CALL;
   5575    insn->size = 0;  /* does not matter */
   5576    insn->variant.helper_call.details = helper_call;
   5577 
   5578    helper_call->cond = cond;
   5579    helper_call->target = target;
   5580    helper_call->num_args = num_args;
   5581    helper_call->name = name;
   5582    helper_call->rloc = rloc;
   5583 
   5584    vassert(is_sane_RetLoc(rloc));
   5585 
   5586    return insn;
   5587 }
   5588 
   5589 
   5590 s390_insn *
   5591 s390_insn_bfp_triop(UChar size, s390_bfp_triop_t tag, HReg dst, HReg op2,
   5592                     HReg op3)
   5593 {
   5594    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5595 
   5596    vassert(size == 4 || size == 8);
   5597 
   5598    insn->tag  = S390_INSN_BFP_TRIOP;
   5599    insn->size = size;
   5600    insn->variant.bfp_triop.tag = tag;
   5601    insn->variant.bfp_triop.dst = dst;
   5602    insn->variant.bfp_triop.op2 = op2;
   5603    insn->variant.bfp_triop.op3 = op3;
   5604 
   5605    return insn;
   5606 }
   5607 
   5608 
   5609 s390_insn *
   5610 s390_insn_bfp_binop(UChar size, s390_bfp_binop_t tag, HReg dst, HReg op2)
   5611 {
   5612    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5613 
   5614    vassert(size == 4 || size == 8);
   5615 
   5616    insn->tag  = S390_INSN_BFP_BINOP;
   5617    insn->size = size;
   5618    insn->variant.bfp_binop.tag = tag;
   5619    insn->variant.bfp_binop.dst_hi = dst;
   5620    insn->variant.bfp_binop.op2_hi = op2;
   5621    insn->variant.bfp_binop.dst_lo = INVALID_HREG;
   5622    insn->variant.bfp_binop.op2_lo = INVALID_HREG;
   5623 
   5624    return insn;
   5625 }
   5626 
   5627 
   5628 s390_insn *
   5629 s390_insn_bfp_unop(UChar size, s390_bfp_unop_t tag, HReg dst, HReg op)
   5630 {
   5631    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5632 
   5633    vassert(size == 4 || size == 8);
   5634 
   5635    insn->tag  = S390_INSN_BFP_UNOP;
   5636    insn->size = size;
   5637    insn->variant.bfp_unop.tag = tag;
   5638    insn->variant.bfp_unop.dst_hi = dst;
   5639    insn->variant.bfp_unop.op_hi  = op;
   5640    insn->variant.bfp_unop.dst_lo = INVALID_HREG;
   5641    insn->variant.bfp_unop.op_lo  = INVALID_HREG;
   5642 
   5643    return insn;
   5644 }
   5645 
   5646 
   5647 s390_insn *
   5648 s390_insn_bfp_compare(UChar size, HReg dst, HReg op1, HReg op2)
   5649 {
   5650    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5651 
   5652    vassert(size == 4 || size == 8);
   5653 
   5654    insn->tag  = S390_INSN_BFP_COMPARE;
   5655    insn->size = size;
   5656    insn->variant.bfp_compare.dst = dst;
   5657    insn->variant.bfp_compare.op1_hi = op1;
   5658    insn->variant.bfp_compare.op2_hi = op2;
   5659    insn->variant.bfp_compare.op1_lo = INVALID_HREG;
   5660    insn->variant.bfp_compare.op2_lo = INVALID_HREG;
   5661 
   5662    return insn;
   5663 }
   5664 
   5665 
   5666 s390_insn *
   5667 s390_insn_bfp_convert(UChar size, s390_bfp_conv_t tag, HReg dst, HReg op,
   5668                       s390_bfp_round_t rounding_mode)
   5669 {
   5670    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5671 
   5672    vassert(size == 4 || size == 8);
   5673 
   5674    insn->tag  = S390_INSN_BFP_CONVERT;
   5675    insn->size = size;
   5676    insn->variant.bfp_convert.tag = tag;
   5677    insn->variant.bfp_convert.dst_hi = dst;
   5678    insn->variant.bfp_convert.op_hi  = op;
   5679    insn->variant.bfp_convert.dst_lo = INVALID_HREG;
   5680    insn->variant.bfp_convert.op_lo  = INVALID_HREG;
   5681    insn->variant.bfp_convert.rounding_mode = rounding_mode;
   5682 
   5683    return insn;
   5684 }
   5685 
   5686 
   5687 /* Check validity of a register pair for 128-bit FP. Valid register
   5688    pairs are (0,2), (1,3), (4, 6), (5, 7), (8, 10), (9, 11), (12, 14),
   5689    and (13, 15). */
   5690 static Bool
   5691 is_valid_fp128_regpair(HReg hi, HReg lo)
   5692 {
   5693    UInt hi_regno = hregNumber(hi);
   5694    UInt lo_regno = hregNumber(lo);
   5695 
   5696    if (lo_regno != hi_regno + 2) return False;
   5697    if ((hi_regno & 0x2) != 0) return False;
   5698 
   5699    return True;
   5700 }
   5701 
   5702 s390_insn *
   5703 s390_insn_bfp128_binop(UChar size, s390_bfp_binop_t tag, HReg dst_hi,
   5704                        HReg dst_lo, HReg op2_hi, HReg op2_lo)
   5705 {
   5706    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5707 
   5708    vassert(size == 16);
   5709    vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   5710    vassert(is_valid_fp128_regpair(op2_hi, op2_lo));
   5711 
   5712    insn->tag  = S390_INSN_BFP_BINOP;
   5713    insn->size = size;
   5714    insn->variant.bfp_binop.tag = tag;
   5715    insn->variant.bfp_binop.dst_hi = dst_hi;
   5716    insn->variant.bfp_binop.dst_lo = dst_lo;
   5717    insn->variant.bfp_binop.op2_hi = op2_hi;
   5718    insn->variant.bfp_binop.op2_lo = op2_lo;
   5719 
   5720    return insn;
   5721 }
   5722 
   5723 
   5724 s390_insn *
   5725 s390_insn_bfp128_unop(UChar size, s390_bfp_unop_t tag, HReg dst_hi,
   5726                       HReg dst_lo, HReg op_hi, HReg op_lo)
   5727 {
   5728    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5729 
   5730    vassert(size == 16);
   5731    vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   5732    vassert(is_valid_fp128_regpair(op_hi, op_lo));
   5733 
   5734    insn->tag  = S390_INSN_BFP_UNOP;
   5735    insn->size = size;
   5736    insn->variant.bfp_unop.tag = tag;
   5737    insn->variant.bfp_unop.dst_hi = dst_hi;
   5738    insn->variant.bfp_unop.dst_lo = dst_lo;
   5739    insn->variant.bfp_unop.op_hi = op_hi;
   5740    insn->variant.bfp_unop.op_lo = op_lo;
   5741 
   5742    return insn;
   5743 }
   5744 
   5745 
   5746 s390_insn *
   5747 s390_insn_bfp128_compare(UChar size, HReg dst, HReg op1_hi, HReg op1_lo,
   5748                          HReg op2_hi, HReg op2_lo)
   5749 {
   5750    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5751 
   5752    vassert(size == 16);
   5753    vassert(is_valid_fp128_regpair(op1_hi, op1_lo));
   5754    vassert(is_valid_fp128_regpair(op2_hi, op2_lo));
   5755 
   5756    insn->tag  = S390_INSN_BFP_COMPARE;
   5757    insn->size = size;
   5758    insn->variant.bfp_compare.dst = dst;
   5759    insn->variant.bfp_compare.op1_hi = op1_hi;
   5760    insn->variant.bfp_compare.op1_lo = op1_lo;
   5761    insn->variant.bfp_compare.op2_hi = op2_hi;
   5762    insn->variant.bfp_compare.op2_lo = op2_lo;
   5763 
   5764    return insn;
   5765 }
   5766 
   5767 
   5768 s390_insn *
   5769 s390_insn_bfp128_convert(UChar size, s390_bfp_conv_t tag, HReg dst_hi,
   5770                          HReg dst_lo, HReg op_hi, HReg op_lo,
   5771                          s390_bfp_round_t rounding_mode)
   5772 {
   5773    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5774 
   5775    if (size == 16) {
   5776       /* From smaller or equal size to 16 bytes */
   5777       vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   5778       vassert(hregIsInvalid(op_lo)
   5779               || is_valid_fp128_regpair(op_hi, op_lo));
   5780    } else {
   5781       /* From 16 bytes to smaller size */
   5782       vassert(is_valid_fp128_regpair(op_hi, op_lo));
   5783    }
   5784 
   5785    insn->tag  = S390_INSN_BFP_CONVERT;
   5786    insn->size = size;
   5787    insn->variant.bfp_convert.tag = tag;
   5788    insn->variant.bfp_convert.dst_hi = dst_hi;
   5789    insn->variant.bfp_convert.dst_lo = dst_lo;
   5790    insn->variant.bfp_convert.op_hi = op_hi;
   5791    insn->variant.bfp_convert.op_lo = op_lo;
   5792    insn->variant.bfp_convert.rounding_mode = rounding_mode;
   5793 
   5794    return insn;
   5795 }
   5796 
   5797 
   5798 s390_insn *
   5799 s390_insn_bfp128_convert_to(UChar size, s390_bfp_conv_t tag, HReg dst_hi,
   5800                             HReg dst_lo, HReg op)
   5801 {
   5802    /* Conversion to bfp128 never requires a rounding mode. Provide default
   5803       rounding mode. It will not be used when emitting insns. */
   5804    s390_bfp_round_t rounding_mode = S390_BFP_ROUND_NEAREST_EVEN;
   5805 
   5806    return s390_insn_bfp128_convert(size, tag, dst_hi, dst_lo, op,
   5807                                    INVALID_HREG, rounding_mode);
   5808 }
   5809 
   5810 
   5811 s390_insn *
   5812 s390_insn_bfp128_convert_from(UChar size, s390_bfp_conv_t tag, HReg dst_hi,
   5813                               HReg dst_lo, HReg op_hi, HReg op_lo,
   5814                               s390_bfp_round_t rounding_mode)
   5815 {
   5816    return s390_insn_bfp128_convert(size, tag, dst_hi, dst_lo, op_hi, op_lo,
   5817                                    rounding_mode);
   5818 }
   5819 
   5820 
   5821 s390_insn *
   5822 s390_insn_dfp_binop(UChar size, s390_dfp_binop_t tag, HReg dst, HReg op2,
   5823                     HReg op3, s390_dfp_round_t rounding_mode)
   5824 {
   5825    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5826    s390_dfp_binop *dfp_binop = LibVEX_Alloc_inline(sizeof(s390_dfp_binop));
   5827 
   5828    vassert(size == 8);
   5829 
   5830    insn->tag  = S390_INSN_DFP_BINOP;
   5831    insn->size = size;
   5832    insn->variant.dfp_binop.details = dfp_binop;
   5833 
   5834    dfp_binop->tag = tag;
   5835    dfp_binop->dst_hi = dst;
   5836    dfp_binop->op2_hi = op2;
   5837    dfp_binop->op3_hi = op3;
   5838    dfp_binop->dst_lo = INVALID_HREG;
   5839    dfp_binop->op2_lo = INVALID_HREG;
   5840    dfp_binop->op3_lo = INVALID_HREG;
   5841    dfp_binop->rounding_mode = rounding_mode;
   5842 
   5843    return insn;
   5844 }
   5845 
   5846 
   5847 s390_insn *
   5848 s390_insn_dfp_unop(UChar size, s390_dfp_unop_t tag, HReg dst, HReg op)
   5849 {
   5850    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5851 
   5852    vassert(size == 8);
   5853 
   5854    insn->tag  = S390_INSN_DFP_UNOP;
   5855    insn->size = size;
   5856    insn->variant.dfp_unop.tag = tag;
   5857    insn->variant.dfp_unop.dst_hi = dst;
   5858    insn->variant.dfp_unop.op_hi  = op;
   5859    insn->variant.dfp_unop.dst_lo = INVALID_HREG;
   5860    insn->variant.dfp_unop.op_lo  = INVALID_HREG;
   5861 
   5862    return insn;
   5863 }
   5864 
   5865 
   5866 s390_insn *
   5867 s390_insn_dfp_intop(UChar size, s390_dfp_intop_t tag, HReg dst, HReg op2,
   5868                     HReg op3)
   5869 {
   5870    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5871 
   5872    vassert(size == 8);
   5873 
   5874    insn->tag  = S390_INSN_DFP_INTOP;
   5875    insn->size = size;
   5876    insn->variant.dfp_intop.tag = tag;
   5877    insn->variant.dfp_intop.dst_hi = dst;
   5878    insn->variant.dfp_intop.op2    = op2;
   5879    insn->variant.dfp_intop.op3_hi = op3;
   5880    insn->variant.dfp_intop.dst_lo = INVALID_HREG;
   5881    insn->variant.dfp_intop.op3_lo = INVALID_HREG;
   5882 
   5883    return insn;
   5884 }
   5885 
   5886 
   5887 s390_insn *
   5888 s390_insn_dfp_compare(UChar size, s390_dfp_cmp_t tag, HReg dst,
   5889                       HReg op1, HReg op2)
   5890 {
   5891    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5892 
   5893    vassert(size == 8);
   5894 
   5895    insn->tag  = S390_INSN_DFP_COMPARE;
   5896    insn->size = size;
   5897    insn->variant.dfp_compare.tag = tag;
   5898    insn->variant.dfp_compare.dst = dst;
   5899    insn->variant.dfp_compare.op1_hi = op1;
   5900    insn->variant.dfp_compare.op2_hi = op2;
   5901    insn->variant.dfp_compare.op1_lo = INVALID_HREG;
   5902    insn->variant.dfp_compare.op2_lo = INVALID_HREG;
   5903 
   5904    return insn;
   5905 }
   5906 
   5907 
   5908 s390_insn *
   5909 s390_insn_dfp_convert(UChar size, s390_dfp_conv_t tag, HReg dst, HReg op,
   5910                       s390_dfp_round_t rounding_mode)
   5911 {
   5912    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5913 
   5914    vassert(size == 4 || size == 8);
   5915 
   5916    insn->tag  = S390_INSN_DFP_CONVERT;
   5917    insn->size = size;
   5918    insn->variant.dfp_convert.tag = tag;
   5919    insn->variant.dfp_convert.dst_hi = dst;
   5920    insn->variant.dfp_convert.op_hi  = op;
   5921    insn->variant.dfp_convert.dst_lo = INVALID_HREG;
   5922    insn->variant.dfp_convert.op_lo  = INVALID_HREG;
   5923    insn->variant.dfp_convert.rounding_mode = rounding_mode;
   5924 
   5925    return insn;
   5926 }
   5927 
   5928 
   5929 s390_insn *
   5930 s390_insn_dfp_reround(UChar size, HReg dst, HReg op2, HReg op3,
   5931                       s390_dfp_round_t rounding_mode)
   5932 {
   5933    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5934 
   5935    vassert(size == 8);
   5936 
   5937    insn->tag  = S390_INSN_DFP_REROUND;
   5938    insn->size = size;
   5939    insn->variant.dfp_reround.dst_hi = dst;
   5940    insn->variant.dfp_reround.op2 = op2;
   5941    insn->variant.dfp_reround.op3_hi = op3;
   5942    insn->variant.dfp_reround.dst_lo = INVALID_HREG;
   5943    insn->variant.dfp_reround.op3_lo = INVALID_HREG;
   5944    insn->variant.dfp_reround.rounding_mode = rounding_mode;
   5945 
   5946    return insn;
   5947 }
   5948 
   5949 
   5950 s390_insn *
   5951 s390_insn_fp_convert(UChar size, s390_fp_conv_t tag, HReg dst, HReg op,
   5952                      HReg r1, s390_dfp_round_t rounding_mode)
   5953 {
   5954    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5955    s390_fp_convert *fp_convert = LibVEX_Alloc_inline(sizeof(s390_fp_convert));
   5956 
   5957    vassert(size == 4 || size == 8);
   5958 
   5959    insn->tag  = S390_INSN_FP_CONVERT;
   5960    insn->size = size;
   5961    insn->variant.fp_convert.details = fp_convert;
   5962 
   5963    fp_convert->tag = tag;
   5964    fp_convert->dst_hi = dst;
   5965    fp_convert->op_hi  = op;
   5966    fp_convert->r1 = r1;
   5967    fp_convert->dst_lo = INVALID_HREG;
   5968    fp_convert->op_lo  = INVALID_HREG;
   5969    fp_convert->rounding_mode = rounding_mode;
   5970 
   5971    return insn;
   5972 }
   5973 
   5974 
   5975 s390_insn *
   5976 s390_insn_fp128_convert(UChar size, s390_fp_conv_t tag, HReg dst_hi,
   5977                         HReg dst_lo, HReg op_hi, HReg op_lo, HReg r1,
   5978                         s390_dfp_round_t rounding_mode)
   5979 {
   5980    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   5981    s390_fp_convert *fp_convert = LibVEX_Alloc_inline(sizeof(s390_fp_convert));
   5982 
   5983    vassert(size == 16);
   5984 
   5985    insn->tag  = S390_INSN_FP_CONVERT;
   5986    insn->size = size;
   5987    insn->variant.fp_convert.details = fp_convert;
   5988 
   5989    fp_convert->tag = tag;
   5990    fp_convert->dst_hi = dst_hi;
   5991    fp_convert->dst_lo = dst_lo;
   5992    fp_convert->op_hi  = op_hi;
   5993    fp_convert->r1 = r1;
   5994    fp_convert->op_lo  = op_lo;
   5995    fp_convert->rounding_mode = rounding_mode;
   5996 
   5997    return insn;
   5998 }
   5999 
   6000 
   6001 s390_insn *
   6002 s390_insn_dfp128_binop(UChar size, s390_dfp_binop_t tag, HReg dst_hi,
   6003                        HReg dst_lo, HReg op2_hi, HReg op2_lo, HReg op3_hi,
   6004                        HReg op3_lo, s390_dfp_round_t rounding_mode)
   6005 {
   6006    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6007    s390_dfp_binop *dfp_binop = LibVEX_Alloc_inline(sizeof(s390_dfp_binop));
   6008 
   6009    vassert(size == 16);
   6010    vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   6011    vassert(is_valid_fp128_regpair(op2_hi, op2_lo));
   6012    vassert(is_valid_fp128_regpair(op3_hi, op3_lo));
   6013 
   6014    insn->tag  = S390_INSN_DFP_BINOP;
   6015    insn->size = size;
   6016    insn->variant.dfp_binop.details = dfp_binop;
   6017 
   6018    dfp_binop->tag = tag;
   6019    dfp_binop->dst_hi = dst_hi;
   6020    dfp_binop->dst_lo = dst_lo;
   6021    dfp_binop->op2_hi = op2_hi;
   6022    dfp_binop->op2_lo = op2_lo;
   6023    dfp_binop->op3_hi = op3_hi;
   6024    dfp_binop->op3_lo = op3_lo;
   6025    dfp_binop->rounding_mode = rounding_mode;
   6026 
   6027    return insn;
   6028 }
   6029 
   6030 
   6031 s390_insn *
   6032 s390_insn_dfp128_unop(UChar size, s390_dfp_unop_t tag, HReg dst,
   6033                       HReg op_hi, HReg op_lo)
   6034 {
   6035    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6036 
   6037    /* destination is an 8 byte integer value */
   6038    vassert(size == 8);
   6039    vassert(is_valid_fp128_regpair(op_hi, op_lo));
   6040 
   6041    insn->tag  = S390_INSN_DFP_UNOP;
   6042    insn->size = size;
   6043    insn->variant.dfp_unop.tag = tag;
   6044    insn->variant.dfp_unop.dst_hi = dst;
   6045    insn->variant.dfp_unop.dst_lo = INVALID_HREG;
   6046    insn->variant.dfp_unop.op_hi = op_hi;
   6047    insn->variant.dfp_unop.op_lo = op_lo;
   6048 
   6049    return insn;
   6050 }
   6051 
   6052 
   6053 s390_insn *
   6054 s390_insn_dfp128_intop(UChar size, s390_dfp_intop_t tag, HReg dst_hi,
   6055                        HReg dst_lo, HReg op2, HReg op3_hi, HReg op3_lo)
   6056 {
   6057    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6058 
   6059    vassert(size == 16);
   6060    vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   6061    vassert(is_valid_fp128_regpair(op3_hi, op3_lo));
   6062 
   6063    insn->tag  = S390_INSN_DFP_INTOP;
   6064    insn->size = size;
   6065    insn->variant.dfp_intop.tag = tag;
   6066    insn->variant.dfp_intop.dst_hi = dst_hi;
   6067    insn->variant.dfp_intop.dst_lo = dst_lo;
   6068    insn->variant.dfp_intop.op2    = op2;
   6069    insn->variant.dfp_intop.op3_hi = op3_hi;
   6070    insn->variant.dfp_intop.op3_lo = op3_lo;
   6071 
   6072    return insn;
   6073 }
   6074 
   6075 
   6076 s390_insn *
   6077 s390_insn_dfp128_compare(UChar size, s390_dfp_cmp_t tag, HReg dst, HReg op1_hi,
   6078                          HReg op1_lo, HReg op2_hi, HReg op2_lo)
   6079 {
   6080    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6081 
   6082    vassert(size == 16);
   6083    vassert(is_valid_fp128_regpair(op1_hi, op1_lo));
   6084    vassert(is_valid_fp128_regpair(op2_hi, op2_lo));
   6085 
   6086    insn->tag  = S390_INSN_DFP_COMPARE;
   6087    insn->size = size;
   6088    insn->variant.dfp_compare.tag = tag;
   6089    insn->variant.dfp_compare.dst = dst;
   6090    insn->variant.dfp_compare.op1_hi = op1_hi;
   6091    insn->variant.dfp_compare.op1_lo = op1_lo;
   6092    insn->variant.dfp_compare.op2_hi = op2_hi;
   6093    insn->variant.dfp_compare.op2_lo = op2_lo;
   6094 
   6095    return insn;
   6096 }
   6097 
   6098 
   6099 static s390_insn *
   6100 s390_insn_dfp128_convert(UChar size, s390_dfp_conv_t tag, HReg dst_hi,
   6101                          HReg dst_lo, HReg op_hi, HReg op_lo,
   6102                          s390_dfp_round_t rounding_mode)
   6103 {
   6104    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6105 
   6106    if (size == 16) {
   6107       /* From smaller size to 16 bytes */
   6108       vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   6109       vassert(hregIsInvalid(op_lo));
   6110    } else {
   6111       /* From 16 bytes to smaller size */
   6112       vassert(is_valid_fp128_regpair(op_hi, op_lo));
   6113    }
   6114 
   6115    insn->tag  = S390_INSN_DFP_CONVERT;
   6116    insn->size = size;
   6117    insn->variant.dfp_convert.tag = tag;
   6118    insn->variant.dfp_convert.dst_hi = dst_hi;
   6119    insn->variant.dfp_convert.dst_lo = dst_lo;
   6120    insn->variant.dfp_convert.op_hi = op_hi;
   6121    insn->variant.dfp_convert.op_lo = op_lo;
   6122    insn->variant.dfp_convert.rounding_mode = rounding_mode;
   6123 
   6124    return insn;
   6125 }
   6126 
   6127 
   6128 s390_insn *
   6129 s390_insn_dfp128_convert_to(UChar size, s390_dfp_conv_t tag, HReg dst_hi,
   6130                             HReg dst_lo, HReg op)
   6131 {
   6132    /* Conversion to dfp128 never requires a rounding mode. Provide default
   6133       rounding mode. It will not be used when emitting insns. */
   6134    s390_dfp_round_t rounding_mode = S390_DFP_ROUND_NEAREST_EVEN_4;
   6135 
   6136    return s390_insn_dfp128_convert(size, tag, dst_hi, dst_lo, op,
   6137                                    INVALID_HREG, rounding_mode);
   6138 }
   6139 
   6140 
   6141 s390_insn *
   6142 s390_insn_dfp128_convert_from(UChar size, s390_dfp_conv_t tag, HReg dst_hi,
   6143                               HReg dst_lo, HReg op_hi, HReg op_lo,
   6144                               s390_dfp_round_t rounding_mode)
   6145 {
   6146    return s390_insn_dfp128_convert(size, tag, dst_hi, dst_lo, op_hi, op_lo,
   6147                                    rounding_mode);
   6148 }
   6149 
   6150 
   6151 s390_insn *
   6152 s390_insn_dfp128_reround(UChar size, HReg dst_hi, HReg dst_lo, HReg op2,
   6153                          HReg op3_hi, HReg op3_lo,
   6154                          s390_dfp_round_t rounding_mode)
   6155 {
   6156    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6157 
   6158    vassert(size == 16);
   6159    vassert(is_valid_fp128_regpair(dst_hi, dst_lo));
   6160    vassert(is_valid_fp128_regpair(op3_hi, op3_lo));
   6161 
   6162    insn->tag  = S390_INSN_DFP_REROUND;
   6163    insn->size = size;
   6164    insn->variant.dfp_reround.dst_hi = dst_hi;
   6165    insn->variant.dfp_reround.dst_lo = dst_lo;
   6166    insn->variant.dfp_reround.op2    = op2;
   6167    insn->variant.dfp_reround.op3_hi = op3_hi;
   6168    insn->variant.dfp_reround.op3_lo = op3_lo;
   6169    insn->variant.dfp_reround.rounding_mode = rounding_mode;
   6170 
   6171    return insn;
   6172 }
   6173 
   6174 
   6175 s390_insn *
   6176 s390_insn_mfence(void)
   6177 {
   6178    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6179 
   6180    insn->tag  = S390_INSN_MFENCE;
   6181    insn->size = 0;   /* not needed */
   6182 
   6183    return insn;
   6184 }
   6185 
   6186 
   6187 s390_insn *
   6188 s390_insn_mimm(UChar size, s390_amode *dst, ULong value)
   6189 {
   6190    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6191 
   6192    /* This insn will be mapped to insns that require base register
   6193       plus 12-bit displacement */
   6194    vassert(dst->tag == S390_AMODE_B12);
   6195 
   6196    insn->tag  = S390_INSN_MIMM;
   6197    insn->size = size;
   6198    insn->variant.mimm.dst = dst;
   6199    insn->variant.mimm.value = value;
   6200 
   6201    return insn;
   6202 }
   6203 
   6204 
   6205 s390_insn *
   6206 s390_insn_madd(UChar size, s390_amode *dst, UChar delta, ULong value)
   6207 {
   6208    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6209 
   6210    vassert(size == 4 || size == 8);
   6211 
   6212    /* This insn will be mapped to an ASI or AGSI so we can only allow base
   6213       register plus 12-bit / 20-bit displacement. */
   6214    vassert(dst->tag == S390_AMODE_B12 || dst->tag == S390_AMODE_B20);
   6215    /* ASI and AGSI require the GIE facility */
   6216    vassert(s390_host_has_gie);
   6217 
   6218    insn->tag  = S390_INSN_MADD;
   6219    insn->size = size;
   6220    insn->variant.madd.dst   = dst;
   6221    insn->variant.madd.delta = delta;
   6222    insn->variant.madd.value = value;
   6223 
   6224    return insn;
   6225 }
   6226 
   6227 
   6228 s390_insn *
   6229 s390_insn_set_fpc_bfprm(UChar size, HReg mode)
   6230 {
   6231    vassert(size == 4);
   6232 
   6233    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6234 
   6235    insn->tag  = S390_INSN_SET_FPC_BFPRM;
   6236    insn->size = size;
   6237    insn->variant.set_fpc_bfprm.mode = mode;
   6238 
   6239    return insn;
   6240 }
   6241 
   6242 
   6243 s390_insn *
   6244 s390_insn_set_fpc_dfprm(UChar size, HReg mode)
   6245 {
   6246    vassert(size == 4);
   6247 
   6248    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6249 
   6250    insn->tag  = S390_INSN_SET_FPC_DFPRM;
   6251    insn->size = size;
   6252    insn->variant.set_fpc_dfprm.mode = mode;
   6253 
   6254    return insn;
   6255 }
   6256 
   6257 
   6258 s390_insn *
   6259 s390_insn_xdirect(s390_cc_t cond, Addr64 dst, s390_amode *guest_IA,
   6260                   Bool to_fast_entry)
   6261 {
   6262    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6263 
   6264    vassert(guest_IA->tag == S390_AMODE_B12);
   6265 
   6266    insn->tag  = S390_INSN_XDIRECT;
   6267    insn->size = 0;   /* does not matter */
   6268 
   6269    insn->variant.xdirect.cond = cond;
   6270    insn->variant.xdirect.dst = dst;
   6271    insn->variant.xdirect.guest_IA = guest_IA;
   6272    insn->variant.xdirect.to_fast_entry = to_fast_entry;
   6273 
   6274    return insn;
   6275 }
   6276 
   6277 
   6278 s390_insn *
   6279 s390_insn_xindir(s390_cc_t cond, HReg dst, s390_amode *guest_IA)
   6280 {
   6281    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6282 
   6283    vassert(guest_IA->tag == S390_AMODE_B12);
   6284 
   6285    insn->tag  = S390_INSN_XINDIR;
   6286    insn->size = 0;   /* does not matter */
   6287 
   6288    insn->variant.xindir.cond = cond;
   6289    insn->variant.xindir.dst = dst;
   6290    insn->variant.xindir.guest_IA = guest_IA;
   6291 
   6292    return insn;
   6293 }
   6294 
   6295 
   6296 s390_insn *
   6297 s390_insn_xassisted(s390_cc_t cond, HReg dst, s390_amode *guest_IA,
   6298                     IRJumpKind kind)
   6299 {
   6300    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6301 
   6302    vassert(guest_IA->tag == S390_AMODE_B12);
   6303 
   6304    insn->tag  = S390_INSN_XASSISTED;
   6305    insn->size = 0;   /* does not matter */
   6306 
   6307    insn->variant.xassisted.cond = cond;
   6308    insn->variant.xassisted.dst = dst;
   6309    insn->variant.xassisted.guest_IA = guest_IA;
   6310    insn->variant.xassisted.kind = kind;
   6311 
   6312    return insn;
   6313 }
   6314 
   6315 
   6316 s390_insn *
   6317 s390_insn_evcheck(s390_amode *counter, s390_amode *fail_addr)
   6318 {
   6319    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6320 
   6321    vassert(counter->tag   == S390_AMODE_B12);
   6322    vassert(fail_addr->tag == S390_AMODE_B12);
   6323 
   6324    insn->tag  = S390_INSN_EVCHECK;
   6325    insn->size = 0;   /* does not matter */
   6326 
   6327    insn->variant.evcheck.counter = counter;
   6328    insn->variant.evcheck.fail_addr = fail_addr;
   6329 
   6330    return insn;
   6331 }
   6332 
   6333 
   6334 s390_insn *
   6335 s390_insn_profinc(void)
   6336 {
   6337    s390_insn *insn = LibVEX_Alloc_inline(sizeof(s390_insn));
   6338 
   6339    insn->tag  = S390_INSN_PROFINC;
   6340    insn->size = 0;   /* does not matter */
   6341 
   6342    return insn;
   6343 }
   6344 
   6345 
   6346 /*---------------------------------------------------------------*/
   6347 /*--- Debug print                                             ---*/
   6348 /*---------------------------------------------------------------*/
   6349 
   6350 static const HChar *
   6351 s390_cc_as_string(s390_cc_t cc)
   6352 {
   6353    switch (cc) {
   6354    case S390_CC_NEVER:  return "never";
   6355    case S390_CC_OVFL:   return "overflow";
   6356    case S390_CC_H:      return "greater than";     /* A > B ; high */
   6357    case S390_CC_NLE:    return "not low or equal";
   6358    case S390_CC_L:      return "less than";        /* A < B ; low */
   6359    case S390_CC_NHE:    return "not high or equal";
   6360    case S390_CC_LH:     return "low or high";
   6361    case S390_CC_NE:     return "not equal";        /* A != B ; not zero */
   6362    case S390_CC_E:      return "equal";            /* A == B ; zero */
   6363    case S390_CC_NLH:    return "not low or high";
   6364    case S390_CC_HE:     return "greater or equal"; /* A >= B ; high or equal*/
   6365    case S390_CC_NL:     return "not low";          /* not low */
   6366    case S390_CC_LE:     return "less or equal";    /* A <= B ; low or equal */
   6367    case S390_CC_NH:     return "not high";
   6368    case S390_CC_NO:     return "not overflow";
   6369    case S390_CC_ALWAYS: return "always";
   6370    default:
   6371       vpanic("s390_cc_as_string");
   6372    }
   6373 }
   6374 
   6375 
   6376 static const HChar *
   6377 s390_jump_kind_as_string(IRJumpKind kind)
   6378 {
   6379    switch (kind) {
   6380    case Ijk_Boring:      return "Boring";
   6381    case Ijk_Call:        return "Call";
   6382    case Ijk_Ret:         return "Return";
   6383    case Ijk_ClientReq:   return "ClientReq";
   6384    case Ijk_Yield:       return "Yield";
   6385    case Ijk_EmWarn:      return "EmWarn";
   6386    case Ijk_EmFail:      return "EmFail";
   6387    case Ijk_NoDecode:    return "NoDecode";
   6388    case Ijk_MapFail:     return "MapFail";
   6389    case Ijk_InvalICache: return "Invalidate";
   6390    case Ijk_NoRedir:     return "NoRedir";
   6391    case Ijk_SigTRAP:     return "SigTRAP";
   6392    case Ijk_SigSEGV:     return "SigSEGV";
   6393    case Ijk_SigBUS:      return "SigBUS";
   6394    case Ijk_Sys_syscall: return "Sys_syscall";
   6395    default:
   6396       vpanic("s390_jump_kind_as_string");
   6397    }
   6398 }
   6399 
   6400 
   6401 /* Helper function for writing out a V insn */
   6402 static void
   6403 s390_sprintf(HChar *buf, const HChar *fmt, ...)
   6404 {
   6405    HChar *p;
   6406    ULong value;
   6407    va_list args;
   6408    va_start(args, fmt);
   6409 
   6410    p = buf;
   6411    for ( ; *fmt; ++fmt) {
   6412       Int c = *fmt;
   6413 
   6414       if (c != '%') {
   6415          *p++ = c;
   6416          continue;
   6417       }
   6418 
   6419       c = *++fmt;  /* next char */
   6420       switch (c) {
   6421       case '%':
   6422          *p++ = c;   /* %% */
   6423          continue;
   6424 
   6425       case 's':     /* %s */
   6426          p += vex_sprintf(p, "%s", va_arg(args, HChar *));
   6427          continue;
   6428 
   6429       case 'M':     /* %M = mnemonic */
   6430          p += vex_sprintf(p, "%-8s", va_arg(args, HChar *));
   6431          continue;
   6432 
   6433       case 'R':     /* %R = register */
   6434          p += vex_sprintf(p, "%s", s390_hreg_as_string(va_arg(args, HReg)));
   6435          continue;
   6436 
   6437       case 'A':     /* %A = amode */
   6438          p += vex_sprintf(p, "%s",
   6439                           s390_amode_as_string(va_arg(args, s390_amode *)));
   6440          continue;
   6441 
   6442       case 'G':     /* %G = guest state @ offset */
   6443          p += vex_sprintf(p, "guest[%u]", va_arg(args, UInt));
   6444          continue;
   6445 
   6446       case 'C':     /* %C = condition code */
   6447          p += vex_sprintf(p, "%s", s390_cc_as_string(va_arg(args, s390_cc_t)));
   6448          continue;
   6449 
   6450       case 'J':     /* &J = jump kind */
   6451          p += vex_sprintf(p, "%s",
   6452                           s390_jump_kind_as_string(va_arg(args, IRJumpKind)));
   6453          continue;
   6454 
   6455       case 'L': {   /* %L = argument list in helper call*/
   6456          UInt i, num_args;
   6457 
   6458          num_args = va_arg(args, UInt);
   6459 
   6460          for (i = 0; i < num_args; ++i) {
   6461             if (i != 0) p += vex_sprintf(p, ", ");
   6462             p += vex_sprintf(p, "r%u", s390_gprno_from_arg_index(i));
   6463          }
   6464          continue;
   6465       }
   6466 
   6467       case 'O': {   /* %O = RMI operand */
   6468          s390_opnd_RMI *op = va_arg(args, s390_opnd_RMI *);
   6469 
   6470          switch (op->tag) {
   6471          case S390_OPND_REG:
   6472             p += vex_sprintf(p, "%s", s390_hreg_as_string(op->variant.reg));
   6473             continue;
   6474 
   6475          case S390_OPND_AMODE:
   6476             p += vex_sprintf(p, "%s", s390_amode_as_string(op->variant.am));
   6477             continue;
   6478 
   6479          case S390_OPND_IMMEDIATE:
   6480             value = op->variant.imm;
   6481             goto print_value;
   6482 
   6483          default:
   6484             goto fail;
   6485          }
   6486       }
   6487 
   6488       case 'I':     /* %I = immediate value */
   6489          value = va_arg(args, ULong);
   6490          goto print_value;
   6491 
   6492       print_value:
   6493          if ((Long)value < 0)
   6494             p += vex_sprintf(p, "%lld", (Long)value);
   6495          else if (value < 100)
   6496             p += vex_sprintf(p, "%llu", value);
   6497          else
   6498             p += vex_sprintf(p, "0x%llx", value);
   6499          continue;
   6500 
   6501       default:
   6502          goto fail;
   6503       }
   6504    }
   6505    *p = '\0';
   6506    va_end(args);
   6507 
   6508    return;
   6509 
   6510  fail: vpanic("s390_printf");
   6511 }
   6512 
   6513 
   6514 /* Decompile the given insn into a static buffer and return it */
   6515 const HChar *
   6516 s390_insn_as_string(const s390_insn *insn)
   6517 {
   6518    static HChar buf[300];  // large enough
   6519    const HChar *op;
   6520    HChar *p;
   6521 
   6522    buf[0] = '\0';
   6523 
   6524    switch (insn->tag) {
   6525    case S390_INSN_LOAD:
   6526       s390_sprintf(buf, "%M %R,%A", "v-load", insn->variant.load.dst,
   6527                    insn->variant.load.src);
   6528       break;
   6529 
   6530    case S390_INSN_STORE:
   6531       s390_sprintf(buf, "%M %R,%A", "v-store", insn->variant.store.src,
   6532                    insn->variant.store.dst);
   6533       break;
   6534 
   6535    case S390_INSN_MOVE:
   6536       s390_sprintf(buf, "%M %R,%R", "v-move", insn->variant.move.dst,
   6537                    insn->variant.move.src);
   6538       break;
   6539 
   6540    case S390_INSN_MEMCPY:
   6541       s390_sprintf(buf, "%M %A,%A", "v-memcpy", insn->variant.memcpy.dst,
   6542                    insn->variant.memcpy.src);
   6543       break;
   6544 
   6545    case S390_INSN_COND_MOVE:
   6546       s390_sprintf(buf, "%M if (%C) %R,%O", "v-move",
   6547                    insn->variant.cond_move.cond, insn->variant.cond_move.dst,
   6548                    &insn->variant.cond_move.src);
   6549       break;
   6550 
   6551    case S390_INSN_LOAD_IMMEDIATE:
   6552       s390_sprintf(buf, "%M %R,%I", "v-loadi", insn->variant.load_immediate.dst,
   6553                    insn->variant.load_immediate.value);
   6554       break;
   6555 
   6556    case S390_INSN_ALU:
   6557       switch (insn->variant.alu.tag) {
   6558       case S390_ALU_ADD:  op = "v-add";  break;
   6559       case S390_ALU_SUB:  op = "v-sub";  break;
   6560       case S390_ALU_MUL:  op = "v-mul";  break;
   6561       case S390_ALU_AND:  op = "v-and";  break;
   6562       case S390_ALU_OR:   op = "v-or";   break;
   6563       case S390_ALU_XOR:  op = "v-xor";  break;
   6564       case S390_ALU_LSH:  op = "v-lsh";  break;
   6565       case S390_ALU_RSH:  op = "v-rsh";  break;
   6566       case S390_ALU_RSHA: op = "v-rsha"; break;
   6567       default: goto fail;
   6568       }
   6569       s390_sprintf(buf, "%M %R,%O", op, insn->variant.alu.dst, /* also op1 */
   6570                    &insn->variant.alu.op2);
   6571       break;
   6572 
   6573    case S390_INSN_SMUL:
   6574    case S390_INSN_UMUL:
   6575       if (insn->tag == S390_INSN_SMUL) {
   6576          op = "v-muls";
   6577       } else {
   6578          op = "v-mulu";
   6579       }
   6580       s390_sprintf(buf, "%M %R,%O", op, insn->variant.mul.dst_hi,
   6581                    &insn->variant.mul.op2);
   6582       break;
   6583 
   6584    case S390_INSN_SDIV:
   6585    case S390_INSN_UDIV:
   6586       if (insn->tag == S390_INSN_SDIV) {
   6587          op = "v-divs";
   6588       } else {
   6589          op = "v-divu";
   6590       }
   6591       s390_sprintf(buf, "%M %R,%O", op, insn->variant.div.op1_hi,
   6592                    &insn->variant.div.op2);
   6593       break;
   6594 
   6595    case S390_INSN_DIVS:
   6596       s390_sprintf(buf, "%M %R,%O", "v-divsi", insn->variant.divs.op1,
   6597                    &insn->variant.divs.op2);
   6598       break;
   6599 
   6600    case S390_INSN_CLZ:
   6601       s390_sprintf(buf, "%M %R,%O", "v-clz", insn->variant.clz.num_bits,
   6602                    &insn->variant.clz.src);
   6603       break;
   6604 
   6605    case S390_INSN_UNOP:
   6606       switch (insn->variant.unop.tag) {
   6607       case S390_ZERO_EXTEND_8:
   6608       case S390_ZERO_EXTEND_16:
   6609       case S390_ZERO_EXTEND_32:
   6610          op = "v-zerox";
   6611          break;
   6612 
   6613       case S390_SIGN_EXTEND_8:
   6614       case S390_SIGN_EXTEND_16:
   6615       case S390_SIGN_EXTEND_32:
   6616          op = "v-signx";
   6617          break;
   6618 
   6619       case S390_NEGATE:
   6620          op = "v-neg";
   6621          break;
   6622 
   6623       default:
   6624          goto fail;
   6625       }
   6626       s390_sprintf(buf, "%M %R,%O", op, insn->variant.unop.dst,
   6627                    &insn->variant.unop.src);
   6628       break;
   6629 
   6630    case S390_INSN_TEST:
   6631       s390_sprintf(buf, "%M %O", "v-test", &insn->variant.test.src);
   6632       break;
   6633 
   6634    case S390_INSN_CC2BOOL:
   6635       s390_sprintf(buf, "%M %R,%C", "v-cc2b", insn->variant.cc2bool.dst,
   6636                    insn->variant.cc2bool.cond);
   6637       break;
   6638 
   6639    case S390_INSN_CAS:
   6640       s390_sprintf(buf, "%M %R,%A,%R,%R", "v-cas", insn->variant.cas.op1,
   6641                    insn->variant.cas.op2, insn->variant.cas.op3,
   6642                    insn->variant.cas.old_mem);
   6643       break;
   6644 
   6645    case S390_INSN_CDAS: {
   6646       s390_cdas *cdas = insn->variant.cdas.details;
   6647 
   6648       s390_sprintf(buf, "%M %R,%R,%A,%R,%R,%R,%R", "v-cdas",
   6649                    cdas->op1_high, cdas->op1_low, cdas->op2, cdas->op3_high,
   6650                    cdas->op3_low, cdas->old_mem_high, cdas->old_mem_low);
   6651       break;
   6652    }
   6653 
   6654    case S390_INSN_COMPARE:
   6655       if (insn->variant.compare.signed_comparison) {
   6656          op = "v-cmps";
   6657       } else {
   6658          op = "v-cmpu";
   6659       }
   6660       s390_sprintf(buf, "%M %R,%O", op, insn->variant.compare.src1,
   6661                    &insn->variant.compare.src2);
   6662       break;
   6663 
   6664    case S390_INSN_HELPER_CALL: {
   6665       s390_helper_call *helper_call = insn->variant.helper_call.details;
   6666       s390_sprintf(buf, "%M if (%C) %s{%I}(%L)", "v-call",
   6667                    helper_call->cond,
   6668                    helper_call->name,
   6669                    helper_call->target,
   6670                    helper_call->num_args);
   6671       return buf;   /* avoid printing "size = ..." which is meaningless */
   6672    }
   6673 
   6674    case S390_INSN_BFP_TRIOP:
   6675       switch (insn->variant.bfp_triop.tag) {
   6676       case S390_BFP_MADD:  op = "v-fmadd";  break;
   6677       case S390_BFP_MSUB:  op = "v-fmsub";  break;
   6678       default: goto fail;
   6679       }
   6680       s390_sprintf(buf, "%M %R,%R,%R", op,
   6681                    insn->variant.bfp_triop.dst  /* op1 same as dst */,
   6682                    insn->variant.bfp_triop.op2, insn->variant.bfp_triop.op3);
   6683       break;
   6684 
   6685    case S390_INSN_BFP_BINOP:
   6686       switch (insn->variant.bfp_binop.tag) {
   6687       case S390_BFP_ADD:      op = "v-fadd";  break;
   6688       case S390_BFP_SUB:      op = "v-fsub";  break;
   6689       case S390_BFP_MUL:      op = "v-fmul";  break;
   6690       case S390_BFP_DIV:      op = "v-fdiv";  break;
   6691       default: goto fail;
   6692       }
   6693       s390_sprintf(buf, "%M %R,%R", op,
   6694                    insn->variant.bfp_binop.dst_hi  /* op1 same as dst */,
   6695                    insn->variant.bfp_binop.op2_hi);
   6696       break;
   6697 
   6698    case S390_INSN_BFP_COMPARE:
   6699       s390_sprintf(buf, "%M %R,%R,%R", "v-fcmp", insn->variant.bfp_compare.dst,
   6700                    insn->variant.bfp_compare.op1_hi,
   6701                    insn->variant.bfp_compare.op2_hi);
   6702       break;
   6703 
   6704    case S390_INSN_BFP_UNOP:
   6705       switch (insn->variant.bfp_unop.tag) {
   6706       case S390_BFP_ABS:         op = "v-fabs";  break;
   6707       case S390_BFP_NABS:        op = "v-fnabs"; break;
   6708       case S390_BFP_NEG:         op = "v-fneg";  break;
   6709       case S390_BFP_SQRT:        op = "v-fsqrt"; break;
   6710       default: goto fail;
   6711       }
   6712       s390_sprintf(buf, "%M %R,%R", op, insn->variant.bfp_unop.dst_hi,
   6713                    insn->variant.bfp_unop.op_hi);
   6714       break;
   6715 
   6716    case S390_INSN_BFP_CONVERT:
   6717       switch (insn->variant.bfp_convert.tag) {
   6718       case S390_BFP_I32_TO_F32:
   6719       case S390_BFP_I32_TO_F64:
   6720       case S390_BFP_I32_TO_F128:
   6721       case S390_BFP_I64_TO_F32:
   6722       case S390_BFP_I64_TO_F64:
   6723       case S390_BFP_I64_TO_F128: op = "v-i2f"; break;
   6724       case S390_BFP_U32_TO_F32:
   6725       case S390_BFP_U32_TO_F64:
   6726       case S390_BFP_U32_TO_F128:
   6727       case S390_BFP_U64_TO_F32:
   6728       case S390_BFP_U64_TO_F64:
   6729       case S390_BFP_U64_TO_F128: op = "v-u2f"; break;
   6730       case S390_BFP_F32_TO_I32:
   6731       case S390_BFP_F32_TO_I64:
   6732       case S390_BFP_F64_TO_I32:
   6733       case S390_BFP_F64_TO_I64:
   6734       case S390_BFP_F128_TO_I32:
   6735       case S390_BFP_F128_TO_I64: op = "v-f2i"; break;
   6736       case S390_BFP_F32_TO_U32:
   6737       case S390_BFP_F32_TO_U64:
   6738       case S390_BFP_F64_TO_U32:
   6739       case S390_BFP_F64_TO_U64:
   6740       case S390_BFP_F128_TO_U32:
   6741       case S390_BFP_F128_TO_U64: op = "v-f2u"; break;
   6742       case S390_BFP_F32_TO_F64:
   6743       case S390_BFP_F32_TO_F128:
   6744       case S390_BFP_F64_TO_F32:
   6745       case S390_BFP_F64_TO_F128:
   6746       case S390_BFP_F128_TO_F32:
   6747       case S390_BFP_F128_TO_F64: op = "v-f2f"; break;
   6748       case S390_BFP_F32_TO_F32I:
   6749       case S390_BFP_F64_TO_F64I:
   6750       case S390_BFP_F128_TO_F128I: op = "v-f2fi"; break;
   6751       default: goto fail;
   6752       }
   6753       s390_sprintf(buf, "%M %R,%R", op, insn->variant.bfp_convert.dst_hi,
   6754                    insn->variant.bfp_convert.op_hi);
   6755       break;
   6756 
   6757    case S390_INSN_DFP_BINOP: {
   6758       s390_dfp_binop *dfp_binop = insn->variant.dfp_binop.details;
   6759 
   6760       switch (dfp_binop->tag) {
   6761       case S390_DFP_ADD:  op = "v-dadd";  break;
   6762       case S390_DFP_SUB:  op = "v-dsub";  break;
   6763       case S390_DFP_MUL:  op = "v-dmul";  break;
   6764       case S390_DFP_DIV:  op = "v-ddiv";  break;
   6765       case S390_DFP_QUANTIZE:  op = "v-dqua";  break;
   6766       default: goto fail;
   6767       }
   6768       s390_sprintf(buf, "%M %R,%R,%R", op, dfp_binop->dst_hi,
   6769                    dfp_binop->op2_hi, dfp_binop->op3_hi);
   6770       break;
   6771    }
   6772 
   6773    case S390_INSN_DFP_UNOP:
   6774       switch (insn->variant.dfp_unop.tag) {
   6775       case S390_DFP_EXTRACT_EXP_D64:
   6776       case S390_DFP_EXTRACT_EXP_D128:  op = "v-d2exp";  break;
   6777       case S390_DFP_EXTRACT_SIG_D64:
   6778       case S390_DFP_EXTRACT_SIG_D128:  op = "v-d2sig";  break;
   6779       default: goto fail;
   6780       }
   6781       s390_sprintf(buf, "%M %R,%R", op, insn->variant.dfp_unop.dst_hi,
   6782                    insn->variant.dfp_unop.op_hi);
   6783       break;
   6784 
   6785    case S390_INSN_DFP_INTOP:
   6786       switch (insn->variant.dfp_intop.tag) {
   6787       case S390_DFP_SHIFT_LEFT:  op = "v-dshl"; break;
   6788       case S390_DFP_SHIFT_RIGHT: op = "v-dshr"; break;
   6789       case S390_DFP_INSERT_EXP:  op = "v-diexp"; break;
   6790       default: goto fail;
   6791       }
   6792       s390_sprintf(buf, "%M %R,%R,%R", op, insn->variant.dfp_intop.dst_hi,
   6793                    insn->variant.dfp_intop.op2,
   6794                    insn->variant.dfp_intop.op3_hi);
   6795       break;
   6796 
   6797    case S390_INSN_DFP_COMPARE:
   6798       switch (insn->variant.dfp_compare.tag) {
   6799       case S390_DFP_COMPARE:     op = "v-dcmp"; break;
   6800       case S390_DFP_COMPARE_EXP: op = "v-dcmpexp"; break;
   6801       default: goto fail;
   6802       }
   6803       s390_sprintf(buf, "%M %R,%R,%R", op, insn->variant.dfp_compare.dst,
   6804                    insn->variant.dfp_compare.op1_hi,
   6805                    insn->variant.dfp_compare.op2_hi);
   6806       break;
   6807 
   6808    case S390_INSN_DFP_CONVERT:
   6809       switch (insn->variant.dfp_convert.tag) {
   6810       case S390_DFP_D32_TO_D64:
   6811       case S390_DFP_D64_TO_D32:
   6812       case S390_DFP_D64_TO_D128:
   6813       case S390_DFP_D128_TO_D64: op = "v-d2d"; break;
   6814       case S390_DFP_I32_TO_D64:
   6815       case S390_DFP_I32_TO_D128:
   6816       case S390_DFP_I64_TO_D64:
   6817       case S390_DFP_I64_TO_D128: op = "v-i2d"; break;
   6818       case S390_DFP_U32_TO_D64:
   6819       case S390_DFP_U32_TO_D128:
   6820       case S390_DFP_U64_TO_D64:
   6821       case S390_DFP_U64_TO_D128: op = "v-u2d"; break;
   6822       case S390_DFP_D64_TO_I32:
   6823       case S390_DFP_D128_TO_I32:
   6824       case S390_DFP_D64_TO_I64:
   6825       case S390_DFP_D128_TO_I64: op = "v-d2i"; break;
   6826       case S390_DFP_D64_TO_U32:
   6827       case S390_DFP_D64_TO_U64:
   6828       case S390_DFP_D128_TO_U32:
   6829       case S390_DFP_D128_TO_U64: op = "v-d2u"; break;
   6830       default: goto fail;
   6831       }
   6832       s390_sprintf(buf, "%M %R,%R", op, insn->variant.dfp_convert.dst_hi,
   6833                    insn->variant.dfp_convert.op_hi);
   6834       break;
   6835 
   6836    case S390_INSN_DFP_REROUND:
   6837       s390_sprintf(buf, "%M %R,%R,%R", "v-drrnd",
   6838                    insn->variant.dfp_reround.dst_hi,
   6839                    insn->variant.dfp_reround.op2,
   6840                    insn->variant.dfp_reround.op3_hi);
   6841       break;
   6842 
   6843    case S390_INSN_FP_CONVERT: {
   6844       s390_fp_convert *fp_convert = insn->variant.fp_convert.details;
   6845 
   6846       switch (fp_convert->tag) {
   6847       case S390_FP_F32_TO_D32:
   6848       case S390_FP_F32_TO_D64:
   6849       case S390_FP_F32_TO_D128:
   6850       case S390_FP_F64_TO_D32:
   6851       case S390_FP_F64_TO_D64:
   6852       case S390_FP_F64_TO_D128:
   6853       case S390_FP_F128_TO_D32:
   6854       case S390_FP_F128_TO_D64:
   6855       case S390_FP_F128_TO_D128: op = "v-f2d"; break;
   6856       case S390_FP_D32_TO_F32:
   6857       case S390_FP_D32_TO_F64:
   6858       case S390_FP_D32_TO_F128:
   6859       case S390_FP_D64_TO_F32:
   6860       case S390_FP_D64_TO_F64:
   6861       case S390_FP_D64_TO_F128:
   6862       case S390_FP_D128_TO_F32:
   6863       case S390_FP_D128_TO_F64:
   6864       case S390_FP_D128_TO_F128: op = "v-d2f"; break;
   6865       default: goto fail;
   6866       }
   6867       s390_sprintf(buf, "%M %R,%R", op, fp_convert->dst_hi,
   6868                    fp_convert->op_hi);
   6869       break;
   6870    }
   6871 
   6872    case S390_INSN_MFENCE:
   6873       s390_sprintf(buf, "%M", "v-mfence");
   6874       return buf;   /* avoid printing "size = ..." which is meaningless */
   6875 
   6876    case S390_INSN_MIMM:
   6877       s390_sprintf(buf, "%M %A,%I", "v-mimm", insn->variant.mimm.dst,
   6878                    insn->variant.mimm.value);
   6879       break;
   6880 
   6881    case S390_INSN_MADD:
   6882       s390_sprintf(buf, "%M %A += %I  (= %I)", "v-madd",
   6883                    insn->variant.madd.dst,
   6884                    (Long)(Char)insn->variant.madd.delta,
   6885                    insn->variant.madd.value);
   6886       break;
   6887 
   6888    case S390_INSN_SET_FPC_BFPRM:
   6889       s390_sprintf(buf, "%M %R", "v-set-fpc-bfprm",
   6890                    insn->variant.set_fpc_bfprm.mode);
   6891       break;
   6892 
   6893    case S390_INSN_SET_FPC_DFPRM:
   6894       s390_sprintf(buf, "%M %R", "v-set-fpc-dfprm",
   6895                    insn->variant.set_fpc_dfprm.mode);
   6896       break;
   6897 
   6898    case S390_INSN_EVCHECK:
   6899       s390_sprintf(buf, "%M counter = %A, fail-addr = %A", "v-evcheck",
   6900                    insn->variant.evcheck.counter,
   6901                    insn->variant.evcheck.fail_addr);
   6902       return buf;   /* avoid printing "size = ..." which is meaningless */
   6903 
   6904    case S390_INSN_PROFINC:
   6905       s390_sprintf(buf, "%M", "v-profinc");
   6906       return buf;   /* avoid printing "size = ..." which is meaningless */
   6907 
   6908    case S390_INSN_XDIRECT:
   6909       s390_sprintf(buf, "%M if (%C) %A = %I  %s", "v-xdirect",
   6910                    insn->variant.xdirect.cond,
   6911                    insn->variant.xdirect.guest_IA,
   6912                    insn->variant.xdirect.dst,
   6913                    insn->variant.xdirect.to_fast_entry ? "fast" : "slow");
   6914       return buf;   /* avoid printing "size = ..." which is meaningless */
   6915 
   6916    case S390_INSN_XINDIR:
   6917       s390_sprintf(buf, "%M if (%C) %A = %R", "v-xindir",
   6918                    insn->variant.xindir.cond,
   6919                    insn->variant.xindir.guest_IA,
   6920                    insn->variant.xindir.dst);
   6921       return buf;   /* avoid printing "size = ..." which is meaningless */
   6922 
   6923    case S390_INSN_XASSISTED:
   6924       s390_sprintf(buf, "%M if (%C) %J %A = %R", "v-xassisted",
   6925                    insn->variant.xassisted.cond,
   6926                    insn->variant.xassisted.kind,
   6927                    insn->variant.xassisted.guest_IA,
   6928                    insn->variant.xassisted.dst);
   6929       return buf;   /* avoid printing "size = ..." which is meaningless */
   6930 
   6931    default: goto fail;
   6932    }
   6933 
   6934    /* Write out how many bytes are involved in the operation */
   6935 
   6936    {
   6937       UInt len, i;
   6938 
   6939       for (p = buf; *p; ++p)
   6940          continue;
   6941 
   6942       len = p - buf;
   6943 
   6944       if (len < 32) {
   6945          for (i = len; i < 32; ++i)
   6946             p += vex_sprintf(p, " ");
   6947       } else {
   6948          p += vex_sprintf(p, "\t");
   6949       }
   6950    }
   6951 
   6952    /* Special cases first */
   6953    switch (insn->tag) {
   6954    case S390_INSN_UNOP:
   6955       switch (insn->variant.unop.tag) {
   6956       case S390_SIGN_EXTEND_8:
   6957       case S390_ZERO_EXTEND_8:  p += vex_sprintf(p, "1 -> "); goto common;
   6958       case S390_SIGN_EXTEND_16:
   6959       case S390_ZERO_EXTEND_16: p += vex_sprintf(p, "2 -> "); goto common;
   6960       case S390_SIGN_EXTEND_32:
   6961       case S390_ZERO_EXTEND_32: p += vex_sprintf(p, "4 -> "); goto common;
   6962       default:
   6963          goto common;
   6964       }
   6965 
   6966    case S390_INSN_BFP_CONVERT:
   6967       switch (insn->variant.bfp_convert.tag) {
   6968       case S390_BFP_I32_TO_F32:
   6969       case S390_BFP_I32_TO_F64:
   6970       case S390_BFP_I32_TO_F128:
   6971       case S390_BFP_U32_TO_F32:
   6972       case S390_BFP_U32_TO_F64:
   6973       case S390_BFP_U32_TO_F128:
   6974       case S390_BFP_F32_TO_I32:
   6975       case S390_BFP_F32_TO_I64:
   6976       case S390_BFP_F32_TO_U32:
   6977       case S390_BFP_F32_TO_U64:
   6978       case S390_BFP_F32_TO_F64:
   6979       case S390_BFP_F32_TO_F128: p += vex_sprintf(p, "4 -> "); goto common;
   6980       case S390_BFP_I64_TO_F32:
   6981       case S390_BFP_I64_TO_F64:
   6982       case S390_BFP_I64_TO_F128:
   6983       case S390_BFP_U64_TO_F32:
   6984       case S390_BFP_U64_TO_F64:
   6985       case S390_BFP_U64_TO_F128:
   6986       case S390_BFP_F64_TO_I32:
   6987       case S390_BFP_F64_TO_I64:
   6988       case S390_BFP_F64_TO_U32:
   6989       case S390_BFP_F64_TO_U64:
   6990       case S390_BFP_F64_TO_F32:
   6991       case S390_BFP_F64_TO_F128: p += vex_sprintf(p, "8 -> "); goto common;
   6992       case S390_BFP_F128_TO_I32:
   6993       case S390_BFP_F128_TO_I64:
   6994       case S390_BFP_F128_TO_U32:
   6995       case S390_BFP_F128_TO_U64:
   6996       case S390_BFP_F128_TO_F32:
   6997       case S390_BFP_F128_TO_F64: p += vex_sprintf(p, "16 -> "); goto common;
   6998       default:
   6999          goto common;
   7000       }
   7001 
   7002    case S390_INSN_DFP_CONVERT:
   7003       switch (insn->variant.dfp_convert.tag) {
   7004       case S390_DFP_D32_TO_D64:
   7005       case S390_DFP_I32_TO_D64:
   7006       case S390_DFP_I32_TO_D128:
   7007       case S390_DFP_U32_TO_D64:
   7008       case S390_DFP_U32_TO_D128: p += vex_sprintf(p, "4 -> "); goto common;
   7009       case S390_DFP_D64_TO_D32:
   7010       case S390_DFP_D64_TO_D128:
   7011       case S390_DFP_I64_TO_D64:
   7012       case S390_DFP_I64_TO_D128:
   7013       case S390_DFP_U64_TO_D64:
   7014       case S390_DFP_U64_TO_D128:
   7015       case S390_DFP_D64_TO_I32:
   7016       case S390_DFP_D64_TO_I64:
   7017       case S390_DFP_D64_TO_U32:
   7018       case S390_DFP_D64_TO_U64:  p += vex_sprintf(p, "8 -> "); goto common;
   7019       case S390_DFP_D128_TO_D64:
   7020       case S390_DFP_D128_TO_I32:
   7021       case S390_DFP_D128_TO_I64:
   7022       case S390_DFP_D128_TO_U32:
   7023       case S390_DFP_D128_TO_U64: p += vex_sprintf(p, "16 -> "); goto common;
   7024       default:
   7025          goto common;
   7026       }
   7027 
   7028    case S390_INSN_FP_CONVERT: {
   7029       s390_fp_convert *fp_convert = insn->variant.fp_convert.details;
   7030 
   7031       switch (fp_convert->tag) {
   7032       case S390_FP_F32_TO_D32:
   7033       case S390_FP_F32_TO_D64:
   7034       case S390_FP_F32_TO_D128:
   7035       case S390_FP_D32_TO_F32:
   7036       case S390_FP_D32_TO_F64:
   7037       case S390_FP_D32_TO_F128:  p += vex_sprintf(p, "4 -> "); goto common;
   7038       case S390_FP_F64_TO_D32:
   7039       case S390_FP_F64_TO_D64:
   7040       case S390_FP_F64_TO_D128:
   7041       case S390_FP_D64_TO_F32:
   7042       case S390_FP_D64_TO_F64:
   7043       case S390_FP_D64_TO_F128:  p += vex_sprintf(p, "8 -> "); goto common;
   7044       case S390_FP_F128_TO_D32:
   7045       case S390_FP_F128_TO_D64:
   7046       case S390_FP_F128_TO_D128:
   7047       case S390_FP_D128_TO_F32:
   7048       case S390_FP_D128_TO_F64:
   7049       case S390_FP_D128_TO_F128: p += vex_sprintf(p, "16 -> "); goto common;
   7050       default:
   7051          goto common;
   7052       }
   7053    }
   7054 
   7055    default:
   7056       goto common;
   7057    }
   7058 
   7059    /* Common case */
   7060  common:
   7061    vex_sprintf(p, "%u bytes", (UInt)insn->size);
   7062 
   7063    return buf;
   7064 
   7065  fail: vpanic("s390_insn_as_string");
   7066 }
   7067 
   7068 
   7069 
   7070 /* Load NUM bytes from memory into register REG using addressing mode AM. */
   7071 static UChar *
   7072 s390_emit_load_mem(UChar *p, UInt num, UChar reg, const s390_amode *am)
   7073 {
   7074    UInt b = hregNumber(am->b);
   7075    UInt x = hregNumber(am->x);  /* 0 for B12 and B20 */
   7076    UInt d = am->d;
   7077 
   7078    switch (am->tag) {
   7079    case S390_AMODE_B12:
   7080    case S390_AMODE_BX12:
   7081       switch (num) {
   7082       case 1: return s390_emit_IC(p, reg, x, b, d);
   7083       case 2: return s390_emit_LH(p, reg, x, b, d);
   7084       case 4: return s390_emit_L(p, reg, x, b, d);
   7085       case 8: return s390_emit_LG(p, reg, x, b, DISP20(d));
   7086       default: goto fail;
   7087       }
   7088       break;
   7089 
   7090    case S390_AMODE_B20:
   7091    case S390_AMODE_BX20:
   7092       switch (num) {
   7093       case 1: return s390_emit_ICY(p, reg, x, b, DISP20(d));
   7094       case 2: return s390_emit_LHY(p, reg, x, b, DISP20(d));
   7095       case 4: return s390_emit_LY(p, reg, x, b, DISP20(d));
   7096       case 8: return s390_emit_LG(p, reg, x, b, DISP20(d));
   7097       default: goto fail;
   7098       }
   7099       break;
   7100 
   7101    default: goto fail;
   7102    }
   7103 
   7104  fail:
   7105    vpanic("s390_emit_load_mem");
   7106 }
   7107 
   7108 
   7109 /* Load condition code into register REG */
   7110 static UChar *
   7111 s390_emit_load_cc(UChar *p, UChar reg)
   7112 {
   7113    p = s390_emit_LGHI(p, reg, 0);  /* Clear out, cc not affected */
   7114    p = s390_emit_IPM(p, reg, reg);
   7115    /* Shift 28 bits to the right --> [0,1,2,3] */
   7116    return s390_emit_SRL(p, reg, 0, 28); /* REG = cc */
   7117 }
   7118 
   7119 
   7120 /*---------------------------------------------------------------*/
   7121 /*--- Code generation                                         ---*/
   7122 /*---------------------------------------------------------------*/
   7123 
   7124 /* Do not load more bytes than requested. */
   7125 static UChar *
   7126 s390_insn_load_emit(UChar *buf, const s390_insn *insn)
   7127 {
   7128    UInt r, x, b, d;
   7129    const s390_amode *src;
   7130 
   7131    src = insn->variant.load.src;
   7132 
   7133    r = hregNumber(insn->variant.load.dst);
   7134 
   7135    if (hregClass(insn->variant.load.dst) == HRcFlt64) {
   7136       b = hregNumber(src->b);
   7137       x = hregNumber(src->x);  /* 0 for B12 and B20 */
   7138       d = src->d;
   7139 
   7140       switch (insn->size) {
   7141 
   7142       case 4:
   7143          switch (src->tag) {
   7144          case S390_AMODE_B12:
   7145          case S390_AMODE_BX12:
   7146             return s390_emit_LE(buf, r, x, b, d);
   7147 
   7148          case S390_AMODE_B20:
   7149          case S390_AMODE_BX20:
   7150             return s390_emit_LEY(buf, r, x, b, DISP20(d));
   7151          }
   7152          break;
   7153 
   7154       case 8:
   7155          switch (src->tag) {
   7156          case S390_AMODE_B12:
   7157          case S390_AMODE_BX12:
   7158             return s390_emit_LD(buf, r, x, b, d);
   7159 
   7160          case S390_AMODE_B20:
   7161          case S390_AMODE_BX20:
   7162             return s390_emit_LDY(buf, r, x, b, DISP20(d));
   7163          }
   7164          break;
   7165       }
   7166       vpanic("s390_insn_load_emit");
   7167    }
   7168 
   7169    /* Integer stuff */
   7170    return s390_emit_load_mem(buf, insn->size, r, src);
   7171 }
   7172 
   7173 
   7174 static UChar *
   7175 s390_insn_store_emit(UChar *buf, const s390_insn *insn)
   7176 {
   7177    UInt r, x, b, d;
   7178    const s390_amode *dst;
   7179 
   7180    dst = insn->variant.store.dst;
   7181 
   7182    r = hregNumber(insn->variant.store.src);
   7183    b = hregNumber(dst->b);
   7184    x = hregNumber(dst->x);  /* 0 for B12 and B20 */
   7185    d = dst->d;
   7186 
   7187    if (hregClass(insn->variant.store.src) == HRcFlt64) {
   7188       switch (insn->size) {
   7189 
   7190       case 4:
   7191          switch (dst->tag) {
   7192          case S390_AMODE_B12:
   7193          case S390_AMODE_BX12:
   7194             return s390_emit_STE(buf, r, x, b, d);
   7195 
   7196          case S390_AMODE_B20:
   7197          case S390_AMODE_BX20:
   7198             return s390_emit_STEY(buf, r, x, b, DISP20(d));
   7199          }
   7200          break;
   7201 
   7202       case 8:
   7203          switch (dst->tag) {
   7204          case S390_AMODE_B12:
   7205          case S390_AMODE_BX12:
   7206             return s390_emit_STD(buf, r, x, b, d);
   7207 
   7208          case S390_AMODE_B20:
   7209          case S390_AMODE_BX20:
   7210             return s390_emit_STDY(buf, r, x, b, DISP20(d));
   7211          }
   7212          break;
   7213       }
   7214       vpanic("s390_insn_store_emit");
   7215    }
   7216 
   7217    /* Integer stuff */
   7218    switch (insn->size) {
   7219    case 1:
   7220       switch (dst->tag) {
   7221       case S390_AMODE_B12:
   7222       case S390_AMODE_BX12:
   7223          return s390_emit_STC(buf, r, x, b, d);
   7224 
   7225       case S390_AMODE_B20:
   7226       case S390_AMODE_BX20:
   7227          return s390_emit_STCY(buf, r, x, b, DISP20(d));
   7228       }
   7229       break;
   7230 
   7231    case 2:
   7232       switch (dst->tag) {
   7233       case S390_AMODE_B12:
   7234       case S390_AMODE_BX12:
   7235          return s390_emit_STH(buf, r, x, b, d);
   7236 
   7237       case S390_AMODE_B20:
   7238       case S390_AMODE_BX20:
   7239          return s390_emit_STHY(buf, r, x, b, DISP20(d));
   7240       }
   7241       break;
   7242 
   7243    case 4:
   7244       switch (dst->tag) {
   7245       case S390_AMODE_B12:
   7246       case S390_AMODE_BX12:
   7247          return s390_emit_ST(buf, r, x, b, d);
   7248 
   7249       case S390_AMODE_B20:
   7250       case S390_AMODE_BX20:
   7251          return s390_emit_STY(buf, r, x, b, DISP20(d));
   7252       }
   7253       break;
   7254 
   7255    case 8:
   7256       return s390_emit_STG(buf, r, x, b, DISP20(d));
   7257 
   7258    default:
   7259       break;
   7260    }
   7261 
   7262    vpanic("s390_insn_store_emit");
   7263 }
   7264 
   7265 
   7266 static UChar *
   7267 s390_insn_move_emit(UChar *buf, const s390_insn *insn)
   7268 {
   7269    UInt dst, src;
   7270    HRegClass dst_class, src_class;
   7271 
   7272    dst = hregNumber(insn->variant.move.dst);
   7273    src = hregNumber(insn->variant.move.src);
   7274 
   7275    dst_class = hregClass(insn->variant.move.dst);
   7276    src_class = hregClass(insn->variant.move.src);
   7277 
   7278    if (dst_class == src_class) {
   7279       if (dst_class == HRcInt64)
   7280          return s390_emit_LGR(buf, dst, src);
   7281       if (dst_class == HRcFlt64)
   7282          return s390_emit_LDR(buf, dst, src);
   7283    } else {
   7284       if (dst_class == HRcFlt64 && src_class == HRcInt64) {
   7285          if (insn->size == 4) {
   7286             buf = s390_emit_SLLG(buf, R0, src, 0, DISP20(32)); /* r0 = src << 32 */
   7287             return s390_emit_LDGRw(buf, dst, R0);
   7288          } else {
   7289             return s390_emit_LDGRw(buf, dst, src);
   7290          }
   7291       }
   7292       if (dst_class == HRcInt64 && src_class == HRcFlt64) {
   7293          if (insn->size == 4) {
   7294             buf = s390_emit_LGDRw(buf, dst, src);
   7295             return s390_emit_SRLG(buf, dst, dst, 0, DISP20(32)); /* dst >>= 32 */
   7296          } else {
   7297             return s390_emit_LGDRw(buf, dst, src);
   7298          }
   7299       }
   7300       /* A move between floating point registers and general purpose
   7301          registers of different size should never occur and indicates
   7302          an error elsewhere. */
   7303    }
   7304 
   7305    vpanic("s390_insn_move_emit");
   7306 }
   7307 
   7308 
   7309 static UChar *
   7310 s390_insn_memcpy_emit(UChar *buf, const s390_insn *insn)
   7311 {
   7312    s390_amode *dst = insn->variant.memcpy.dst;
   7313    s390_amode *src = insn->variant.memcpy.src;
   7314 
   7315    return s390_emit_MVC(buf, insn->size - 1, hregNumber(dst->b), dst->d,
   7316                         hregNumber(src->b), src->d);
   7317 }
   7318 
   7319 
   7320 static UChar *
   7321 s390_insn_load_immediate_emit(UChar *buf, const s390_insn *insn)
   7322 {
   7323    UInt  r;
   7324    ULong value = insn->variant.load_immediate.value;
   7325 
   7326    r = hregNumber(insn->variant.load_immediate.dst);
   7327 
   7328    if (hregClass(insn->variant.load_immediate.dst) == HRcFlt64) {
   7329       vassert(value == 0);
   7330       switch (insn->size) {
   7331       case 4: return s390_emit_LZER(buf, r, value);
   7332       case 8: return s390_emit_LZDR(buf, r, value);
   7333       }
   7334       vpanic("s390_insn_load_immediate_emit");
   7335    }
   7336 
   7337    switch (insn->size) {
   7338    case 1:
   7339    case 2:
   7340       /* Load the immediate values as a 4 byte value. That does not hurt as
   7341          those extra bytes will not be looked at. Fall through .... */
   7342    case 4:
   7343       return s390_emit_load_32imm(buf, r, value);
   7344 
   7345    case 8:
   7346       return s390_emit_load_64imm(buf, r, value);
   7347    }
   7348 
   7349    vpanic("s390_insn_load_immediate_emit");
   7350 }
   7351 
   7352 
   7353 /* There is no easy way to do ALU operations on 1-byte or 2-byte operands.
   7354    So we simply perform a 4-byte operation. Doing so uses possibly undefined
   7355    bits and produces an undefined result in those extra bit positions. But
   7356    upstream does not look at those positions, so this is OK. */
   7357 static UChar *
   7358 s390_insn_alu_emit(UChar *buf, const s390_insn *insn)
   7359 {
   7360    s390_opnd_RMI op2;
   7361    UInt dst;
   7362 
   7363    dst = hregNumber(insn->variant.alu.dst);
   7364    op2 = insn->variant.alu.op2;
   7365 
   7366    /* Second operand is in a register */
   7367    if (op2.tag == S390_OPND_REG) {
   7368       UInt r2 = hregNumber(op2.variant.reg);
   7369 
   7370       switch (insn->size) {
   7371       case 1:
   7372       case 2:
   7373       case 4:
   7374          switch (insn->variant.alu.tag) {
   7375          case S390_ALU_ADD:  return s390_emit_AR(buf, dst, r2);
   7376          case S390_ALU_SUB:  return s390_emit_SR(buf, dst, r2);
   7377          case S390_ALU_MUL:  return s390_emit_MSR(buf, dst, r2);
   7378          case S390_ALU_AND:  return s390_emit_NR(buf, dst, r2);
   7379          case S390_ALU_OR:   return s390_emit_OR(buf, dst, r2);
   7380          case S390_ALU_XOR:  return s390_emit_XR(buf, dst, r2);
   7381          case S390_ALU_LSH:  return s390_emit_SLL(buf, dst, r2, 0);
   7382          case S390_ALU_RSH:  return s390_emit_SRL(buf, dst, r2, 0);
   7383          case S390_ALU_RSHA: return s390_emit_SRA(buf, dst, r2, 0);
   7384          }
   7385          goto fail;
   7386 
   7387       case 8:
   7388          switch (insn->variant.alu.tag) {
   7389          case S390_ALU_ADD:  return s390_emit_AGR(buf, dst, r2);
   7390          case S390_ALU_SUB:  return s390_emit_SGR(buf, dst, r2);
   7391          case S390_ALU_MUL:  return s390_emit_MSGR(buf, dst, r2);
   7392          case S390_ALU_AND:  return s390_emit_NGR(buf, dst, r2);
   7393          case S390_ALU_OR:   return s390_emit_OGR(buf, dst, r2);
   7394          case S390_ALU_XOR:  return s390_emit_XGR(buf, dst, r2);
   7395          case S390_ALU_LSH:  return s390_emit_SLLG(buf, dst, dst, r2, DISP20(0));
   7396          case S390_ALU_RSH:  return s390_emit_SRLG(buf, dst, dst, r2, DISP20(0));
   7397          case S390_ALU_RSHA: return s390_emit_SRAG(buf, dst, dst, r2, DISP20(0));
   7398          }
   7399          goto fail;
   7400       }
   7401       goto fail;
   7402    }
   7403 
   7404    /* 2nd operand is in memory */
   7405    if (op2.tag == S390_OPND_AMODE) {
   7406       UInt b, x, d;
   7407       const s390_amode *src = op2.variant.am;
   7408 
   7409       b = hregNumber(src->b);
   7410       x = hregNumber(src->x);  /* 0 for B12 and B20 */
   7411       d = src->d;
   7412 
   7413       /* Shift operands are special here as there are no opcodes that
   7414          allow a memory operand. So we first load the 2nd operand into
   7415          some register. R0 is used to save restore the contents of the
   7416          chosen register.. */
   7417 
   7418       if (insn->variant.alu.tag == S390_ALU_LSH ||
   7419           insn->variant.alu.tag == S390_ALU_RSH ||
   7420           insn->variant.alu.tag == S390_ALU_RSHA) {
   7421          UInt b2;
   7422 
   7423          /* Choose a register (other than DST or R0) into which to stick the
   7424             shift amount. The following works because r15 is reserved and
   7425             thusly dst != 15. */
   7426          vassert(dst != 15);  /* extra paranoia */
   7427          b2 = (dst + 1) % 16;
   7428 
   7429          buf = s390_emit_LGR(buf, R0, b2);  /* save */
   7430 
   7431          /* Loading SRC to B2 does not modify R0. */
   7432          buf = s390_emit_load_mem(buf, insn->size, b2, src);
   7433 
   7434          if (insn->size == 8) {
   7435             switch (insn->variant.alu.tag) {
   7436             case S390_ALU_LSH:
   7437                buf = s390_emit_SLLG(buf, dst, dst, b2, DISP20(0));
   7438                break;
   7439             case S390_ALU_RSH:
   7440                buf = s390_emit_SRLG(buf, dst, dst, b2, DISP20(0));
   7441                break;
   7442             case S390_ALU_RSHA:
   7443                buf = s390_emit_SRAG(buf, dst, dst, b2, DISP20(0));
   7444                break;
   7445             default: /* unreachable */
   7446                goto fail;
   7447             }
   7448          } else {
   7449             switch (insn->variant.alu.tag) {
   7450             case S390_ALU_LSH:
   7451                buf = s390_emit_SLL(buf, dst, b2, 0);
   7452                break;
   7453             case S390_ALU_RSH:
   7454                buf = s390_emit_SRL(buf, dst, b2, 0);
   7455                break;
   7456             case S390_ALU_RSHA:
   7457                buf = s390_emit_SRA(buf, dst, b2, 0);
   7458                break;
   7459             default: /* unreachable */
   7460                goto fail;
   7461             }
   7462          }
   7463          return s390_emit_LGR(buf, b2, R0);  /* restore */
   7464       }
   7465 
   7466       switch (insn->size) {
   7467       case 1:
   7468          /* Move the byte from memory into scratch register r0 */
   7469          buf = s390_emit_load_mem(buf, 1, R0, src);
   7470 
   7471          switch (insn->variant.alu.tag) {
   7472          case S390_ALU_ADD: return s390_emit_AR(buf, dst, R0);
   7473          case S390_ALU_SUB: return s390_emit_SR(buf, dst, R0);
   7474          case S390_ALU_MUL: return s390_emit_MSR(buf, dst, R0);
   7475          case S390_ALU_AND: return s390_emit_NR(buf, dst, R0);
   7476          case S390_ALU_OR:  return s390_emit_OR(buf, dst, R0);
   7477          case S390_ALU_XOR: return s390_emit_XR(buf, dst, R0);
   7478          case S390_ALU_LSH:
   7479          case S390_ALU_RSH:
   7480          case S390_ALU_RSHA: ; /* avoid GCC warning */
   7481          }
   7482          goto fail;
   7483 
   7484       case 2:
   7485          switch (src->tag) {
   7486          case S390_AMODE_B12:
   7487          case S390_AMODE_BX12:
   7488             switch (insn->variant.alu.tag) {
   7489             case S390_ALU_ADD:
   7490                return s390_emit_AH(buf, dst, x, b, d);
   7491 
   7492             case S390_ALU_SUB:
   7493                return s390_emit_SH(buf, dst, x, b, d);
   7494 
   7495             case S390_ALU_MUL:
   7496                return s390_emit_MH(buf, dst, x, b, d);
   7497 
   7498                /* For bitwise operations: Move two bytes from memory into scratch
   7499                   register r0; then perform operation */
   7500             case S390_ALU_AND:
   7501                buf = s390_emit_LH(buf, R0, x, b, d);
   7502                return s390_emit_NR(buf, dst, R0);
   7503 
   7504             case S390_ALU_OR:
   7505                buf = s390_emit_LH(buf, R0, x, b, d);
   7506                return s390_emit_OR(buf, dst, R0);
   7507 
   7508             case S390_ALU_XOR:
   7509                buf = s390_emit_LH(buf, R0, x, b, d);
   7510                return s390_emit_XR(buf, dst, R0);
   7511 
   7512             case S390_ALU_LSH:
   7513             case S390_ALU_RSH:
   7514             case S390_ALU_RSHA: ; /* avoid GCC warning */
   7515             }
   7516             goto fail;
   7517 
   7518          case S390_AMODE_B20:
   7519          case S390_AMODE_BX20:
   7520             switch (insn->variant.alu.tag) {
   7521             case S390_ALU_ADD:
   7522                return s390_emit_AHY(buf, dst, x, b, DISP20(d));
   7523 
   7524             case S390_ALU_SUB:
   7525                return s390_emit_SHY(buf, dst, x, b, DISP20(d));
   7526 
   7527             case S390_ALU_MUL:
   7528                return s390_emit_MHYw(buf, dst, x, b, DISP20(d));
   7529 
   7530                /* For bitwise operations: Move two bytes from memory into scratch
   7531                   register r0; then perform operation */
   7532             case S390_ALU_AND:
   7533                buf = s390_emit_LHY(buf, R0, x, b, DISP20(d));
   7534                return s390_emit_NR(buf, dst, R0);
   7535 
   7536             case S390_ALU_OR:
   7537                buf = s390_emit_LHY(buf, R0, x, b, DISP20(d));
   7538                return s390_emit_OR(buf, dst, R0);
   7539 
   7540             case S390_ALU_XOR:
   7541                buf = s390_emit_LHY(buf, R0, x, b, DISP20(d));
   7542                return s390_emit_XR(buf, dst, R0);
   7543 
   7544             case S390_ALU_LSH:
   7545             case S390_ALU_RSH:
   7546             case S390_ALU_RSHA: ; /* avoid GCC warning */
   7547             }
   7548             goto fail;
   7549          }
   7550          goto fail;
   7551 
   7552       case 4:
   7553          switch (src->tag) {
   7554          case S390_AMODE_B12:
   7555          case S390_AMODE_BX12:
   7556             switch (insn->variant.alu.tag) {
   7557             case S390_ALU_ADD: return s390_emit_A(buf, dst, x, b, d);
   7558             case S390_ALU_SUB: return s390_emit_S(buf, dst, x, b, d);
   7559             case S390_ALU_MUL: return s390_emit_MS(buf, dst, x, b, d);
   7560             case S390_ALU_AND: return s390_emit_N(buf, dst, x, b, d);
   7561             case S390_ALU_OR:  return s390_emit_O(buf, dst, x, b, d);
   7562             case S390_ALU_XOR: return s390_emit_X(buf, dst, x, b, d);
   7563             case S390_ALU_LSH:
   7564             case S390_ALU_RSH:
   7565             case S390_ALU_RSHA: ; /* avoid GCC warning */
   7566             }
   7567             goto fail;
   7568 
   7569          case S390_AMODE_B20:
   7570          case S390_AMODE_BX20:
   7571             switch (insn->variant.alu.tag) {
   7572             case S390_ALU_ADD: return s390_emit_AY(buf, dst, x, b, DISP20(d));
   7573             case S390_ALU_SUB: return s390_emit_SY(buf, dst, x, b, DISP20(d));
   7574             case S390_ALU_MUL: return s390_emit_MSY(buf, dst, x, b, DISP20(d));
   7575             case S390_ALU_AND: return s390_emit_NY(buf, dst, x, b, DISP20(d));
   7576             case S390_ALU_OR:  return s390_emit_OY(buf, dst, x, b, DISP20(d));
   7577             case S390_ALU_XOR: return s390_emit_XY(buf, dst, x, b, DISP20(d));
   7578             case S390_ALU_LSH:
   7579             case S390_ALU_RSH:
   7580             case S390_ALU_RSHA: ; /* avoid GCC warning */
   7581             }
   7582             goto fail;
   7583          }
   7584          goto fail;
   7585 
   7586       case 8:
   7587          switch (insn->variant.alu.tag) {
   7588          case S390_ALU_ADD: return s390_emit_AG(buf, dst, x, b, DISP20(d));
   7589          case S390_ALU_SUB: return s390_emit_SG(buf, dst, x, b, DISP20(d));
   7590          case S390_ALU_MUL: return s390_emit_MSG(buf, dst, x, b, DISP20(d));
   7591          case S390_ALU_AND: return s390_emit_NG(buf, dst, x, b, DISP20(d));
   7592          case S390_ALU_OR:  return s390_emit_OG(buf, dst, x, b, DISP20(d));
   7593          case S390_ALU_XOR: return s390_emit_XG(buf, dst, x, b, DISP20(d));
   7594          case S390_ALU_LSH:
   7595          case S390_ALU_RSH:
   7596          case S390_ALU_RSHA: ; /* avoid GCC warning */
   7597          }
   7598          goto fail;
   7599       }
   7600       goto fail;
   7601    }
   7602 
   7603    /* 2nd operand is an immediate value */
   7604    if (op2.tag == S390_OPND_IMMEDIATE) {
   7605       ULong value;
   7606 
   7607       /* No masking of the value is required as it is not sign extended */
   7608       value = op2.variant.imm;
   7609 
   7610       switch (insn->size) {
   7611       case 1:
   7612       case 2:
   7613          /* There is no 1-byte opcode. Do the computation in
   7614             2 bytes. The extra byte will be ignored. */
   7615          switch (insn->variant.alu.tag) {
   7616          case S390_ALU_ADD:
   7617             return s390_emit_AHI(buf, dst, value);
   7618 
   7619          case S390_ALU_SUB:
   7620             return s390_emit_SLFIw(buf, dst, value);
   7621 
   7622          case S390_ALU_MUL:
   7623             return s390_emit_MHI(buf, dst, value);
   7624 
   7625          case S390_ALU_AND: return s390_emit_NILL(buf, dst, value);
   7626          case S390_ALU_OR:  return s390_emit_OILL(buf, dst, value);
   7627          case S390_ALU_XOR:
   7628             /* There is no XILL instruction.  Load the immediate value into
   7629                R0 and combine with the destination register. */
   7630             buf = s390_emit_LHI(buf, R0, value);
   7631             return s390_emit_XR(buf, dst, R0);
   7632 
   7633          case S390_ALU_LSH:
   7634             return s390_emit_SLL(buf, dst, 0, value);
   7635 
   7636          case S390_ALU_RSH:
   7637             return s390_emit_SRL(buf, dst, 0, value);
   7638 
   7639          case S390_ALU_RSHA:
   7640             return s390_emit_SRA(buf, dst, 0, value);
   7641          }
   7642          goto fail;
   7643 
   7644       case 4:
   7645          switch (insn->variant.alu.tag) {
   7646          case S390_ALU_ADD:
   7647             if (uint_fits_signed_16bit(value)) {
   7648                return s390_emit_AHI(buf, dst, value);
   7649             }
   7650             return s390_emit_AFIw(buf, dst, value);
   7651 
   7652          case S390_ALU_SUB:  return s390_emit_SLFIw(buf, dst, value);
   7653          case S390_ALU_MUL:  return s390_emit_MSFIw(buf, dst, value);
   7654          case S390_ALU_AND:  return s390_emit_NILFw(buf, dst, value);
   7655          case S390_ALU_OR:   return s390_emit_OILFw(buf, dst, value);
   7656          case S390_ALU_XOR:  return s390_emit_XILFw(buf, dst, value);
   7657          case S390_ALU_LSH:  return s390_emit_SLL(buf, dst, 0, value);
   7658          case S390_ALU_RSH:  return s390_emit_SRL(buf, dst, 0, value);
   7659          case S390_ALU_RSHA: return s390_emit_SRA(buf, dst, 0, value);
   7660          }
   7661          goto fail;
   7662 
   7663       case 8:
   7664          switch (insn->variant.alu.tag) {
   7665          case S390_ALU_ADD:
   7666             if (ulong_fits_signed_16bit(value)) {
   7667                return s390_emit_AGHI(buf, dst, value);
   7668             }
   7669             if (ulong_fits_signed_32bit(value) && s390_host_has_eimm) {
   7670                return s390_emit_AGFI(buf, dst, value);
   7671             }
   7672             /* Load constant into R0 then add */
   7673             buf = s390_emit_load_64imm(buf, R0, value);
   7674             return s390_emit_AGR(buf, dst, R0);
   7675 
   7676          case S390_ALU_SUB:
   7677             if (ulong_fits_unsigned_32bit(value)) {
   7678                return s390_emit_SLGFIw(buf, dst, value);
   7679             }
   7680             /* Load value into R0; then subtract from destination reg */
   7681             buf = s390_emit_load_64imm(buf, R0, value);
   7682             return s390_emit_SGR(buf, dst, R0);
   7683 
   7684          case S390_ALU_MUL:
   7685             if (ulong_fits_signed_32bit(value) && s390_host_has_gie) {
   7686                return s390_emit_MSGFI(buf, dst, value);
   7687             }
   7688             /* Load constant into R0 then add */
   7689             buf = s390_emit_load_64imm(buf, R0, value);
   7690             return s390_emit_MSGR(buf, dst, R0);
   7691 
   7692             /* Do it in two steps: upper half [0:31] and lower half [32:63] */
   7693          case S390_ALU_AND:
   7694             if (s390_host_has_eimm) {
   7695                buf  = s390_emit_NIHF(buf, dst, value >> 32);
   7696                return s390_emit_NILF(buf, dst, value & 0xFFFFFFFF);
   7697             }
   7698             /* Load value into R0; then combine with destination reg */
   7699             buf = s390_emit_load_64imm(buf, R0, value);
   7700             return s390_emit_NGR(buf, dst, R0);
   7701 
   7702          case S390_ALU_OR:
   7703             if (s390_host_has_eimm) {
   7704                buf  = s390_emit_OIHF(buf, dst, value >> 32);
   7705                return s390_emit_OILF(buf, dst, value & 0xFFFFFFFF);
   7706             }
   7707             /* Load value into R0; then combine with destination reg */
   7708             buf = s390_emit_load_64imm(buf, R0, value);
   7709             return s390_emit_OGR(buf, dst, R0);
   7710 
   7711          case S390_ALU_XOR:
   7712             if (s390_host_has_eimm) {
   7713                buf  = s390_emit_XIHF(buf, dst, value >> 32);
   7714                return s390_emit_XILF(buf, dst, value & 0xFFFFFFFF);
   7715             }
   7716             /* Load value into R0; then combine with destination reg */
   7717             buf = s390_emit_load_64imm(buf, R0, value);
   7718             return s390_emit_XGR(buf, dst, R0);
   7719 
   7720             /* No special considerations for long displacement here. Only the six
   7721                least significant bits of VALUE will be taken; all other bits are
   7722                ignored. So the DH2 bits are irrelevant and do not influence the
   7723                shift operation, independent of whether long-displacement is available
   7724                or not. */
   7725          case S390_ALU_LSH:  return s390_emit_SLLG(buf, dst, dst, 0, DISP20(value));
   7726          case S390_ALU_RSH:  return s390_emit_SRLG(buf, dst, dst, 0, DISP20(value));
   7727          case S390_ALU_RSHA: return s390_emit_SRAG(buf, dst, dst, 0, DISP20(value));
   7728          }
   7729          goto fail;
   7730       }
   7731       goto fail;
   7732    }
   7733 
   7734  fail:
   7735    vpanic("s390_insn_alu_emit");
   7736 }
   7737 
   7738 
   7739 static UChar *
   7740 s390_widen_emit(UChar *buf, const s390_insn *insn, UInt from_size,
   7741                 Bool sign_extend)
   7742 {
   7743    s390_opnd_RMI opnd = insn->variant.unop.src;
   7744 
   7745    switch (opnd.tag) {
   7746    case S390_OPND_REG: {
   7747       UChar r1 = hregNumber(insn->variant.unop.dst);
   7748       UChar r2 = hregNumber(opnd.variant.reg);
   7749 
   7750       switch (from_size) {
   7751       case 1:
   7752          /* Widening to a half-word is implemented like widening to a word
   7753             because the upper half-word will not be looked at. */
   7754          if (insn->size == 4 || insn->size == 2) {  /* 8 --> 32    8 --> 16 */
   7755             if (sign_extend)
   7756                return s390_emit_LBRw(buf, r1, r2);
   7757             else
   7758                return s390_emit_LLCRw(buf, r1, r2);
   7759          }
   7760          if (insn->size == 8) {  /* 8 --> 64 */
   7761             if (sign_extend)
   7762                return s390_emit_LGBRw(buf, r1, r2);
   7763             else
   7764                return s390_emit_LLGCRw(buf, r1, r2);
   7765          }
   7766          goto fail;
   7767 
   7768       case 2:
   7769          if (insn->size == 4) {  /* 16 --> 32 */
   7770             if (sign_extend)
   7771                return s390_emit_LHRw(buf, r1, r2);
   7772             else
   7773                return s390_emit_LLHRw(buf, r1, r2);
   7774          }
   7775          if (insn->size == 8) {  /* 16 --> 64 */
   7776             if (sign_extend)
   7777                return s390_emit_LGHRw(buf, r1, r2);
   7778             else
   7779                return s390_emit_LLGHRw(buf, r1, r2);
   7780          }
   7781          goto fail;
   7782 
   7783       case 4:
   7784          if (insn->size == 8) {  /* 32 --> 64 */
   7785             if (sign_extend)
   7786                return s390_emit_LGFR(buf, r1, r2);
   7787             else
   7788                return s390_emit_LLGFR(buf, r1, r2);
   7789          }
   7790          goto fail;
   7791 
   7792       default: /* unexpected "from" size */
   7793          goto fail;
   7794       }
   7795    }
   7796 
   7797    case S390_OPND_AMODE: {
   7798       UChar r1 = hregNumber(insn->variant.unop.dst);
   7799       const s390_amode *src = opnd.variant.am;
   7800       UChar b = hregNumber(src->b);
   7801       UChar x = hregNumber(src->x);
   7802       Int   d = src->d;
   7803 
   7804       switch (from_size) {
   7805       case 1:
   7806          if (insn->size == 4 || insn->size == 2) {
   7807             if (sign_extend)
   7808                return s390_emit_LB(buf, r1, x, b, DISP20(d));
   7809             else
   7810                return s390_emit_LLCw(buf, r1, x, b, DISP20(d));
   7811          }
   7812          if (insn->size == 8) {
   7813             if (sign_extend)
   7814                return s390_emit_LGB(buf, r1, x, b, DISP20(d));
   7815             else
   7816                return s390_emit_LLGC(buf, r1, x, b, DISP20(d));
   7817          }
   7818          goto fail;
   7819 
   7820       case 2:
   7821          if (insn->size == 4) {  /* 16 --> 32 */
   7822             if (sign_extend == 0)
   7823                return s390_emit_LLHw(buf, r1, x, b, DISP20(d));
   7824 
   7825             switch (src->tag) {
   7826             case S390_AMODE_B12:
   7827             case S390_AMODE_BX12:
   7828                return s390_emit_LH(buf, r1, x, b, d);
   7829 
   7830             case S390_AMODE_B20:
   7831             case S390_AMODE_BX20:
   7832                return s390_emit_LHY(buf, r1, x, b, DISP20(d));
   7833             }
   7834             goto fail;
   7835          }
   7836          if (insn->size == 8) {  /* 16 --> 64 */
   7837             if (sign_extend)
   7838                return s390_emit_LGH(buf, r1, x, b, DISP20(d));
   7839             else
   7840                return s390_emit_LLGH(buf, r1, x, b, DISP20(d));
   7841          }
   7842          goto fail;
   7843 
   7844       case 4:
   7845          if (insn->size == 8) {  /* 32 --> 64 */
   7846             if (sign_extend)
   7847                return s390_emit_LGF(buf, r1, x, b, DISP20(d));
   7848             else
   7849                return s390_emit_LLGF(buf, r1, x, b, DISP20(d));
   7850          }
   7851          goto fail;
   7852 
   7853       default: /* unexpected "from" size */
   7854          goto fail;
   7855       }
   7856    }
   7857 
   7858    case S390_OPND_IMMEDIATE: {
   7859       UChar r1 = hregNumber(insn->variant.unop.dst);
   7860       ULong value = opnd.variant.imm;
   7861 
   7862       switch (from_size) {
   7863       case 1:
   7864          if (insn->size == 4 || insn->size == 2) {  /* 8 --> 32   8 --> 16 */
   7865             if (sign_extend) {
   7866                /* host can do the sign extension to 16-bit; LHI does the rest */
   7867                return s390_emit_LHI(buf, r1, (Short)(Char)(UChar)value);
   7868             } else {
   7869                return s390_emit_LHI(buf, r1, value);
   7870             }
   7871          }
   7872          if (insn->size == 8) {  /* 8 --> 64 */
   7873             if (sign_extend) {
   7874                /* host can do the sign extension to 16-bit; LGHI does the rest */
   7875                return s390_emit_LGHI(buf, r1, (Short)(Char)(UChar)value);
   7876             } else {
   7877                return s390_emit_LGHI(buf, r1, value);
   7878             }
   7879          }
   7880          goto fail;
   7881 
   7882       case 2:
   7883          if (insn->size == 4) {  /* 16 --> 32 */
   7884             return s390_emit_LHI(buf, r1, value);
   7885          }
   7886          if (insn->size == 8) {  /* 16 --> 64 */
   7887             if (sign_extend)
   7888                return s390_emit_LGHI(buf, r1, value);
   7889             else
   7890                return s390_emit_LLILL(buf, r1, value);
   7891          }
   7892          goto fail;
   7893 
   7894       case 4:
   7895          if (insn->size == 8) {  /* 32 --> 64 */
   7896             if (sign_extend)
   7897                return s390_emit_LGFIw(buf, r1, value);
   7898             else
   7899                return s390_emit_LLILFw(buf, r1, value);
   7900          }
   7901          goto fail;
   7902 
   7903       default: /* unexpected "from" size */
   7904          goto fail;
   7905       }
   7906    }
   7907    }
   7908 
   7909  fail:
   7910    vpanic("s390_widen_emit");
   7911 }
   7912 
   7913 
   7914 static UChar *
   7915 s390_negate_emit(UChar *buf, const s390_insn *insn)
   7916 {
   7917    s390_opnd_RMI opnd;
   7918 
   7919    opnd = insn->variant.unop.src;
   7920 
   7921    switch (opnd.tag) {
   7922    case S390_OPND_REG: {
   7923       UChar r1 = hregNumber(insn->variant.unop.dst);
   7924       UChar r2 = hregNumber(opnd.variant.reg);
   7925 
   7926       switch (insn->size) {
   7927       case 1:
   7928       case 2:
   7929       case 4:
   7930          return s390_emit_LCR(buf, r1, r2);
   7931 
   7932       case 8:
   7933          return s390_emit_LCGR(buf, r1, r2);
   7934 
   7935       default:
   7936          goto fail;
   7937       }
   7938    }
   7939 
   7940    case S390_OPND_AMODE: {
   7941       UChar r1 = hregNumber(insn->variant.unop.dst);
   7942 
   7943       /* Load bytes into scratch register R0, then negate */
   7944       buf = s390_emit_load_mem(buf, insn->size, R0, opnd.variant.am);
   7945 
   7946       switch (insn->size) {
   7947       case 1:
   7948       case 2:
   7949       case 4:
   7950          return s390_emit_LCR(buf, r1, R0);
   7951 
   7952       case 8:
   7953          return s390_emit_LCGR(buf, r1, R0);
   7954 
   7955       default:
   7956          goto fail;
   7957       }
   7958    }
   7959 
   7960    case S390_OPND_IMMEDIATE: {
   7961       UChar r1 = hregNumber(insn->variant.unop.dst);
   7962       ULong value = opnd.variant.imm;
   7963 
   7964       value = ~value + 1;   /* two's complement */
   7965 
   7966       switch (insn->size) {
   7967       case 1:
   7968       case 2:
   7969          /* Load the immediate values as a 4 byte value. That does not hurt as
   7970             those extra bytes will not be looked at. Fall through .... */
   7971       case 4:
   7972          return s390_emit_load_32imm(buf, r1, value);
   7973 
   7974       case 8:
   7975          return s390_emit_load_64imm(buf, r1, value);
   7976 
   7977       default:
   7978          goto fail;
   7979       }
   7980    }
   7981    }
   7982 
   7983  fail:
   7984    vpanic("s390_negate_emit");
   7985 }
   7986 
   7987 
   7988 static UChar *
   7989 s390_insn_unop_emit(UChar *buf, const s390_insn *insn)
   7990 {
   7991    switch (insn->variant.unop.tag) {
   7992    case S390_ZERO_EXTEND_8:  return s390_widen_emit(buf, insn, 1, 0);
   7993    case S390_ZERO_EXTEND_16: return s390_widen_emit(buf, insn, 2, 0);
   7994    case S390_ZERO_EXTEND_32: return s390_widen_emit(buf, insn, 4, 0);
   7995 
   7996    case S390_SIGN_EXTEND_8:  return s390_widen_emit(buf, insn, 1, 1);
   7997    case S390_SIGN_EXTEND_16: return s390_widen_emit(buf, insn, 2, 1);
   7998    case S390_SIGN_EXTEND_32: return s390_widen_emit(buf, insn, 4, 1);
   7999 
   8000    case S390_NEGATE:         return s390_negate_emit(buf, insn);
   8001    }
   8002 
   8003    vpanic("s390_insn_unop_emit");
   8004 }
   8005 
   8006 
   8007 /* Only 4-byte and 8-byte operands are handled. 1-byte and 2-byte
   8008    comparisons will have been converted to 4-byte comparisons in
   8009    s390_isel_cc and should not occur here. */
   8010 static UChar *
   8011 s390_insn_test_emit(UChar *buf, const s390_insn *insn)
   8012 {
   8013    s390_opnd_RMI opnd;
   8014 
   8015    opnd = insn->variant.test.src;
   8016 
   8017    switch (opnd.tag) {
   8018    case S390_OPND_REG: {
   8019       UInt reg = hregNumber(opnd.variant.reg);
   8020 
   8021       switch (insn->size) {
   8022       case 4:
   8023          return s390_emit_LTR(buf, reg, reg);
   8024 
   8025       case 8:
   8026          return s390_emit_LTGR(buf, reg, reg);
   8027 
   8028       default:
   8029          goto fail;
   8030       }
   8031    }
   8032 
   8033    case S390_OPND_AMODE: {
   8034       const s390_amode *am = opnd.variant.am;
   8035       UChar b = hregNumber(am->b);
   8036       UChar x = hregNumber(am->x);
   8037       Int   d = am->d;
   8038 
   8039       switch (insn->size) {
   8040       case 4:
   8041          return s390_emit_LTw(buf, R0, x, b, DISP20(d));
   8042 
   8043       case 8:
   8044          return s390_emit_LTGw(buf, R0, x, b, DISP20(d));
   8045 
   8046       default:
   8047          goto fail;
   8048       }
   8049    }
   8050 
   8051    case S390_OPND_IMMEDIATE: {
   8052       ULong value = opnd.variant.imm;
   8053 
   8054       switch (insn->size) {
   8055       case 4:
   8056          buf = s390_emit_load_32imm(buf, R0, value);
   8057          return s390_emit_LTR(buf, R0, R0);
   8058 
   8059       case 8:
   8060          buf = s390_emit_load_64imm(buf, R0, value);
   8061          return s390_emit_LTGR(buf, R0, R0);
   8062 
   8063       default:
   8064          goto fail;
   8065       }
   8066    }
   8067 
   8068    default:
   8069       goto fail;
   8070    }
   8071 
   8072  fail:
   8073    vpanic("s390_insn_test_emit");
   8074 }
   8075 
   8076 
   8077 static UChar *
   8078 s390_insn_cc2bool_emit(UChar *buf, const s390_insn *insn)
   8079 {
   8080    UChar r1 = hregNumber(insn->variant.cc2bool.dst);
   8081    s390_cc_t cond = insn->variant.cc2bool.cond;
   8082 
   8083    /* Make the destination register be 1 or 0, depending on whether
   8084       the relevant condition holds. A 64-bit value is computed. */
   8085    if (cond == S390_CC_ALWAYS)
   8086       return s390_emit_LGHI(buf, r1, 1);  /* r1 = 1 */
   8087 
   8088    buf = s390_emit_load_cc(buf, r1);                 /* r1 = cc */
   8089    buf = s390_emit_LGHI(buf, R0, cond);              /* r0 = mask */
   8090    buf = s390_emit_SLLG(buf, r1, R0, r1, DISP20(0)); /* r1 = mask << cc */
   8091    buf = s390_emit_SRLG(buf, r1, r1, 0,  DISP20(3)); /* r1 = r1 >> 3 */
   8092    buf = s390_emit_NILL(buf, r1, 1);                 /* r1 = r1 & 0x1 */
   8093 
   8094    return buf;
   8095 }
   8096 
   8097 
   8098 /* Only 4-byte and 8-byte operands are handled. */
   8099 static UChar *
   8100 s390_insn_cas_emit(UChar *buf, const s390_insn *insn)
   8101 {
   8102    UChar r1, r3, b, old;
   8103    Int d;
   8104    s390_amode *am;
   8105 
   8106    r1 = hregNumber(insn->variant.cas.op1); /* expected value */
   8107    r3 = hregNumber(insn->variant.cas.op3);
   8108    old= hregNumber(insn->variant.cas.old_mem);
   8109    am = insn->variant.cas.op2;
   8110    b  = hregNumber(am->b);
   8111    d  = am->d;
   8112 
   8113    vassert(am->tag == S390_AMODE_B12 || am->tag == S390_AMODE_B20);
   8114 
   8115    switch (insn->size) {
   8116    case 4:
   8117       /* r1 must not be overwritten. So copy it to R0 and let CS clobber it */
   8118       buf = s390_emit_LR(buf, R0, r1);
   8119       if (am->tag == S390_AMODE_B12)
   8120          buf = s390_emit_CS(buf, R0, r3, b, d);
   8121       else
   8122          buf = s390_emit_CSY(buf, R0, r3, b, DISP20(d));
   8123       /* Now copy R0 which has the old memory value to OLD */
   8124       return s390_emit_LR(buf, old, R0);
   8125 
   8126    case 8:
   8127       /* r1 must not be overwritten. So copy it to R0 and let CS clobber it */
   8128       buf = s390_emit_LGR(buf, R0, r1);
   8129       buf = s390_emit_CSG(buf, R0, r3, b, DISP20(d));
   8130       /* Now copy R0 which has the old memory value to OLD */
   8131       return s390_emit_LGR(buf, old, R0);
   8132 
   8133    default:
   8134       goto fail;
   8135    }
   8136 
   8137  fail:
   8138    vpanic("s390_insn_cas_emit");
   8139 }
   8140 
   8141 
   8142 /* Only 4-byte and 8-byte operands are handled. */
   8143 static UChar *
   8144 s390_insn_cdas_emit(UChar *buf, const s390_insn *insn)
   8145 {
   8146    UChar r1, r1p1, r3, /*r3p1,*/ b, old_high, old_low, scratch;
   8147    Int d;
   8148    s390_amode *am;
   8149    s390_cdas *cdas = insn->variant.cdas.details;
   8150 
   8151    r1   = hregNumber(cdas->op1_high); /* expected value */
   8152    r1p1 = hregNumber(cdas->op1_low);  /* expected value */
   8153    r3   = hregNumber(cdas->op3_high);
   8154    /* r3p1 = hregNumber(cdas->op3_low); */ /* unused */
   8155    old_high = hregNumber(cdas->old_mem_high);
   8156    old_low  = hregNumber(cdas->old_mem_low);
   8157    scratch  = hregNumber(cdas->scratch);
   8158    am = cdas->op2;
   8159    b  = hregNumber(am->b);
   8160    d  = am->d;
   8161 
   8162    vassert(scratch == 1);
   8163    vassert(am->tag == S390_AMODE_B12 || am->tag == S390_AMODE_B20);
   8164 
   8165    switch (insn->size) {
   8166    case 4:
   8167       /* r1, r1+1 must not be overwritten. So copy them to R0,scratch
   8168          and let CDS/CDSY clobber it */
   8169       buf = s390_emit_LR(buf, R0, r1);
   8170       buf = s390_emit_LR(buf, scratch, r1p1);
   8171 
   8172       if (am->tag == S390_AMODE_B12)
   8173          buf = s390_emit_CDS(buf, R0, r3, b, d);
   8174       else
   8175          buf = s390_emit_CDSY(buf, R0, r3, b, DISP20(d));
   8176 
   8177       /* Now copy R0,scratch which has the old memory value to OLD */
   8178       buf = s390_emit_LR(buf, old_high, R0);
   8179       buf = s390_emit_LR(buf, old_low,  scratch);
   8180       return buf;
   8181 
   8182    case 8:
   8183       /* r1, r1+1 must not be overwritten. So copy them to R0,scratch
   8184          and let CDSG clobber it */
   8185       buf = s390_emit_LGR(buf, R0, r1);
   8186       buf = s390_emit_LGR(buf, scratch, r1p1);
   8187 
   8188       buf = s390_emit_CDSG(buf, R0, r3, b, DISP20(d));
   8189 
   8190       /* Now copy R0,scratch which has the old memory value to OLD */
   8191       buf = s390_emit_LGR(buf, old_high, R0);
   8192       buf = s390_emit_LGR(buf, old_low,  scratch);
   8193       return buf;
   8194 
   8195    default:
   8196       goto fail;
   8197    }
   8198 
   8199  fail:
   8200    vpanic("s390_insn_cdas_emit");
   8201 }
   8202 
   8203 
   8204 /* Only 4-byte and 8-byte comparisons are handled. 1-byte and 2-byte
   8205    comparisons will have been converted to 4-byte comparisons in
   8206    s390_isel_cc and should not occur here. */
   8207 static UChar *
   8208 s390_insn_compare_emit(UChar *buf, const s390_insn *insn)
   8209 {
   8210    s390_opnd_RMI op2;
   8211    HReg op1;
   8212    Bool signed_comparison;
   8213 
   8214    op1 = insn->variant.compare.src1;
   8215    op2 = insn->variant.compare.src2;
   8216    signed_comparison = insn->variant.compare.signed_comparison;
   8217 
   8218    switch (op2.tag) {
   8219    case S390_OPND_REG: {
   8220       UInt r1 = hregNumber(op1);
   8221       UInt r2 = hregNumber(op2.variant.reg);
   8222 
   8223       switch (insn->size) {
   8224       case 4:
   8225          if (signed_comparison)
   8226             return s390_emit_CR(buf, r1, r2);
   8227          else
   8228             return s390_emit_CLR(buf, r1, r2);
   8229 
   8230       case 8:
   8231          if (signed_comparison)
   8232             return s390_emit_CGR(buf, r1, r2);
   8233          else
   8234             return s390_emit_CLGR(buf, r1, r2);
   8235 
   8236       default:
   8237          goto fail;
   8238       }
   8239    }
   8240 
   8241    case S390_OPND_AMODE: {
   8242       UChar r1 = hregNumber(op1);
   8243       const s390_amode *am = op2.variant.am;
   8244       UChar b = hregNumber(am->b);
   8245       UChar x = hregNumber(am->x);
   8246       Int   d = am->d;
   8247 
   8248       switch (insn->size) {
   8249       case 4:
   8250          switch (am->tag) {
   8251          case S390_AMODE_B12:
   8252          case S390_AMODE_BX12:
   8253             if (signed_comparison)
   8254                return s390_emit_C(buf, r1, x, b, d);
   8255             else
   8256                return s390_emit_CL(buf, r1, x, b, d);
   8257 
   8258          case S390_AMODE_B20:
   8259          case S390_AMODE_BX20:
   8260             if (signed_comparison)
   8261                return s390_emit_CY(buf, r1, x, b, DISP20(d));
   8262             else
   8263                return s390_emit_CLY(buf, r1, x, b, DISP20(d));
   8264          }
   8265          goto fail;
   8266 
   8267       case 8:
   8268          if (signed_comparison)
   8269             return s390_emit_CG(buf, r1, x, b, DISP20(d));
   8270          else
   8271             return s390_emit_CLG(buf, r1, x, b, DISP20(d));
   8272 
   8273       default:
   8274          goto fail;
   8275       }
   8276    }
   8277 
   8278    case S390_OPND_IMMEDIATE: {
   8279       UChar r1 = hregNumber(op1);
   8280       ULong value = op2.variant.imm;
   8281 
   8282       switch (insn->size) {
   8283       case 4:
   8284          if (signed_comparison)
   8285             return s390_emit_CFIw(buf, r1, value);
   8286          else
   8287             return s390_emit_CLFIw(buf, r1, value);
   8288 
   8289       case 8:
   8290          if (s390_host_has_eimm) {
   8291             if (signed_comparison) {
   8292                if (ulong_fits_signed_32bit(value))
   8293                   return s390_emit_CGFI(buf, r1, value);
   8294             } else {
   8295                if (ulong_fits_unsigned_32bit(value))
   8296                   return s390_emit_CLGFI(buf, r1, value);
   8297             }
   8298          }
   8299          buf = s390_emit_load_64imm(buf, R0, value);
   8300          if (signed_comparison)
   8301             return s390_emit_CGR(buf, r1, R0);
   8302          else
   8303             return s390_emit_CLGR(buf, r1, R0);
   8304 
   8305       default:
   8306          goto fail;
   8307       }
   8308    }
   8309 
   8310    default:
   8311       goto fail;
   8312    }
   8313 
   8314  fail:
   8315    vpanic("s390_insn_compare_emit");
   8316 }
   8317 
   8318 
   8319 static UChar *
   8320 s390_insn_mul_emit(UChar *buf, const s390_insn *insn)
   8321 {
   8322    s390_opnd_RMI op2;
   8323    UChar r1;
   8324    Bool signed_multiply;
   8325 
   8326    /* The register number identifying the register pair */
   8327    r1  = hregNumber(insn->variant.mul.dst_hi);
   8328 
   8329    op2 = insn->variant.mul.op2;
   8330    signed_multiply = insn->tag == S390_INSN_SMUL;
   8331 
   8332    switch (op2.tag) {
   8333    case S390_OPND_REG: {
   8334       UInt r2 = hregNumber(op2.variant.reg);
   8335 
   8336       switch (insn->size) {
   8337       case 1:
   8338       case 2:
   8339       case 4:
   8340          if (signed_multiply)
   8341             return s390_emit_MR(buf, r1, r2);
   8342          else
   8343             return s390_emit_MLR(buf, r1, r2);
   8344 
   8345       case 8:
   8346          if (signed_multiply)
   8347             vpanic("s390_insn_mul_emit");
   8348          else
   8349             return s390_emit_MLGR(buf, r1, r2);
   8350 
   8351       default:
   8352          goto fail;
   8353       }
   8354    }
   8355 
   8356    case S390_OPND_AMODE: {
   8357       const s390_amode *am = op2.variant.am;
   8358       UChar b = hregNumber(am->b);
   8359       UChar x = hregNumber(am->x);
   8360       Int   d = am->d;
   8361 
   8362       switch (insn->size) {
   8363       case 1:
   8364       case 2:
   8365          /* Load bytes into scratch register R0, then multiply */
   8366          buf = s390_emit_load_mem(buf, insn->size, R0, am);
   8367          if (signed_multiply)
   8368             return s390_emit_MR(buf, r1, R0);
   8369          else
   8370             return s390_emit_MLR(buf, r1, R0);
   8371 
   8372       case 4:
   8373          switch (am->tag) {
   8374          case S390_AMODE_B12:
   8375          case S390_AMODE_BX12:
   8376             if (signed_multiply)
   8377                return s390_emit_M(buf, r1, x, b, d);
   8378             else
   8379                return s390_emit_ML(buf, r1, x, b, DISP20(d));
   8380 
   8381          case S390_AMODE_B20:
   8382          case S390_AMODE_BX20:
   8383             if (signed_multiply)
   8384                return s390_emit_MFYw(buf, r1, x, b, DISP20(d));
   8385             else
   8386                return s390_emit_ML(buf, r1, x, b, DISP20(d));
   8387          }
   8388          goto fail;
   8389 
   8390       case 8:
   8391          if (signed_multiply)
   8392             vpanic("s390_insn_mul_emit");
   8393          else
   8394             return s390_emit_MLG(buf, r1, x, b, DISP20(d));
   8395 
   8396       default:
   8397          goto fail;
   8398       }
   8399    }
   8400 
   8401    case S390_OPND_IMMEDIATE: {
   8402       ULong value = op2.variant.imm;
   8403 
   8404       switch (insn->size) {
   8405       case 1:
   8406       case 2:
   8407       case 4:
   8408          buf = s390_emit_load_32imm(buf, R0, value);
   8409          if (signed_multiply)
   8410             return s390_emit_MR(buf, r1, R0);
   8411          else
   8412             return s390_emit_MLR(buf, r1, R0);
   8413 
   8414       case 8:
   8415          buf = s390_emit_load_64imm(buf, R0, value);
   8416          if (signed_multiply)
   8417             vpanic("s390_insn_mul_emit");
   8418          else
   8419             return s390_emit_MLGR(buf, r1, R0);
   8420 
   8421       default:
   8422          goto fail;
   8423       }
   8424    }
   8425 
   8426    default:
   8427       goto fail;
   8428    }
   8429 
   8430  fail:
   8431    vpanic("s390_insn_mul_emit");
   8432 }
   8433 
   8434 
   8435 static UChar *
   8436 s390_insn_div_emit(UChar *buf, const s390_insn *insn)
   8437 {
   8438    s390_opnd_RMI op2;
   8439    UChar r1;
   8440    Bool signed_divide;
   8441 
   8442    r1  = hregNumber(insn->variant.div.op1_hi);
   8443    op2 = insn->variant.div.op2;
   8444    signed_divide = insn->tag == S390_INSN_SDIV;
   8445 
   8446    switch (op2.tag) {
   8447    case S390_OPND_REG: {
   8448       UInt r2 = hregNumber(op2.variant.reg);
   8449 
   8450       switch (insn->size) {
   8451       case 4:
   8452          if (signed_divide)
   8453             return s390_emit_DR(buf, r1, r2);
   8454          else
   8455             return s390_emit_DLR(buf, r1, r2);
   8456 
   8457       case 8:
   8458          if (signed_divide)
   8459             vpanic("s390_insn_div_emit");
   8460          else
   8461             return s390_emit_DLGR(buf, r1, r2);
   8462 
   8463       default:
   8464          goto fail;
   8465       }
   8466    }
   8467 
   8468    case S390_OPND_AMODE: {
   8469       const s390_amode *am = op2.variant.am;
   8470       UChar b = hregNumber(am->b);
   8471       UChar x = hregNumber(am->x);
   8472       Int   d = am->d;
   8473 
   8474       switch (insn->size) {
   8475       case 4:
   8476          switch (am->tag) {
   8477          case S390_AMODE_B12:
   8478          case S390_AMODE_BX12:
   8479             if (signed_divide)
   8480                return s390_emit_D(buf, r1, x, b, d);
   8481             else
   8482                return s390_emit_DL(buf, r1, x, b, DISP20(d));
   8483 
   8484          case S390_AMODE_B20:
   8485          case S390_AMODE_BX20:
   8486             if (signed_divide) {
   8487                buf = s390_emit_LY(buf, R0, x, b, DISP20(d));
   8488                return s390_emit_DR(buf, r1, R0);
   8489             } else
   8490                return s390_emit_DL(buf, r1, x, b, DISP20(d));
   8491          }
   8492          goto fail;
   8493 
   8494       case 8:
   8495          if (signed_divide)
   8496             vpanic("s390_insn_div_emit");
   8497          else
   8498             return s390_emit_DLG(buf, r1, x, b, DISP20(d));
   8499 
   8500       default:
   8501          goto fail;
   8502       }
   8503    }
   8504 
   8505    case S390_OPND_IMMEDIATE: {
   8506       ULong value = op2.variant.imm;
   8507 
   8508       switch (insn->size) {
   8509       case 4:
   8510          buf = s390_emit_load_32imm(buf, R0, value);
   8511          if (signed_divide)
   8512             return s390_emit_DR(buf, r1, R0);
   8513          else
   8514             return s390_emit_DLR(buf, r1, R0);
   8515 
   8516       case 8:
   8517          buf = s390_emit_load_64imm(buf, R0, value);
   8518          if (signed_divide)
   8519             vpanic("s390_insn_div_emit");
   8520          else
   8521             return s390_emit_DLGR(buf, r1, R0);
   8522 
   8523       default:
   8524          goto fail;
   8525       }
   8526    }
   8527 
   8528    default:
   8529       goto fail;
   8530    }
   8531 
   8532  fail:
   8533    vpanic("s390_insn_div_emit");
   8534 }
   8535 
   8536 
   8537 static UChar *
   8538 s390_insn_divs_emit(UChar *buf, const s390_insn *insn)
   8539 {
   8540    s390_opnd_RMI op2;
   8541    UChar r1;
   8542 
   8543    r1  = hregNumber(insn->variant.divs.rem);
   8544    op2 = insn->variant.divs.op2;
   8545 
   8546    switch (op2.tag) {
   8547    case S390_OPND_REG: {
   8548       UInt r2 = hregNumber(op2.variant.reg);
   8549 
   8550       return s390_emit_DSGR(buf, r1, r2);
   8551    }
   8552 
   8553    case S390_OPND_AMODE: {
   8554       const s390_amode *am = op2.variant.am;
   8555       UChar b = hregNumber(am->b);
   8556       UChar x = hregNumber(am->x);
   8557       Int   d = am->d;
   8558 
   8559       return s390_emit_DSG(buf, r1, x, b, DISP20(d));
   8560    }
   8561 
   8562    case S390_OPND_IMMEDIATE: {
   8563       ULong value = op2.variant.imm;
   8564 
   8565       buf = s390_emit_load_64imm(buf, R0, value);
   8566       return s390_emit_DSGR(buf, r1, R0);
   8567    }
   8568 
   8569    default:
   8570       goto fail;
   8571    }
   8572 
   8573  fail:
   8574    vpanic("s390_insn_divs_emit");
   8575 }
   8576 
   8577 
   8578 static UChar *
   8579 s390_insn_clz_emit(UChar *buf, const s390_insn *insn)
   8580 {
   8581    s390_opnd_RMI src;
   8582    UChar r1, r1p1, r2, *p;
   8583 
   8584    r1   = hregNumber(insn->variant.clz.num_bits);
   8585    r1p1 = hregNumber(insn->variant.clz.clobber);
   8586 
   8587    vassert((r1 & 0x1) == 0);
   8588    vassert(r1p1 == r1 + 1);
   8589 
   8590    p = buf;
   8591    src = insn->variant.clz.src;
   8592 
   8593    /* Get operand and move it to r2 */
   8594    switch (src.tag) {
   8595    case S390_OPND_REG:
   8596       r2 = hregNumber(src.variant.reg);
   8597       break;
   8598 
   8599    case S390_OPND_AMODE: {
   8600       const s390_amode *am = src.variant.am;
   8601       UChar b = hregNumber(am->b);
   8602       UChar x = hregNumber(am->x);
   8603       Int   d = am->d;
   8604 
   8605       p  = s390_emit_LG(p, R0, x, b, DISP20(d));
   8606       r2 = R0;
   8607       break;
   8608    }
   8609 
   8610    case S390_OPND_IMMEDIATE: {
   8611       ULong value = src.variant.imm;
   8612 
   8613       p  = s390_emit_load_64imm(p, R0, value);
   8614       r2 = R0;
   8615       break;
   8616    }
   8617 
   8618    default:
   8619       goto fail;
   8620    }
   8621 
   8622    /* Use FLOGR if you can */
   8623    if (s390_host_has_eimm) {
   8624       return s390_emit_FLOGR(p, r1, r2);
   8625    }
   8626 
   8627    /*
   8628       r0 = r2;
   8629       r1 = 64;
   8630       while (r0 != 0) {
   8631         r1 -= 1;
   8632         r0 >>= 1;
   8633       }
   8634    */
   8635    p = s390_emit_LTGR(p, R0, r2);
   8636    p = s390_emit_LLILL(p, r1,  64);
   8637 
   8638    p = s390_emit_BRC(p, S390_CC_E, (4 + 4 + 6 + 4 + 4)/ 2);  /* 4 bytes */
   8639    p = s390_emit_AGHI(p, r1, (UShort)-1);         /* r1  -= 1;  4 bytes */
   8640    p = s390_emit_SRLG(p, R0, R0, R0, DISP20(1));  /* r0 >>= 1;  6 bytes */
   8641    p = s390_emit_LTGR(p, R0, R0);                 /* set cc     4 bytes */
   8642    p = s390_emit_BRC(p, S390_CC_NE,               /*            4 bytes */
   8643                      (UShort)(-(4 + 6 + 4) / 2));
   8644    return p;
   8645 
   8646  fail:
   8647    vpanic("s390_insn_clz_emit");
   8648 }
   8649 
   8650 
   8651 /* Returns a value == BUF to denote failure, != BUF to denote success. */
   8652 static UChar *
   8653 s390_insn_helper_call_emit(UChar *buf, const s390_insn *insn)
   8654 {
   8655    s390_cc_t cond;
   8656    ULong target;
   8657    UChar *ptmp = buf;
   8658    s390_helper_call *helper_call = insn->variant.helper_call.details;
   8659 
   8660    cond = helper_call->cond;
   8661    target = helper_call->target;
   8662 
   8663    if (cond != S390_CC_ALWAYS
   8664        && helper_call->rloc.pri != RLPri_None) {
   8665       /* The call might not happen (it isn't unconditional) and it
   8666          returns a result.  In this case we will need to generate a
   8667          control flow diamond to put 0x555..555 in the return
   8668          register(s) in the case where the call doesn't happen.  If
   8669          this ever becomes necessary, maybe copy code from the ARM
   8670          equivalent.  Until that day, just give up. */
   8671       return buf; /* To denote failure. */
   8672    }
   8673 
   8674    if (cond != S390_CC_ALWAYS) {
   8675       /* So we have something like this
   8676          if (cond) call X;
   8677          Y: ...
   8678          We convert this into
   8679          if (! cond) goto Y;        // BRC opcode; 4 bytes
   8680          call X;
   8681          Y:
   8682       */
   8683       /* 4 bytes (a BRC insn) to be filled in here */
   8684       buf += 4;
   8685    }
   8686 
   8687    /* Load the target address into a register, that
   8688       (a) is not used for passing parameters to the helper and
   8689       (b) can be clobbered by the callee
   8690       (c) is not special to the BASR insn
   8691       r1 is the only choice.
   8692       Also, need to arrange for the return address be put into the
   8693       link-register */
   8694    buf = s390_emit_load_64imm(buf, 1, target);
   8695 
   8696    /* Stash away the client's FPC register because the helper might change it. */
   8697    buf = s390_emit_STFPC(buf, S390_REGNO_STACK_POINTER, S390_OFFSET_SAVED_FPC_C);
   8698 
   8699    buf = s390_emit_BASR(buf, S390_REGNO_LINK_REGISTER, 1);      // call helper
   8700 
   8701    buf = s390_emit_LFPC(buf, S390_REGNO_STACK_POINTER,          // restore FPC
   8702                         S390_OFFSET_SAVED_FPC_C);
   8703 
   8704    if (cond != S390_CC_ALWAYS) {
   8705       Int delta = buf - ptmp;
   8706 
   8707       delta >>= 1;  /* immediate constant is #half-words */
   8708       vassert(delta > 0 && delta < (1 << 16));
   8709       s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
   8710    }
   8711 
   8712    return buf;
   8713 }
   8714 
   8715 
   8716 static UChar *
   8717 s390_insn_cond_move_emit(UChar *buf, const s390_insn *insn)
   8718 {
   8719    HReg dst;
   8720    s390_opnd_RMI src;
   8721    s390_cc_t cond;
   8722    UChar *p, *ptmp = 0;   /* avoid compiler warnings */
   8723 
   8724    cond = insn->variant.cond_move.cond;
   8725    dst  = insn->variant.cond_move.dst;
   8726    src  = insn->variant.cond_move.src;
   8727 
   8728    if (cond == S390_CC_NEVER) return buf;
   8729 
   8730    p = buf;
   8731 
   8732    if (s390_host_has_lsc) {
   8733       /* LOCx is not the preferred way to implement an unconditional load. */
   8734       if (cond != S390_CC_ALWAYS) goto use_branch_insn;
   8735 
   8736       switch (src.tag) {
   8737       case S390_OPND_REG:
   8738          return s390_emit_LOCGR(p, cond, hregNumber(dst),
   8739                                 hregNumber(src.variant.reg));
   8740 
   8741       case S390_OPND_AMODE: {
   8742          const s390_amode *am = src.variant.am;
   8743 
   8744          /* We cannot use LOCx for loads less than 4 bytes. In that case
   8745             load into R0 and then use LOCGR. Do the same if the amode uses
   8746             an index register. */
   8747          if (insn->size < 4 ||
   8748              am->tag == S390_AMODE_BX12 || am->tag == S390_AMODE_BX20) {
   8749             p = s390_emit_load_mem(p, insn->size, R0, am);
   8750             p = s390_emit_LOCGR(p, cond, hregNumber(dst), R0);
   8751             return p;
   8752          }
   8753 
   8754          vassert(am->tag == S390_AMODE_B12 || am->tag == S390_AMODE_B20);
   8755          vassert(insn->size == 4 || insn->size == 8);
   8756 
   8757          UInt b = hregNumber(am->b);
   8758          UInt d = am->d;
   8759 
   8760          if (insn->size == 4) {
   8761             return s390_emit_LOC(p, hregNumber(dst), cond, b, DISP20(d));
   8762          }
   8763          return s390_emit_LOCG(p, hregNumber(dst), cond, b, DISP20(d));
   8764       }
   8765 
   8766       case S390_OPND_IMMEDIATE: {
   8767          ULong value = src.variant.imm;
   8768 
   8769          /* Load value into R0, then use LOCGR */
   8770          if (insn->size <= 4) {
   8771             p = s390_emit_load_32imm(p, R0, value);
   8772             return s390_emit_LOCGR(p, cond, hregNumber(dst), R0);
   8773          }
   8774 
   8775          vassert(insn->size == 8);
   8776          p = s390_emit_load_64imm(p, R0, value);
   8777          return s390_emit_LOCGR(p, cond, hregNumber(dst), R0);
   8778       }
   8779       }
   8780    }
   8781 
   8782 use_branch_insn:
   8783    /* Branch (if cond fails) over move instrs */
   8784    if (cond != S390_CC_ALWAYS) {
   8785       /* Don't know how many bytes to jump over yet.
   8786          Make space for a BRC instruction (4 bytes) and fill in later. */
   8787       ptmp = p;   /*  to be filled in here */
   8788       p += 4;
   8789    }
   8790 
   8791    // cond true: move src => dst
   8792 
   8793    switch (src.tag) {
   8794    case S390_OPND_REG:
   8795       p = s390_emit_LGR(p, hregNumber(dst), hregNumber(src.variant.reg));
   8796       break;
   8797 
   8798    case S390_OPND_AMODE:
   8799       p = s390_emit_load_mem(p, insn->size, hregNumber(dst), src.variant.am);
   8800       break;
   8801 
   8802    case S390_OPND_IMMEDIATE: {
   8803       ULong value = src.variant.imm;
   8804       UInt  r = hregNumber(dst);
   8805 
   8806       switch (insn->size) {
   8807       case 1:
   8808       case 2:
   8809          /* Load the immediate values as a 4 byte value. That does not hurt as
   8810             those extra bytes will not be looked at. Fall through .... */
   8811       case 4:
   8812          p = s390_emit_load_32imm(p, r, value);
   8813          break;
   8814 
   8815       case 8:
   8816          p = s390_emit_load_64imm(p, r, value);
   8817          break;
   8818       }
   8819       break;
   8820    }
   8821 
   8822    default:
   8823       goto fail;
   8824    }
   8825 
   8826    if (cond != S390_CC_ALWAYS) {
   8827       Int delta = p - ptmp;
   8828 
   8829       delta >>= 1;  /* immediate constant is #half-words */
   8830       vassert(delta > 0 && delta < (1 << 16));
   8831       s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
   8832    }
   8833 
   8834    return p;
   8835 
   8836  fail:
   8837    vpanic("s390_insn_cond_move_emit");
   8838 }
   8839 
   8840 
   8841 static UChar *
   8842 s390_insn_bfp_triop_emit(UChar *buf, const s390_insn *insn)
   8843 {
   8844    UInt r1 = hregNumber(insn->variant.bfp_triop.dst);
   8845    UInt r2 = hregNumber(insn->variant.bfp_triop.op2);
   8846    UInt r3 = hregNumber(insn->variant.bfp_triop.op3);
   8847 
   8848    switch (insn->size) {
   8849    case 4:
   8850       switch (insn->variant.bfp_triop.tag) {
   8851       case S390_BFP_MADD:  return s390_emit_MAEBR(buf, r1, r3, r2);
   8852       case S390_BFP_MSUB:  return s390_emit_MSEBR(buf, r1, r3, r2);
   8853       default:  goto fail;
   8854       }
   8855       break;
   8856 
   8857    case 8:
   8858       switch (insn->variant.bfp_triop.tag) {
   8859       case S390_BFP_MADD:  return s390_emit_MADBR(buf, r1, r3, r2);
   8860       case S390_BFP_MSUB:  return s390_emit_MSDBR(buf, r1, r3, r2);
   8861       default:  goto fail;
   8862       }
   8863       break;
   8864 
   8865    default:  goto fail;
   8866    }
   8867 
   8868  fail:
   8869    vpanic("s390_insn_bfp_triop_emit");
   8870 }
   8871 
   8872 
   8873 static UChar *
   8874 s390_insn_bfp_binop_emit(UChar *buf, const s390_insn *insn)
   8875 {
   8876    UInt r1 = hregNumber(insn->variant.bfp_binop.dst_hi);
   8877    UInt r2 = hregNumber(insn->variant.bfp_binop.op2_hi);
   8878 
   8879    switch (insn->size) {
   8880    case 4:
   8881       switch (insn->variant.bfp_binop.tag) {
   8882       case S390_BFP_ADD:     return s390_emit_AEBR(buf, r1, r2);
   8883       case S390_BFP_SUB:     return s390_emit_SEBR(buf, r1, r2);
   8884       case S390_BFP_MUL:     return s390_emit_MEEBR(buf, r1, r2);
   8885       case S390_BFP_DIV:     return s390_emit_DEBR(buf, r1, r2);
   8886       default:  goto fail;
   8887       }
   8888       break;
   8889 
   8890    case 8:
   8891       switch (insn->variant.bfp_binop.tag) {
   8892       case S390_BFP_ADD:     return s390_emit_ADBR(buf, r1, r2);
   8893       case S390_BFP_SUB:     return s390_emit_SDBR(buf, r1, r2);
   8894       case S390_BFP_MUL:     return s390_emit_MDBR(buf, r1, r2);
   8895       case S390_BFP_DIV:     return s390_emit_DDBR(buf, r1, r2);
   8896       default:  goto fail;
   8897       }
   8898       break;
   8899 
   8900    case 16:
   8901       switch (insn->variant.bfp_binop.tag) {
   8902       case S390_BFP_ADD:     return s390_emit_AXBR(buf, r1, r2);
   8903       case S390_BFP_SUB:     return s390_emit_SXBR(buf, r1, r2);
   8904       case S390_BFP_MUL:     return s390_emit_MXBR(buf, r1, r2);
   8905       case S390_BFP_DIV:     return s390_emit_DXBR(buf, r1, r2);
   8906       default:  goto fail;
   8907       }
   8908       break;
   8909 
   8910    default:  goto fail;
   8911    }
   8912 
   8913  fail:
   8914    vpanic("s390_insn_bfp_binop_emit");
   8915 }
   8916 
   8917 
   8918 static UChar *
   8919 s390_insn_bfp_unop_emit(UChar *buf, const s390_insn *insn)
   8920 {
   8921    UInt  r1 = hregNumber(insn->variant.bfp_unop.dst_hi);
   8922    UInt  r2 = hregNumber(insn->variant.bfp_unop.op_hi);
   8923 
   8924    switch (insn->variant.bfp_unop.tag) {
   8925    case S390_BFP_ABS:
   8926       switch (insn->size) {
   8927       case 4:   return s390_emit_LPEBR(buf, r1, r2);
   8928       case 8:   return s390_emit_LPDBR(buf, r1, r2);
   8929       case 16:  return s390_emit_LPXBR(buf, r1, r2);
   8930       default:  goto fail;
   8931       }
   8932       break;
   8933 
   8934    case S390_BFP_NABS:
   8935       switch (insn->size) {
   8936       case 4:   return s390_emit_LNEBR(buf, r1, r2);
   8937       case 8:   return s390_emit_LNDBR(buf, r1, r2);
   8938       case 16:  return s390_emit_LNXBR(buf, r1, r2);
   8939       default:  goto fail;
   8940       }
   8941       break;
   8942 
   8943    case S390_BFP_NEG:
   8944       switch (insn->size) {
   8945       case 4:   return s390_emit_LCEBR(buf, r1, r2);
   8946       case 8:   return s390_emit_LCDBR(buf, r1, r2);
   8947       case 16:  return s390_emit_LCXBR(buf, r1, r2);
   8948       default:  goto fail;
   8949       }
   8950       break;
   8951 
   8952    case S390_BFP_SQRT:
   8953       switch (insn->size) {
   8954       case 4:   return s390_emit_SQEBR(buf, r1, r2);
   8955       case 8:   return s390_emit_SQDBR(buf, r1, r2);
   8956       case 16:  return s390_emit_SQXBR(buf, r1, r2);
   8957       default:  goto fail;
   8958       }
   8959       break;
   8960 
   8961    default: goto fail;
   8962    }
   8963 
   8964  fail:
   8965    vpanic("s390_insn_bfp_unop_emit");
   8966 }
   8967 
   8968 
   8969 static UChar *
   8970 s390_insn_bfp_convert_emit(UChar *buf, const s390_insn *insn)
   8971 {
   8972    UInt  r1 = hregNumber(insn->variant.bfp_convert.dst_hi);
   8973    UInt  r2 = hregNumber(insn->variant.bfp_convert.op_hi);
   8974    s390_bfp_round_t m3 = insn->variant.bfp_convert.rounding_mode;
   8975    /* The IEEE-inexact-exception control is not modelled. So the
   8976       m4 field is 0 (which is what GCC does, too) */
   8977    const UInt m4 = 0;
   8978 
   8979    switch (insn->variant.bfp_convert.tag) {
   8980       /* Convert to fixed */
   8981    case S390_BFP_F32_TO_I32:  return s390_emit_CFEBR(buf, m3, r1, r2);
   8982    case S390_BFP_F64_TO_I32:  return s390_emit_CFDBR(buf, m3, r1, r2);
   8983    case S390_BFP_F128_TO_I32: return s390_emit_CFXBR(buf, m3, r1, r2);
   8984    case S390_BFP_F32_TO_I64:  return s390_emit_CGEBR(buf, m3, r1, r2);
   8985    case S390_BFP_F64_TO_I64:  return s390_emit_CGDBR(buf, m3, r1, r2);
   8986    case S390_BFP_F128_TO_I64: return s390_emit_CGXBR(buf, m3, r1, r2);
   8987 
   8988       /* Convert to logical */
   8989    case S390_BFP_F32_TO_U32:  return s390_emit_CLFEBR(buf, m3, m4, r1, r2);
   8990    case S390_BFP_F64_TO_U32:  return s390_emit_CLFDBR(buf, m3, m4, r1, r2);
   8991    case S390_BFP_F128_TO_U32: return s390_emit_CLFXBR(buf, m3, m4, r1, r2);
   8992    case S390_BFP_F32_TO_U64:  return s390_emit_CLGEBR(buf, m3, m4, r1, r2);
   8993    case S390_BFP_F64_TO_U64:  return s390_emit_CLGDBR(buf, m3, m4, r1, r2);
   8994    case S390_BFP_F128_TO_U64: return s390_emit_CLGXBR(buf, m3, m4, r1, r2);
   8995 
   8996       /* Convert from fixed */
   8997    case S390_BFP_I32_TO_F32:  return s390_emit_CEFBRA(buf, m3, m4, r1, r2);
   8998    case S390_BFP_I32_TO_F64:  return s390_emit_CDFBRA(buf,  0, m4, r1, r2);
   8999    case S390_BFP_I32_TO_F128: return s390_emit_CXFBRA(buf,  0, m4, r1, r2);
   9000    case S390_BFP_I64_TO_F32:  return s390_emit_CEGBRA(buf, m3, m4, r1, r2);
   9001    case S390_BFP_I64_TO_F64:  return s390_emit_CDGBRA(buf, m3, m4, r1, r2);
   9002    case S390_BFP_I64_TO_F128: return s390_emit_CXGBRA(buf,  0, m4, r1, r2);
   9003 
   9004       /* Convert from logical */
   9005    case S390_BFP_U32_TO_F32:  return s390_emit_CELFBR(buf, m3, m4, r1, r2);
   9006    case S390_BFP_U32_TO_F64:  return s390_emit_CDLFBR(buf, m3, m4, r1, r2);
   9007    case S390_BFP_U32_TO_F128: return s390_emit_CXLFBR(buf, m3, m4, r1, r2);
   9008    case S390_BFP_U64_TO_F32:  return s390_emit_CELGBR(buf, m3, m4, r1, r2);
   9009    case S390_BFP_U64_TO_F64:  return s390_emit_CDLGBR(buf, m3, m4, r1, r2);
   9010    case S390_BFP_U64_TO_F128: return s390_emit_CXLGBR(buf, m3, m4, r1, r2);
   9011 
   9012       /* Load lengthened */
   9013    case S390_BFP_F32_TO_F64:  return s390_emit_LDEBR(buf, r1, r2);
   9014    case S390_BFP_F32_TO_F128: return s390_emit_LXEBR(buf, r1, r2);
   9015    case S390_BFP_F64_TO_F128: return s390_emit_LXDBR(buf, r1, r2);
   9016 
   9017       /* Load rounded */
   9018    case S390_BFP_F64_TO_F32:  return s390_emit_LEDBRA(buf, m3, m4, r1, r2);
   9019    case S390_BFP_F128_TO_F32: return s390_emit_LEXBRA(buf, m3, m4, r1, r2);
   9020    case S390_BFP_F128_TO_F64: return s390_emit_LDXBRA(buf, m3, m4, r1, r2);
   9021 
   9022       /* Load FP integer */
   9023    case S390_BFP_F32_TO_F32I: return s390_emit_FIEBRA(buf, m3, m4, r1, r2);
   9024    case S390_BFP_F64_TO_F64I: return s390_emit_FIDBRA(buf, m3, m4, r1, r2);
   9025    case S390_BFP_F128_TO_F128I: return s390_emit_FIXBRA(buf, m3, m4, r1, r2);
   9026 
   9027    default: goto fail;
   9028    }
   9029 
   9030  fail:
   9031    vpanic("s390_insn_bfp_convert_emit");
   9032 }
   9033 
   9034 
   9035 static UChar *
   9036 s390_insn_bfp_compare_emit(UChar *buf, const s390_insn *insn)
   9037 {
   9038    UInt dst = hregNumber(insn->variant.bfp_compare.dst);
   9039    UInt r1  = hregNumber(insn->variant.bfp_compare.op1_hi);
   9040    UInt r2  = hregNumber(insn->variant.bfp_compare.op2_hi);
   9041 
   9042    switch (insn->size) {
   9043    case 4:  buf = s390_emit_CEBR(buf, r1, r2); break;
   9044    case 8:  buf = s390_emit_CDBR(buf, r1, r2); break;
   9045    case 16: buf = s390_emit_CXBR(buf, r1, r2); break;
   9046    default:  goto fail;
   9047    }
   9048 
   9049    return s390_emit_load_cc(buf, dst);  /* Load condition code into DST */
   9050 
   9051  fail:
   9052    vpanic("s390_insn_bfp_compare_emit");
   9053 }
   9054 
   9055 
   9056 static UChar *
   9057 s390_insn_dfp_binop_emit(UChar *buf, const s390_insn *insn)
   9058 {
   9059    s390_dfp_binop *dfp_binop = insn->variant.dfp_binop.details;
   9060 
   9061    UInt r1 = hregNumber(dfp_binop->dst_hi);
   9062    UInt r2 = hregNumber(dfp_binop->op2_hi);
   9063    UInt r3 = hregNumber(dfp_binop->op3_hi);
   9064    s390_dfp_round_t m4 = dfp_binop->rounding_mode;
   9065 
   9066    switch (insn->size) {
   9067    case 8:
   9068       switch (dfp_binop->tag) {
   9069       case S390_DFP_ADD: return s390_emit_ADTRA(buf, r3, m4, r1, r2);
   9070       case S390_DFP_SUB: return s390_emit_SDTRA(buf, r3, m4, r1, r2);
   9071       case S390_DFP_MUL: return s390_emit_MDTRA(buf, r3, m4, r1, r2);
   9072       case S390_DFP_DIV: return s390_emit_DDTRA(buf, r3, m4, r1, r2);
   9073       case S390_DFP_QUANTIZE: return s390_emit_QADTR(buf, r3, m4, r1, r2);
   9074       default:  goto fail;
   9075       }
   9076       break;
   9077 
   9078    case 16:
   9079       switch (dfp_binop->tag) {
   9080       case S390_DFP_ADD:     return s390_emit_AXTRA(buf, r3, m4, r1, r2);
   9081       case S390_DFP_SUB:     return s390_emit_SXTRA(buf, r3, m4, r1, r2);
   9082       case S390_DFP_MUL:     return s390_emit_MXTRA(buf, r3, m4, r1, r2);
   9083       case S390_DFP_DIV:     return s390_emit_DXTRA(buf, r3, m4, r1, r2);
   9084       case S390_DFP_QUANTIZE: return s390_emit_QAXTR(buf, r3, m4, r1, r2);
   9085       default:  goto fail;
   9086       }
   9087       break;
   9088 
   9089    default:  goto fail;
   9090    }
   9091 
   9092  fail:
   9093    vpanic("s390_insn_dfp_binop_emit");
   9094 }
   9095 
   9096 
   9097 static UChar *
   9098 s390_insn_dfp_reround_emit(UChar *buf, const s390_insn *insn)
   9099 {
   9100    UInt r1 = hregNumber(insn->variant.dfp_reround.dst_hi);
   9101    UInt r2 = hregNumber(insn->variant.dfp_reround.op2);
   9102    UInt r3 = hregNumber(insn->variant.dfp_reround.op3_hi);
   9103    s390_dfp_round_t m4 = insn->variant.dfp_reround.rounding_mode;
   9104 
   9105    switch (insn->size) {
   9106    case 8:
   9107       return s390_emit_RRDTR(buf, r3, m4, r1, r2);
   9108 
   9109    case 16:
   9110       return s390_emit_RRXTR(buf, r3, m4, r1, r2);
   9111 
   9112    default: goto fail;
   9113    }
   9114  fail:
   9115    vpanic("s390_insn_dfp_reround_emit");
   9116 }
   9117 
   9118 
   9119 static UChar *
   9120 s390_insn_dfp_unop_emit(UChar *buf, const s390_insn *insn)
   9121 {
   9122    UInt  r1 = hregNumber(insn->variant.dfp_unop.dst_hi);
   9123    UInt  r2 = hregNumber(insn->variant.dfp_unop.op_hi);
   9124 
   9125    switch (insn->variant.dfp_unop.tag) {
   9126    case S390_DFP_EXTRACT_EXP_D64:  return s390_emit_EEDTR(buf, r1, r2); break;
   9127    case S390_DFP_EXTRACT_EXP_D128: return s390_emit_EEXTR(buf, r1, r2); break;
   9128    case S390_DFP_EXTRACT_SIG_D64:  return s390_emit_ESDTR(buf, r1, r2); break;
   9129    case S390_DFP_EXTRACT_SIG_D128: return s390_emit_ESXTR(buf, r1, r2); break;
   9130    default: goto fail;
   9131    }
   9132  fail:
   9133    vpanic("s390_insn_dfp_unop_emit");
   9134 }
   9135 
   9136 
   9137 static UChar *
   9138 s390_insn_dfp_intop_emit(UChar *buf, const s390_insn *insn)
   9139 {
   9140    UInt r1 = hregNumber(insn->variant.dfp_intop.dst_hi);
   9141    UInt r2 = hregNumber(insn->variant.dfp_intop.op2);
   9142    UInt r3 = hregNumber(insn->variant.dfp_intop.op3_hi);
   9143 
   9144    switch (insn->size) {
   9145    case 8:
   9146       switch (insn->variant.dfp_intop.tag) {
   9147       case S390_DFP_SHIFT_LEFT:  return s390_emit_SLDT(buf, r3, r1, r2);
   9148       case S390_DFP_SHIFT_RIGHT: return s390_emit_SRDT(buf, r3, r1, r2);
   9149       case S390_DFP_INSERT_EXP:  return s390_emit_IEDTR(buf, r3, r1, r2);
   9150       default:  goto fail;
   9151       }
   9152       break;
   9153 
   9154    case 16:
   9155       switch (insn->variant.dfp_intop.tag) {
   9156       case S390_DFP_SHIFT_LEFT:  return s390_emit_SLXT(buf, r3, r1, r2);
   9157       case S390_DFP_SHIFT_RIGHT: return s390_emit_SRXT(buf, r3, r1, r2);
   9158       case S390_DFP_INSERT_EXP:  return s390_emit_IEXTR(buf, r3, r1, r2);
   9159       default:  goto fail;
   9160       }
   9161       break;
   9162 
   9163    default: goto fail;
   9164    }
   9165 
   9166  fail:
   9167    vpanic("s390_insn_dfp_intop_emit");
   9168 }
   9169 
   9170 
   9171 static UChar *
   9172 s390_insn_dfp_compare_emit(UChar *buf, const s390_insn *insn)
   9173 {
   9174    UInt dst = hregNumber(insn->variant.dfp_compare.dst);
   9175    UInt r1  = hregNumber(insn->variant.dfp_compare.op1_hi);
   9176    UInt r2  = hregNumber(insn->variant.dfp_compare.op2_hi);
   9177 
   9178    switch (insn->size) {
   9179    case 8:
   9180       switch(insn->variant.dfp_compare.tag) {
   9181       case S390_DFP_COMPARE:     buf = s390_emit_CDTR(buf, r1, r2); break;
   9182       case S390_DFP_COMPARE_EXP: buf = s390_emit_CEDTR(buf, r1, r2); break;
   9183       default: goto fail;
   9184       }
   9185       break;
   9186 
   9187    case 16:
   9188       switch(insn->variant.dfp_compare.tag) {
   9189       case S390_DFP_COMPARE:     buf = s390_emit_CXTR(buf, r1, r2); break;
   9190       case S390_DFP_COMPARE_EXP: buf = s390_emit_CEXTR(buf, r1, r2); break;
   9191       default: goto fail;
   9192       }
   9193       break;
   9194 
   9195    default:  goto fail;
   9196    }
   9197 
   9198    return s390_emit_load_cc(buf, dst);  /* Load condition code into DST */
   9199 
   9200  fail:
   9201    vpanic("s390_insn_dfp_compare_emit");
   9202 }
   9203 
   9204 
   9205 static UChar *
   9206 s390_insn_dfp_convert_emit(UChar *buf, const s390_insn *insn)
   9207 {
   9208    UInt  r1 = hregNumber(insn->variant.dfp_convert.dst_hi);
   9209    UInt  r2 = hregNumber(insn->variant.dfp_convert.op_hi);
   9210    s390_dfp_round_t m3 = insn->variant.dfp_convert.rounding_mode;
   9211    /* The IEEE-inexact-exception control is not modelled. So the
   9212       m4 field is 0 (which is what GCC does, too) */
   9213    const UInt m4 = 0;
   9214 
   9215    switch (insn->variant.dfp_convert.tag) {
   9216 
   9217       /* Convert to fixed */
   9218    case S390_DFP_D64_TO_I32:  return s390_emit_CFDTR(buf, m3, m4, r1, r2);
   9219    case S390_DFP_D128_TO_I32: return s390_emit_CFXTR(buf, m3, m4, r1, r2);
   9220    case S390_DFP_D64_TO_I64:  return s390_emit_CGDTR(buf, m3, m4, r1, r2);
   9221    case S390_DFP_D128_TO_I64: return s390_emit_CGXTR(buf, m3, m4, r1, r2);
   9222 
   9223       /* Convert to logical */
   9224    case S390_DFP_D64_TO_U32:  return s390_emit_CLFDTR(buf, m3, m4, r1, r2);
   9225    case S390_DFP_D128_TO_U32: return s390_emit_CLFXTR(buf, m3, m4, r1, r2);
   9226    case S390_DFP_D64_TO_U64:  return s390_emit_CLGDTR(buf, m3, m4, r1, r2);
   9227    case S390_DFP_D128_TO_U64: return s390_emit_CLGXTR(buf, m3, m4, r1, r2);
   9228 
   9229       /* Convert from fixed */
   9230    case S390_DFP_I32_TO_D64:  return s390_emit_CDFTR(buf, 0, m4, r1, r2);
   9231    case S390_DFP_I32_TO_D128: return s390_emit_CXFTR(buf, 0, m4, r1, r2);
   9232    case S390_DFP_I64_TO_D64:  return s390_emit_CDGTRA(buf, m3, m4, r1, r2);
   9233    case S390_DFP_I64_TO_D128: return s390_emit_CXGTR(buf, 0, m4, r1, r2);
   9234 
   9235       /* Convert from logical */
   9236    case S390_DFP_U32_TO_D64:  return s390_emit_CDLFTR(buf, m3, m4, r1, r2);
   9237    case S390_DFP_U64_TO_D64:  return s390_emit_CDLGTR(buf, m3, m4, r1, r2);
   9238    case S390_DFP_U32_TO_D128: return s390_emit_CXLFTR(buf, m3, m4, r1, r2);
   9239    case S390_DFP_U64_TO_D128: return s390_emit_CXLGTR(buf, m3, m4, r1, r2);
   9240 
   9241       /* Load lengthened */
   9242    case S390_DFP_D32_TO_D64:   return s390_emit_LDETR(buf, m4, r1, r2);
   9243    case S390_DFP_D64_TO_D128:  return s390_emit_LXDTR(buf, m4, r1, r2);
   9244 
   9245       /* Load rounded */
   9246    case S390_DFP_D64_TO_D32:   return s390_emit_LEDTR(buf, m3, m4, r1, r2);
   9247    case S390_DFP_D128_TO_D64:  return s390_emit_LDXTR(buf, m3, m4, r1, r2);
   9248 
   9249    default: goto fail;
   9250    }
   9251 
   9252  fail:
   9253    vpanic("s390_insn_dfp_convert_emit");
   9254 }
   9255 
   9256 
   9257 static UChar *
   9258 s390_insn_fp_convert_emit(UChar *buf, const s390_insn *insn)
   9259 {
   9260    UInt pfpo;
   9261    s390_fp_convert *fp_convert = insn->variant.fp_convert.details;
   9262    s390_dfp_round_t rm = fp_convert->rounding_mode;
   9263 
   9264    vassert(rm < 2 || rm > 7);
   9265 
   9266    switch (fp_convert->tag) {
   9267    case S390_FP_F32_TO_D32:   pfpo = S390_PFPO_F32_TO_D32   << 8; break;
   9268    case S390_FP_F32_TO_D64:   pfpo = S390_PFPO_F32_TO_D64   << 8; break;
   9269    case S390_FP_F32_TO_D128:  pfpo = S390_PFPO_F32_TO_D128  << 8; break;
   9270    case S390_FP_F64_TO_D32:   pfpo = S390_PFPO_F64_TO_D32   << 8; break;
   9271    case S390_FP_F64_TO_D64:   pfpo = S390_PFPO_F64_TO_D64   << 8; break;
   9272    case S390_FP_F64_TO_D128:  pfpo = S390_PFPO_F64_TO_D128  << 8; break;
   9273    case S390_FP_F128_TO_D32:  pfpo = S390_PFPO_F128_TO_D32  << 8; break;
   9274    case S390_FP_F128_TO_D64:  pfpo = S390_PFPO_F128_TO_D64  << 8; break;
   9275    case S390_FP_F128_TO_D128: pfpo = S390_PFPO_F128_TO_D128 << 8; break;
   9276    case S390_FP_D32_TO_F32:   pfpo = S390_PFPO_D32_TO_F32   << 8; break;
   9277    case S390_FP_D32_TO_F64:   pfpo = S390_PFPO_D32_TO_F64   << 8; break;
   9278    case S390_FP_D32_TO_F128:  pfpo = S390_PFPO_D32_TO_F128  << 8; break;
   9279    case S390_FP_D64_TO_F32:   pfpo = S390_PFPO_D64_TO_F32   << 8; break;
   9280    case S390_FP_D64_TO_F64:   pfpo = S390_PFPO_D64_TO_F64   << 8; break;
   9281    case S390_FP_D64_TO_F128:  pfpo = S390_PFPO_D64_TO_F128  << 8; break;
   9282    case S390_FP_D128_TO_F32:  pfpo = S390_PFPO_D128_TO_F32  << 8; break;
   9283    case S390_FP_D128_TO_F64:  pfpo = S390_PFPO_D128_TO_F64  << 8; break;
   9284    case S390_FP_D128_TO_F128: pfpo = S390_PFPO_D128_TO_F128 << 8; break;
   9285    default: goto fail;
   9286    }
   9287 
   9288    pfpo = pfpo | rm;
   9289    buf = s390_emit_load_32imm(buf, R0, pfpo);
   9290    buf = s390_emit_PFPO(buf);
   9291    return buf;
   9292 
   9293  fail:
   9294    vpanic("s390_insn_fp_convert_emit");
   9295 }
   9296 
   9297 
   9298 static UChar *
   9299 s390_insn_mfence_emit(UChar *buf, const s390_insn *insn)
   9300 {
   9301    return s390_emit_BCR(buf, 0xF, 0x0);
   9302 }
   9303 
   9304 
   9305 static UChar *
   9306 s390_insn_mimm_emit(UChar *buf, const s390_insn *insn)
   9307 {
   9308    s390_amode *am = insn->variant.mimm.dst;
   9309    UChar b = hregNumber(am->b);
   9310    Int   d = am->d;
   9311    ULong value = insn->variant.mimm.value;
   9312 
   9313    if (value == 0) {
   9314       return s390_emit_XC(buf, insn->size - 1, b, d, b, d);
   9315    }
   9316 
   9317    if (insn->size == 1) {
   9318       return s390_emit_MVI(buf, value & 0xFF, b, d);
   9319    }
   9320 
   9321    if (s390_host_has_gie && ulong_fits_signed_16bit(value)) {
   9322       value &= 0xFFFF;
   9323       switch (insn->size) {
   9324       case 2: return s390_emit_MVHHI(buf, b, d, value);
   9325       case 4: return s390_emit_MVHI(buf,  b, d, value);
   9326       case 8: return s390_emit_MVGHI(buf, b, d, value);
   9327       }
   9328    } else {
   9329       // Load value to R0, then store.
   9330       switch (insn->size) {
   9331       case 2:
   9332          buf = s390_emit_LHI(buf, R0, value & 0xFFFF);
   9333          return s390_emit_STH(buf, R0, 0, b, d);
   9334       case 4:
   9335          buf = s390_emit_load_32imm(buf, R0, value);
   9336          return s390_emit_ST(buf, R0, 0, b, d);
   9337       case 8:
   9338          buf = s390_emit_load_64imm(buf, R0, value);
   9339          return s390_emit_STG(buf, R0, 0, b, DISP20(d));
   9340       }
   9341    }
   9342 
   9343    vpanic("s390_insn_mimm_emit");
   9344 }
   9345 
   9346 
   9347 static UChar *
   9348 s390_insn_madd_emit(UChar *buf, const s390_insn *insn)
   9349 {
   9350    s390_amode *am = insn->variant.madd.dst;
   9351    UChar b = hregNumber(am->b);
   9352    Int   d = am->d;
   9353 
   9354    if (insn->size == 4) {
   9355       return s390_emit_ASI(buf, insn->variant.madd.delta, b, DISP20(d));
   9356    }
   9357 
   9358    return s390_emit_AGSI(buf, insn->variant.madd.delta, b, DISP20(d));
   9359 }
   9360 
   9361 
   9362 static UChar *
   9363 s390_insn_set_fpc_bfprm_emit(UChar *buf, const s390_insn *insn)
   9364 {
   9365    UInt mode = hregNumber(insn->variant.set_fpc_bfprm.mode);
   9366 
   9367    /* Copy FPC from guest state to R0 and OR in the new rounding mode */
   9368    buf = s390_emit_L(buf, R0, 0, S390_REGNO_GUEST_STATE_POINTER,
   9369                      S390X_GUEST_OFFSET(guest_fpc));   // r0 = guest_fpc
   9370 
   9371    buf = s390_emit_NILL(buf, R0, 0xFFF8); /* Clear out right-most 3 bits */
   9372    buf = s390_emit_OR(buf, R0, mode);     /* OR in the new rounding mode */
   9373    buf = s390_emit_SFPC(buf, R0);         /* Load FPC register from R0 */
   9374 
   9375    return buf;
   9376 }
   9377 
   9378 
   9379 static UChar *
   9380 s390_insn_set_fpc_dfprm_emit(UChar *buf, const s390_insn *insn)
   9381 {
   9382    UInt mode = hregNumber(insn->variant.set_fpc_dfprm.mode);
   9383 
   9384    /* Copy FPC from guest state to R0 and OR in the new rounding mode */
   9385    buf = s390_emit_L(buf, R0, 0, S390_REGNO_GUEST_STATE_POINTER,
   9386                      S390X_GUEST_OFFSET(guest_fpc));   // r0 = guest_fpc
   9387 
   9388    /* DFP rounding mode is set at bit position 25:27 in FPC register */
   9389    buf = s390_emit_NILL(buf, R0, 0xFF8F); /* Clear out 25:27 bits */
   9390    buf = s390_emit_SLL(buf, mode, 0, 4);  /* bring mode to 25:27 bits */
   9391    buf = s390_emit_OR(buf, R0, mode);     /* OR in the new rounding mode */
   9392    buf = s390_emit_SFPC(buf, R0);         /* Load FPC register from R0 */
   9393 
   9394    return buf;
   9395 }
   9396 
   9397 
   9398 /* Define convenience functions needed for translation chaining.
   9399    Any changes need to be applied to the functions in concert. */
   9400 
   9401 static __inline__ Bool
   9402 s390_insn_is_BRCL(const UChar *p, UChar condition)
   9403 {
   9404    return p[0] == 0xc0 && p[1] == ((condition << 4) | 0x04);
   9405 }
   9406 
   9407 static __inline__ Bool
   9408 s390_insn_is_BR(const UChar *p, UChar reg)
   9409 {
   9410    return p[0] == 0x07 && p[1] == (0xF0 | reg);  /* BCR 15,reg */
   9411 }
   9412 
   9413 
   9414 /* The length of the BASR insn */
   9415 #define S390_BASR_LEN  2
   9416 
   9417 
   9418 /* Load the 64-bit VALUE into REG. Note that this function must NOT
   9419    optimise the generated code by looking at the value. I.e. using
   9420    LGHI if value == 0 would be very wrong. */
   9421 static UChar *
   9422 s390_tchain_load64(UChar *buf, UChar regno, ULong value)
   9423 {
   9424    UChar *begin = buf;
   9425 
   9426    if (s390_host_has_eimm) {
   9427       /* Do it in two steps: upper half [0:31] and lower half [32:63] */
   9428       buf = s390_emit_IIHF(buf, regno, value >> 32);
   9429       buf = s390_emit_IILF(buf, regno, value & 0xFFFFFFFF);
   9430    } else {
   9431       buf = s390_emit_IILL(buf, regno, value & 0xFFFF);
   9432       value >>= 16;
   9433       buf = s390_emit_IILH(buf, regno, value & 0xFFFF);
   9434       value >>= 16;
   9435       buf = s390_emit_IIHL(buf, regno, value & 0xFFFF);
   9436       value >>= 16;
   9437       buf = s390_emit_IIHH(buf, regno, value & 0xFFFF);
   9438    }
   9439 
   9440    vassert(buf - begin == s390_tchain_load64_len());
   9441 
   9442    return buf;
   9443 }
   9444 
   9445 /* Return number of bytes generated by s390_tchain_load64 */
   9446 static UInt
   9447 s390_tchain_load64_len(void)
   9448 {
   9449    if (s390_host_has_eimm) {
   9450       return 6 + 6;      /* IIHF + IILF */
   9451    }
   9452    return 4 + 4 + 4 + 4; /* IIHH + IIHL + IILH + IILL */
   9453 }
   9454 
   9455 /* Verify that CODE is the code sequence generated by s390_tchain_load64
   9456    to load VALUE into REGNO. Return pointer to the byte following the
   9457    insn sequence. */
   9458 static const UChar *
   9459 s390_tchain_verify_load64(const UChar *code, UChar regno, ULong value)
   9460 {
   9461    UInt regmask = regno << 4;
   9462    UInt hw;
   9463 
   9464    if (s390_host_has_eimm) {
   9465       /* Check for IIHF */
   9466       vassert(code[0]  ==  0xC0);
   9467       vassert(code[1]  == (0x08 | regmask));
   9468       vassert(*(const UInt *)&code[2] == (value >> 32));
   9469       /* Check for IILF */
   9470       vassert(code[6]  ==  0xC0);
   9471       vassert(code[7]  == (0x09 | regmask));
   9472       vassert(*(const UInt *)&code[8] == (value & 0xFFFFFFFF));
   9473    } else {
   9474       /* Check for IILL */
   9475       hw = value & 0xFFFF;
   9476       vassert(code[0]  ==  0xA5);
   9477       vassert(code[1]  == (0x03 | regmask));
   9478       vassert(code[2]  == (hw >> 8));
   9479       vassert(code[3]  == (hw & 0xFF));
   9480 
   9481       /* Check for IILH */
   9482       hw = (value >> 16) & 0xFFFF;
   9483       vassert(code[4]  ==  0xA5);
   9484       vassert(code[5]  == (0x02 | regmask));
   9485       vassert(code[6]  == (hw >> 8));
   9486       vassert(code[7]  == (hw & 0xFF));
   9487 
   9488       /* Check for IIHL */
   9489       hw = (value >> 32) & 0xFFFF;
   9490       vassert(code[8]  ==  0xA5);
   9491       vassert(code[9]  == (0x01 | regmask));
   9492       vassert(code[10] == (hw >> 8));
   9493       vassert(code[11] == (hw & 0xFF));
   9494 
   9495       /* Check for IIHH */
   9496       hw = (value >> 48) & 0xFFFF;
   9497       vassert(code[12] ==  0xA5);
   9498       vassert(code[13] == (0x00 | regmask));
   9499       vassert(code[14] == (hw >> 8));
   9500       vassert(code[15] == (hw & 0xFF));
   9501    }
   9502 
   9503    return code + s390_tchain_load64_len();
   9504 }
   9505 
   9506 /* CODE points to the code sequence as generated by s390_tchain_load64.
   9507    Change the loaded value to IMM64. Return pointer to the byte following
   9508    the patched code sequence. */
   9509 static UChar *
   9510 s390_tchain_patch_load64(UChar *code, ULong imm64)
   9511 {
   9512    if (s390_host_has_eimm) {
   9513       /* Patch IIHF */
   9514       *(UInt *)&code[2] = imm64 >> 32;
   9515       /* Patch IILF */
   9516       *(UInt *)&code[8] = imm64 & 0xFFFFFFFF;
   9517    } else {
   9518       code[3]  = imm64 & 0xFF; imm64 >>= 8;
   9519       code[2]  = imm64 & 0xFF; imm64 >>= 8;
   9520       code[7]  = imm64 & 0xFF; imm64 >>= 8;
   9521       code[6]  = imm64 & 0xFF; imm64 >>= 8;
   9522       code[11] = imm64 & 0xFF; imm64 >>= 8;
   9523       code[10] = imm64 & 0xFF; imm64 >>= 8;
   9524       code[15] = imm64 & 0xFF; imm64 >>= 8;
   9525       code[14] = imm64 & 0xFF; imm64 >>= 8;
   9526    }
   9527 
   9528    return code + s390_tchain_load64_len();
   9529 }
   9530 
   9531 
   9532 /* NB: what goes on here has to be very closely coordinated with the
   9533    chainXDirect_S390 and unchainXDirect_S390 below. */
   9534 static UChar *
   9535 s390_insn_xdirect_emit(UChar *buf, const s390_insn *insn,
   9536                        const void *disp_cp_chain_me_to_slowEP,
   9537                        const void *disp_cp_chain_me_to_fastEP)
   9538 {
   9539    /* We're generating chain-me requests here, so we need to be
   9540       sure this is actually allowed -- no-redir translations can't
   9541       use chain-me's.  Hence: */
   9542    vassert(disp_cp_chain_me_to_slowEP != NULL);
   9543    vassert(disp_cp_chain_me_to_fastEP != NULL);
   9544 
   9545    /* Use ptmp for backpatching conditional jumps. */
   9546    UChar *ptmp = buf;
   9547 
   9548    /* First off, if this is conditional, create a conditional
   9549       jump over the rest of it. */
   9550    s390_cc_t cond = insn->variant.xdirect.cond;
   9551 
   9552    if (cond != S390_CC_ALWAYS) {
   9553       /* So we have something like this
   9554          if (cond) do_xdirect;
   9555          Y: ...
   9556          We convert this into
   9557          if (! cond) goto Y;        // BRC opcode; 4 bytes
   9558          do_xdirect;
   9559          Y:
   9560       */
   9561       /* 4 bytes (a BRC insn) to be filled in here */
   9562       buf += 4;
   9563    }
   9564 
   9565    /* Update the guest IA. */
   9566    buf = s390_emit_load_64imm(buf, R0, insn->variant.xdirect.dst);
   9567 
   9568    const s390_amode *amode = insn->variant.xdirect.guest_IA;
   9569    vassert(amode->tag == S390_AMODE_B12);
   9570    UInt b = hregNumber(amode->b);
   9571    UInt d = amode->d;
   9572 
   9573    buf = s390_emit_STG(buf, R0, 0, b, DISP20(d));
   9574 
   9575    /* Load the chosen entry point into the scratch reg */
   9576    const void *disp_cp_chain_me;
   9577 
   9578    disp_cp_chain_me =
   9579       insn->variant.xdirect.to_fast_entry ? disp_cp_chain_me_to_fastEP
   9580                                           : disp_cp_chain_me_to_slowEP;
   9581    /* Get the address of the beginning of the load64 code sequence into %r1.
   9582       Do not change the register! This is part of the protocol with the
   9583       dispatcher. */
   9584    buf = s390_emit_BASR(buf, 1, R0);
   9585 
   9586    /* --- FIRST PATCHABLE BYTE follows (must not modify %r1) --- */
   9587    Addr64 addr = (Addr)disp_cp_chain_me;
   9588    buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH, addr);
   9589 
   9590    /* goto *tchain_scratch */
   9591    buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
   9592 
   9593    /* --- END of PATCHABLE BYTES --- */
   9594 
   9595    /* Fix up the conditional jump, if there was one. */
   9596    if (cond != S390_CC_ALWAYS) {
   9597       Int delta = buf - ptmp;
   9598 
   9599       delta >>= 1;  /* immediate constant is #half-words */
   9600       vassert(delta > 0 && delta < (1 << 16));
   9601       s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
   9602    }
   9603 
   9604    return buf;
   9605 }
   9606 
   9607 /* Return the number of patchable bytes from an xdirect insn. */
   9608 static UInt
   9609 s390_xdirect_patchable_len(void)
   9610 {
   9611    return s390_tchain_load64_len() + S390_BASR_LEN;
   9612 }
   9613 
   9614 
   9615 static UChar *
   9616 s390_insn_xindir_emit(UChar *buf, const s390_insn *insn,
   9617                       const void *disp_cp_xindir)
   9618 {
   9619    /* We're generating transfers that could lead indirectly to a
   9620       chain-me, so we need to be sure this is actually allowed --
   9621       no-redir translations are not allowed to reach normal
   9622       translations without going through the scheduler.  That means
   9623       no XDirects or XIndirs out from no-redir translations.
   9624       Hence: */
   9625    vassert(disp_cp_xindir != NULL);
   9626 
   9627    /* Use ptmp for backpatching conditional jumps. */
   9628    UChar *ptmp = buf;
   9629 
   9630    /* First off, if this is conditional, create a conditional
   9631       jump over the rest of it. */
   9632    s390_cc_t cond = insn->variant.xdirect.cond;
   9633 
   9634    if (cond != S390_CC_ALWAYS) {
   9635       /* So we have something like this
   9636          if (cond) do_xdirect;
   9637          Y: ...
   9638          We convert this into
   9639          if (! cond) goto Y;        // BRC opcode; 4 bytes
   9640          do_xdirect;
   9641          Y:
   9642       */
   9643       /* 4 bytes (a BRC insn) to be filled in here */
   9644       buf += 4;
   9645    }
   9646 
   9647    /* Update the guest IA with the address in xdirect.dst. */
   9648    const s390_amode *amode = insn->variant.xindir.guest_IA;
   9649 
   9650    vassert(amode->tag == S390_AMODE_B12);
   9651    UInt b = hregNumber(amode->b);
   9652    UInt d = amode->d;
   9653    UInt regno = hregNumber(insn->variant.xindir.dst);
   9654 
   9655    buf = s390_emit_STG(buf, regno, 0, b, DISP20(d));
   9656 
   9657    /* load tchain_scratch, #disp_indir */
   9658    buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH,
   9659                             (Addr)disp_cp_xindir);
   9660    /* goto *tchain_direct */
   9661    buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
   9662 
   9663    /* Fix up the conditional jump, if there was one. */
   9664    if (cond != S390_CC_ALWAYS) {
   9665       Int delta = buf - ptmp;
   9666 
   9667       delta >>= 1;  /* immediate constant is #half-words */
   9668       vassert(delta > 0 && delta < (1 << 16));
   9669       s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
   9670    }
   9671 
   9672    return buf;
   9673 }
   9674 
   9675 static UChar *
   9676 s390_insn_xassisted_emit(UChar *buf, const s390_insn *insn,
   9677                          const void *disp_cp_xassisted)
   9678 {
   9679    /* Use ptmp for backpatching conditional jumps. */
   9680    UChar *ptmp = buf;
   9681 
   9682    /* First off, if this is conditional, create a conditional
   9683       jump over the rest of it. */
   9684    s390_cc_t cond = insn->variant.xdirect.cond;
   9685 
   9686    if (cond != S390_CC_ALWAYS) {
   9687       /* So we have something like this
   9688          if (cond) do_xdirect;
   9689          Y: ...
   9690          We convert this into
   9691          if (! cond) goto Y;        // BRC opcode; 4 bytes
   9692          do_xdirect;
   9693          Y:
   9694       */
   9695       /* 4 bytes (a BRC insn) to be filled in here */
   9696       buf += 4;
   9697    }
   9698 
   9699    /* Update the guest IA with the address in xassisted.dst. */
   9700    const s390_amode *amode = insn->variant.xassisted.guest_IA;
   9701 
   9702    vassert(amode->tag == S390_AMODE_B12);
   9703    UInt b = hregNumber(amode->b);
   9704    UInt d = amode->d;
   9705    UInt regno = hregNumber(insn->variant.xassisted.dst);
   9706 
   9707    buf = s390_emit_STG(buf, regno, 0, b, DISP20(d));
   9708 
   9709    UInt trcval = 0;
   9710 
   9711    switch (insn->variant.xassisted.kind) {
   9712    case Ijk_ClientReq:   trcval = VEX_TRC_JMP_CLIENTREQ;   break;
   9713    case Ijk_Sys_syscall: trcval = VEX_TRC_JMP_SYS_SYSCALL; break;
   9714    case Ijk_Yield:       trcval = VEX_TRC_JMP_YIELD;       break;
   9715    case Ijk_EmWarn:      trcval = VEX_TRC_JMP_EMWARN;      break;
   9716    case Ijk_EmFail:      trcval = VEX_TRC_JMP_EMFAIL;      break;
   9717    case Ijk_MapFail:     trcval = VEX_TRC_JMP_MAPFAIL;     break;
   9718    case Ijk_NoDecode:    trcval = VEX_TRC_JMP_NODECODE;    break;
   9719    case Ijk_InvalICache: trcval = VEX_TRC_JMP_INVALICACHE; break;
   9720    case Ijk_NoRedir:     trcval = VEX_TRC_JMP_NOREDIR;     break;
   9721    case Ijk_SigTRAP:     trcval = VEX_TRC_JMP_SIGTRAP;     break;
   9722    case Ijk_SigSEGV:     trcval = VEX_TRC_JMP_SIGSEGV;     break;
   9723    case Ijk_Boring:      trcval = VEX_TRC_JMP_BORING;      break;
   9724       /* We don't expect to see the following being assisted. */
   9725    case Ijk_Ret:
   9726    case Ijk_Call:
   9727       /* fallthrough */
   9728    default:
   9729       ppIRJumpKind(insn->variant.xassisted.kind);
   9730       vpanic("s390_insn_xassisted_emit: unexpected jump kind");
   9731    }
   9732 
   9733    vassert(trcval != 0);
   9734 
   9735    /* guest_state_pointer = trcval */
   9736    buf = s390_emit_LGHI(buf, S390_REGNO_GUEST_STATE_POINTER, trcval);
   9737 
   9738    /* load tchain_scratch, #disp_assisted */
   9739    buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH,
   9740                             (Addr)disp_cp_xassisted);
   9741 
   9742    /* goto *tchain_direct */
   9743    buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
   9744 
   9745    /* Fix up the conditional jump, if there was one. */
   9746    if (cond != S390_CC_ALWAYS) {
   9747       Int delta = buf - ptmp;
   9748 
   9749       delta >>= 1;  /* immediate constant is #half-words */
   9750       vassert(delta > 0 && delta < (1 << 16));
   9751       s390_emit_BRC(ptmp, s390_cc_invert(cond), delta);
   9752    }
   9753 
   9754    return buf;
   9755 }
   9756 
   9757 
   9758 /* Pseudo code:
   9759 
   9760    guest_state[host_EvC_COUNTER] -= 1;
   9761    if (guest_state[host_EvC_COUNTER] >= 0) goto nofail;
   9762    goto guest_state[host_EvC_FAILADDR];
   9763    nofail: ;
   9764 
   9765    The dispatch counter is a 32-bit value. */
   9766 static UChar *
   9767 s390_insn_evcheck_emit(UChar *buf, const s390_insn *insn,
   9768                        VexEndness endness_host)
   9769 {
   9770    s390_amode *amode;
   9771    UInt b, d;
   9772    UChar *code_begin, *code_end;
   9773 
   9774    code_begin = buf;
   9775 
   9776    amode = insn->variant.evcheck.counter;
   9777    vassert(amode->tag == S390_AMODE_B12);
   9778    b = hregNumber(amode->b);
   9779    d = amode->d;
   9780 
   9781    /* Decrement the dispatch counter in the guest state */
   9782    if (s390_host_has_gie) {
   9783       buf = s390_emit_ASI(buf, -1, b, DISP20(d));   /* 6 bytes */
   9784    } else {
   9785       buf = s390_emit_LHI(buf, R0, -1);             /* 4 bytes */
   9786       buf = s390_emit_A(buf, R0, 0, b, d);          /* 4 bytes */
   9787       buf = s390_emit_ST(buf, R0, 0, b, d);         /* 4 bytes */
   9788    }
   9789 
   9790    /* Jump over the next insn if >= 0 */
   9791    buf = s390_emit_BRC(buf, S390_CC_HE, (4 + 6 + 2) / 2);  /* 4 bytes */
   9792 
   9793    /* Computed goto to fail_address */
   9794    amode = insn->variant.evcheck.fail_addr;
   9795    b = hregNumber(amode->b);
   9796    d = amode->d;
   9797    buf = s390_emit_LG(buf, S390_REGNO_TCHAIN_SCRATCH, 0, b, DISP20(d));  /* 6 bytes */
   9798    buf = s390_emit_BCR(buf, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);  /* 2 bytes */
   9799 
   9800    code_end = buf;
   9801 
   9802    /* Make sure the size of the generated code is identical to the size
   9803       returned by evCheckSzB_S390 */
   9804    vassert(evCheckSzB_S390() == code_end - code_begin);
   9805 
   9806    return buf;
   9807 }
   9808 
   9809 
   9810 static UChar *
   9811 s390_insn_profinc_emit(UChar *buf,
   9812                        const s390_insn *insn __attribute__((unused)))
   9813 {
   9814    /* Generate a code template to increment a memory location whose
   9815       address will be known later as an immediate value. This code
   9816       template will be patched once the memory location is known.
   9817       For now we do this with address == 0. */
   9818    buf = s390_tchain_load64(buf, S390_REGNO_TCHAIN_SCRATCH, 0);
   9819    if (s390_host_has_gie) {
   9820       buf = s390_emit_AGSI(buf, 1, S390_REGNO_TCHAIN_SCRATCH, DISP20(0));
   9821    } else {
   9822       buf = s390_emit_LGHI(buf, R0, 1);
   9823       buf = s390_emit_AG( buf, R0, 0, S390_REGNO_TCHAIN_SCRATCH, DISP20(0));
   9824       buf = s390_emit_STG(buf, R0, 0, S390_REGNO_TCHAIN_SCRATCH, DISP20(0));
   9825    }
   9826 
   9827    return buf;
   9828 }
   9829 
   9830 
   9831 Int
   9832 emit_S390Instr(Bool *is_profinc, UChar *buf, Int nbuf, const s390_insn *insn,
   9833                Bool mode64, VexEndness endness_host,
   9834                const void *disp_cp_chain_me_to_slowEP,
   9835                const void *disp_cp_chain_me_to_fastEP,
   9836                const void *disp_cp_xindir,
   9837                const void *disp_cp_xassisted)
   9838 {
   9839    UChar *end;
   9840 
   9841    /* Used to be 48 bytes. Make sure it stays low */
   9842    vassert(sizeof(s390_insn) == 32);
   9843 
   9844    switch (insn->tag) {
   9845    case S390_INSN_LOAD:
   9846       end = s390_insn_load_emit(buf, insn);
   9847       break;
   9848 
   9849    case S390_INSN_STORE:
   9850       end = s390_insn_store_emit(buf, insn);
   9851       break;
   9852 
   9853    case S390_INSN_MOVE:
   9854       end = s390_insn_move_emit(buf, insn);
   9855       break;
   9856 
   9857    case S390_INSN_MEMCPY:
   9858       end = s390_insn_memcpy_emit(buf, insn);
   9859       break;
   9860 
   9861    case S390_INSN_COND_MOVE:
   9862       end = s390_insn_cond_move_emit(buf, insn);
   9863       break;
   9864 
   9865    case S390_INSN_LOAD_IMMEDIATE:
   9866       end = s390_insn_load_immediate_emit(buf, insn);
   9867       break;
   9868 
   9869    case S390_INSN_ALU:
   9870       end = s390_insn_alu_emit(buf, insn);
   9871       break;
   9872 
   9873    case S390_INSN_SMUL:
   9874    case S390_INSN_UMUL:
   9875       end = s390_insn_mul_emit(buf, insn);
   9876       break;
   9877 
   9878    case S390_INSN_SDIV:
   9879    case S390_INSN_UDIV:
   9880       end = s390_insn_div_emit(buf, insn);
   9881       break;
   9882 
   9883    case S390_INSN_DIVS:
   9884       end = s390_insn_divs_emit(buf, insn);
   9885       break;
   9886 
   9887    case S390_INSN_CLZ:
   9888       end = s390_insn_clz_emit(buf, insn);
   9889       break;
   9890 
   9891    case S390_INSN_UNOP:
   9892       end = s390_insn_unop_emit(buf, insn);
   9893       break;
   9894 
   9895    case S390_INSN_TEST:
   9896       end = s390_insn_test_emit(buf, insn);
   9897       break;
   9898 
   9899    case S390_INSN_CC2BOOL:
   9900       end = s390_insn_cc2bool_emit(buf, insn);
   9901       break;
   9902 
   9903    case S390_INSN_CAS:
   9904       end = s390_insn_cas_emit(buf, insn);
   9905       break;
   9906 
   9907    case S390_INSN_CDAS:
   9908       end = s390_insn_cdas_emit(buf, insn);
   9909       break;
   9910 
   9911    case S390_INSN_COMPARE:
   9912       end = s390_insn_compare_emit(buf, insn);
   9913       break;
   9914 
   9915    case S390_INSN_HELPER_CALL:
   9916       end = s390_insn_helper_call_emit(buf, insn);
   9917       if (end == buf) goto fail;
   9918       break;
   9919 
   9920    case S390_INSN_BFP_TRIOP:
   9921       end = s390_insn_bfp_triop_emit(buf, insn);
   9922       break;
   9923 
   9924    case S390_INSN_BFP_BINOP:
   9925       end = s390_insn_bfp_binop_emit(buf, insn);
   9926       break;
   9927 
   9928    case S390_INSN_BFP_UNOP:
   9929       end = s390_insn_bfp_unop_emit(buf, insn);
   9930       break;
   9931 
   9932    case S390_INSN_BFP_COMPARE:
   9933       end = s390_insn_bfp_compare_emit(buf, insn);
   9934       break;
   9935 
   9936    case S390_INSN_BFP_CONVERT:
   9937       end = s390_insn_bfp_convert_emit(buf, insn);
   9938       break;
   9939 
   9940    case S390_INSN_DFP_BINOP:
   9941       end = s390_insn_dfp_binop_emit(buf, insn);
   9942       break;
   9943 
   9944    case S390_INSN_DFP_UNOP:
   9945       end = s390_insn_dfp_unop_emit(buf, insn);
   9946       break;
   9947 
   9948    case S390_INSN_DFP_INTOP:
   9949       end = s390_insn_dfp_intop_emit(buf, insn);
   9950       break;
   9951 
   9952    case S390_INSN_DFP_COMPARE:
   9953       end = s390_insn_dfp_compare_emit(buf, insn);
   9954       break;
   9955 
   9956    case S390_INSN_DFP_CONVERT:
   9957       end = s390_insn_dfp_convert_emit(buf, insn);
   9958       break;
   9959 
   9960    case S390_INSN_DFP_REROUND:
   9961       end = s390_insn_dfp_reround_emit(buf, insn);
   9962       break;
   9963 
   9964    case S390_INSN_FP_CONVERT:
   9965       end = s390_insn_fp_convert_emit(buf, insn);
   9966       break;
   9967 
   9968    case S390_INSN_MFENCE:
   9969       end = s390_insn_mfence_emit(buf, insn);
   9970       break;
   9971 
   9972    case S390_INSN_MIMM:
   9973       end = s390_insn_mimm_emit(buf, insn);
   9974       break;
   9975 
   9976    case S390_INSN_MADD:
   9977       end = s390_insn_madd_emit(buf, insn);
   9978       break;
   9979 
   9980    case S390_INSN_SET_FPC_BFPRM:
   9981       end = s390_insn_set_fpc_bfprm_emit(buf, insn);
   9982       break;
   9983 
   9984    case S390_INSN_SET_FPC_DFPRM:
   9985       end = s390_insn_set_fpc_dfprm_emit(buf, insn);
   9986       break;
   9987 
   9988    case S390_INSN_PROFINC:
   9989       end = s390_insn_profinc_emit(buf, insn);
   9990       /* Tell the caller .. */
   9991       vassert(*is_profinc == False);
   9992       *is_profinc = True;
   9993       break;
   9994 
   9995    case S390_INSN_EVCHECK:
   9996       end = s390_insn_evcheck_emit(buf, insn, endness_host);
   9997       break;
   9998 
   9999    case S390_INSN_XDIRECT:
   10000       end = s390_insn_xdirect_emit(buf, insn, disp_cp_chain_me_to_slowEP,
   10001                                    disp_cp_chain_me_to_fastEP);
   10002       break;
   10003 
   10004    case S390_INSN_XINDIR:
   10005       end = s390_insn_xindir_emit(buf, insn, disp_cp_xindir);
   10006       break;
   10007 
   10008    case S390_INSN_XASSISTED:
   10009       end = s390_insn_xassisted_emit(buf, insn, disp_cp_xassisted);
   10010       break;
   10011 
   10012    fail:
   10013    default:
   10014       vpanic("emit_S390Instr");
   10015    }
   10016 
   10017    vassert(end - buf <= nbuf);
   10018 
   10019    return end - buf;
   10020 }
   10021 
   10022 
   10023 /* Return the number of bytes emitted for an S390_INSN_EVCHECK.
   10024    See s390_insn_evcheck_emit */
   10025 Int
   10026 evCheckSzB_S390(void)
   10027 {
   10028    return s390_host_has_gie ? 18 : 24;
   10029 }
   10030 
   10031 
   10032 /* Patch the counter address into CODE_TO_PATCH as previously
   10033    generated by s390_insn_profinc_emit. */
   10034 VexInvalRange
   10035 patchProfInc_S390(VexEndness endness_host,
   10036                   void *code_to_patch, const ULong *location_of_counter)
   10037 {
   10038    vassert(sizeof(ULong *) == 8);
   10039 
   10040    s390_tchain_verify_load64(code_to_patch, S390_REGNO_TCHAIN_SCRATCH, 0);
   10041 
   10042    UChar *p = s390_tchain_patch_load64(code_to_patch,
   10043                                        (Addr)location_of_counter);
   10044 
   10045    UInt len = p - (UChar *)code_to_patch;
   10046    VexInvalRange vir = { (HWord)code_to_patch, len };
   10047    return vir;
   10048 }
   10049 
   10050 
   10051 /* NB: what goes on here has to be very closely coordinated with the
   10052    s390_insn_xdirect_emit code above. */
   10053 VexInvalRange
   10054 chainXDirect_S390(VexEndness endness_host,
   10055                   void *place_to_chain,
   10056                   const void *disp_cp_chain_me_EXPECTED,
   10057                   const void *place_to_jump_to)
   10058 {
   10059    vassert(endness_host == VexEndnessBE);
   10060 
   10061    /* What we're expecting to see @ PLACE_TO_CHAIN is:
   10062 
   10063         load  tchain_scratch, #disp_cp_chain_me_EXPECTED
   10064         goto *tchain_scratch
   10065    */
   10066    const UChar *next;
   10067    next = s390_tchain_verify_load64(place_to_chain, S390_REGNO_TCHAIN_SCRATCH,
   10068                                     (Addr)disp_cp_chain_me_EXPECTED);
   10069    vassert(s390_insn_is_BR(next, S390_REGNO_TCHAIN_SCRATCH));
   10070 
   10071    /* And what we want to change it to is either:
   10072         (general case):
   10073 
   10074           load  tchain_scratch, #place_to_jump_to
   10075           goto *tchain_scratch
   10076 
   10077       ---OR---
   10078 
   10079         in the case where the displacement is small enough
   10080 
   10081           BRCL delta       where delta is in half-words
   10082           invalid opcodes
   10083 
   10084       In both cases the replacement has the same length as the original.
   10085       To remain sane & verifiable,
   10086       (1) limit the displacement for the short form to
   10087           (say) +/- one billion, so as to avoid wraparound
   10088           off-by-ones
   10089       (2) even if the short form is applicable, once every (say)
   10090           1024 times use the long form anyway, so as to maintain
   10091           verifiability
   10092    */
   10093 
   10094    /* This is the delta we need to put into a BRCL insn. Note, that the
   10095       offset in BRCL is in half-words. Hence division by 2. */
   10096    Long delta =
   10097       (Long)((const UChar *)place_to_jump_to - (const UChar *)place_to_chain) / 2;
   10098    Bool shortOK = delta >= -1000*1000*1000 && delta < 1000*1000*1000;
   10099 
   10100    static UInt shortCTR = 0; /* DO NOT MAKE NON-STATIC */
   10101    if (shortOK) {
   10102       shortCTR++; // thread safety bleh
   10103       if (0 == (shortCTR & 0x3FF)) {
   10104          shortOK = False;
   10105          if (0)
   10106             vex_printf("QQQ chainXDirect_S390: shortCTR = %u, "
   10107                        "using long jmp\n", shortCTR);
   10108       }
   10109    }
   10110 
   10111    /* And make the modifications. */
   10112    UChar *p = (UChar *)place_to_chain;
   10113    if (shortOK) {
   10114       p = s390_emit_BRCL(p, S390_CC_ALWAYS, delta);  /* 6 bytes */
   10115 
   10116       /* Make sure that BRCL fits into the patchable part of an xdirect
   10117          code sequence */
   10118       vassert(6 <= s390_xdirect_patchable_len());
   10119 
   10120       /* Fill remaining bytes with 0x00 (invalid opcode) */
   10121       Int i;
   10122       for (i = 0; i < s390_xdirect_patchable_len() - 6; ++i)
   10123          p[i] = 0x00;
   10124    } else {
   10125       /*
   10126           load  tchain_scratch, #place_to_jump_to
   10127           goto *tchain_scratch
   10128       */
   10129       Addr64 addr = (Addr)place_to_jump_to;
   10130       p = s390_tchain_load64(p, S390_REGNO_TCHAIN_SCRATCH, addr);
   10131       /* There is not need to emit a BCR here, as it is already there. */
   10132    }
   10133 
   10134    UInt len = p - (UChar *)place_to_chain;
   10135    VexInvalRange vir = { (HWord)place_to_chain, len };
   10136    return vir;
   10137 }
   10138 
   10139 
   10140 /* NB: what goes on here has to be very closely coordinated with the
   10141    s390_insn_xdirect_emit code above. */
   10142 VexInvalRange
   10143 unchainXDirect_S390(VexEndness endness_host,
   10144                     void *place_to_unchain,
   10145                     const void *place_to_jump_to_EXPECTED,
   10146                     const void *disp_cp_chain_me)
   10147 {
   10148    vassert(endness_host == VexEndnessBE);
   10149 
   10150    /* What we're expecting to see @ PLACE_TO_UNCHAIN:
   10151 
   10152           load  tchain_scratch, #place_to_jump_to_EXPECTED
   10153           goto *tchain_scratch
   10154 
   10155       ---OR---
   10156         in the case where the displacement falls within 32 bits
   10157 
   10158           BRCL delta
   10159           invalid opcodes
   10160    */
   10161    UChar *p = place_to_unchain;
   10162 
   10163    Bool uses_short_form = False;
   10164 
   10165    if (s390_insn_is_BRCL(p, S390_CC_ALWAYS)) {
   10166       /* Looks like the short form */
   10167       Int num_hw = *(Int *)&p[2];
   10168       Int delta = 2 *num_hw;
   10169 
   10170       vassert(p + delta == place_to_jump_to_EXPECTED);
   10171 
   10172       Int i;
   10173       for (i = 0; i < s390_xdirect_patchable_len() - 6; ++i)
   10174          vassert(p[6+i] == 0x00);
   10175       uses_short_form = True;
   10176    } else {
   10177       /* Should be the long form */
   10178       const UChar *next;
   10179 
   10180       next = s390_tchain_verify_load64(p, S390_REGNO_TCHAIN_SCRATCH,
   10181                                        (Addr)place_to_jump_to_EXPECTED);
   10182       /* Check for BR *tchain_scratch */
   10183       vassert(s390_insn_is_BR(next, S390_REGNO_TCHAIN_SCRATCH));
   10184    }
   10185 
   10186    /* And what we want to change it to is:
   10187 
   10188         load  tchain_scratch, #disp_cp_chain_me
   10189         goto *tchain_scratch
   10190    */
   10191 
   10192    /* Get the address of the beginning of the load64 code sequence into %r1.
   10193       Do not change the register! This is part of the protocol with the
   10194       dispatcher.
   10195       Note: the incoming argument PLACE_TO_CHAIN points to the beginning of the
   10196       load64 insn sequence. That sequence is prefixed with a BASR to get its
   10197       address (see s390_insn_xdirect_emit).  */
   10198    p = s390_emit_BASR(p - S390_BASR_LEN, 1, R0);
   10199 
   10200    Addr64 addr = (Addr)disp_cp_chain_me;
   10201    p = s390_tchain_load64(p, S390_REGNO_TCHAIN_SCRATCH, addr);
   10202 
   10203    /* Emit the BCR in case the short form was used. In case of the long
   10204       form, the BCR is already there. */
   10205    if (uses_short_form)
   10206       s390_emit_BCR(p, S390_CC_ALWAYS, S390_REGNO_TCHAIN_SCRATCH);
   10207 
   10208    UInt len = p - (UChar *)place_to_unchain;
   10209    VexInvalRange vir = { (HWord)place_to_unchain, len };
   10210    return vir;
   10211 }
   10212 
   10213 /*---------------------------------------------------------------*/
   10214 /*--- end                                    host_s390_defs.c ---*/
   10215 /*---------------------------------------------------------------*/
   10216