Home | History | Annotate | Download | only in priv
      1 
      2 /*---------------------------------------------------------------*/
      3 /*--- begin                             host_generic_simd64.c ---*/
      4 /*---------------------------------------------------------------*/
      5 
      6 /*
      7    This file is part of Valgrind, a dynamic binary instrumentation
      8    framework.
      9 
     10    Copyright (C) 2004-2013 OpenWorks LLP
     11       info (at) open-works.net
     12 
     13    This program is free software; you can redistribute it and/or
     14    modify it under the terms of the GNU General Public License as
     15    published by the Free Software Foundation; either version 2 of the
     16    License, or (at your option) any later version.
     17 
     18    This program is distributed in the hope that it will be useful, but
     19    WITHOUT ANY WARRANTY; without even the implied warranty of
     20    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     21    General Public License for more details.
     22 
     23    You should have received a copy of the GNU General Public License
     24    along with this program; if not, write to the Free Software
     25    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
     26    02110-1301, USA.
     27 
     28    The GNU General Public License is contained in the file COPYING.
     29 
     30    Neither the names of the U.S. Department of Energy nor the
     31    University of California nor the names of its contributors may be
     32    used to endorse or promote products derived from this software
     33    without prior written permission.
     34 */
     35 
     36 /* Generic helper functions for doing 64-bit SIMD arithmetic in cases
     37    where the instruction selectors cannot generate code in-line.
     38    These are purely back-end entities and cannot be seen/referenced
     39    from IR.  There are also helpers for 32-bit arithmetic in here. */
     40 
     41 #include "libvex_basictypes.h"
     42 #include "main_util.h"              // LIKELY, UNLIKELY
     43 #include "host_generic_simd64.h"
     44 
     45 
     46 
     47 /* Tuple/select functions for 32x2 vectors. */
     48 
     49 static inline ULong mk32x2 ( UInt w1, UInt w0 ) {
     50    return (((ULong)w1) << 32) | ((ULong)w0);
     51 }
     52 
     53 static inline UInt sel32x2_1 ( ULong w64 ) {
     54    return 0xFFFFFFFF & toUInt(w64 >> 32);
     55 }
     56 static inline UInt sel32x2_0 ( ULong w64 ) {
     57    return 0xFFFFFFFF & toUInt(w64);
     58 }
     59 
     60 
     61 /* Tuple/select functions for 16x4 vectors.  gcc is pretty hopeless
     62    with 64-bit shifts so we give it a hand. */
     63 
     64 static inline ULong mk16x4 ( UShort w3, UShort w2,
     65                              UShort w1, UShort w0 ) {
     66    UInt hi32 = (((UInt)w3) << 16) | ((UInt)w2);
     67    UInt lo32 = (((UInt)w1) << 16) | ((UInt)w0);
     68    return mk32x2(hi32, lo32);
     69 }
     70 
     71 static inline UShort sel16x4_3 ( ULong w64 ) {
     72    UInt hi32 = toUInt(w64 >> 32);
     73    return toUShort(0xFFFF & (hi32 >> 16));
     74 }
     75 static inline UShort sel16x4_2 ( ULong w64 ) {
     76    UInt hi32 = toUInt(w64 >> 32);
     77    return toUShort(0xFFFF & hi32);
     78 }
     79 static inline UShort sel16x4_1 ( ULong w64 ) {
     80    UInt lo32 = (UInt)w64;
     81    return toUShort(0xFFFF & (lo32 >> 16));
     82 }
     83 static inline UShort sel16x4_0 ( ULong w64 ) {
     84    UInt lo32 = (UInt)w64;
     85    return toUShort(0xFFFF & lo32);
     86 }
     87 
     88 
     89 /* Tuple/select functions for 8x8 vectors. */
     90 
     91 static inline ULong mk8x8 ( UChar w7, UChar w6,
     92                             UChar w5, UChar w4,
     93                             UChar w3, UChar w2,
     94                             UChar w1, UChar w0 ) {
     95    UInt hi32 =   (((UInt)w7) << 24) | (((UInt)w6) << 16)
     96                | (((UInt)w5) << 8)  | (((UInt)w4) << 0);
     97    UInt lo32 =   (((UInt)w3) << 24) | (((UInt)w2) << 16)
     98                | (((UInt)w1) << 8)  | (((UInt)w0) << 0);
     99    return mk32x2(hi32, lo32);
    100 }
    101 
    102 static inline UChar sel8x8_7 ( ULong w64 ) {
    103    UInt hi32 = toUInt(w64 >> 32);
    104    return toUChar(0xFF & (hi32 >> 24));
    105 }
    106 static inline UChar sel8x8_6 ( ULong w64 ) {
    107    UInt hi32 = toUInt(w64 >> 32);
    108    return toUChar(0xFF & (hi32 >> 16));
    109 }
    110 static inline UChar sel8x8_5 ( ULong w64 ) {
    111    UInt hi32 = toUInt(w64 >> 32);
    112    return toUChar(0xFF & (hi32 >> 8));
    113 }
    114 static inline UChar sel8x8_4 ( ULong w64 ) {
    115    UInt hi32 = toUInt(w64 >> 32);
    116    return toUChar(0xFF & (hi32 >> 0));
    117 }
    118 static inline UChar sel8x8_3 ( ULong w64 ) {
    119    UInt lo32 = (UInt)w64;
    120    return toUChar(0xFF & (lo32 >> 24));
    121 }
    122 static inline UChar sel8x8_2 ( ULong w64 ) {
    123    UInt lo32 = (UInt)w64;
    124    return toUChar(0xFF & (lo32 >> 16));
    125 }
    126 static inline UChar sel8x8_1 ( ULong w64 ) {
    127    UInt lo32 = (UInt)w64;
    128    return toUChar(0xFF & (lo32 >> 8));
    129 }
    130 static inline UChar sel8x8_0 ( ULong w64 ) {
    131    UInt lo32 = (UInt)w64;
    132    return toUChar(0xFF & (lo32 >> 0));
    133 }
    134 
    135 static inline UChar index8x8 ( ULong w64, UChar ix ) {
    136    ix &= 7;
    137    return toUChar((w64 >> (8*ix)) & 0xFF);
    138 }
    139 
    140 
    141 /* Scalar helpers. */
    142 
    143 static inline Int qadd32S ( Int xx, Int yy )
    144 {
    145    Long t = ((Long)xx) + ((Long)yy);
    146    const Long loLim = -0x80000000LL;
    147    const Long hiLim =  0x7FFFFFFFLL;
    148    if (t < loLim) t = loLim;
    149    if (t > hiLim) t = hiLim;
    150    return (Int)t;
    151 }
    152 
    153 static inline Short qadd16S ( Short xx, Short yy )
    154 {
    155    Int t = ((Int)xx) + ((Int)yy);
    156    if (t < -32768) t = -32768;
    157    if (t > 32767)  t = 32767;
    158    return (Short)t;
    159 }
    160 
    161 static inline Char qadd8S ( Char xx, Char yy )
    162 {
    163    Int t = ((Int)xx) + ((Int)yy);
    164    if (t < -128) t = -128;
    165    if (t > 127)  t = 127;
    166    return (Char)t;
    167 }
    168 
    169 static inline UShort qadd16U ( UShort xx, UShort yy )
    170 {
    171    UInt t = ((UInt)xx) + ((UInt)yy);
    172    if (t > 0xFFFF) t = 0xFFFF;
    173    return (UShort)t;
    174 }
    175 
    176 static inline UChar qadd8U ( UChar xx, UChar yy )
    177 {
    178    UInt t = ((UInt)xx) + ((UInt)yy);
    179    if (t > 0xFF) t = 0xFF;
    180    return (UChar)t;
    181 }
    182 
    183 static inline Int qsub32S ( Int xx, Int yy )
    184 {
    185    Long t = ((Long)xx) - ((Long)yy);
    186    const Long loLim = -0x80000000LL;
    187    const Long hiLim =  0x7FFFFFFFLL;
    188    if (t < loLim) t = loLim;
    189    if (t > hiLim) t = hiLim;
    190    return (Int)t;
    191 }
    192 
    193 static inline Short qsub16S ( Short xx, Short yy )
    194 {
    195    Int t = ((Int)xx) - ((Int)yy);
    196    if (t < -32768) t = -32768;
    197    if (t > 32767)  t = 32767;
    198    return (Short)t;
    199 }
    200 
    201 static inline Char qsub8S ( Char xx, Char yy )
    202 {
    203    Int t = ((Int)xx) - ((Int)yy);
    204    if (t < -128) t = -128;
    205    if (t > 127)  t = 127;
    206    return (Char)t;
    207 }
    208 
    209 static inline UShort qsub16U ( UShort xx, UShort yy )
    210 {
    211    Int t = ((Int)xx) - ((Int)yy);
    212    if (t < 0)      t = 0;
    213    if (t > 0xFFFF) t = 0xFFFF;
    214    return (UShort)t;
    215 }
    216 
    217 static inline UChar qsub8U ( UChar xx, UChar yy )
    218 {
    219    Int t = ((Int)xx) - ((Int)yy);
    220    if (t < 0)    t = 0;
    221    if (t > 0xFF) t = 0xFF;
    222    return (UChar)t;
    223 }
    224 
    225 static inline Short mul16 ( Short xx, Short yy )
    226 {
    227    Int t = ((Int)xx) * ((Int)yy);
    228    return (Short)t;
    229 }
    230 
    231 static inline Int mul32 ( Int xx, Int yy )
    232 {
    233    Int t = ((Int)xx) * ((Int)yy);
    234    return (Int)t;
    235 }
    236 
    237 static inline Short mulhi16S ( Short xx, Short yy )
    238 {
    239    Int t = ((Int)xx) * ((Int)yy);
    240    t >>=/*s*/ 16;
    241    return (Short)t;
    242 }
    243 
    244 static inline UShort mulhi16U ( UShort xx, UShort yy )
    245 {
    246    UInt t = ((UInt)xx) * ((UInt)yy);
    247    t >>=/*u*/ 16;
    248    return (UShort)t;
    249 }
    250 
    251 static inline UInt cmpeq32 ( UInt xx, UInt yy )
    252 {
    253    return xx==yy ? 0xFFFFFFFF : 0;
    254 }
    255 
    256 static inline UShort cmpeq16 ( UShort xx, UShort yy )
    257 {
    258    return toUShort(xx==yy ? 0xFFFF : 0);
    259 }
    260 
    261 static inline UChar cmpeq8 ( UChar xx, UChar yy )
    262 {
    263    return toUChar(xx==yy ? 0xFF : 0);
    264 }
    265 
    266 static inline UInt cmpgt32S ( Int xx, Int yy )
    267 {
    268    return xx>yy ? 0xFFFFFFFF : 0;
    269 }
    270 
    271 static inline UShort cmpgt16S ( Short xx, Short yy )
    272 {
    273    return toUShort(xx>yy ? 0xFFFF : 0);
    274 }
    275 
    276 static inline UChar cmpgt8S ( Char xx, Char yy )
    277 {
    278    return toUChar(xx>yy ? 0xFF : 0);
    279 }
    280 
    281 static inline UInt cmpnez32 ( UInt xx )
    282 {
    283    return xx==0 ? 0 : 0xFFFFFFFF;
    284 }
    285 
    286 static inline UShort cmpnez16 ( UShort xx )
    287 {
    288    return toUShort(xx==0 ? 0 : 0xFFFF);
    289 }
    290 
    291 static inline UChar cmpnez8 ( UChar xx )
    292 {
    293    return toUChar(xx==0 ? 0 : 0xFF);
    294 }
    295 
    296 static inline Short qnarrow32Sto16S ( UInt xx0 )
    297 {
    298    Int xx = (Int)xx0;
    299    if (xx < -32768) xx = -32768;
    300    if (xx > 32767)  xx = 32767;
    301    return (Short)xx;
    302 }
    303 
    304 static inline Char qnarrow16Sto8S ( UShort xx0 )
    305 {
    306    Short xx = (Short)xx0;
    307    if (xx < -128) xx = -128;
    308    if (xx > 127)  xx = 127;
    309    return (Char)xx;
    310 }
    311 
    312 static inline UChar qnarrow16Sto8U ( UShort xx0 )
    313 {
    314    Short xx = (Short)xx0;
    315    if (xx < 0)   xx = 0;
    316    if (xx > 255) xx = 255;
    317    return (UChar)xx;
    318 }
    319 
    320 static inline UShort narrow32to16 ( UInt xx )
    321 {
    322    return (UShort)xx;
    323 }
    324 
    325 static inline UChar narrow16to8 ( UShort xx )
    326 {
    327    return (UChar)xx;
    328 }
    329 
    330 /* shifts: we don't care about out-of-range ones, since
    331    that is dealt with at a higher level. */
    332 
    333 static inline UChar shl8 ( UChar v, UInt n )
    334 {
    335    return toUChar(v << n);
    336 }
    337 
    338 static inline UChar sar8 ( UChar v, UInt n )
    339 {
    340    return toUChar(((Char)v) >> n);
    341 }
    342 
    343 static inline UShort shl16 ( UShort v, UInt n )
    344 {
    345    return toUShort(v << n);
    346 }
    347 
    348 static inline UShort shr16 ( UShort v, UInt n )
    349 {
    350    return toUShort((((UShort)v) >> n));
    351 }
    352 
    353 static inline UShort sar16 ( UShort v, UInt n )
    354 {
    355    return toUShort(((Short)v) >> n);
    356 }
    357 
    358 static inline UInt shl32 ( UInt v, UInt n )
    359 {
    360    return v << n;
    361 }
    362 
    363 static inline UInt shr32 ( UInt v, UInt n )
    364 {
    365    return (((UInt)v) >> n);
    366 }
    367 
    368 static inline UInt sar32 ( UInt v, UInt n )
    369 {
    370    return ((Int)v) >> n;
    371 }
    372 
    373 static inline UChar avg8U ( UChar xx, UChar yy )
    374 {
    375    UInt xxi = (UInt)xx;
    376    UInt yyi = (UInt)yy;
    377    UInt r   = (xxi + yyi + 1) >> 1;
    378    return (UChar)r;
    379 }
    380 
    381 static inline UShort avg16U ( UShort xx, UShort yy )
    382 {
    383    UInt xxi = (UInt)xx;
    384    UInt yyi = (UInt)yy;
    385    UInt r   = (xxi + yyi + 1) >> 1;
    386    return (UShort)r;
    387 }
    388 
    389 static inline Short max16S ( Short xx, Short yy )
    390 {
    391    return toUShort((xx > yy) ? xx : yy);
    392 }
    393 
    394 static inline UChar max8U ( UChar xx, UChar yy )
    395 {
    396    return toUChar((xx > yy) ? xx : yy);
    397 }
    398 
    399 static inline Short min16S ( Short xx, Short yy )
    400 {
    401    return toUShort((xx < yy) ? xx : yy);
    402 }
    403 
    404 static inline UChar min8U ( UChar xx, UChar yy )
    405 {
    406    return toUChar((xx < yy) ? xx : yy);
    407 }
    408 
    409 static inline UShort hadd16U ( UShort xx, UShort yy )
    410 {
    411    UInt xxi = (UInt)xx;
    412    UInt yyi = (UInt)yy;
    413    UInt r   = (xxi + yyi) >> 1;
    414    return (UShort)r;
    415 }
    416 
    417 static inline Short hadd16S ( Short xx, Short yy )
    418 {
    419    Int xxi = (Int)xx;
    420    Int yyi = (Int)yy;
    421    Int r   = (xxi + yyi) >> 1;
    422    return (Short)r;
    423 }
    424 
    425 static inline UShort hsub16U ( UShort xx, UShort yy )
    426 {
    427    UInt xxi = (UInt)xx;
    428    UInt yyi = (UInt)yy;
    429    UInt r   = (xxi - yyi) >> 1;
    430    return (UShort)r;
    431 }
    432 
    433 static inline Short hsub16S ( Short xx, Short yy )
    434 {
    435    Int xxi = (Int)xx;
    436    Int yyi = (Int)yy;
    437    Int r   = (xxi - yyi) >> 1;
    438    return (Short)r;
    439 }
    440 
    441 static inline UChar hadd8U ( UChar xx, UChar yy )
    442 {
    443    UInt xxi = (UInt)xx;
    444    UInt yyi = (UInt)yy;
    445    UInt r   = (xxi + yyi) >> 1;
    446    return (UChar)r;
    447 }
    448 
    449 static inline Char hadd8S ( Char xx, Char yy )
    450 {
    451    Int xxi = (Int)xx;
    452    Int yyi = (Int)yy;
    453    Int r   = (xxi + yyi) >> 1;
    454    return (Char)r;
    455 }
    456 
    457 static inline UChar hsub8U ( UChar xx, UChar yy )
    458 {
    459    UInt xxi = (UInt)xx;
    460    UInt yyi = (UInt)yy;
    461    UInt r   = (xxi - yyi) >> 1;
    462    return (UChar)r;
    463 }
    464 
    465 static inline Char hsub8S ( Char xx, Char yy )
    466 {
    467    Int xxi = (Int)xx;
    468    Int yyi = (Int)yy;
    469    Int r   = (xxi - yyi) >> 1;
    470    return (Char)r;
    471 }
    472 
    473 static inline UInt absdiff8U ( UChar xx, UChar yy )
    474 {
    475    UInt xxu = (UChar)xx;
    476    UInt yyu = (UChar)yy;
    477    return xxu >= yyu  ? xxu - yyu  : yyu - xxu;
    478 }
    479 
    480 /* ----------------------------------------------------- */
    481 /* Start of the externally visible functions.  These simply
    482    implement the corresponding IR primops. */
    483 /* ----------------------------------------------------- */
    484 
    485 /* ------------ Normal addition ------------ */
    486 
    487 ULong h_generic_calc_Add32x2 ( ULong xx, ULong yy )
    488 {
    489    return mk32x2(
    490              sel32x2_1(xx) + sel32x2_1(yy),
    491              sel32x2_0(xx) + sel32x2_0(yy)
    492           );
    493 }
    494 
    495 ULong h_generic_calc_Add16x4 ( ULong xx, ULong yy )
    496 {
    497    return mk16x4(
    498              toUShort( sel16x4_3(xx) + sel16x4_3(yy) ),
    499              toUShort( sel16x4_2(xx) + sel16x4_2(yy) ),
    500              toUShort( sel16x4_1(xx) + sel16x4_1(yy) ),
    501              toUShort( sel16x4_0(xx) + sel16x4_0(yy) )
    502           );
    503 }
    504 
    505 ULong h_generic_calc_Add8x8 ( ULong xx, ULong yy )
    506 {
    507    return mk8x8(
    508              toUChar( sel8x8_7(xx) + sel8x8_7(yy) ),
    509              toUChar( sel8x8_6(xx) + sel8x8_6(yy) ),
    510              toUChar( sel8x8_5(xx) + sel8x8_5(yy) ),
    511              toUChar( sel8x8_4(xx) + sel8x8_4(yy) ),
    512              toUChar( sel8x8_3(xx) + sel8x8_3(yy) ),
    513              toUChar( sel8x8_2(xx) + sel8x8_2(yy) ),
    514              toUChar( sel8x8_1(xx) + sel8x8_1(yy) ),
    515              toUChar( sel8x8_0(xx) + sel8x8_0(yy) )
    516           );
    517 }
    518 
    519 /* ------------ Saturating addition ------------ */
    520 
    521 ULong h_generic_calc_QAdd16Sx4 ( ULong xx, ULong yy )
    522 {
    523    return mk16x4(
    524              qadd16S( sel16x4_3(xx), sel16x4_3(yy) ),
    525              qadd16S( sel16x4_2(xx), sel16x4_2(yy) ),
    526              qadd16S( sel16x4_1(xx), sel16x4_1(yy) ),
    527              qadd16S( sel16x4_0(xx), sel16x4_0(yy) )
    528           );
    529 }
    530 
    531 ULong h_generic_calc_QAdd8Sx8 ( ULong xx, ULong yy )
    532 {
    533    return mk8x8(
    534              qadd8S( sel8x8_7(xx), sel8x8_7(yy) ),
    535              qadd8S( sel8x8_6(xx), sel8x8_6(yy) ),
    536              qadd8S( sel8x8_5(xx), sel8x8_5(yy) ),
    537              qadd8S( sel8x8_4(xx), sel8x8_4(yy) ),
    538              qadd8S( sel8x8_3(xx), sel8x8_3(yy) ),
    539              qadd8S( sel8x8_2(xx), sel8x8_2(yy) ),
    540              qadd8S( sel8x8_1(xx), sel8x8_1(yy) ),
    541              qadd8S( sel8x8_0(xx), sel8x8_0(yy) )
    542           );
    543 }
    544 
    545 ULong h_generic_calc_QAdd16Ux4 ( ULong xx, ULong yy )
    546 {
    547    return mk16x4(
    548              qadd16U( sel16x4_3(xx), sel16x4_3(yy) ),
    549              qadd16U( sel16x4_2(xx), sel16x4_2(yy) ),
    550              qadd16U( sel16x4_1(xx), sel16x4_1(yy) ),
    551              qadd16U( sel16x4_0(xx), sel16x4_0(yy) )
    552           );
    553 }
    554 
    555 ULong h_generic_calc_QAdd8Ux8 ( ULong xx, ULong yy )
    556 {
    557    return mk8x8(
    558              qadd8U( sel8x8_7(xx), sel8x8_7(yy) ),
    559              qadd8U( sel8x8_6(xx), sel8x8_6(yy) ),
    560              qadd8U( sel8x8_5(xx), sel8x8_5(yy) ),
    561              qadd8U( sel8x8_4(xx), sel8x8_4(yy) ),
    562              qadd8U( sel8x8_3(xx), sel8x8_3(yy) ),
    563              qadd8U( sel8x8_2(xx), sel8x8_2(yy) ),
    564              qadd8U( sel8x8_1(xx), sel8x8_1(yy) ),
    565              qadd8U( sel8x8_0(xx), sel8x8_0(yy) )
    566           );
    567 }
    568 
    569 /* ------------ Normal subtraction ------------ */
    570 
    571 ULong h_generic_calc_Sub32x2 ( ULong xx, ULong yy )
    572 {
    573    return mk32x2(
    574              sel32x2_1(xx) - sel32x2_1(yy),
    575              sel32x2_0(xx) - sel32x2_0(yy)
    576           );
    577 }
    578 
    579 ULong h_generic_calc_Sub16x4 ( ULong xx, ULong yy )
    580 {
    581    return mk16x4(
    582              toUShort( sel16x4_3(xx) - sel16x4_3(yy) ),
    583              toUShort( sel16x4_2(xx) - sel16x4_2(yy) ),
    584              toUShort( sel16x4_1(xx) - sel16x4_1(yy) ),
    585              toUShort( sel16x4_0(xx) - sel16x4_0(yy) )
    586           );
    587 }
    588 
    589 ULong h_generic_calc_Sub8x8 ( ULong xx, ULong yy )
    590 {
    591    return mk8x8(
    592              toUChar( sel8x8_7(xx) - sel8x8_7(yy) ),
    593              toUChar( sel8x8_6(xx) - sel8x8_6(yy) ),
    594              toUChar( sel8x8_5(xx) - sel8x8_5(yy) ),
    595              toUChar( sel8x8_4(xx) - sel8x8_4(yy) ),
    596              toUChar( sel8x8_3(xx) - sel8x8_3(yy) ),
    597              toUChar( sel8x8_2(xx) - sel8x8_2(yy) ),
    598              toUChar( sel8x8_1(xx) - sel8x8_1(yy) ),
    599              toUChar( sel8x8_0(xx) - sel8x8_0(yy) )
    600           );
    601 }
    602 
    603 /* ------------ Saturating subtraction ------------ */
    604 
    605 ULong h_generic_calc_QSub16Sx4 ( ULong xx, ULong yy )
    606 {
    607    return mk16x4(
    608              qsub16S( sel16x4_3(xx), sel16x4_3(yy) ),
    609              qsub16S( sel16x4_2(xx), sel16x4_2(yy) ),
    610              qsub16S( sel16x4_1(xx), sel16x4_1(yy) ),
    611              qsub16S( sel16x4_0(xx), sel16x4_0(yy) )
    612           );
    613 }
    614 
    615 ULong h_generic_calc_QSub8Sx8 ( ULong xx, ULong yy )
    616 {
    617    return mk8x8(
    618              qsub8S( sel8x8_7(xx), sel8x8_7(yy) ),
    619              qsub8S( sel8x8_6(xx), sel8x8_6(yy) ),
    620              qsub8S( sel8x8_5(xx), sel8x8_5(yy) ),
    621              qsub8S( sel8x8_4(xx), sel8x8_4(yy) ),
    622              qsub8S( sel8x8_3(xx), sel8x8_3(yy) ),
    623              qsub8S( sel8x8_2(xx), sel8x8_2(yy) ),
    624              qsub8S( sel8x8_1(xx), sel8x8_1(yy) ),
    625              qsub8S( sel8x8_0(xx), sel8x8_0(yy) )
    626           );
    627 }
    628 
    629 ULong h_generic_calc_QSub16Ux4 ( ULong xx, ULong yy )
    630 {
    631    return mk16x4(
    632              qsub16U( sel16x4_3(xx), sel16x4_3(yy) ),
    633              qsub16U( sel16x4_2(xx), sel16x4_2(yy) ),
    634              qsub16U( sel16x4_1(xx), sel16x4_1(yy) ),
    635              qsub16U( sel16x4_0(xx), sel16x4_0(yy) )
    636           );
    637 }
    638 
    639 ULong h_generic_calc_QSub8Ux8 ( ULong xx, ULong yy )
    640 {
    641    return mk8x8(
    642              qsub8U( sel8x8_7(xx), sel8x8_7(yy) ),
    643              qsub8U( sel8x8_6(xx), sel8x8_6(yy) ),
    644              qsub8U( sel8x8_5(xx), sel8x8_5(yy) ),
    645              qsub8U( sel8x8_4(xx), sel8x8_4(yy) ),
    646              qsub8U( sel8x8_3(xx), sel8x8_3(yy) ),
    647              qsub8U( sel8x8_2(xx), sel8x8_2(yy) ),
    648              qsub8U( sel8x8_1(xx), sel8x8_1(yy) ),
    649              qsub8U( sel8x8_0(xx), sel8x8_0(yy) )
    650           );
    651 }
    652 
    653 /* ------------ Multiplication ------------ */
    654 
    655 ULong h_generic_calc_Mul16x4 ( ULong xx, ULong yy )
    656 {
    657    return mk16x4(
    658              mul16( sel16x4_3(xx), sel16x4_3(yy) ),
    659              mul16( sel16x4_2(xx), sel16x4_2(yy) ),
    660              mul16( sel16x4_1(xx), sel16x4_1(yy) ),
    661              mul16( sel16x4_0(xx), sel16x4_0(yy) )
    662           );
    663 }
    664 
    665 ULong h_generic_calc_Mul32x2 ( ULong xx, ULong yy )
    666 {
    667    return mk32x2(
    668              mul32( sel32x2_1(xx), sel32x2_1(yy) ),
    669              mul32( sel32x2_0(xx), sel32x2_0(yy) )
    670           );
    671 }
    672 
    673 ULong h_generic_calc_MulHi16Sx4 ( ULong xx, ULong yy )
    674 {
    675    return mk16x4(
    676              mulhi16S( sel16x4_3(xx), sel16x4_3(yy) ),
    677              mulhi16S( sel16x4_2(xx), sel16x4_2(yy) ),
    678              mulhi16S( sel16x4_1(xx), sel16x4_1(yy) ),
    679              mulhi16S( sel16x4_0(xx), sel16x4_0(yy) )
    680           );
    681 }
    682 
    683 ULong h_generic_calc_MulHi16Ux4 ( ULong xx, ULong yy )
    684 {
    685    return mk16x4(
    686              mulhi16U( sel16x4_3(xx), sel16x4_3(yy) ),
    687              mulhi16U( sel16x4_2(xx), sel16x4_2(yy) ),
    688              mulhi16U( sel16x4_1(xx), sel16x4_1(yy) ),
    689              mulhi16U( sel16x4_0(xx), sel16x4_0(yy) )
    690           );
    691 }
    692 
    693 /* ------------ Comparison ------------ */
    694 
    695 ULong h_generic_calc_CmpEQ32x2 ( ULong xx, ULong yy )
    696 {
    697    return mk32x2(
    698              cmpeq32( sel32x2_1(xx), sel32x2_1(yy) ),
    699              cmpeq32( sel32x2_0(xx), sel32x2_0(yy) )
    700           );
    701 }
    702 
    703 ULong h_generic_calc_CmpEQ16x4 ( ULong xx, ULong yy )
    704 {
    705    return mk16x4(
    706              cmpeq16( sel16x4_3(xx), sel16x4_3(yy) ),
    707              cmpeq16( sel16x4_2(xx), sel16x4_2(yy) ),
    708              cmpeq16( sel16x4_1(xx), sel16x4_1(yy) ),
    709              cmpeq16( sel16x4_0(xx), sel16x4_0(yy) )
    710           );
    711 }
    712 
    713 ULong h_generic_calc_CmpEQ8x8 ( ULong xx, ULong yy )
    714 {
    715    return mk8x8(
    716              cmpeq8( sel8x8_7(xx), sel8x8_7(yy) ),
    717              cmpeq8( sel8x8_6(xx), sel8x8_6(yy) ),
    718              cmpeq8( sel8x8_5(xx), sel8x8_5(yy) ),
    719              cmpeq8( sel8x8_4(xx), sel8x8_4(yy) ),
    720              cmpeq8( sel8x8_3(xx), sel8x8_3(yy) ),
    721              cmpeq8( sel8x8_2(xx), sel8x8_2(yy) ),
    722              cmpeq8( sel8x8_1(xx), sel8x8_1(yy) ),
    723              cmpeq8( sel8x8_0(xx), sel8x8_0(yy) )
    724           );
    725 }
    726 
    727 ULong h_generic_calc_CmpGT32Sx2 ( ULong xx, ULong yy )
    728 {
    729    return mk32x2(
    730              cmpgt32S( sel32x2_1(xx), sel32x2_1(yy) ),
    731              cmpgt32S( sel32x2_0(xx), sel32x2_0(yy) )
    732           );
    733 }
    734 
    735 ULong h_generic_calc_CmpGT16Sx4 ( ULong xx, ULong yy )
    736 {
    737    return mk16x4(
    738              cmpgt16S( sel16x4_3(xx), sel16x4_3(yy) ),
    739              cmpgt16S( sel16x4_2(xx), sel16x4_2(yy) ),
    740              cmpgt16S( sel16x4_1(xx), sel16x4_1(yy) ),
    741              cmpgt16S( sel16x4_0(xx), sel16x4_0(yy) )
    742           );
    743 }
    744 
    745 ULong h_generic_calc_CmpGT8Sx8 ( ULong xx, ULong yy )
    746 {
    747    return mk8x8(
    748              cmpgt8S( sel8x8_7(xx), sel8x8_7(yy) ),
    749              cmpgt8S( sel8x8_6(xx), sel8x8_6(yy) ),
    750              cmpgt8S( sel8x8_5(xx), sel8x8_5(yy) ),
    751              cmpgt8S( sel8x8_4(xx), sel8x8_4(yy) ),
    752              cmpgt8S( sel8x8_3(xx), sel8x8_3(yy) ),
    753              cmpgt8S( sel8x8_2(xx), sel8x8_2(yy) ),
    754              cmpgt8S( sel8x8_1(xx), sel8x8_1(yy) ),
    755              cmpgt8S( sel8x8_0(xx), sel8x8_0(yy) )
    756           );
    757 }
    758 
    759 ULong h_generic_calc_CmpNEZ32x2 ( ULong xx )
    760 {
    761    return mk32x2(
    762              cmpnez32( sel32x2_1(xx) ),
    763              cmpnez32( sel32x2_0(xx) )
    764           );
    765 }
    766 
    767 ULong h_generic_calc_CmpNEZ16x4 ( ULong xx )
    768 {
    769    return mk16x4(
    770              cmpnez16( sel16x4_3(xx) ),
    771              cmpnez16( sel16x4_2(xx) ),
    772              cmpnez16( sel16x4_1(xx) ),
    773              cmpnez16( sel16x4_0(xx) )
    774           );
    775 }
    776 
    777 ULong h_generic_calc_CmpNEZ8x8 ( ULong xx )
    778 {
    779    return mk8x8(
    780              cmpnez8( sel8x8_7(xx) ),
    781              cmpnez8( sel8x8_6(xx) ),
    782              cmpnez8( sel8x8_5(xx) ),
    783              cmpnez8( sel8x8_4(xx) ),
    784              cmpnez8( sel8x8_3(xx) ),
    785              cmpnez8( sel8x8_2(xx) ),
    786              cmpnez8( sel8x8_1(xx) ),
    787              cmpnez8( sel8x8_0(xx) )
    788           );
    789 }
    790 
    791 /* ------------ Saturating narrowing ------------ */
    792 
    793 ULong h_generic_calc_QNarrowBin32Sto16Sx4 ( ULong aa, ULong bb )
    794 {
    795    UInt d = sel32x2_1(aa);
    796    UInt c = sel32x2_0(aa);
    797    UInt b = sel32x2_1(bb);
    798    UInt a = sel32x2_0(bb);
    799    return mk16x4(
    800              qnarrow32Sto16S(d),
    801              qnarrow32Sto16S(c),
    802              qnarrow32Sto16S(b),
    803              qnarrow32Sto16S(a)
    804           );
    805 }
    806 
    807 ULong h_generic_calc_QNarrowBin16Sto8Sx8 ( ULong aa, ULong bb )
    808 {
    809    UShort h = sel16x4_3(aa);
    810    UShort g = sel16x4_2(aa);
    811    UShort f = sel16x4_1(aa);
    812    UShort e = sel16x4_0(aa);
    813    UShort d = sel16x4_3(bb);
    814    UShort c = sel16x4_2(bb);
    815    UShort b = sel16x4_1(bb);
    816    UShort a = sel16x4_0(bb);
    817    return mk8x8(
    818              qnarrow16Sto8S(h),
    819              qnarrow16Sto8S(g),
    820              qnarrow16Sto8S(f),
    821              qnarrow16Sto8S(e),
    822              qnarrow16Sto8S(d),
    823              qnarrow16Sto8S(c),
    824              qnarrow16Sto8S(b),
    825              qnarrow16Sto8S(a)
    826           );
    827 }
    828 
    829 ULong h_generic_calc_QNarrowBin16Sto8Ux8 ( ULong aa, ULong bb )
    830 {
    831    UShort h = sel16x4_3(aa);
    832    UShort g = sel16x4_2(aa);
    833    UShort f = sel16x4_1(aa);
    834    UShort e = sel16x4_0(aa);
    835    UShort d = sel16x4_3(bb);
    836    UShort c = sel16x4_2(bb);
    837    UShort b = sel16x4_1(bb);
    838    UShort a = sel16x4_0(bb);
    839    return mk8x8(
    840              qnarrow16Sto8U(h),
    841              qnarrow16Sto8U(g),
    842              qnarrow16Sto8U(f),
    843              qnarrow16Sto8U(e),
    844              qnarrow16Sto8U(d),
    845              qnarrow16Sto8U(c),
    846              qnarrow16Sto8U(b),
    847              qnarrow16Sto8U(a)
    848           );
    849 }
    850 
    851 /* ------------ Truncating narrowing ------------ */
    852 
    853 ULong h_generic_calc_NarrowBin32to16x4 ( ULong aa, ULong bb )
    854 {
    855    UInt d = sel32x2_1(aa);
    856    UInt c = sel32x2_0(aa);
    857    UInt b = sel32x2_1(bb);
    858    UInt a = sel32x2_0(bb);
    859    return mk16x4(
    860              narrow32to16(d),
    861              narrow32to16(c),
    862              narrow32to16(b),
    863              narrow32to16(a)
    864           );
    865 }
    866 
    867 ULong h_generic_calc_NarrowBin16to8x8 ( ULong aa, ULong bb )
    868 {
    869    UShort h = sel16x4_3(aa);
    870    UShort g = sel16x4_2(aa);
    871    UShort f = sel16x4_1(aa);
    872    UShort e = sel16x4_0(aa);
    873    UShort d = sel16x4_3(bb);
    874    UShort c = sel16x4_2(bb);
    875    UShort b = sel16x4_1(bb);
    876    UShort a = sel16x4_0(bb);
    877    return mk8x8(
    878              narrow16to8(h),
    879              narrow16to8(g),
    880              narrow16to8(f),
    881              narrow16to8(e),
    882              narrow16to8(d),
    883              narrow16to8(c),
    884              narrow16to8(b),
    885              narrow16to8(a)
    886           );
    887 }
    888 
    889 /* ------------ Interleaving ------------ */
    890 
    891 ULong h_generic_calc_InterleaveHI8x8 ( ULong aa, ULong bb )
    892 {
    893    return mk8x8(
    894              sel8x8_7(aa),
    895              sel8x8_7(bb),
    896              sel8x8_6(aa),
    897              sel8x8_6(bb),
    898              sel8x8_5(aa),
    899              sel8x8_5(bb),
    900              sel8x8_4(aa),
    901              sel8x8_4(bb)
    902           );
    903 }
    904 
    905 ULong h_generic_calc_InterleaveLO8x8 ( ULong aa, ULong bb )
    906 {
    907    return mk8x8(
    908              sel8x8_3(aa),
    909              sel8x8_3(bb),
    910              sel8x8_2(aa),
    911              sel8x8_2(bb),
    912              sel8x8_1(aa),
    913              sel8x8_1(bb),
    914              sel8x8_0(aa),
    915              sel8x8_0(bb)
    916           );
    917 }
    918 
    919 ULong h_generic_calc_InterleaveHI16x4 ( ULong aa, ULong bb )
    920 {
    921    return mk16x4(
    922              sel16x4_3(aa),
    923              sel16x4_3(bb),
    924              sel16x4_2(aa),
    925              sel16x4_2(bb)
    926           );
    927 }
    928 
    929 ULong h_generic_calc_InterleaveLO16x4 ( ULong aa, ULong bb )
    930 {
    931    return mk16x4(
    932              sel16x4_1(aa),
    933              sel16x4_1(bb),
    934              sel16x4_0(aa),
    935              sel16x4_0(bb)
    936           );
    937 }
    938 
    939 ULong h_generic_calc_InterleaveHI32x2 ( ULong aa, ULong bb )
    940 {
    941    return mk32x2(
    942              sel32x2_1(aa),
    943              sel32x2_1(bb)
    944           );
    945 }
    946 
    947 ULong h_generic_calc_InterleaveLO32x2 ( ULong aa, ULong bb )
    948 {
    949    return mk32x2(
    950              sel32x2_0(aa),
    951              sel32x2_0(bb)
    952           );
    953 }
    954 
    955 /* ------------ Concatenation ------------ */
    956 
    957 ULong h_generic_calc_CatOddLanes16x4 ( ULong aa, ULong bb )
    958 {
    959    return mk16x4(
    960              sel16x4_3(aa),
    961              sel16x4_1(aa),
    962              sel16x4_3(bb),
    963              sel16x4_1(bb)
    964           );
    965 }
    966 
    967 ULong h_generic_calc_CatEvenLanes16x4 ( ULong aa, ULong bb )
    968 {
    969    return mk16x4(
    970              sel16x4_2(aa),
    971              sel16x4_0(aa),
    972              sel16x4_2(bb),
    973              sel16x4_0(bb)
    974           );
    975 }
    976 
    977 /* misc hack looking for a proper home */
    978 ULong h_generic_calc_Perm8x8 ( ULong aa, ULong bb )
    979 {
    980    return mk8x8(
    981              index8x8(aa, sel8x8_7(bb)),
    982              index8x8(aa, sel8x8_6(bb)),
    983              index8x8(aa, sel8x8_5(bb)),
    984              index8x8(aa, sel8x8_4(bb)),
    985              index8x8(aa, sel8x8_3(bb)),
    986              index8x8(aa, sel8x8_2(bb)),
    987              index8x8(aa, sel8x8_1(bb)),
    988              index8x8(aa, sel8x8_0(bb))
    989           );
    990 }
    991 
    992 /* ------------ Shifting ------------ */
    993 /* Note that because these primops are undefined if the shift amount
    994    equals or exceeds the lane width, the shift amount is masked so
    995    that the scalar shifts are always in range.  In fact, given the
    996    semantics of these primops (ShlN16x4, etc) it is an error if in
    997    fact we are ever given an out-of-range shift amount.
    998 */
    999 ULong h_generic_calc_ShlN32x2 ( ULong xx, UInt nn )
   1000 {
   1001    /* vassert(nn < 32); */
   1002    nn &= 31;
   1003    return mk32x2(
   1004              shl32( sel32x2_1(xx), nn ),
   1005              shl32( sel32x2_0(xx), nn )
   1006           );
   1007 }
   1008 
   1009 ULong h_generic_calc_ShlN16x4 ( ULong xx, UInt nn )
   1010 {
   1011    /* vassert(nn < 16); */
   1012    nn &= 15;
   1013    return mk16x4(
   1014              shl16( sel16x4_3(xx), nn ),
   1015              shl16( sel16x4_2(xx), nn ),
   1016              shl16( sel16x4_1(xx), nn ),
   1017              shl16( sel16x4_0(xx), nn )
   1018           );
   1019 }
   1020 
   1021 ULong h_generic_calc_ShlN8x8  ( ULong xx, UInt nn )
   1022 {
   1023    /* vassert(nn < 8); */
   1024    nn &= 7;
   1025    return mk8x8(
   1026              shl8( sel8x8_7(xx), nn ),
   1027              shl8( sel8x8_6(xx), nn ),
   1028              shl8( sel8x8_5(xx), nn ),
   1029              shl8( sel8x8_4(xx), nn ),
   1030              shl8( sel8x8_3(xx), nn ),
   1031              shl8( sel8x8_2(xx), nn ),
   1032              shl8( sel8x8_1(xx), nn ),
   1033              shl8( sel8x8_0(xx), nn )
   1034           );
   1035 }
   1036 
   1037 ULong h_generic_calc_ShrN32x2 ( ULong xx, UInt nn )
   1038 {
   1039    /* vassert(nn < 32); */
   1040    nn &= 31;
   1041    return mk32x2(
   1042              shr32( sel32x2_1(xx), nn ),
   1043              shr32( sel32x2_0(xx), nn )
   1044           );
   1045 }
   1046 
   1047 ULong h_generic_calc_ShrN16x4 ( ULong xx, UInt nn )
   1048 {
   1049    /* vassert(nn < 16); */
   1050    nn &= 15;
   1051    return mk16x4(
   1052              shr16( sel16x4_3(xx), nn ),
   1053              shr16( sel16x4_2(xx), nn ),
   1054              shr16( sel16x4_1(xx), nn ),
   1055              shr16( sel16x4_0(xx), nn )
   1056           );
   1057 }
   1058 
   1059 ULong h_generic_calc_SarN32x2 ( ULong xx, UInt nn )
   1060 {
   1061    /* vassert(nn < 32); */
   1062    nn &= 31;
   1063    return mk32x2(
   1064              sar32( sel32x2_1(xx), nn ),
   1065              sar32( sel32x2_0(xx), nn )
   1066           );
   1067 }
   1068 
   1069 ULong h_generic_calc_SarN16x4 ( ULong xx, UInt nn )
   1070 {
   1071    /* vassert(nn < 16); */
   1072    nn &= 15;
   1073    return mk16x4(
   1074              sar16( sel16x4_3(xx), nn ),
   1075              sar16( sel16x4_2(xx), nn ),
   1076              sar16( sel16x4_1(xx), nn ),
   1077              sar16( sel16x4_0(xx), nn )
   1078           );
   1079 }
   1080 
   1081 ULong h_generic_calc_SarN8x8 ( ULong xx, UInt nn )
   1082 {
   1083    /* vassert(nn < 8); */
   1084    nn &= 7;
   1085    return mk8x8(
   1086              sar8( sel8x8_7(xx), nn ),
   1087              sar8( sel8x8_6(xx), nn ),
   1088              sar8( sel8x8_5(xx), nn ),
   1089              sar8( sel8x8_4(xx), nn ),
   1090              sar8( sel8x8_3(xx), nn ),
   1091              sar8( sel8x8_2(xx), nn ),
   1092              sar8( sel8x8_1(xx), nn ),
   1093              sar8( sel8x8_0(xx), nn )
   1094           );
   1095 }
   1096 
   1097 /* ------------ Averaging ------------ */
   1098 
   1099 ULong h_generic_calc_Avg8Ux8 ( ULong xx, ULong yy )
   1100 {
   1101    return mk8x8(
   1102              avg8U( sel8x8_7(xx), sel8x8_7(yy) ),
   1103              avg8U( sel8x8_6(xx), sel8x8_6(yy) ),
   1104              avg8U( sel8x8_5(xx), sel8x8_5(yy) ),
   1105              avg8U( sel8x8_4(xx), sel8x8_4(yy) ),
   1106              avg8U( sel8x8_3(xx), sel8x8_3(yy) ),
   1107              avg8U( sel8x8_2(xx), sel8x8_2(yy) ),
   1108              avg8U( sel8x8_1(xx), sel8x8_1(yy) ),
   1109              avg8U( sel8x8_0(xx), sel8x8_0(yy) )
   1110           );
   1111 }
   1112 
   1113 ULong h_generic_calc_Avg16Ux4 ( ULong xx, ULong yy )
   1114 {
   1115    return mk16x4(
   1116              avg16U( sel16x4_3(xx), sel16x4_3(yy) ),
   1117              avg16U( sel16x4_2(xx), sel16x4_2(yy) ),
   1118              avg16U( sel16x4_1(xx), sel16x4_1(yy) ),
   1119              avg16U( sel16x4_0(xx), sel16x4_0(yy) )
   1120           );
   1121 }
   1122 
   1123 /* ------------ max/min ------------ */
   1124 
   1125 ULong h_generic_calc_Max16Sx4 ( ULong xx, ULong yy )
   1126 {
   1127    return mk16x4(
   1128              max16S( sel16x4_3(xx), sel16x4_3(yy) ),
   1129              max16S( sel16x4_2(xx), sel16x4_2(yy) ),
   1130              max16S( sel16x4_1(xx), sel16x4_1(yy) ),
   1131              max16S( sel16x4_0(xx), sel16x4_0(yy) )
   1132           );
   1133 }
   1134 
   1135 ULong h_generic_calc_Max8Ux8 ( ULong xx, ULong yy )
   1136 {
   1137    return mk8x8(
   1138              max8U( sel8x8_7(xx), sel8x8_7(yy) ),
   1139              max8U( sel8x8_6(xx), sel8x8_6(yy) ),
   1140              max8U( sel8x8_5(xx), sel8x8_5(yy) ),
   1141              max8U( sel8x8_4(xx), sel8x8_4(yy) ),
   1142              max8U( sel8x8_3(xx), sel8x8_3(yy) ),
   1143              max8U( sel8x8_2(xx), sel8x8_2(yy) ),
   1144              max8U( sel8x8_1(xx), sel8x8_1(yy) ),
   1145              max8U( sel8x8_0(xx), sel8x8_0(yy) )
   1146           );
   1147 }
   1148 
   1149 ULong h_generic_calc_Min16Sx4 ( ULong xx, ULong yy )
   1150 {
   1151    return mk16x4(
   1152              min16S( sel16x4_3(xx), sel16x4_3(yy) ),
   1153              min16S( sel16x4_2(xx), sel16x4_2(yy) ),
   1154              min16S( sel16x4_1(xx), sel16x4_1(yy) ),
   1155              min16S( sel16x4_0(xx), sel16x4_0(yy) )
   1156           );
   1157 }
   1158 
   1159 ULong h_generic_calc_Min8Ux8 ( ULong xx, ULong yy )
   1160 {
   1161    return mk8x8(
   1162              min8U( sel8x8_7(xx), sel8x8_7(yy) ),
   1163              min8U( sel8x8_6(xx), sel8x8_6(yy) ),
   1164              min8U( sel8x8_5(xx), sel8x8_5(yy) ),
   1165              min8U( sel8x8_4(xx), sel8x8_4(yy) ),
   1166              min8U( sel8x8_3(xx), sel8x8_3(yy) ),
   1167              min8U( sel8x8_2(xx), sel8x8_2(yy) ),
   1168              min8U( sel8x8_1(xx), sel8x8_1(yy) ),
   1169              min8U( sel8x8_0(xx), sel8x8_0(yy) )
   1170           );
   1171 }
   1172 
   1173 UInt h_generic_calc_GetMSBs8x8 ( ULong xx )
   1174 {
   1175    UInt r = 0;
   1176    if (xx & (1ULL << (64-1))) r |= (1<<7);
   1177    if (xx & (1ULL << (56-1))) r |= (1<<6);
   1178    if (xx & (1ULL << (48-1))) r |= (1<<5);
   1179    if (xx & (1ULL << (40-1))) r |= (1<<4);
   1180    if (xx & (1ULL << (32-1))) r |= (1<<3);
   1181    if (xx & (1ULL << (24-1))) r |= (1<<2);
   1182    if (xx & (1ULL << (16-1))) r |= (1<<1);
   1183    if (xx & (1ULL << ( 8-1))) r |= (1<<0);
   1184    return r;
   1185 }
   1186 
   1187 /* ------------ SOME 32-bit SIMD HELPERS TOO ------------ */
   1188 
   1189 /* Tuple/select functions for 16x2 vectors. */
   1190 static inline UInt mk16x2 ( UShort w1, UShort w2 ) {
   1191    return (((UInt)w1) << 16) | ((UInt)w2);
   1192 }
   1193 
   1194 static inline UShort sel16x2_1 ( UInt w32 ) {
   1195    return 0xFFFF & (UShort)(w32 >> 16);
   1196 }
   1197 static inline UShort sel16x2_0 ( UInt w32 ) {
   1198    return 0xFFFF & (UShort)(w32);
   1199 }
   1200 
   1201 static inline UInt mk8x4 ( UChar w3, UChar w2,
   1202                            UChar w1, UChar w0 ) {
   1203    UInt w32 =   (((UInt)w3) << 24) | (((UInt)w2) << 16)
   1204               | (((UInt)w1) << 8)  | (((UInt)w0) << 0);
   1205    return w32;
   1206 }
   1207 
   1208 static inline UChar sel8x4_3 ( UInt w32 ) {
   1209    return toUChar(0xFF & (w32 >> 24));
   1210 }
   1211 static inline UChar sel8x4_2 ( UInt w32 ) {
   1212    return toUChar(0xFF & (w32 >> 16));
   1213 }
   1214 static inline UChar sel8x4_1 ( UInt w32 ) {
   1215    return toUChar(0xFF & (w32 >> 8));
   1216 }
   1217 static inline UChar sel8x4_0 ( UInt w32 ) {
   1218    return toUChar(0xFF & (w32 >> 0));
   1219 }
   1220 
   1221 
   1222 /* ----------------------------------------------------- */
   1223 /* More externally visible functions.  These simply
   1224    implement the corresponding IR primops. */
   1225 /* ----------------------------------------------------- */
   1226 
   1227 /* ------ 16x2 ------ */
   1228 
   1229 UInt h_generic_calc_Add16x2 ( UInt xx, UInt yy )
   1230 {
   1231    return mk16x2( sel16x2_1(xx) + sel16x2_1(yy),
   1232                   sel16x2_0(xx) + sel16x2_0(yy) );
   1233 }
   1234 
   1235 UInt h_generic_calc_Sub16x2 ( UInt xx, UInt yy )
   1236 {
   1237    return mk16x2( sel16x2_1(xx) - sel16x2_1(yy),
   1238                   sel16x2_0(xx) - sel16x2_0(yy) );
   1239 }
   1240 
   1241 UInt h_generic_calc_HAdd16Ux2 ( UInt xx, UInt yy )
   1242 {
   1243    return mk16x2( hadd16U( sel16x2_1(xx), sel16x2_1(yy) ),
   1244                   hadd16U( sel16x2_0(xx), sel16x2_0(yy) ) );
   1245 }
   1246 
   1247 UInt h_generic_calc_HAdd16Sx2 ( UInt xx, UInt yy )
   1248 {
   1249    return mk16x2( hadd16S( sel16x2_1(xx), sel16x2_1(yy) ),
   1250                   hadd16S( sel16x2_0(xx), sel16x2_0(yy) ) );
   1251 }
   1252 
   1253 UInt h_generic_calc_HSub16Ux2 ( UInt xx, UInt yy )
   1254 {
   1255    return mk16x2( hsub16U( sel16x2_1(xx), sel16x2_1(yy) ),
   1256                   hsub16U( sel16x2_0(xx), sel16x2_0(yy) ) );
   1257 }
   1258 
   1259 UInt h_generic_calc_HSub16Sx2 ( UInt xx, UInt yy )
   1260 {
   1261    return mk16x2( hsub16S( sel16x2_1(xx), sel16x2_1(yy) ),
   1262                   hsub16S( sel16x2_0(xx), sel16x2_0(yy) ) );
   1263 }
   1264 
   1265 UInt h_generic_calc_QAdd16Ux2 ( UInt xx, UInt yy )
   1266 {
   1267    return mk16x2( qadd16U( sel16x2_1(xx), sel16x2_1(yy) ),
   1268                   qadd16U( sel16x2_0(xx), sel16x2_0(yy) ) );
   1269 }
   1270 
   1271 UInt h_generic_calc_QAdd16Sx2 ( UInt xx, UInt yy )
   1272 {
   1273    return mk16x2( qadd16S( sel16x2_1(xx), sel16x2_1(yy) ),
   1274                   qadd16S( sel16x2_0(xx), sel16x2_0(yy) ) );
   1275 }
   1276 
   1277 UInt h_generic_calc_QSub16Ux2 ( UInt xx, UInt yy )
   1278 {
   1279    return mk16x2( qsub16U( sel16x2_1(xx), sel16x2_1(yy) ),
   1280                   qsub16U( sel16x2_0(xx), sel16x2_0(yy) ) );
   1281 }
   1282 
   1283 UInt h_generic_calc_QSub16Sx2 ( UInt xx, UInt yy )
   1284 {
   1285    return mk16x2( qsub16S( sel16x2_1(xx), sel16x2_1(yy) ),
   1286                   qsub16S( sel16x2_0(xx), sel16x2_0(yy) ) );
   1287 }
   1288 
   1289 /* ------ 8x4 ------ */
   1290 
   1291 UInt h_generic_calc_Add8x4 ( UInt xx, UInt yy )
   1292 {
   1293    return mk8x4(
   1294              sel8x4_3(xx) + sel8x4_3(yy),
   1295              sel8x4_2(xx) + sel8x4_2(yy),
   1296              sel8x4_1(xx) + sel8x4_1(yy),
   1297              sel8x4_0(xx) + sel8x4_0(yy)
   1298           );
   1299 }
   1300 
   1301 UInt h_generic_calc_Sub8x4 ( UInt xx, UInt yy )
   1302 {
   1303    return mk8x4(
   1304              sel8x4_3(xx) - sel8x4_3(yy),
   1305              sel8x4_2(xx) - sel8x4_2(yy),
   1306              sel8x4_1(xx) - sel8x4_1(yy),
   1307              sel8x4_0(xx) - sel8x4_0(yy)
   1308           );
   1309 }
   1310 
   1311 UInt h_generic_calc_HAdd8Ux4 ( UInt xx, UInt yy )
   1312 {
   1313    return mk8x4(
   1314              hadd8U( sel8x4_3(xx), sel8x4_3(yy) ),
   1315              hadd8U( sel8x4_2(xx), sel8x4_2(yy) ),
   1316              hadd8U( sel8x4_1(xx), sel8x4_1(yy) ),
   1317              hadd8U( sel8x4_0(xx), sel8x4_0(yy) )
   1318           );
   1319 }
   1320 
   1321 UInt h_generic_calc_HAdd8Sx4 ( UInt xx, UInt yy )
   1322 {
   1323    return mk8x4(
   1324              hadd8S( sel8x4_3(xx), sel8x4_3(yy) ),
   1325              hadd8S( sel8x4_2(xx), sel8x4_2(yy) ),
   1326              hadd8S( sel8x4_1(xx), sel8x4_1(yy) ),
   1327              hadd8S( sel8x4_0(xx), sel8x4_0(yy) )
   1328           );
   1329 }
   1330 
   1331 UInt h_generic_calc_HSub8Ux4 ( UInt xx, UInt yy )
   1332 {
   1333    return mk8x4(
   1334              hsub8U( sel8x4_3(xx), sel8x4_3(yy) ),
   1335              hsub8U( sel8x4_2(xx), sel8x4_2(yy) ),
   1336              hsub8U( sel8x4_1(xx), sel8x4_1(yy) ),
   1337              hsub8U( sel8x4_0(xx), sel8x4_0(yy) )
   1338           );
   1339 }
   1340 
   1341 UInt h_generic_calc_HSub8Sx4 ( UInt xx, UInt yy )
   1342 {
   1343    return mk8x4(
   1344              hsub8S( sel8x4_3(xx), sel8x4_3(yy) ),
   1345              hsub8S( sel8x4_2(xx), sel8x4_2(yy) ),
   1346              hsub8S( sel8x4_1(xx), sel8x4_1(yy) ),
   1347              hsub8S( sel8x4_0(xx), sel8x4_0(yy) )
   1348           );
   1349 }
   1350 
   1351 UInt h_generic_calc_QAdd8Ux4 ( UInt xx, UInt yy )
   1352 {
   1353    return mk8x4(
   1354              qadd8U( sel8x4_3(xx), sel8x4_3(yy) ),
   1355              qadd8U( sel8x4_2(xx), sel8x4_2(yy) ),
   1356              qadd8U( sel8x4_1(xx), sel8x4_1(yy) ),
   1357              qadd8U( sel8x4_0(xx), sel8x4_0(yy) )
   1358           );
   1359 }
   1360 
   1361 UInt h_generic_calc_QAdd8Sx4 ( UInt xx, UInt yy )
   1362 {
   1363    return mk8x4(
   1364              qadd8S( sel8x4_3(xx), sel8x4_3(yy) ),
   1365              qadd8S( sel8x4_2(xx), sel8x4_2(yy) ),
   1366              qadd8S( sel8x4_1(xx), sel8x4_1(yy) ),
   1367              qadd8S( sel8x4_0(xx), sel8x4_0(yy) )
   1368           );
   1369 }
   1370 
   1371 UInt h_generic_calc_QSub8Ux4 ( UInt xx, UInt yy )
   1372 {
   1373    return mk8x4(
   1374              qsub8U( sel8x4_3(xx), sel8x4_3(yy) ),
   1375              qsub8U( sel8x4_2(xx), sel8x4_2(yy) ),
   1376              qsub8U( sel8x4_1(xx), sel8x4_1(yy) ),
   1377              qsub8U( sel8x4_0(xx), sel8x4_0(yy) )
   1378           );
   1379 }
   1380 
   1381 UInt h_generic_calc_QSub8Sx4 ( UInt xx, UInt yy )
   1382 {
   1383    return mk8x4(
   1384              qsub8S( sel8x4_3(xx), sel8x4_3(yy) ),
   1385              qsub8S( sel8x4_2(xx), sel8x4_2(yy) ),
   1386              qsub8S( sel8x4_1(xx), sel8x4_1(yy) ),
   1387              qsub8S( sel8x4_0(xx), sel8x4_0(yy) )
   1388           );
   1389 }
   1390 
   1391 UInt h_generic_calc_CmpNEZ16x2 ( UInt xx )
   1392 {
   1393    return mk16x2(
   1394              cmpnez16( sel16x2_1(xx) ),
   1395              cmpnez16( sel16x2_0(xx) )
   1396           );
   1397 }
   1398 
   1399 UInt h_generic_calc_CmpNEZ8x4 ( UInt xx )
   1400 {
   1401    return mk8x4(
   1402              cmpnez8( sel8x4_3(xx) ),
   1403              cmpnez8( sel8x4_2(xx) ),
   1404              cmpnez8( sel8x4_1(xx) ),
   1405              cmpnez8( sel8x4_0(xx) )
   1406           );
   1407 }
   1408 
   1409 UInt h_generic_calc_Sad8Ux4 ( UInt xx, UInt yy )
   1410 {
   1411    return absdiff8U( sel8x4_3(xx), sel8x4_3(yy) )
   1412           + absdiff8U( sel8x4_2(xx), sel8x4_2(yy) )
   1413           + absdiff8U( sel8x4_1(xx), sel8x4_1(yy) )
   1414           + absdiff8U( sel8x4_0(xx), sel8x4_0(yy) );
   1415 }
   1416 
   1417 UInt h_generic_calc_QAdd32S ( UInt xx, UInt yy )
   1418 {
   1419    return qadd32S( xx, yy );
   1420 }
   1421 
   1422 UInt h_generic_calc_QSub32S ( UInt xx, UInt yy )
   1423 {
   1424    return qsub32S( xx, yy );
   1425 }
   1426 
   1427 
   1428 /*------------------------------------------------------------------*/
   1429 /* Decimal Floating Point (DFP) externally visible helper functions */
   1430 /* that implement Iop_BCDtoDPB and Iop_DPBtoBCD                     */
   1431 /*------------------------------------------------------------------*/
   1432 
   1433 #define NOT( x )    ( ( ( x ) == 0) ? 1 : 0)
   1434 #define GET( x, y ) ( ( ( x ) & ( 0x1UL << ( y ) ) ) >> ( y ) )
   1435 #define PUT( x, y ) ( ( x )<< ( y ) )
   1436 
   1437 static ULong dpb_to_bcd( ULong chunk )
   1438 {
   1439    Short a, b, c, d, e, f, g, h, i, j, k, m;
   1440    Short p, q, r, s, t, u, v, w, x, y;
   1441    ULong value;
   1442 
   1443    /* convert 10 bit densely packed BCD to BCD */
   1444    p = GET( chunk, 9 );
   1445    q = GET( chunk, 8 );
   1446    r = GET( chunk, 7 );
   1447    s = GET( chunk, 6 );
   1448    t = GET( chunk, 5 );
   1449    u = GET( chunk, 4 );
   1450    v = GET( chunk, 3 );
   1451    w = GET( chunk, 2 );
   1452    x = GET( chunk, 1 );
   1453    y = GET( chunk, 0 );
   1454 
   1455    /* The BCD bit values are given by the following boolean equations.*/
   1456    a = ( NOT(s) & v & w ) | ( t & v & w & s ) | ( v & w & NOT(x) );
   1457    b = ( p & s & x & NOT(t) ) | ( p & NOT(w) ) | ( p & NOT(v) );
   1458    c = ( q & s & x & NOT(t) ) | ( q & NOT(w) ) | ( q & NOT(v) );
   1459    d = r;
   1460    e = ( v & NOT(w) & x ) | ( s & v & w & x ) | ( NOT(t) & v & x & w );
   1461    f = ( p & t & v & w & x & NOT(s) ) | ( s & NOT(x) & v ) | ( s & NOT(v) );
   1462    g = ( q & t & w & v & x & NOT(s) ) | ( t & NOT(x) & v ) | ( t & NOT(v) );
   1463    h = u;
   1464    i = ( t & v & w & x ) | ( s & v & w & x ) | ( v & NOT(w) & NOT(x) );
   1465    j = ( p & NOT(s) & NOT(t) & w & v ) | ( s & v & NOT(w) & x )
   1466             | ( p & w & NOT(x) & v ) | ( w & NOT(v) );
   1467    k = ( q & NOT(s) & NOT(t) & v & w ) | ( t & v & NOT(w) & x )
   1468             | ( q & v & w & NOT(x) ) | ( x & NOT(v) );
   1469    m = y;
   1470 
   1471    value = PUT(a, 11) | PUT(b, 10) | PUT(c, 9) | PUT(d, 8) | PUT(e, 7)
   1472             | PUT(f, 6) | PUT(g, 5) | PUT(h, 4) | PUT(i, 3) | PUT(j, 2)
   1473             | PUT(k, 1) | PUT(m, 0);
   1474    return value;
   1475 }
   1476 
   1477 static ULong bcd_to_dpb( ULong chunk )
   1478 {
   1479    Short a, b, c, d, e, f, g, h, i, j, k, m;
   1480    Short p, q, r, s, t, u, v, w, x, y;
   1481    ULong value;
   1482    /* Convert a 3 digit BCD value to a 10 bit Densely Packed Binary (DPD) value
   1483     The boolean equations to calculate the value of each of the DPD bit
   1484     is given in Appendix B  of Book 1: Power ISA User Instruction set.  The
   1485     bits for the DPD number are [abcdefghijkm].  The bits for the BCD value
   1486     are [pqrstuvwxy].  The boolean logic equations in psuedo C code are:
   1487     */
   1488    a = GET( chunk, 11 );
   1489    b = GET( chunk, 10 );
   1490    c = GET( chunk, 9 );
   1491    d = GET( chunk, 8 );
   1492    e = GET( chunk, 7 );
   1493    f = GET( chunk, 6 );
   1494    g = GET( chunk, 5 );
   1495    h = GET( chunk, 4 );
   1496    i = GET( chunk, 3 );
   1497    j = GET( chunk, 2 );
   1498    k = GET( chunk, 1 );
   1499    m = GET( chunk, 0 );
   1500 
   1501    p = ( f & a & i & NOT(e) ) | ( j & a & NOT(i) ) | ( b & NOT(a) );
   1502    q = ( g & a & i & NOT(e) ) | ( k & a & NOT(i) ) | ( c & NOT(a) );
   1503    r = d;
   1504    s = ( j & NOT(a) & e & NOT(i) ) | ( f & NOT(i) & NOT(e) )
   1505             | ( f & NOT(a) & NOT(e) ) | ( e & i );
   1506    t = ( k & NOT(a) & e & NOT(i) ) | ( g & NOT(i) & NOT(e) )
   1507             | ( g & NOT(a) & NOT(e) ) | ( a & i );
   1508    u = h;
   1509    v = a | e | i;
   1510    w = ( NOT(e) & j & NOT(i) ) | ( e & i ) | a;
   1511    x = ( NOT(a) & k & NOT(i) ) | ( a & i ) | e;
   1512    y = m;
   1513 
   1514    value = PUT(p, 9) | PUT(q, 8) | PUT(r, 7) | PUT(s, 6) | PUT(t, 5)
   1515             | PUT(u, 4) | PUT(v, 3) | PUT(w, 2) | PUT(x, 1) | y;
   1516 
   1517    return value;
   1518 }
   1519 
   1520 ULong h_calc_DPBtoBCD( ULong dpb )
   1521 {
   1522    ULong result, chunk;
   1523    Int i;
   1524 
   1525    result = 0;
   1526 
   1527    for (i = 0; i < 5; i++) {
   1528       chunk = dpb >> ( 4 - i ) * 10;
   1529       result = result << 12;
   1530       result |= dpb_to_bcd( chunk & 0x3FF );
   1531    }
   1532    return result;
   1533 }
   1534 
   1535 ULong h_calc_BCDtoDPB( ULong bcd )
   1536 {
   1537    ULong result, chunk;
   1538    Int i;
   1539 
   1540    result = 0;
   1541 
   1542    for (i = 0; i < 5; i++) {
   1543       chunk = bcd >> ( 4 - i ) * 12;
   1544       result = result << 10;
   1545       result |= bcd_to_dpb( chunk & 0xFFF );
   1546    }
   1547    return result;
   1548 }
   1549 #undef NOT
   1550 #undef GET
   1551 #undef PUT
   1552 
   1553 
   1554 /* ----------------------------------------------------- */
   1555 /* Signed and unsigned integer division, that behave like
   1556    the ARMv7 UDIV ansd SDIV instructions.
   1557 
   1558    sdiv32 also behaves like 64-bit v8 SDIV on w-regs.
   1559    udiv32 also behaves like 64-bit v8 UDIV on w-regs.
   1560 */
   1561 /* ----------------------------------------------------- */
   1562 
   1563 UInt h_calc_udiv32_w_arm_semantics ( UInt x, UInt y )
   1564 {
   1565    // Division by zero --> zero
   1566    if (UNLIKELY(y == 0)) return 0;
   1567    // C requires rounding towards zero, which is also what we need.
   1568    return x / y;
   1569 }
   1570 
   1571 ULong h_calc_udiv64_w_arm_semantics ( ULong x, ULong y )
   1572 {
   1573    // Division by zero --> zero
   1574    if (UNLIKELY(y == 0)) return 0;
   1575    // C requires rounding towards zero, which is also what we need.
   1576    return x / y;
   1577 }
   1578 
   1579 Int h_calc_sdiv32_w_arm_semantics ( Int x, Int y )
   1580 {
   1581    // Division by zero --> zero
   1582    if (UNLIKELY(y == 0)) return 0;
   1583    // The single case that produces an unrepresentable result
   1584    if (UNLIKELY( ((UInt)x) == ((UInt)0x80000000)
   1585                  && ((UInt)y) == ((UInt)0xFFFFFFFF) ))
   1586       return (Int)(UInt)0x80000000;
   1587    // Else return the result rounded towards zero.  C89 says
   1588    // this is implementation defined (in the signed case), but gcc
   1589    // promises to round towards zero.  Nevertheless, at startup,
   1590    // in main_main.c, do a check for that.
   1591    return x / y;
   1592 }
   1593 
   1594 Long h_calc_sdiv64_w_arm_semantics ( Long x, Long y )
   1595 {
   1596    // Division by zero --> zero
   1597    if (UNLIKELY(y == 0)) return 0;
   1598    // The single case that produces an unrepresentable result
   1599    if (UNLIKELY( ((ULong)x) == ((ULong)0x8000000000000000ULL )
   1600                  && ((ULong)y) == ((ULong)0xFFFFFFFFFFFFFFFFULL ) ))
   1601       return (Long)(ULong)0x8000000000000000ULL;
   1602    // Else return the result rounded towards zero.  C89 says
   1603    // this is implementation defined (in the signed case), but gcc
   1604    // promises to round towards zero.  Nevertheless, at startup,
   1605    // in main_main.c, do a check for that.
   1606    return x / y;
   1607 }
   1608 
   1609 
   1610 /*---------------------------------------------------------------*/
   1611 /*--- end                               host_generic_simd64.c ---*/
   1612 /*---------------------------------------------------------------*/
   1613