Lines Matching defs:env
168 static HReg lookupIRTemp ( ISelEnv* env, IRTemp tmp )
171 vassert(tmp < env->n_vregmap);
172 return env->vregmap[tmp];
176 ISelEnv* env, IRTemp tmp )
179 vassert(tmp < env->n_vregmap);
180 vassert(! hregIsInvalid(env->vregmapHI[tmp]));
181 *vrLO = env->vregmap[tmp];
182 *vrHI = env->vregmapHI[tmp];
185 static void addInstr ( ISelEnv* env, AMD64Instr* instr )
187 addHInstr(env->code, instr);
194 static HReg newVRegI ( ISelEnv* env )
196 HReg reg = mkHReg(True/*virtual reg*/, HRcInt64, 0/*enc*/, env->vreg_ctr);
197 env->vreg_ctr++;
201 static HReg newVRegV ( ISelEnv* env )
203 HReg reg = mkHReg(True/*virtual reg*/, HRcVec128, 0/*enc*/, env->vreg_ctr);
204 env->vreg_ctr++;
219 static AMD64RMI* iselIntExpr_RMI_wrk ( ISelEnv* env, IRExpr* e );
220 static AMD64RMI* iselIntExpr_RMI ( ISelEnv* env, IRExpr* e );
222 static AMD64RI* iselIntExpr_RI_wrk ( ISelEnv* env, IRExpr* e );
223 static AMD64RI* iselIntExpr_RI ( ISelEnv* env, IRExpr* e );
225 static AMD64RM* iselIntExpr_RM_wrk ( ISelEnv* env, IRExpr* e );
226 static AMD64RM* iselIntExpr_RM ( ISelEnv* env, IRExpr* e );
228 static HReg iselIntExpr_R_wrk ( ISelEnv* env, IRExpr* e );
229 static HReg iselIntExpr_R ( ISelEnv* env, IRExpr* e );
231 static AMD64AMode* iselIntExpr_AMode_wrk ( ISelEnv* env, IRExpr* e );
232 static AMD64AMode* iselIntExpr_AMode ( ISelEnv* env, IRExpr* e );
235 ISelEnv* env, IRExpr* e );
237 ISelEnv* env, IRExpr* e );
239 static AMD64CondCode iselCondCode_wrk ( ISelEnv* env, IRExpr* e );
240 static AMD64CondCode iselCondCode ( ISelEnv* env, IRExpr* e );
242 static HReg iselDblExpr_wrk ( ISelEnv* env, IRExpr* e );
243 static HReg iselDblExpr ( ISelEnv* env, IRExpr* e );
245 static HReg iselFltExpr_wrk ( ISelEnv* env, IRExpr* e );
246 static HReg iselFltExpr ( ISelEnv* env, IRExpr* e );
248 static HReg iselVecExpr_wrk ( ISelEnv* env, IRExpr* e );
249 static HReg iselVecExpr ( ISelEnv* env, IRExpr* e );
252 ISelEnv* env, IRExpr* e );
254 ISelEnv* env, IRExpr* e );
328 static void add_to_rsp ( ISelEnv* env, Int n )
331 addInstr(env,
336 static void sub_from_rsp ( ISelEnv* env, Int n )
339 addInstr(env,
345 static void push_uimm64( ISelEnv* env, ULong uimm64 )
351 addInstr( env, AMD64Instr_Push(AMD64RMI_Imm( (UInt)uimm64 )) );
353 HReg tmp = newVRegI(env);
354 addInstr( env, AMD64Instr_Imm64(uimm64, tmp) );
355 addInstr( env, AMD64Instr_Push(AMD64RMI_Reg(tmp)) );
364 static AMD64Instr* iselIntExpr_single_instruction ( ISelEnv* env,
377 vassert(typeOfIRExpr(env->type_env, e) == Ity_I64);
393 HReg src = lookupIRTemp(env, e->Iex.RdTmp.tmp);
410 HReg src = lookupIRTemp(env, e->Iex.Unop.arg->Iex.RdTmp.tmp);
430 ISelEnv* env,
568 vassert(typeOfIRExpr(env->type_env, args[i]) == Ity_I64);
571 = iselIntExpr_single_instruction( env, argregs[i], args[i] );
579 addInstr(env, fastinstrs[i]);
600 r_vecRetAddr = newVRegI(env);
601 sub_from_rsp(env, 16);
602 addInstr(env, mk_iMOVsd_RR( hregAMD64_RSP(), r_vecRetAddr ));
605 r_vecRetAddr = newVRegI(env);
606 sub_from_rsp(env, 32);
607 addInstr(env, mk_iMOVsd_RR( hregAMD64_RSP(), r_vecRetAddr ));
614 tmpregs[i] = newVRegI(env);
615 addInstr(env, mk_iMOVsd_RR( hregAMD64_RBP(), tmpregs[i]));
626 vassert(typeOfIRExpr(env->type_env, args[i]) == Ity_I64);
627 tmpregs[i] = iselIntExpr_R(env, args[i]);
642 cc = iselCondCode( env, guard );
650 addInstr( env, mk_iMOVsd_RR( tmpregs[i], argregs[i] ) );
692 addInstr(env,
702 AMD64AMode* genGuestArrayOffset ( ISelEnv* env, IRRegArray* descr,
723 tmp = newVRegI(env);
724 roff = iselIntExpr_R(env, off);
725 addInstr(env, mk_iMOVsd_RR(roff, tmp));
730 addInstr(env,
733 addInstr(env,
744 void set_SSE_rounding_default ( ISelEnv* env )
751 addInstr(env, AMD64Instr_Push(AMD64RMI_Imm(DEFAULT_MXCSR)));
752 addInstr(env, AMD64Instr_LdMXCSR(zero_rsp));
753 add_to_rsp(env, 8);
759 void set_FPU_rounding_default ( ISelEnv* env )
765 addInstr(env, AMD64Instr_Alu64M(
767 addInstr(env, AMD64Instr_A87LdCW(m8_rsp));
777 void set_SSE_rounding_mode ( ISelEnv* env, IRExpr* mode )
792 HReg reg = newVRegI(env);
794 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV, AMD64RMI_Imm(3), reg));
795 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
796 iselIntExpr_RMI(env, mode), reg));
797 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 13, reg));
798 addInstr(env, AMD64Instr_Alu64R(
800 addInstr(env, AMD64Instr_Push(AMD64RMI_Reg(reg)));
801 addInstr(env, AMD64Instr_LdMXCSR(zero_rsp));
802 add_to_rsp(env, 8);
812 void set_FPU_rounding_mode ( ISelEnv* env, IRExpr* mode )
814 HReg rrm = iselIntExpr_R(env, mode);
815 HReg rrm2 = newVRegI(env);
825 addInstr(env, mk_iMOVsd_RR(rrm, rrm2));
826 addInstr(env, AMD64Instr_Alu64R(Aalu_AND, AMD64RMI_Imm(3), rrm2));
827 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 10, rrm2));
828 addInstr(env, AMD64Instr_Alu64R(Aalu_OR,
830 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV,
832 addInstr(env, AMD64Instr_A87LdCW(m8_rsp));
838 static HReg generate_zeroes_V128 ( ISelEnv* env )
840 HReg dst = newVRegV(env);
841 addInstr(env, AMD64Instr_SseReRg(Asse_XOR, dst, dst));
847 static HReg generate_ones_V128 ( ISelEnv* env )
849 HReg dst = newVRegV(env);
850 addInstr(env, AMD64Instr_SseReRg(Asse_CMPEQ32, dst, dst));
858 static HReg do_sse_NotV128 ( ISelEnv* env, HReg src )
860 HReg dst = generate_ones_V128(env);
861 addInstr(env, AMD64Instr_SseReRg(Asse_XOR, src, dst));
899 static HReg iselIntExpr_R ( ISelEnv* env, IRExpr* e )
901 HReg r = iselIntExpr_R_wrk(env, e);
912 static HReg iselIntExpr_R_wrk ( ISelEnv* env, IRExpr* e )
923 IRType ty = typeOfIRExpr(env->type_env,e);
933 return lookupIRTemp(env, e->Iex.RdTmp.tmp);
938 HReg dst = newVRegI(env);
939 AMD64AMode* amode = iselIntExpr_AMode ( env, e->Iex.Load.addr );
946 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV,
951 addInstr(env, AMD64Instr_LoadEX(4,False,amode,dst));
955 addInstr(env, AMD64Instr_LoadEX(2,False,amode,dst));
959 addInstr(env, AMD64Instr_LoadEX(1,False,amode,dst));
974 HReg dst = newVRegI(env);
975 HReg reg = iselIntExpr_R(env, e->Iex.Binop.arg2);
976 addInstr(env, mk_iMOVsd_RR(reg,dst));
977 addInstr(env, AMD64Instr_Unary64(Aun_NEG,dst));
1001 HReg dst = newVRegI(env);
1002 HReg reg = iselIntExpr_R(env, e->Iex.Binop.arg1);
1003 AMD64RMI* rmi = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
1004 addInstr(env, mk_iMOVsd_RR(reg,dst));
1005 addInstr(env, AMD64Instr_Alu64R(aluOp, rmi, dst));
1021 HReg dst = newVRegI(env);
1024 HReg regL = iselIntExpr_R(env, e->Iex.Binop.arg1);
1025 addInstr(env, mk_iMOVsd_RR(regL,dst));
1034 addInstr(env, AMD64Instr_Alu64R(
1038 addInstr(env, AMD64Instr_Alu64R(
1042 addInstr(env, AMD64Instr_MovxLQ(False, dst, dst));
1045 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 56, dst));
1046 addInstr(env, AMD64Instr_Sh64(Ash_SAR, 56, dst));
1049 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 48, dst));
1050 addInstr(env, AMD64Instr_Sh64(Ash_SAR, 48, dst));
1053 addInstr(env, AMD64Instr_MovxLQ(True, dst, dst));
1070 addInstr(env, AMD64Instr_Sh64(shOp, nshift, dst));
1073 HReg regR = iselIntExpr_R(env, e->Iex.Binop.arg2);
1074 addInstr(env, mk_iMOVsd_RR(regR,hregAMD64_RCX()));
1075 addInstr(env, AMD64Instr_Sh64(shOp, 0/* %cl */, dst));
1223 HReg dst = newVRegI(env);
1224 HReg argL = iselIntExpr_R(env, e->Iex.Binop.arg1);
1225 HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
1227 addInstr(env, AMD64Instr_MovxLQ(False, argR, argR));
1228 addInstr(env, mk_iMOVsd_RR(argL, hregAMD64_RDI()) );
1229 addInstr(env, mk_iMOVsd_RR(argR, hregAMD64_RSI()) );
1230 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn, 2,
1232 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(), dst));
1239 HReg src1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
1240 HReg dst = newVRegI(env);
1241 HReg src2 = iselIntExpr_R(env, e->Iex.Binop.arg2);
1242 addInstr(env, mk_iMOVsd_RR(src1, dst));
1243 addInstr(env, AMD64Instr_Alu32R(Aalu_CMP, AMD64RMI_Reg(src2), dst));
1244 addInstr(env, AMD64Instr_CMov64(Acc_B, src2, dst));
1255 HReg dst = newVRegI(env);
1257 AMD64RM* rmRight = iselIntExpr_RM(env, e->Iex.Binop.arg2);
1260 HReg left64 = iselIntExpr_R(env, e->Iex.Binop.arg1);
1261 addInstr(env, mk_iMOVsd_RR(left64, rdx));
1262 addInstr(env, mk_iMOVsd_RR(left64, rax));
1263 addInstr(env, AMD64Instr_Sh64(Ash_SHR, 32, rdx));
1264 addInstr(env, AMD64Instr_Div(syned, 4, rmRight));
1265 addInstr(env, AMD64Instr_MovxLQ(False, rdx, rdx));
1266 addInstr(env, AMD64Instr_MovxLQ(False, rax, rax));
1267 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 32, rdx));
1268 addInstr(env, mk_iMOVsd_RR(rax, dst));
1269 addInstr(env, AMD64Instr_Alu64R(Aalu_OR, AMD64RMI_Reg(rdx), dst));
1274 HReg hi32 = newVRegI(env);
1275 HReg lo32 = newVRegI(env);
1276 HReg hi32s = iselIntExpr_R(env, e->Iex.Binop.arg1);
1277 HReg lo32s = iselIntExpr_R(env, e->Iex.Binop.arg2);
1278 addInstr(env, mk_iMOVsd_RR(hi32s, hi32));
1279 addInstr(env, mk_iMOVsd_RR(lo32s, lo32));
1280 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 32, hi32));
1281 addInstr(env, AMD64Instr_MovxLQ(False, lo32, lo32));
1282 addInstr(env, AMD64Instr_Alu64R(
1288 HReg hi16 = newVRegI(env);
1289 HReg lo16 = newVRegI(env);
1290 HReg hi16s = iselIntExpr_R(env, e->Iex.Binop.arg1);
1291 HReg lo16s = iselIntExpr_R(env, e->Iex.Binop.arg2);
1292 addInstr(env, mk_iMOVsd_RR(hi16s, hi16));
1293 addInstr(env, mk_iMOVsd_RR(lo16s, lo16));
1294 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 16, hi16));
1295 addInstr(env, AMD64Instr_Alu64R(
1297 addInstr(env, AMD64Instr_Alu64R(
1303 HReg hi8 = newVRegI(env);
1304 HReg lo8 = newVRegI(env);
1305 HReg hi8s = iselIntExpr_R(env, e->Iex.Binop.arg1);
1306 HReg lo8s = iselIntExpr_R(env, e->Iex.Binop.arg2);
1307 addInstr(env, mk_iMOVsd_RR(hi8s, hi8));
1308 addInstr(env, mk_iMOVsd_RR(lo8s, lo8));
1309 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 8, hi8));
1310 addInstr(env, AMD64Instr_Alu64R(
1312 addInstr(env, AMD64Instr_Alu64R(
1323 HReg a32 = newVRegI(env);
1324 HReg b32 = newVRegI(env);
1325 HReg a32s = iselIntExpr_R(env, e->Iex.Binop.arg1);
1326 HReg b32s = iselIntExpr_R(env, e->Iex.Binop.arg2);
1339 addInstr(env, mk_iMOVsd_RR(a32s, a32));
1340 addInstr(env, mk_iMOVsd_RR(b32s, b32));
1341 addInstr(env, AMD64Instr_Sh64(Ash_SHL, shift, a32));
1342 addInstr(env, AMD64Instr_Sh64(Ash_SHL, shift, b32));
1343 addInstr(env, AMD64Instr_Sh64(shr_op, shift, a32));
1344 addInstr(env, AMD64Instr_Sh64(shr_op, shift, b32));
1345 addInstr(env, AMD64Instr_Alu64R(Aalu_MUL, AMD64RMI_Reg(a32), b32));
1350 HReg fL = iselDblExpr(env, e->Iex.Binop.arg1);
1351 HReg fR = iselDblExpr(env, e->Iex.Binop.arg2);
1352 HReg dst = newVRegI(env);
1353 addInstr(env, AMD64Instr_SseUComIS(8,fL,fR,dst));
1356 addInstr(env, AMD64Instr_Alu64R(Aalu_AND, AMD64RMI_Imm(0x45), dst));
1363 HReg rf = iselDblExpr(env, e->Iex.Binop.arg2);
1364 HReg dst = newVRegI(env);
1365 set_SSE_rounding_mode( env, e->Iex.Binop.arg1 );
1366 addInstr(env, AMD64Instr_SseSF2SI( 8, szD, rf, dst ));
1367 set_SSE_rounding_default(env);
1383 HReg dst = newVRegI(env);
1384 HReg src = iselIntExpr_R(env, expr64);
1385 addInstr(env, mk_iMOVsd_RR(src,dst) );
1386 env, AMD64Instr_Alu64R(Aalu_AND,
1398 HReg dst = newVRegI(env);
1399 AMD64AMode* amode = iselIntExpr_AMode ( env, mi.bindee[0] );
1400 addInstr(env, AMD64Instr_LoadEX(1,False,amode,dst));
1411 HReg dst = newVRegI(env);
1412 AMD64AMode* amode = iselIntExpr_AMode ( env, mi.bindee[0] );
1413 addInstr(env, AMD64Instr_LoadEX(2,False,amode,dst));
1437 HReg dst = newVRegI(env);
1438 HReg reg = iselIntExpr_R(env, argL);
1439 AMD64RMI* rmi = iselIntExpr_RMI(env, argR);
1440 addInstr(env, mk_iMOVsd_RR(reg,dst));
1441 addInstr(env, AMD64Instr_Alu32R(aluOp, rmi, dst));
1451 HReg dst = newVRegI(env);
1452 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1453 addInstr(env, AMD64Instr_MovxLQ(e->Iex.Unop.op == Iop_32Sto64,
1459 iselInt128Expr(&rHi,&rLo, env, e->Iex.Unop.arg);
1464 iselInt128Expr(&rHi,&rLo, env, e->Iex.Unop.arg);
1472 HReg dst = newVRegI(env);
1473 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1477 addInstr(env, mk_iMOVsd_RR(src,dst) );
1478 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
1487 HReg dst = newVRegI(env);
1488 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1492 addInstr(env, mk_iMOVsd_RR(src,dst) );
1493 addInstr(env, AMD64Instr_Sh64(Ash_SHL, amt, dst));
1494 addInstr(env, AMD64Instr_Sh64(Ash_SAR, amt, dst));
1501 HReg dst = newVRegI(env);
1502 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1503 addInstr(env, mk_iMOVsd_RR(src,dst) );
1504 addInstr(env, AMD64Instr_Unary64(Aun_NOT,dst));
1510 HReg dst = newVRegI(env);
1511 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1519 addInstr(env, mk_iMOVsd_RR(src,dst) );
1520 addInstr(env, AMD64Instr_Sh64(Ash_SHR, shift, dst));
1526 HReg dst = newVRegI(env);
1527 AMD64CondCode cond = iselCondCode(env, e->Iex.Unop.arg);
1528 addInstr(env, AMD64Instr_Set64(cond,dst));
1536 HReg dst = newVRegI(env);
1537 AMD64CondCode cond = iselCondCode(env, e->Iex.Unop.arg);
1538 addInstr(env, AMD64Instr_Set64(cond,dst));
1539 addInstr(env, AMD64Instr_Sh64(Ash_SHL, 63, dst));
1540 addInstr(env, AMD64Instr_Sh64(Ash_SAR, 63, dst));
1545 HReg dst = newVRegI(env);
1546 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1547 addInstr(env, AMD64Instr_Bsfr64(True,src,dst));
1554 HReg tmp = newVRegI(env);
1555 HReg dst = newVRegI(env);
1556 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1557 addInstr(env, AMD64Instr_Bsfr64(False,src,tmp));
1558 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV,
1560 addInstr(env, AMD64Instr_Alu64R(Aalu_SUB,
1566 HReg dst = newVRegI(env);
1567 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1568 addInstr(env, mk_iMOVsd_RR(src,dst));
1569 addInstr(env, AMD64Instr_Unary64(Aun_NEG,dst));
1570 addInstr(env, AMD64Instr_Alu64R(Aalu_OR,
1572 addInstr(env, AMD64Instr_Sh64(Ash_SAR, 63, dst));
1577 HReg src = newVRegI(env);
1578 HReg dst = newVRegI(env);
1579 HReg pre = iselIntExpr_R(env, e->Iex.Unop.arg);
1580 addInstr(env, mk_iMOVsd_RR(pre,src));
1581 addInstr(env, AMD64Instr_MovxLQ(False, src, src));
1582 addInstr(env, mk_iMOVsd_RR(src,dst));
1583 addInstr(env, AMD64Instr_Unary64(Aun_NEG,dst));
1584 addInstr(env, AMD64Instr_Alu64R(Aalu_OR,
1586 addInstr(env, AMD64Instr_Sh64(Ash_SAR, 63, dst));
1594 HReg dst = newVRegI(env);
1595 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
1596 addInstr(env, mk_iMOVsd_RR(src, dst));
1597 addInstr(env, AMD64Instr_Unary64(Aun_NEG, dst));
1598 addInstr(env, AMD64Instr_Alu64R(Aalu_OR, AMD64RMI_Reg(src), dst));
1603 HReg dst = newVRegI(env);
1604 HReg vec = iselVecExpr(env, e->Iex.Unop.arg);
1606 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vec, rsp_m16));
1607 addInstr(env, AMD64Instr_LoadEX(4, False/*z-widen*/, rsp_m16, dst));
1614 HReg dst = newVRegI(env);
1617 HReg vec = iselVecExpr(env, e->Iex.Unop.arg);
1620 addInstr(env, AMD64Instr_SseLdSt(False/*store*/,
1622 addInstr(env, AMD64Instr_Alu64R( Aalu_MOV,
1630 iselDVecExpr(&vHi, &vLo, env, e->Iex.Unop.arg);
1642 HReg dst = newVRegI(env);
1646 addInstr(env, AMD64Instr_SseLdSt(False/*store*/,
1648 addInstr(env, AMD64Instr_Alu64R( Aalu_MOV,
1658 HReg dst = newVRegI(env);
1659 HReg src = iselDblExpr(env, e->Iex.Unop.arg);
1661 set_SSE_rounding_default(env);
1662 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, src, m8_rsp));
1663 addInstr(env, AMD64Instr_Alu64R(
1673 HReg dst = newVRegI(env);
1674 HReg src = iselFltExpr(env, e->Iex.Unop.arg);
1676 set_SSE_rounding_default(env);
1677 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 4, src, m8_rsp));
1678 addInstr(env, AMD64Instr_LoadEX(4, False/*unsigned*/, m8_rsp, dst ));
1689 return iselIntExpr_R(env, e->Iex.Unop.arg);
1696 HReg dst = newVRegI(env);
1697 HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
1699 addInstr(env, mk_iMOVsd_RR(arg, hregAMD64_RDI()) );
1700 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn,
1707 addInstr(env, AMD64Instr_MovxLQ(False, hregAMD64_RAX(), dst));
1715 HReg dst = newVRegI(env);
1716 HReg vec = iselVecExpr(env, e->Iex.Unop.arg);
1721 addInstr(env, AMD64Instr_SseLdSt(False/*store*/,
1724 addInstr(env, AMD64Instr_Alu64R( Aalu_MOV,
1728 addInstr(env, AMD64Instr_Alu64R( Aalu_MOV,
1731 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn,
1738 addInstr(env, AMD64Instr_MovxLQ(False, hregAMD64_RAX(), dst));
1763 HReg dst = newVRegI(env);
1764 HReg arg = iselIntExpr_R(env, e->Iex.Unop.arg);
1765 addInstr(env, mk_iMOVsd_RR(arg, hregAMD64_RDI()) );
1766 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn, 1,
1768 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(), dst));
1778 env);
1779 addInstr(env, AMD64Instr_Alu64R(
1788 HReg dst = newVRegI(env);
1789 addInstr(env, AMD64Instr_LoadEX(
1802 env, e->Iex.GetI.descr,
1804 HReg dst = newVRegI(env);
1806 addInstr(env, AMD64Instr_LoadEX( 1, False, am, dst ));
1810 addInstr(env, AMD64Instr_Alu64R( Aalu_MOV, AMD64RMI_Mem(am), dst ));
1818 HReg dst = newVRegI(env);
1829 doHelperCall( &addToSp, &rloc, env, NULL/*guard*/,
1838 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(), dst));
1840 addInstr(env, AMD64Instr_MovxLQ(False, hregAMD64_RAX(), dst));
1849 HReg r = newVRegI(env);
1850 addInstr(env, AMD64Instr_Imm64(e->Iex.Const.con->Ico.U64, r));
1853 AMD64RMI* rmi = iselIntExpr_RMI ( env, e );
1854 HReg r = newVRegI(env);
1855 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV, rmi, r));
1862 && typeOfIRExpr(env->type_env,e->Iex.ITE.cond) == Ity_I1) {
1863 HReg r1 = iselIntExpr_R(env, e->Iex.ITE.iftrue);
1864 HReg r0 = iselIntExpr_R(env, e->Iex.ITE.iffalse);
1865 HReg dst = newVRegI(env);
1866 addInstr(env, mk_iMOVsd_RR(r1,dst));
1867 AMD64CondCode cc = iselCondCode(env, e->Iex.ITE.cond);
1868 addInstr(env, AMD64Instr_CMov64(cc ^ 1, r0, dst));
1882 HReg arg1 = iselDblExpr(env, triop->arg2);
1883 HReg arg2 = iselDblExpr(env, triop->arg3);
1884 HReg dst = newVRegI(env);
1885 addInstr(env, AMD64Instr_A87Free(2));
1888 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, arg2, m8_rsp));
1889 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
1892 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, arg1, m8_rsp));
1893 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
1897 addInstr(env, AMD64Instr_A87FpOp(Afp_PREM));
1900 addInstr(env, AMD64Instr_A87FpOp(Afp_PREM1));
1907 addInstr(env, AMD64Instr_A87StSW(m8_rsp));
1908 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV,AMD64RMI_Mem(m8_rsp),dst));
1909 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,AMD64RMI_Imm(0x4700),dst));
1937 static AMD64AMode* iselIntExpr_AMode ( ISelEnv* env, IRExpr* e )
1939 AMD64AMode* am = iselIntExpr_AMode_wrk(env, e);
1945 static AMD64AMode* iselIntExpr_AMode_wrk ( ISelEnv* env, IRExpr* e )
1949 IRType ty = typeOfIRExpr(env->type_env,e);
1977 HReg r1 = iselIntExpr_R(env, expr1);
1978 HReg r2 = iselIntExpr_R(env, expr2);
1993 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
1994 HReg r2 = iselIntExpr_R(env, e->Iex.Binop.arg2->Iex.Binop.arg1 );
2005 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2015 HReg r1 = iselIntExpr_R(env, e);
2026 static AMD64RMI* iselIntExpr_RMI ( ISelEnv* env, IRExpr* e )
2028 AMD64RMI* rmi = iselIntExpr_RMI_wrk(env, e);
2046 static AMD64RMI* iselIntExpr_RMI_wrk ( ISelEnv* env, IRExpr* e )
2048 IRType ty = typeOfIRExpr(env->type_env,e);
2080 AMD64AMode* am = iselIntExpr_AMode(env, e->Iex.Load.addr);
2086 HReg r = iselIntExpr_R ( env, e );
2098 static AMD64RI* iselIntExpr_RI ( ISelEnv* env, IRExpr* e )
2100 AMD64RI* ri = iselIntExpr_RI_wrk(env, e);
2115 static AMD64RI* iselIntExpr_RI_wrk ( ISelEnv* env, IRExpr* e )
2117 IRType ty = typeOfIRExpr(env->type_env,e);
2142 HReg r = iselIntExpr_R ( env, e );
2154 static AMD64RM* iselIntExpr_RM ( ISelEnv* env, IRExpr* e )
2156 AMD64RM* rm = iselIntExpr_RM_wrk(env, e);
2172 static AMD64RM* iselIntExpr_RM_wrk ( ISelEnv* env, IRExpr* e )
2174 IRType ty = typeOfIRExpr(env->type_env,e);
2187 HReg r = iselIntExpr_R ( env, e );
2199 static AMD64CondCode iselCondCode ( ISelEnv* env, IRExpr* e )
2202 return iselCondCode_wrk(env,e);
2206 static AMD64CondCode iselCondCode_wrk ( ISelEnv* env, IRExpr* e )
2211 vassert(typeOfIRExpr(env->type_env,e) == Ity_I1);
2215 HReg r64 = lookupIRTemp(env, e->Iex.RdTmp.tmp);
2216 HReg dst = newVRegI(env);
2217 addInstr(env, mk_iMOVsd_RR(r64,dst));
2218 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,AMD64RMI_Imm(1),dst));
2228 r = newVRegI(env);
2229 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV,AMD64RMI_Imm(0),r));
2230 addInstr(env, AMD64Instr_Alu64R(Aalu_XOR,AMD64RMI_Reg(r),r));
2237 return 1 ^ iselCondCode(env, e->Iex.Unop.arg);
2244 HReg reg = iselIntExpr_R(env, e->Iex.Unop.arg);
2245 addInstr(env, AMD64Instr_Test64(1,reg));
2253 HReg reg = iselIntExpr_R(env, e->Iex.Unop.arg);
2254 addInstr(env, AMD64Instr_Test64(1,reg));
2263 HReg r = iselIntExpr_R(env, e->Iex.Unop.arg);
2264 addInstr(env, AMD64Instr_Test64(0xFF,r));
2273 HReg r = iselIntExpr_R(env, e->Iex.Unop.arg);
2274 addInstr(env, AMD64Instr_Test64(0xFFFF,r));
2283 HReg r1 = iselIntExpr_R(env, e->Iex.Unop.arg);
2285 env, AMD64Instr_Alu32R(Aalu_CMP,rmi2,r1));
2297 HReg r0 = iselIntExpr_R(env, mi.bindee[0]);
2298 AMD64RMI* rmi1 = iselIntExpr_RMI(env, mi.bindee[1]);
2299 HReg tmp = newVRegI(env);
2300 addInstr(env, mk_iMOVsd_RR(r0, tmp));
2301 addInstr(env, AMD64Instr_Alu64R(Aalu_OR,rmi1,tmp));
2309 HReg r1 = iselIntExpr_R(env, e->Iex.Unop.arg);
2311 addInstr(env, AMD64Instr_Alu64R(Aalu_CMP,rmi2,r1));
2324 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2325 addInstr(env, AMD64Instr_Test64(0xFF,r1));
2332 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2333 AMD64RMI* rmi2 = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
2334 HReg r = newVRegI(env);
2335 addInstr(env, mk_iMOVsd_RR(r1,r));
2336 addInstr(env, AMD64Instr_Alu64R(Aalu_XOR,rmi2,r));
2337 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,AMD64RMI_Imm(0xFF),r));
2352 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2353 AMD64RMI* rmi2 = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
2354 HReg r = newVRegI(env);
2355 addInstr(env, mk_iMOVsd_RR(r1,r));
2356 addInstr(env, AMD64Instr_Alu64R(Aalu_XOR,rmi2,r));
2357 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,AMD64RMI_Imm(0xFFFF),r));
2373 HReg tmp = newVRegI(env);
2380 doHelperCall( &addToSp, &rloc, env, NULL/*guard*/,
2387 addInstr(env, AMD64Instr_Imm64(con->Iex.Const.con->Ico.U64, tmp));
2388 addInstr(env, AMD64Instr_Alu64R(Aalu_CMP,
2404 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2405 AMD64RMI* rmi2 = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
2406 addInstr(env, AMD64Instr_Alu64R(Aalu_CMP,rmi2,r1));
2430 HReg r1 = iselIntExpr_R(env, e->Iex.Binop.arg1);
2431 AMD64RMI* rmi2 = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
2432 addInstr(env, AMD64Instr_Alu32R(Aalu_CMP,rmi2,r1));
2460 ISelEnv* env, IRExpr* e )
2462 iselInt128Expr_wrk(rHi, rLo, env, e);
2474 ISelEnv* env, IRExpr* e )
2477 vassert(typeOfIRExpr(env->type_env,e) == Ity_I128);
2481 lookupIRTempPair( rHi, rLo, env, e->Iex.RdTmp.tmp);
2494 HReg tLo = newVRegI(env);
2495 HReg tHi = newVRegI(env);
2497 AMD64RM* rmLeft = iselIntExpr_RM(env, e->Iex.Binop.arg1);
2498 HReg rRight = iselIntExpr_R(env, e->Iex.Binop.arg2);
2499 addInstr(env, mk_iMOVsd_RR(rRight, hregAMD64_RAX()));
2500 addInstr(env, AMD64Instr_MulL(syned, rmLeft));
2502 addInstr(env, mk_iMOVsd_RR(hregAMD64_RDX(), tHi));
2503 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(), tLo));
2515 HReg tLo = newVRegI(env);
2516 HReg tHi = newVRegI(env);
2518 AMD64RM* rmRight = iselIntExpr_RM(env, e->Iex.Binop.arg2);
2519 iselInt128Expr(&sHi,&sLo, env, e->Iex.Binop.arg1);
2520 addInstr(env, mk_iMOVsd_RR(sHi, hregAMD64_RDX()));
2521 addInstr(env, mk_iMOVsd_RR(sLo, hregAMD64_RAX()));
2522 addInstr(env, AMD64Instr_Div(syned, 8, rmRight));
2523 addInstr(env, mk_iMOVsd_RR(hregAMD64_RDX(), tHi));
2524 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(), tLo));
2532 *rHi = iselIntExpr_R(env, e->Iex.Binop.arg1);
2533 *rLo = iselIntExpr_R(env, e->Iex.Binop.arg2);
2553 static HReg iselFltExpr ( ISelEnv* env, IRExpr* e )
2555 HReg r = iselFltExpr_wrk( env, e );
2565 static HReg iselFltExpr_wrk ( ISelEnv* env, IRExpr* e )
2567 IRType ty = typeOfIRExpr(env->type_env,e);
2571 return lookupIRTemp(env, e->Iex.RdTmp.tmp);
2576 HReg res = newVRegV(env);
2578 am = iselIntExpr_AMode(env, e->Iex.Load.addr);
2579 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 4, res, am));
2588 HReg dst = newVRegV(env);
2589 HReg src = iselDblExpr(env, e->Iex.Binop.arg2);
2590 set_SSE_rounding_mode( env, e->Iex.Binop.arg1 );
2591 addInstr(env, AMD64Instr_SseSDSS(True/*D->S*/,src,dst));
2592 set_SSE_rounding_default( env );
2599 HReg res = newVRegV(env);
2600 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 4, res, am ));
2608 HReg dst = newVRegV(env);
2609 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
2611 addInstr(env, AMD64Instr_Store(4, src, m4_rsp));
2612 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 4, dst, m4_rsp ));
2618 HReg arg = iselFltExpr(env, e->Iex.Binop.arg2);
2619 HReg dst = newVRegV(env);
2625 set_FPU_rounding_mode( env, e->Iex.Binop.arg1 );
2627 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 4, arg, m8_rsp));
2628 addInstr(env, AMD64Instr_A87Free(1));
2629 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 4));
2630 addInstr(env, AMD64Instr_A87FpOp(Afp_ROUND));
2631 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, False/*pop*/, 4));
2632 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 4, dst, m8_rsp));
2635 set_FPU_rounding_default( env );
2644 HReg r1 = newVRegI(env);
2645 HReg dst = newVRegV(env);
2646 HReg tmp = newVRegV(env);
2647 HReg src = iselFltExpr(env, e->Iex.Unop.arg);
2649 addInstr(env, mk_vMOVsd_RR(src,tmp));
2650 addInstr(env, AMD64Instr_Push(AMD64RMI_Imm(0)));
2651 addInstr(env, AMD64Instr_Imm64( 1ULL<<31, r1 ));
2652 addInstr(env, AMD64Instr_Push(AMD64RMI_Reg(r1)));
2653 addInstr(env, AMD64Instr_SseLdSt(True, 16, dst, rsp0));
2654 addInstr(env, AMD64Instr_SseReRg(Asse_XOR, tmp, dst));
2655 add_to_rsp(env, 16);
2661 HReg dst = newVRegV(env);
2662 HReg argX = iselFltExpr(env, qop->arg2);
2663 HReg argY = iselFltExpr(env, qop->arg3);
2664 HReg argZ = iselFltExpr(env, qop->arg4);
2668 sub_from_rsp(env, 16);
2675 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, hregAMD64_RSP()),
2677 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(4, hregAMD64_RSP()),
2679 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(8, hregAMD64_RSP()),
2681 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(12, hregAMD64_RSP()),
2688 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 4, argX,
2690 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 4, argY,
2692 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 4, argZ,
2695 addInstr(env, AMD64Instr_Call( Acc_ALWAYS,
2700 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 4, dst,
2703 add_to_rsp(env, 16);
2740 static HReg iselDblExpr ( ISelEnv* env, IRExpr* e )
2742 HReg r = iselDblExpr_wrk( env, e );
2752 static HReg iselDblExpr_wrk ( ISelEnv* env, IRExpr* e )
2754 IRType ty = typeOfIRExpr(env->type_env,e);
2759 return lookupIRTemp(env, e->Iex.RdTmp.tmp);
2764 HReg res = newVRegV(env);
2765 HReg tmp = newVRegI(env);
2779 addInstr(env, AMD64Instr_Imm64(u.u64, tmp));
2780 addInstr(env, AMD64Instr_Push(AMD64RMI_Reg(tmp)));
2781 addInstr(env, AMD64Instr_SseLdSt(
2785 add_to_rsp(env, 8);
2791 HReg res = newVRegV(env);
2793 am = iselIntExpr_AMode(env, e->Iex.Load.addr);
2794 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 8, res, am ));
2801 HReg res = newVRegV(env);
2802 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 8, res, am ));
2809 env, e->Iex.GetI.descr,
2811 HReg res = newVRegV(env);
2812 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 8, res, am ));
2827 HReg dst = newVRegV(env);
2828 HReg argL = iselDblExpr(env, triop->arg2);
2829 HReg argR = iselDblExpr(env, triop->arg3);
2830 addInstr(env, mk_vMOVsd_RR(argL, dst));
2833 addInstr(env, AMD64Instr_Sse64FLo(op, argR, dst));
2840 HReg dst = newVRegV(env);
2841 HReg argX = iselDblExpr(env, qop->arg2);
2842 HReg argY = iselDblExpr(env, qop->arg3);
2843 HReg argZ = iselDblExpr(env, qop->arg4);
2847 sub_from_rsp(env, 32);
2854 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, hregAMD64_RSP()),
2856 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(8, hregAMD64_RSP()),
2858 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(16, hregAMD64_RSP()),
2860 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(24, hregAMD64_RSP()),
2867 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 8, argX,
2869 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 8, argY,
2871 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 8, argZ,
2874 addInstr(env, AMD64Instr_Call( Acc_ALWAYS,
2879 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 8, dst,
2882 add_to_rsp(env, 32);
2888 HReg arg = iselDblExpr(env, e->Iex.Binop.arg2);
2889 HReg dst = newVRegV(env);
2895 set_FPU_rounding_mode( env, e->Iex.Binop.arg1 );
2897 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, arg, m8_rsp));
2898 addInstr(env, AMD64Instr_A87Free(1));
2899 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
2900 addInstr(env, AMD64Instr_A87FpOp(Afp_ROUND));
2901 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, False/*pop*/, 8));
2902 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 8, dst, m8_rsp));
2905 set_FPU_rounding_default( env );
2920 HReg arg1 = iselDblExpr(env, triop->arg2);
2921 HReg arg2 = iselDblExpr(env, triop->arg3);
2922 HReg dst = newVRegV(env);
2926 addInstr(env, AMD64Instr_A87Free(2));
2929 addInstr(env, AMD64Instr_SseLdSt(
2931 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
2934 addInstr(env, AMD64Instr_SseLdSt(
2936 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
2943 addInstr(env, AMD64Instr_A87FpOp(Afp_SCALE));
2946 addInstr(env, AMD64Instr_A87FpOp(Afp_ATAN));
2949 addInstr(env, AMD64Instr_A87FpOp(Afp_YL2X));
2952 addInstr(env, AMD64Instr_A87FpOp(Afp_YL2XP1));
2955 addInstr(env, AMD64Instr_A87FpOp(Afp_PREM));
2958 addInstr(env, AMD64Instr_A87FpOp(Afp_PREM1));
2965 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, False/*pop*/, 8));
2966 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 8, dst, m8_rsp));
2971 HReg dst = newVRegV(env);
2972 HReg src = iselIntExpr_R(env, e->Iex.Binop.arg2);
2973 set_SSE_rounding_mode( env, e->Iex.Binop.arg1 );
2974 addInstr(env, AMD64Instr_SseSI2SF( 8, 8, src, dst ));
2975 set_SSE_rounding_default( env );
2980 HReg dst = newVRegV(env);
2981 HReg src = iselIntExpr_R(env, e->Iex.Unop.arg);
2982 set_SSE_rounding_default( env );
2983 addInstr(env, AMD64Instr_SseSI2SF( 4, 8, src, dst ));
2993 HReg r1 = newVRegI(env);
2994 HReg dst = newVRegV(env);
2995 HReg tmp = newVRegV(env);
2996 HReg src = iselDblExpr(env, e->Iex.Unop.arg);
2998 addInstr(env, mk_vMOVsd_RR(src,tmp));
2999 addInstr(env, AMD64Instr_Push(AMD64RMI_Imm(0)));
3000 addInstr(env, AMD64Instr_Imm64( 1ULL<<63, r1 ));
3001 addInstr(env, AMD64Instr_Push(AMD64RMI_Reg(r1)));
3002 addInstr(env, AMD64Instr_SseLdSt(True, 16, dst, rsp0));
3005 addInstr(env, AMD64Instr_SseReRg(Asse_XOR, tmp, dst));
3007 addInstr(env, AMD64Instr_SseReRg(Asse_ANDN, tmp, dst));
3009 add_to_rsp(env, 16);
3025 HReg arg = iselDblExpr(env, e->Iex.Binop.arg2);
3026 HReg dst = newVRegV(env);
3028 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, arg, m8_rsp));
3029 addInstr(env, AMD64Instr_A87Free(nNeeded));
3030 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, True/*push*/, 8));
3038 addInstr(env, AMD64Instr_A87FpOp(fpop));
3039 addInstr(env, AMD64Instr_A87PushPop(m8_rsp, False/*pop*/, 8));
3040 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 8, dst, m8_rsp));
3048 //.. HReg dst = newVRegF(env);
3049 //.. HReg ri = iselIntExpr_R(env, e->Iex.Unop.arg);
3050 //.. addInstr(env, X86Instr_Push(X86RMI_Reg(ri)));
3051 //.. set_FPU_rounding_default(env);
3052 //.. addInstr(env, X86Instr_FpLdStI(
3055 //.. add_to_esp(env, 4);
3062 HReg dst = newVRegV(env);
3063 AMD64RI* src = iselIntExpr_RI(env, e->Iex.Unop.arg);
3065 set_SSE_rounding_default(env);
3066 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, src, m8_rsp));
3067 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 8, dst, m8_rsp));
3072 HReg f64 = newVRegV(env);
3074 set_SSE_rounding_default(env);
3075 f32 = iselFltExpr(env, e->Iex.Unop.arg);
3076 addInstr(env, AMD64Instr_SseSDSS(False/*S->D*/, f32, f64));
3088 vassert(typeOfIRExpr(env->type_env,e->Iex.ITE.cond) == Ity_I1);
3089 r1 = iselDblExpr(env, e->Iex.ITE.iftrue);
3090 r0 = iselDblExpr(env, e->Iex.ITE.iffalse);
3091 dst = newVRegV(env);
3092 addInstr(env, mk_vMOVsd_RR(r1,dst));
3093 AMD64CondCode cc = iselCondCode(env, e->Iex.ITE.cond);
3094 addInstr(env, AMD64Instr_SseCMov(cc ^ 1, r0, dst));
3107 static HReg iselVecExpr ( ISelEnv* env, IRExpr* e )
3109 HReg r = iselVecExpr_wrk( env, e );
3120 static HReg iselVecExpr_wrk ( ISelEnv* env, IRExpr* e )
3125 IRType ty = typeOfIRExpr(env->type_env,e);
3130 return lookupIRTemp(env, e->Iex.RdTmp.tmp);
3134 HReg dst = newVRegV(env);
3135 addInstr(env, AMD64Instr_SseLdSt(
3146 HReg dst = newVRegV(env);
3147 AMD64AMode* am = iselIntExpr_AMode(env, e->Iex.Load.addr);
3148 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 16, dst, am ));
3153 HReg dst = newVRegV(env);
3157 env);
3160 dst = generate_ones_V128(env);
3165 push_uimm64(env, bitmask8_to_bytemask64(
3168 push_uimm64(env, bitmask8_to_bytemask64(
3171 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 16, dst, rsp0 ));
3172 add_to_rsp(env, 16);
3183 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3184 return do_sse_NotV128(env, arg);
3203 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3204 HReg tmp = generate_zeroes_V128(env);
3205 HReg dst = newVRegV(env);
3206 addInstr(env, AMD64Instr_SseReRg(Asse_CMPEQ32, arg, tmp));
3207 tmp = do_sse_NotV128(env, tmp);
3208 addInstr(env, AMD64Instr_SseShuf(0xB1, tmp, dst));
3209 addInstr(env, AMD64Instr_SseReRg(Asse_OR, tmp, dst));
3218 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3219 HReg tmp = newVRegV(env);
3220 HReg zero = generate_zeroes_V128(env);
3222 addInstr(env, mk_vMOVsd_RR(arg, tmp));
3223 addInstr(env, AMD64Instr_SseReRg(op, zero, tmp));
3224 dst = do_sse_NotV128(env, tmp);
3232 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3233 HReg dst = newVRegV(env);
3234 addInstr(env, AMD64Instr_Sse32Fx4(op, arg, dst));
3249 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3250 HReg dst = newVRegV(env);
3251 addInstr(env, mk_vMOVsd_RR(arg, dst));
3252 addInstr(env, AMD64Instr_Sse32FLo(op, arg, dst));
3265 HReg arg = iselVecExpr(env, e->Iex.Unop.arg);
3266 HReg dst = newVRegV(env);
3267 addInstr(env, mk_vMOVsd_RR(arg, dst));
3268 addInstr(env, AMD64Instr_Sse64FLo(op, arg, dst));
3273 HReg dst = newVRegV(env);
3275 AMD64RI* ri = iselIntExpr_RI(env, e->Iex.Unop.arg);
3276 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, ri, rsp_m32));
3277 addInstr(env, AMD64Instr_SseLdzLO(4, dst, rsp_m32));
3282 HReg dst = newVRegV(env);
3284 AMD64RMI* rmi = iselIntExpr_RMI(env, e->Iex.Unop.arg);
3285 addInstr(env, AMD64Instr_Push(rmi));
3286 addInstr(env, AMD64Instr_SseLdzLO(8, dst, rsp0));
3287 add_to_rsp(env, 8);
3294 iselDVecExpr(&vHi, &vLo, env, e->Iex.Unop.arg);
3309 HReg arg = iselVecExpr(env, e->Iex.Binop.arg2);
3310 HReg dst = newVRegV(env);
3313 addInstr(env, (e->Iex.Binop.op == Iop_Sqrt64Fx2
3321 HReg dst = newVRegV(env);
3322 HReg srcV = iselVecExpr(env, e->Iex.Binop.arg1);
3323 HReg srcI = iselIntExpr_R(env, e->Iex.Binop.arg2);
3325 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, srcV, rsp_m16));
3326 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, AMD64RI_Reg(srcI), rsp_m16));
3327 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, dst, rsp_m16));
3333 HReg dst = newVRegV(env);
3334 HReg srcV = iselVecExpr(env, e->Iex.Binop.arg1);
3335 HReg srcI = iselIntExpr_R(env, e->Iex.Binop.arg2);
3337 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, srcV, rsp_m16));
3338 addInstr(env, AMD64Instr_Store(4, srcI, rsp_m16));
3339 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, dst, rsp_m16));
3347 AMD64RI* qHi = iselIntExpr_RI(env, e->Iex.Binop.arg1);
3348 AMD64RI* qLo = iselIntExpr_RI(env, e->Iex.Binop.arg2);
3349 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, qHi, m8_rsp));
3350 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, qLo, m16_rsp));
3351 HReg dst = newVRegV(env);
3353 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, dst, m16_rsp));
3365 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3366 HReg argR = iselVecExpr(env, e->Iex.Binop.arg2);
3367 HReg dst = newVRegV(env);
3368 addInstr(env, mk_vMOVsd_RR(argL, dst));
3369 addInstr(env, AMD64Instr_Sse32Fx4(op, argR, dst));
3381 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3382 HReg argR = iselVecExpr(env, e->Iex.Binop.arg2);
3383 HReg dst = newVRegV(env);
3384 addInstr(env, mk_vMOVsd_RR(argL, dst));
3385 addInstr(env, AMD64Instr_Sse64Fx2(op, argR, dst));
3400 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3401 HReg argR = iselVecExpr(env, e->Iex.Binop.arg2);
3402 HReg dst = newVRegV(env);
3403 addInstr(env, mk_vMOVsd_RR(argL, dst));
3404 addInstr(env, AMD64Instr_Sse32FLo(op, argR, dst));
3419 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3420 HReg argR = iselVecExpr(env, e->Iex.Binop.arg2);
3421 HReg dst = newVRegV(env);
3422 addInstr(env, mk_vMOVsd_RR(argL, dst));
3423 addInstr(env, AMD64Instr_Sse64FLo(op, argR, dst));
3487 HReg arg1 = iselVecExpr(env, e->Iex.Binop.arg1);
3488 HReg arg2 = iselVecExpr(env, e->Iex.Binop.arg2);
3489 HReg dst = newVRegV(env);
3491 addInstr(env, mk_vMOVsd_RR(arg2, dst));
3492 addInstr(env, AMD64Instr_SseReRg(op, arg1, dst));
3494 addInstr(env, mk_vMOVsd_RR(arg1, dst));
3495 addInstr(env, AMD64Instr_SseReRg(op, arg2, dst));
3509 HReg greg = iselVecExpr(env, e->Iex.Binop.arg1);
3510 AMD64RMI* rmi = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
3512 HReg ereg = newVRegV(env);
3513 HReg dst = newVRegV(env);
3514 addInstr(env, AMD64Instr_Push(AMD64RMI_Imm(0)));
3515 addInstr(env, AMD64Instr_Push(rmi));
3516 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, ereg, rsp0));
3517 addInstr(env, mk_vMOVsd_RR(greg, dst));
3518 addInstr(env, AMD64Instr_SseReRg(op, ereg, dst));
3519 add_to_rsp(env, 16);
3560 HReg dst = newVRegV(env);
3561 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3562 HReg argR = iselVecExpr(env, e->Iex.Binop.arg2);
3563 HReg argp = newVRegI(env);
3565 sub_from_rsp(env, 112);
3567 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(48, hregAMD64_RSP()),
3570 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
3578 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, argp),
3580 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(16, argp),
3582 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(32, argp),
3588 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argL,
3590 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argR,
3593 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn,
3597 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dst,
3600 add_to_rsp(env, 112);
3612 HReg dst = newVRegV(env);
3613 HReg argL = iselVecExpr(env, e->Iex.Binop.arg1);
3614 HReg argR = iselIntExpr_R(env, e->Iex.Binop.arg2);
3615 HReg argp = newVRegI(env);
3617 sub_from_rsp(env, 112);
3619 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(48, hregAMD64_RSP()),
3622 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
3629 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, argp),
3631 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(16, argp),
3636 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argL,
3639 addInstr(env, mk_iMOVsd_RR(argR, hregAMD64_RDX()));
3642 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn,
3646 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dst,
3649 add_to_rsp(env, 112);
3668 HReg argL = iselVecExpr(env, triop->arg2);
3669 HReg argR = iselVecExpr(env, triop->arg3);
3670 HReg dst = newVRegV(env);
3671 addInstr(env, mk_vMOVsd_RR(argL, dst));
3674 addInstr(env, AMD64Instr_Sse64Fx2(op, argR, dst));
3684 HReg argL = iselVecExpr(env, triop->arg2);
3685 HReg argR = iselVecExpr(env, triop->arg3);
3686 HReg dst = newVRegV(env);
3687 addInstr(env, mk_vMOVsd_RR(argL, dst));
3690 addInstr(env, AMD64Instr_Sse32Fx4(op, argR, dst));
3700 HReg r1 = iselVecExpr(env, e->Iex.ITE.iftrue);
3701 HReg r0 = iselVecExpr(env, e->Iex.ITE.iffalse);
3702 HReg dst = newVRegV(env);
3703 addInstr(env, mk_vMOVsd_RR(r1,dst));
3704 AMD64CondCode cc = iselCondCode(env, e->Iex.ITE.cond);
3705 addInstr(env, AMD64Instr_SseCMov(cc ^ 1, r0, dst));
3711 LibVEX_ppVexHwCaps(VexArchAMD64, env->hwcaps));
3722 ISelEnv* env, IRExpr* e )
3724 iselDVecExpr_wrk( rHi, rLo, env, e );
3737 ISelEnv* env, IRExpr* e )
3741 IRType ty = typeOfIRExpr(env->type_env,e);
3748 lookupIRTempPair( rHi, rLo, env, e->Iex.RdTmp.tmp);
3753 HReg vHi = newVRegV(env);
3754 HReg vLo = newVRegV(env);
3758 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vLo, am0));
3759 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vHi, am16));
3766 HReg vHi = newVRegV(env);
3767 HReg vLo = newVRegV(env);
3768 HReg rA = iselIntExpr_R(env, e->Iex.Load.addr);
3771 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vLo, am0));
3772 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vHi, am16));
3782 HReg vHi = generate_zeroes_V128(env);
3783 HReg vLo = newVRegV(env);
3784 addInstr(env, mk_vMOVsd_RR(vHi, vLo));
3799 iselDVecExpr(&argHi, &argLo, env, e->Iex.Unop.arg);
3800 *rHi = do_sse_NotV128(env, argHi);
3801 *rLo = do_sse_NotV128(env, argLo);
3811 iselDVecExpr(&argHi, &argLo, env, e->Iex.Unop.arg);
3812 HReg dstHi = newVRegV(env);
3813 HReg dstLo = newVRegV(env);
3814 addInstr(env, AMD64Instr_Sse32Fx4(op, argHi, dstHi));
3815 addInstr(env, AMD64Instr_Sse32Fx4(op, argLo, dstLo));
3825 iselDVecExpr(&argHi, &argLo, env, e->Iex.Unop.arg);
3826 HReg dstHi = newVRegV(env);
3827 HReg dstLo = newVRegV(env);
3828 addInstr(env, AMD64Instr_Sse64Fx2(op, argHi, dstHi));
3829 addInstr(env, AMD64Instr_Sse64Fx2(op, argLo, dstLo));
3841 iselDVecExpr(&argHi, &argLo, env, e->Iex.Unop.arg);
3842 HReg tmpHi = generate_zeroes_V128(env);
3843 HReg tmpLo = newVRegV(env);
3844 addInstr(env, mk_vMOVsd_RR(tmpHi, tmpLo));
3845 HReg dstHi = newVRegV(env);
3846 HReg dstLo = newVRegV(env);
3847 addInstr(env, AMD64Instr_SseReRg(Asse_CMPEQ32, argHi, tmpHi));
3848 addInstr(env, AMD64Instr_SseReRg(Asse_CMPEQ32, argLo, tmpLo));
3849 tmpHi = do_sse_NotV128(env, tmpHi);
3850 tmpLo = do_sse_NotV128(env, tmpLo);
3851 addInstr(env, AMD64Instr_SseShuf(0xB1, tmpHi, dstHi));
3852 addInstr(env, AMD64Instr_SseShuf(0xB1, tmpLo, dstLo));
3853 addInstr(env, AMD64Instr_SseReRg(Asse_OR, tmpHi, dstHi));
3854 addInstr(env, AMD64Instr_SseReRg(Asse_OR, tmpLo, dstLo));
3866 iselDVecExpr(&argHi, &argLo, env, e->Iex.Unop.arg);
3867 HReg tmpHi = newVRegV(env);
3868 HReg tmpLo = newVRegV(env);
3869 HReg zero = generate_zeroes_V128(env);
3871 addInstr(env, mk_vMOVsd_RR(argHi, tmpHi));
3872 addInstr(env, mk_vMOVsd_RR(argLo, tmpLo));
3873 addInstr(env, AMD64Instr_SseReRg(op, zero, tmpHi));
3874 addInstr(env, AMD64Instr_SseReRg(op, zero, tmpLo));
3875 dstHi = do_sse_NotV128(env, tmpHi);
3876 dstLo = do_sse_NotV128(env, tmpLo);
3895 iselDVecExpr(&argLhi, &argLlo, env, e->Iex.Binop.arg1);
3896 iselDVecExpr(&argRhi, &argRlo, env, e->Iex.Binop.arg2);
3897 HReg dstHi = newVRegV(env);
3898 HReg dstLo = newVRegV(env);
3899 addInstr(env, mk_vMOVsd_RR(argLhi, dstHi));
3900 addInstr(env, mk_vMOVsd_RR(argLlo, dstLo));
3901 addInstr(env, AMD64Instr_Sse64Fx2(op, argRhi, dstHi));
3902 addInstr(env, AMD64Instr_Sse64Fx2(op, argRlo, dstLo));
3913 iselDVecExpr(&argLhi, &argLlo, env, e->Iex.Binop.arg1);
3914 iselDVecExpr(&argRhi, &argRlo, env, e->Iex.Binop.arg2);
3915 HReg dstHi = newVRegV(env);
3916 HReg dstLo = newVRegV(env);
3917 addInstr(env, mk_vMOVsd_RR(argLhi, dstHi));
3918 addInstr(env, mk_vMOVsd_RR(argLlo, dstLo));
3919 addInstr(env, AMD64Instr_Sse32Fx4(op, argRhi, dstHi));
3920 addInstr(env, AMD64Instr_Sse32Fx4(op, argRlo, dstLo));
3963 iselDVecExpr(&argLhi, &argLlo, env, e->Iex.Binop.arg1);
3964 iselDVecExpr(&argRhi, &argRlo, env, e->Iex.Binop.arg2);
3965 HReg dstHi = newVRegV(env);
3966 HReg dstLo = newVRegV(env);
3967 addInstr(env, mk_vMOVsd_RR(argLhi, dstHi));
3968 addInstr(env, mk_vMOVsd_RR(argLlo, dstLo));
3969 addInstr(env, AMD64Instr_SseReRg(op, argRhi, dstHi));
3970 addInstr(env, AMD64Instr_SseReRg(op, argRlo, dstLo));
3986 iselDVecExpr(&gregHi, &gregLo, env, e->Iex.Binop.arg1);
3987 AMD64RMI* rmi = iselIntExpr_RMI(env, e->Iex.Binop.arg2);
3989 HReg ereg = newVRegV(env);
3990 HReg dstHi = newVRegV(env);
3991 HReg dstLo = newVRegV(env);
3992 addInstr(env, AMD64Instr_Push(AMD64RMI_Imm(0)));
3993 addInstr(env, AMD64Instr_Push(rmi));
3994 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, ereg, rsp0));
3995 addInstr(env, mk_vMOVsd_RR(gregHi, dstHi));
3996 addInstr(env, AMD64Instr_SseReRg(op, ereg, dstHi));
3997 addInstr(env, mk_vMOVsd_RR(gregLo, dstLo));
3998 addInstr(env, AMD64Instr_SseReRg(op, ereg, dstLo));
3999 add_to_rsp(env, 16);
4006 *rHi = iselVecExpr(env, e->Iex.Binop.arg1);
4007 *rLo = iselVecExpr(env, e->Iex.Binop.arg2);
4037 HReg dstHi = newVRegV(env);
4038 HReg dstLo = newVRegV(env);
4040 iselDVecExpr(&argLhi, &argLlo, env, e->Iex.Binop.arg1);
4041 iselDVecExpr(&argRhi, &argRlo, env, e->Iex.Binop.arg2);
4042 HReg argp = newVRegI(env);
4044 sub_from_rsp(env, 160);
4046 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(48, hregAMD64_RSP()),
4049 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
4057 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, argp),
4059 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(16, argp),
4061 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(32, argp),
4067 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argLhi,
4069 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argRhi,
4075 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argLlo,
4077 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argRlo,
4080 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn, 3,
4087 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(48, argp),
4089 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(64, argp),
4091 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(80, argp),
4094 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn, 3,
4098 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dstHi,
4100 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dstLo,
4103 add_to_rsp(env, 160);
4115 HReg dstHi = newVRegV(env);
4116 HReg dstLo = newVRegV(env);
4118 iselDVecExpr(&argLhi, &argLlo, env, e->Iex.Binop.arg1);
4119 iselDVecExpr(&argRhi, &argRlo, env, e->Iex.Binop.arg2);
4120 HReg argp = newVRegI(env);
4122 sub_from_rsp(env, 160);
4124 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(48, hregAMD64_RSP()),
4127 addInstr(env, AMD64Instr_Alu64R(Aalu_AND,
4135 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(0, argp),
4137 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(32, argp),
4139 addInstr(env, AMD64Instr_Lea64(AMD64AMode_IR(64, argp),
4147 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argLlo,
4149 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argLhi,
4151 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argRlo,
4153 addInstr(env, AMD64Instr_SseLdSt(False/*!isLoad*/, 16, argRhi,
4156 addInstr(env, AMD64Instr_Call( Acc_ALWAYS, (ULong)fn, 3,
4160 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dstLo,
4162 addInstr(env, AMD64Instr_SseLdSt(True/*isLoad*/, 16, dstHi,
4165 add_to_rsp(env, 160);
4187 iselDVecExpr(&argLhi, &argLlo, env, triop->arg2);
4188 iselDVecExpr(&argRhi, &argRlo, env, triop->arg3);
4189 HReg dstHi = newVRegV(env);
4190 HReg dstLo = newVRegV(env);
4191 addInstr(env, mk_vMOVsd_RR(argLhi, dstHi));
4192 addInstr(env, mk_vMOVsd_RR(argLlo, dstLo));
4195 addInstr(env, AMD64Instr_Sse64Fx2(op, argRhi, dstHi));
4196 addInstr(env, AMD64Instr_Sse64Fx2(op, argRlo, dstLo));
4209 iselDVecExpr(&argLhi, &argLlo, env, triop->arg2);
4210 iselDVecExpr(&argRhi, &argRlo, env, triop->arg3);
4211 HReg dstHi = newVRegV(env);
4212 HReg dstLo = newVRegV(env);
4213 addInstr(env, mk_vMOVsd_RR(argLhi, dstHi));
4214 addInstr(env, mk_vMOVsd_RR(argLlo, dstLo));
4217 addInstr(env, AMD64Instr_Sse32Fx4(op, argRhi, dstHi));
4218 addInstr(env, AMD64Instr_Sse32Fx4(op, argRlo, dstLo));
4232 HReg vHi = newVRegV(env);
4233 HReg vLo = newVRegV(env);
4238 AMD64RI* q3 = iselIntExpr_RI(env, e->Iex.Qop.details->arg1);
4239 AMD64RI* q2 = iselIntExpr_RI(env, e->Iex.Qop.details->arg2);
4240 AMD64RI* q1 = iselIntExpr_RI(env, e->Iex.Qop.details->arg3);
4241 AMD64RI* q0 = iselIntExpr_RI(env, e->Iex.Qop.details->arg4);
4243 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, q3, m8_rsp));
4244 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, q2, m16_rsp));
4245 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vHi, m16_rsp));
4247 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, q1, m8_rsp));
4248 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV, q0, m16_rsp));
4249 addInstr(env, AMD64Instr_SseLdSt(True/*load*/, 16, vLo, m16_rsp));
4257 iselDVecExpr(&r1Hi, &r1Lo, env, e->Iex.ITE.iftrue);
4258 iselDVecExpr(&r0Hi, &r0Lo, env, e->Iex.ITE.iffalse);
4259 HReg dstHi = newVRegV(env);
4260 HReg dstLo = newVRegV(env);
4261 addInstr(env, mk_vMOVsd_RR(r1Hi,dstHi));
4262 addInstr(env, mk_vMOVsd_RR(r1Lo,dstLo));
4263 AMD64CondCode cc = iselCondCode(env, e->Iex.ITE.cond);
4264 addInstr(env, AMD64Instr_SseCMov(cc ^ 1, r0Hi, dstHi));
4265 addInstr(env, AMD64Instr_SseCMov(cc ^ 1, r0Lo, dstLo));
4273 LibVEX_ppVexHwCaps(VexArchAMD64, env->hwcaps));
4283 static void iselStmt ( ISelEnv* env, IRStmt* stmt )
4308 AMD64AMode* amAddr = iselIntExpr_AMode(env, lg->addr);
4309 HReg rAlt = iselIntExpr_R(env, lg->alt);
4310 HReg rDst = lookupIRTemp(env, lg->dst);
4313 addInstr(env, mk_iMOVsd_RR(rAlt, rDst));
4314 AMD64CondCode cc = iselCondCode(env, lg->guard);
4315 addInstr(env, AMD64Instr_CLoad(cc, szB, amAddr, rDst));
4326 switch (typeOfIRExpr(env->type_env, sg->data)) {
4334 AMD64AMode* amAddr = iselIntExpr_AMode(env, sg->addr);
4335 HReg rSrc = iselIntExpr_R(env, sg->data);
4336 AMD64CondCode cc = iselCondCode(env, sg->guard);
4337 addInstr(env, AMD64Instr_CStore(cc, szB, rSrc, amAddr));
4343 IRType tya = typeOfIRExpr(env->type_env, stmt->Ist.Store.addr);
4344 IRType tyd = typeOfIRExpr(env->type_env, stmt->Ist.Store.data);
4351 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr);
4352 AMD64RI* ri = iselIntExpr_RI(env, stmt->Ist.Store.data);
4353 addInstr(env, AMD64Instr_Alu64M(Aalu_MOV,ri,am));
4357 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr);
4358 HReg r = iselIntExpr_R(env, stmt->Ist.Store.data);
4359 addInstr(env, AMD64Instr_Store(
4365 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr);
4366 HReg r = iselDblExpr(env, stmt->Ist.Store.data);
4367 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 8, r, am));
4371 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr);
4372 HReg r = iselFltExpr(env, stmt->Ist.Store.data);
4373 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 4, r, am));
4377 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.Store.addr);
4378 HReg r = iselVecExpr(env, stmt->Ist.Store.data);
4379 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, r, am));
4383 HReg rA = iselIntExpr_R(env, stmt->Ist.Store.addr);
4387 iselDVecExpr(&vHi, &vLo, env, stmt->Ist.Store.data);
4388 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vLo, am0));
4389 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vHi, am16));
4397 IRType ty = typeOfIRExpr(env->type_env, stmt->Ist.Put.data);
4401 AMD64RI* ri = iselIntExpr_RI(env, stmt->Ist.Put.data);
4402 addInstr(env,
4412 HReg r = iselIntExpr_R(env, stmt->Ist.Put.data);
4413 addInstr(env, AMD64Instr_Store(
4421 HReg f32 = iselFltExpr(env, stmt->Ist.Put.data);
4423 set_SSE_rounding_default(env); /* paranoia */
4424 addInstr(env, AMD64Instr_SseLdSt( False/*store*/, 4, f32, am ));
4428 HReg f64 = iselDblExpr(env, stmt->Ist.Put.data);
4431 addInstr(env, AMD64Instr_SseLdSt( False/*store*/, 8, f64, am ));
4435 HReg vec = iselVecExpr(env, stmt->Ist.Put.data);
4438 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vec, am));
4443 iselDVecExpr(&vHi, &vLo, env, stmt->Ist.Put.data);
4447 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vLo, am0));
4448 addInstr(env, AMD64Instr_SseLdSt(False/*store*/, 16, vHi, am16));
4460 env, puti->descr,
4463 IRType ty = typeOfIRExpr(env->type_env, puti->data);
4465 HReg val = iselDblExpr(env, puti->data);
4466 addInstr(env, AMD64Instr_SseLdSt( False/*store*/, 8, val, am ));
4470 HReg r = iselIntExpr_R(env, puti->data);
4471 addInstr(env, AMD64Instr_Store( 1, r, am ));
4475 AMD64RI* ri = iselIntExpr_RI(env, puti->data);
4476 addInstr(env, AMD64Instr_Alu64M( Aalu_MOV, ri, am ));
4485 IRType ty = typeOfIRTemp(env->type_env, tmp);
4496 AMD64AMode* am = iselIntExpr_AMode(env, stmt->Ist.WrTmp.data);
4497 HReg dst = lookupIRTemp(env, tmp);
4503 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV, AMD64RMI_Reg(src), dst));
4505 addInstr(env, AMD64Instr_Lea64(am,dst));
4512 AMD64RMI* rmi = iselIntExpr_RMI(env, stmt->Ist.WrTmp.data);
4513 HReg dst = lookupIRTemp(env, tmp);
4514 addInstr(env, AMD64Instr_Alu64R(Aalu_MOV,rmi,dst));
4519 iselInt128Expr(&rHi,&rLo, env, stmt->Ist.WrTmp.data);
4520 lookupIRTempPair( &dstHi, &dstLo, env, tmp);
4521 addInstr(env, mk_iMOVsd_RR(rHi,dstHi) );
4522 addInstr(env, mk_iMOVsd_RR(rLo,dstLo) );
4526 AMD64CondCode cond = iselCondCode(env, stmt->Ist.WrTmp.data);
4527 HReg dst = lookupIRTemp(env, tmp);
4528 addInstr(env, AMD64Instr_Set64(cond, dst));
4532 HReg dst = lookupIRTemp(env, tmp);
4533 HReg src = iselDblExpr(env, stmt->Ist.WrTmp.data);
4534 addInstr(env, mk_vMOVsd_RR(src, dst));
4538 HReg dst = lookupIRTemp(env, tmp);
4539 HReg src = iselFltExpr(env, stmt->Ist.WrTmp.data);
4540 addInstr(env, mk_vMOVsd_RR(src, dst));
4544 HReg dst = lookupIRTemp(env, tmp);
4545 HReg src = iselVecExpr(env, stmt->Ist.WrTmp.data);
4546 addInstr(env, mk_vMOVsd_RR(src, dst));
4551 iselDVecExpr(&rHi,&rLo, env, stmt->Ist.WrTmp.data);
4552 lookupIRTempPair( &dstHi, &dstLo, env, tmp);
4553 addInstr(env, mk_vMOVsd_RR(rHi,dstHi) );
4554 addInstr(env, mk_vMOVsd_RR(rLo,dstLo) );
4567 retty = typeOfIRTemp(env->type_env, d->tmp);
4587 doHelperCall( &addToSp, &rloc, env, d->guard, d->cee, retty, d->args );
4604 HReg dst = lookupIRTemp(env, d->tmp);
4605 addInstr(env, mk_iMOVsd_RR(hregAMD64_RAX(),dst) );
4615 HReg dst = lookupIRTemp(env, d->tmp);
4617 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 16, dst, am ));
4618 add_to_rsp(env, addToSp);
4626 lookupIRTempPair(&dstHi, &dstLo, env, d->tmp);
4628 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 16, dstLo, amLo ));
4630 addInstr(env, AMD64Instr_SseLdSt( True/*load*/, 16, dstHi, amHi ));
4631 add_to_rsp(env, addToSp);
4645 addInstr(env, AMD64Instr_MFence());
4658 IRType ty = typeOfIRExpr(env->type_env, cas->dataLo);
4660 AMD64AMode* am = iselIntExpr_AMode(env, cas->addr);
4661 HReg rData = iselIntExpr_R(env, cas->dataLo);
4662 HReg rExpd = iselIntExpr_R(env, cas->expdLo);
4663 HReg rOld = lookupIRTemp(env, cas->oldLo);
4666 addInstr(env, mk_iMOVsd_RR(rExpd, rOld));
4667 addInstr(env, mk_iMOVsd_RR(rExpd, hregAMD64_RAX()));
4668 addInstr(env, mk_iMOVsd_RR(rData, hregAMD64_RBX()));
4676 addInstr(env, AMD64Instr_ACAS(am, sz));
4677 addInstr(env, AMD64Instr_CMov64(Acc_NZ, hregAMD64_RAX(), rOld));
4683 IRType ty = typeOfIRExpr(env->type_env, cas->dataLo);
4687 AMD64AMode* am = iselIntExpr_AMode(env, cas->addr);
4688 HReg rDataHi = iselIntExpr_R(env, cas->dataHi);
4689 HReg rDataLo = iselIntExpr_R(env, cas->dataLo);
4690 HReg rExpdHi = iselIntExpr_R(env, cas->expdHi);
4691 HReg rExpdLo = iselIntExpr_R(env, cas->expdLo);
4692 HReg rOldHi = lookupIRTemp(env, cas->oldHi);
4693 HReg rOldLo = lookupIRTemp(env, cas->oldLo);
4696 if (!(env->hwcaps & VEX_HWCAPS_AMD64_CX16))
4708 addInstr(env, mk_iMOVsd_RR(rExpdHi, rOldHi));
4709 addInstr(env, mk_iMOVsd_RR(rExpdLo, rOldLo));
4710 addInstr(env, mk_iMOVsd_RR(rExpdHi, hregAMD64_RDX()));
4711 addInstr(env, mk_iMOVsd_RR(rExpdLo, hregAMD64_RAX()));
4712 addInstr(env, mk_iMOVsd_RR(rDataHi, hregAMD64_RCX()));
4713 addInstr(env, mk_iMOVsd_RR(rDataLo, hregAMD64_RBX()));
4714 addInstr(env, AMD64Instr_DACAS(am, sz));
4715 addInstr(env
4716 addInstr(env, AMD64Instr_CMov64(Acc_NZ, hregAMD64_RAX(), rOldLo));
4742 AMD64CondCode cc = iselCondCode(env, stmt->Ist.Exit.guard);
4748 if (env->chainingAllowed) {
4753 = ((Addr64)stmt->Ist.Exit.dst->Ico.U64) > env->max_ga;
4755 addInstr(env, AMD64Instr_XDirect(stmt->Ist.Exit.dst->Ico.U64,
4761 HReg r = iselIntExpr_R(env, IRExpr_Const(stmt->Ist.Exit.dst));
4762 addInstr(env, AMD64Instr_XAssisted(r, amRIP, cc, Ijk_Boring));
4780 HReg r = iselIntExpr_R(env, IRExpr_Const(stmt->Ist.Exit.dst));
4781 addInstr(env, AMD64Instr_XAssisted(r, amRIP, cc, stmt->Ist.Exit.jk));
4804 static void iselNext ( ISelEnv* env,
4822 if (env->chainingAllowed) {
4827 = ((Addr64)cdst->Ico.U64) > env->max_ga;
4829 addInstr(env, AMD64Instr_XDirect(cdst->Ico.U64,
4837 HReg r = iselIntExpr_R(env, next);
4838 addInstr(env, AMD64Instr_XAssisted(r, amRIP, Acc_ALWAYS,
4848 HReg r = iselIntExpr_R(env, next);
4850 if (env->chainingAllowed) {
4851 addInstr(env, AMD64Instr_XIndir(r, amRIP, Acc_ALWAYS));
4853 addInstr(env, AMD64Instr_XAssisted(r, amRIP, Acc_ALWAYS,
4874 HReg r = iselIntExpr_R(env, next);
4876 addInstr(env, AMD64Instr_XAssisted(r, amRIP, Acc_ALWAYS, jk));
4910 ISelEnv* env;
4929 env = LibVEX_Alloc_inline(sizeof(ISelEnv));
4930 env->vreg_ctr = 0;
4933 env->code = newHInstrArray();
4935 /* Copy BB's type env. */
4936 env->type_env = bb->tyenv;
4940 env->n_vregmap = bb->tyenv->types_used;
4941 env->vregmap = LibVEX_Alloc_inline(env->n_vregmap * sizeof(HReg));
4942 env->vregmapHI = LibVEX_Alloc_inline(env->n_vregmap * sizeof(HReg));
4945 env->chainingAllowed = chainingAllowed;
4946 env->hwcaps = hwcaps_host;
4947 env->max_ga = max_ga;
4952 for (i = 0; i < env->n_vregmap; i++) {
4976 env->vregmap[i] = hreg;
4977 env->vregmapHI[i] = hregHI;
4979 env->vreg_ctr = j;
4984 addInstr(env, AMD64Instr_EvCheck(amCounter, amFailAddr));
4991 addInstr(env, AMD64Instr_ProfInc());
4997 iselStmt(env, bb->stmts[i]);
4999 iselNext(env, bb->next, bb->jumpkind, bb->offsIP);
5002 env->code->n_vregs = env->vreg_ctr;
5003 return env->code;
