1 /************************************************************************** 2 * 3 * Copyright (C) 1999-2005 Brian Paul All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included 13 * in all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN 19 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 * 22 **************************************************************************/ 23 24 #include "pipe/p_config.h" 25 #include "util/u_cpu_detect.h" 26 27 #if defined(PIPE_ARCH_X86) || (defined(PIPE_ARCH_X86_64) && !defined(__MINGW32__)) 28 29 #include "pipe/p_compiler.h" 30 #include "util/u_debug.h" 31 #include "util/u_pointer.h" 32 33 #include "rtasm_execmem.h" 34 #include "rtasm_x86sse.h" 35 36 #define DISASSEM 0 37 #define X86_TWOB 0x0f 38 39 40 #define DUMP_SSE 0 41 42 43 void x86_print_reg( struct x86_reg reg ) 44 { 45 if (reg.mod != mod_REG) 46 debug_printf( "[" ); 47 48 switch( reg.file ) { 49 case file_REG32: 50 switch( reg.idx ) { 51 case reg_AX: debug_printf( "EAX" ); break; 52 case reg_CX: debug_printf( "ECX" ); break; 53 case reg_DX: debug_printf( "EDX" ); break; 54 case reg_BX: debug_printf( "EBX" ); break; 55 case reg_SP: debug_printf( "ESP" ); break; 56 case reg_BP: debug_printf( "EBP" ); break; 57 case reg_SI: debug_printf( "ESI" ); break; 58 case reg_DI: debug_printf( "EDI" ); break; 59 } 60 break; 61 case file_MMX: 62 debug_printf( "MMX%u", reg.idx ); 63 break; 64 case file_XMM: 65 debug_printf( "XMM%u", reg.idx ); 66 break; 67 case file_x87: 68 debug_printf( "fp%u", reg.idx ); 69 break; 70 } 71 72 if (reg.mod == mod_DISP8 || 73 reg.mod == mod_DISP32) 74 debug_printf("+%d", reg.disp); 75 76 if (reg.mod != mod_REG) 77 debug_printf( "]" ); 78 } 79 80 #if DUMP_SSE 81 82 #define DUMP_START() debug_printf( "\n" ) 83 #define DUMP_END() debug_printf( "\n" ) 84 85 #define DUMP() do { \ 86 const char *foo = __FUNCTION__; \ 87 while (*foo && *foo != '_') \ 88 foo++; \ 89 if (*foo) \ 90 foo++; \ 91 debug_printf( "\n%4x %14s ", p->csr - p->store, foo ); \ 92 } while (0) 93 94 #define DUMP_I( I ) do { \ 95 DUMP(); \ 96 debug_printf( "%u", I ); \ 97 } while( 0 ) 98 99 #define DUMP_R( R0 ) do { \ 100 DUMP(); \ 101 x86_print_reg( R0 ); \ 102 } while( 0 ) 103 104 #define DUMP_RR( R0, R1 ) do { \ 105 DUMP(); \ 106 x86_print_reg( R0 ); \ 107 debug_printf( ", " ); \ 108 x86_print_reg( R1 ); \ 109 } while( 0 ) 110 111 #define DUMP_RI( R0, I ) do { \ 112 DUMP(); \ 113 x86_print_reg( R0 ); \ 114 debug_printf( ", %u", I ); \ 115 } while( 0 ) 116 117 #define DUMP_RRI( R0, R1, I ) do { \ 118 DUMP(); \ 119 x86_print_reg( R0 ); \ 120 debug_printf( ", " ); \ 121 x86_print_reg( R1 ); \ 122 debug_printf( ", %u", I ); \ 123 } while( 0 ) 124 125 #else 126 127 #define DUMP_START() 128 #define DUMP_END() 129 #define DUMP( ) 130 #define DUMP_I( I ) 131 #define DUMP_R( R0 ) 132 #define DUMP_RR( R0, R1 ) 133 #define DUMP_RI( R0, I ) 134 #define DUMP_RRI( R0, R1, I ) 135 136 #endif 137 138 139 static void do_realloc( struct x86_function *p ) 140 { 141 if (p->store == p->error_overflow) { 142 p->csr = p->store; 143 } 144 else if (p->size == 0) { 145 p->size = 1024; 146 p->store = rtasm_exec_malloc(p->size); 147 p->csr = p->store; 148 } 149 else { 150 uintptr_t used = pointer_to_uintptr( p->csr ) - pointer_to_uintptr( p->store ); 151 unsigned char *tmp = p->store; 152 p->size *= 2; 153 p->store = rtasm_exec_malloc(p->size); 154 155 if (p->store) { 156 memcpy(p->store, tmp, used); 157 p->csr = p->store + used; 158 } 159 else { 160 p->csr = p->store; 161 } 162 163 rtasm_exec_free(tmp); 164 } 165 166 if (p->store == NULL) { 167 p->store = p->csr = p->error_overflow; 168 p->size = sizeof(p->error_overflow); 169 } 170 } 171 172 /* Emit bytes to the instruction stream: 173 */ 174 static unsigned char *reserve( struct x86_function *p, int bytes ) 175 { 176 if (p->csr + bytes - p->store > (int) p->size) 177 do_realloc(p); 178 179 { 180 unsigned char *csr = p->csr; 181 p->csr += bytes; 182 return csr; 183 } 184 } 185 186 187 188 static void emit_1b( struct x86_function *p, char b0 ) 189 { 190 char *csr = (char *)reserve(p, 1); 191 *csr = b0; 192 } 193 194 static void emit_1i( struct x86_function *p, int i0 ) 195 { 196 int *icsr = (int *)reserve(p, sizeof(i0)); 197 *icsr = i0; 198 } 199 200 static void emit_1ub( struct x86_function *p, unsigned char b0 ) 201 { 202 unsigned char *csr = reserve(p, 1); 203 *csr++ = b0; 204 } 205 206 static void emit_2ub( struct x86_function *p, unsigned char b0, unsigned char b1 ) 207 { 208 unsigned char *csr = reserve(p, 2); 209 *csr++ = b0; 210 *csr++ = b1; 211 } 212 213 static void emit_3ub( struct x86_function *p, unsigned char b0, unsigned char b1, unsigned char b2 ) 214 { 215 unsigned char *csr = reserve(p, 3); 216 *csr++ = b0; 217 *csr++ = b1; 218 *csr++ = b2; 219 } 220 221 222 /* Build a modRM byte + possible displacement. No treatment of SIB 223 * indexing. BZZT - no way to encode an absolute address. 224 * 225 * This is the "/r" field in the x86 manuals... 226 */ 227 static void emit_modrm( struct x86_function *p, 228 struct x86_reg reg, 229 struct x86_reg regmem ) 230 { 231 unsigned char val = 0; 232 233 assert(reg.mod == mod_REG); 234 235 /* TODO: support extended x86-64 registers */ 236 assert(reg.idx < 8); 237 assert(regmem.idx < 8); 238 239 val |= regmem.mod << 6; /* mod field */ 240 val |= reg.idx << 3; /* reg field */ 241 val |= regmem.idx; /* r/m field */ 242 243 emit_1ub(p, val); 244 245 /* Oh-oh we've stumbled into the SIB thing. 246 */ 247 if (regmem.file == file_REG32 && 248 regmem.idx == reg_SP && 249 regmem.mod != mod_REG) { 250 emit_1ub(p, 0x24); /* simplistic! */ 251 } 252 253 switch (regmem.mod) { 254 case mod_REG: 255 case mod_INDIRECT: 256 break; 257 case mod_DISP8: 258 emit_1b(p, (char) regmem.disp); 259 break; 260 case mod_DISP32: 261 emit_1i(p, regmem.disp); 262 break; 263 default: 264 assert(0); 265 break; 266 } 267 } 268 269 /* Emits the "/0".."/7" specialized versions of the modrm ("/r") bytes. 270 */ 271 static void emit_modrm_noreg( struct x86_function *p, 272 unsigned op, 273 struct x86_reg regmem ) 274 { 275 struct x86_reg dummy = x86_make_reg(file_REG32, op); 276 emit_modrm(p, dummy, regmem); 277 } 278 279 /* Many x86 instructions have two opcodes to cope with the situations 280 * where the destination is a register or memory reference 281 * respectively. This function selects the correct opcode based on 282 * the arguments presented. 283 */ 284 static void emit_op_modrm( struct x86_function *p, 285 unsigned char op_dst_is_reg, 286 unsigned char op_dst_is_mem, 287 struct x86_reg dst, 288 struct x86_reg src ) 289 { 290 switch (dst.mod) { 291 case mod_REG: 292 emit_1ub(p, op_dst_is_reg); 293 emit_modrm(p, dst, src); 294 break; 295 case mod_INDIRECT: 296 case mod_DISP32: 297 case mod_DISP8: 298 assert(src.mod == mod_REG); 299 emit_1ub(p, op_dst_is_mem); 300 emit_modrm(p, src, dst); 301 break; 302 default: 303 assert(0); 304 break; 305 } 306 } 307 308 309 310 311 312 313 314 /* Create and manipulate registers and regmem values: 315 */ 316 struct x86_reg x86_make_reg( enum x86_reg_file file, 317 enum x86_reg_name idx ) 318 { 319 struct x86_reg reg; 320 321 reg.file = file; 322 reg.idx = idx; 323 reg.mod = mod_REG; 324 reg.disp = 0; 325 326 return reg; 327 } 328 329 struct x86_reg x86_make_disp( struct x86_reg reg, 330 int disp ) 331 { 332 assert(reg.file == file_REG32); 333 334 if (reg.mod == mod_REG) 335 reg.disp = disp; 336 else 337 reg.disp += disp; 338 339 if (reg.disp == 0 && reg.idx != reg_BP) 340 reg.mod = mod_INDIRECT; 341 else if (reg.disp <= 127 && reg.disp >= -128) 342 reg.mod = mod_DISP8; 343 else 344 reg.mod = mod_DISP32; 345 346 return reg; 347 } 348 349 struct x86_reg x86_deref( struct x86_reg reg ) 350 { 351 return x86_make_disp(reg, 0); 352 } 353 354 struct x86_reg x86_get_base_reg( struct x86_reg reg ) 355 { 356 return x86_make_reg( reg.file, reg.idx ); 357 } 358 359 int x86_get_label( struct x86_function *p ) 360 { 361 return p->csr - p->store; 362 } 363 364 365 366 /*********************************************************************** 367 * x86 instructions 368 */ 369 370 371 void x64_rexw(struct x86_function *p) 372 { 373 if(x86_target(p) != X86_32) 374 emit_1ub(p, 0x48); 375 } 376 377 void x86_jcc( struct x86_function *p, 378 enum x86_cc cc, 379 int label ) 380 { 381 int offset = label - (x86_get_label(p) + 2); 382 DUMP_I(cc); 383 384 if (offset < 0) { 385 /*assert(p->csr - p->store > -offset);*/ 386 if (p->csr - p->store <= -offset) { 387 /* probably out of memory (using the error_overflow buffer) */ 388 return; 389 } 390 } 391 392 if (offset <= 127 && offset >= -128) { 393 emit_1ub(p, 0x70 + cc); 394 emit_1b(p, (char) offset); 395 } 396 else { 397 offset = label - (x86_get_label(p) + 6); 398 emit_2ub(p, 0x0f, 0x80 + cc); 399 emit_1i(p, offset); 400 } 401 } 402 403 /* Always use a 32bit offset for forward jumps: 404 */ 405 int x86_jcc_forward( struct x86_function *p, 406 enum x86_cc cc ) 407 { 408 DUMP_I(cc); 409 emit_2ub(p, 0x0f, 0x80 + cc); 410 emit_1i(p, 0); 411 return x86_get_label(p); 412 } 413 414 int x86_jmp_forward( struct x86_function *p) 415 { 416 DUMP(); 417 emit_1ub(p, 0xe9); 418 emit_1i(p, 0); 419 return x86_get_label(p); 420 } 421 422 int x86_call_forward( struct x86_function *p) 423 { 424 DUMP(); 425 426 emit_1ub(p, 0xe8); 427 emit_1i(p, 0); 428 return x86_get_label(p); 429 } 430 431 /* Fixup offset from forward jump: 432 */ 433 void x86_fixup_fwd_jump( struct x86_function *p, 434 int fixup ) 435 { 436 *(int *)(p->store + fixup - 4) = x86_get_label(p) - fixup; 437 } 438 439 void x86_jmp( struct x86_function *p, int label) 440 { 441 DUMP_I( label ); 442 emit_1ub(p, 0xe9); 443 emit_1i(p, label - x86_get_label(p) - 4); 444 } 445 446 void x86_call( struct x86_function *p, struct x86_reg reg) 447 { 448 DUMP_R( reg ); 449 emit_1ub(p, 0xff); 450 emit_modrm_noreg(p, 2, reg); 451 } 452 453 454 void x86_mov_reg_imm( struct x86_function *p, struct x86_reg dst, int imm ) 455 { 456 DUMP_RI( dst, imm ); 457 assert(dst.file == file_REG32); 458 assert(dst.mod == mod_REG); 459 emit_1ub(p, 0xb8 + dst.idx); 460 emit_1i(p, imm); 461 } 462 463 void x86_mov_imm( struct x86_function *p, struct x86_reg dst, int imm ) 464 { 465 DUMP_RI( dst, imm ); 466 if(dst.mod == mod_REG) 467 x86_mov_reg_imm(p, dst, imm); 468 else 469 { 470 emit_1ub(p, 0xc7); 471 emit_modrm_noreg(p, 0, dst); 472 emit_1i(p, imm); 473 } 474 } 475 476 void x86_mov16_imm( struct x86_function *p, struct x86_reg dst, uint16_t imm ) 477 { 478 DUMP_RI( dst, imm ); 479 emit_1ub(p, 0x66); 480 if(dst.mod == mod_REG) 481 { 482 emit_1ub(p, 0xb8 + dst.idx); 483 emit_2ub(p, imm & 0xff, imm >> 8); 484 } 485 else 486 { 487 emit_1ub(p, 0xc7); 488 emit_modrm_noreg(p, 0, dst); 489 emit_2ub(p, imm & 0xff, imm >> 8); 490 } 491 } 492 493 void x86_mov8_imm( struct x86_function *p, struct x86_reg dst, uint8_t imm ) 494 { 495 DUMP_RI( dst, imm ); 496 if(dst.mod == mod_REG) 497 { 498 emit_1ub(p, 0xb0 + dst.idx); 499 emit_1ub(p, imm); 500 } 501 else 502 { 503 emit_1ub(p, 0xc6); 504 emit_modrm_noreg(p, 0, dst); 505 emit_1ub(p, imm); 506 } 507 } 508 509 /** 510 * Immediate group 1 instructions. 511 */ 512 static INLINE void 513 x86_group1_imm( struct x86_function *p, 514 unsigned op, struct x86_reg dst, int imm ) 515 { 516 assert(dst.file == file_REG32); 517 assert(dst.mod == mod_REG); 518 if(-0x80 <= imm && imm < 0x80) { 519 emit_1ub(p, 0x83); 520 emit_modrm_noreg(p, op, dst); 521 emit_1b(p, (char)imm); 522 } 523 else { 524 emit_1ub(p, 0x81); 525 emit_modrm_noreg(p, op, dst); 526 emit_1i(p, imm); 527 } 528 } 529 530 void x86_add_imm( struct x86_function *p, struct x86_reg dst, int imm ) 531 { 532 DUMP_RI( dst, imm ); 533 x86_group1_imm(p, 0, dst, imm); 534 } 535 536 void x86_or_imm( struct x86_function *p, struct x86_reg dst, int imm ) 537 { 538 DUMP_RI( dst, imm ); 539 x86_group1_imm(p, 1, dst, imm); 540 } 541 542 void x86_and_imm( struct x86_function *p, struct x86_reg dst, int imm ) 543 { 544 DUMP_RI( dst, imm ); 545 x86_group1_imm(p, 4, dst, imm); 546 } 547 548 void x86_sub_imm( struct x86_function *p, struct x86_reg dst, int imm ) 549 { 550 DUMP_RI( dst, imm ); 551 x86_group1_imm(p, 5, dst, imm); 552 } 553 554 void x86_xor_imm( struct x86_function *p, struct x86_reg dst, int imm ) 555 { 556 DUMP_RI( dst, imm ); 557 x86_group1_imm(p, 6, dst, imm); 558 } 559 560 void x86_cmp_imm( struct x86_function *p, struct x86_reg dst, int imm ) 561 { 562 DUMP_RI( dst, imm ); 563 x86_group1_imm(p, 7, dst, imm); 564 } 565 566 567 void x86_push( struct x86_function *p, 568 struct x86_reg reg ) 569 { 570 DUMP_R( reg ); 571 if (reg.mod == mod_REG) 572 emit_1ub(p, 0x50 + reg.idx); 573 else 574 { 575 emit_1ub(p, 0xff); 576 emit_modrm_noreg(p, 6, reg); 577 } 578 579 580 p->stack_offset += sizeof(void*); 581 } 582 583 void x86_push_imm32( struct x86_function *p, 584 int imm32 ) 585 { 586 DUMP_I( imm32 ); 587 emit_1ub(p, 0x68); 588 emit_1i(p, imm32); 589 590 p->stack_offset += sizeof(void*); 591 } 592 593 594 void x86_pop( struct x86_function *p, 595 struct x86_reg reg ) 596 { 597 DUMP_R( reg ); 598 assert(reg.mod == mod_REG); 599 emit_1ub(p, 0x58 + reg.idx); 600 p->stack_offset -= sizeof(void*); 601 } 602 603 void x86_inc( struct x86_function *p, 604 struct x86_reg reg ) 605 { 606 DUMP_R( reg ); 607 if(x86_target(p) == X86_32 && reg.mod == mod_REG) 608 { 609 emit_1ub(p, 0x40 + reg.idx); 610 return; 611 } 612 emit_1ub(p, 0xff); 613 emit_modrm_noreg(p, 0, reg); 614 } 615 616 void x86_dec( struct x86_function *p, 617 struct x86_reg reg ) 618 { 619 DUMP_R( reg ); 620 if(x86_target(p) == X86_32 && reg.mod == mod_REG) 621 { 622 emit_1ub(p, 0x48 + reg.idx); 623 return; 624 } 625 emit_1ub(p, 0xff); 626 emit_modrm_noreg(p, 1, reg); 627 } 628 629 void x86_ret( struct x86_function *p ) 630 { 631 DUMP(); 632 assert(p->stack_offset == 0); 633 emit_1ub(p, 0xc3); 634 } 635 636 void x86_retw( struct x86_function *p, unsigned short imm ) 637 { 638 DUMP(); 639 emit_3ub(p, 0xc2, imm & 0xff, (imm >> 8) & 0xff); 640 } 641 642 void x86_sahf( struct x86_function *p ) 643 { 644 DUMP(); 645 emit_1ub(p, 0x9e); 646 } 647 648 void x86_mov( struct x86_function *p, 649 struct x86_reg dst, 650 struct x86_reg src ) 651 { 652 DUMP_RR( dst, src ); 653 /* special hack for reading arguments until we support x86-64 registers everywhere */ 654 if(src.mod == mod_REG && dst.mod == mod_REG && (src.idx >= 8 || dst.idx >= 8)) 655 { 656 uint8_t rex = 0x40; 657 if(dst.idx >= 8) 658 { 659 rex |= 4; 660 dst.idx -= 8; 661 } 662 if(src.idx >= 8) 663 { 664 rex |= 1; 665 src.idx -= 8; 666 } 667 emit_1ub(p, rex); 668 } 669 emit_op_modrm( p, 0x8b, 0x89, dst, src ); 670 } 671 672 void x86_mov16( struct x86_function *p, 673 struct x86_reg dst, 674 struct x86_reg src ) 675 { 676 DUMP_RR( dst, src ); 677 emit_1ub(p, 0x66); 678 emit_op_modrm( p, 0x8b, 0x89, dst, src ); 679 } 680 681 void x86_mov8( struct x86_function *p, 682 struct x86_reg dst, 683 struct x86_reg src ) 684 { 685 DUMP_RR( dst, src ); 686 emit_op_modrm( p, 0x8a, 0x88, dst, src ); 687 } 688 689 void x64_mov64( struct x86_function *p, 690 struct x86_reg dst, 691 struct x86_reg src ) 692 { 693 uint8_t rex = 0x48; 694 DUMP_RR( dst, src ); 695 assert(x86_target(p) != X86_32); 696 697 /* special hack for reading arguments until we support x86-64 registers everywhere */ 698 if(src.mod == mod_REG && dst.mod == mod_REG && (src.idx >= 8 || dst.idx >= 8)) 699 { 700 if(dst.idx >= 8) 701 { 702 rex |= 4; 703 dst.idx -= 8; 704 } 705 if(src.idx >= 8) 706 { 707 rex |= 1; 708 src.idx -= 8; 709 } 710 } 711 emit_1ub(p, rex); 712 emit_op_modrm( p, 0x8b, 0x89, dst, src ); 713 } 714 715 void x86_movzx8(struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 716 { 717 DUMP_RR( dst, src ); 718 emit_2ub(p, 0x0f, 0xb6); 719 emit_modrm(p, dst, src); 720 } 721 722 void x86_movzx16(struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 723 { 724 DUMP_RR( dst, src ); 725 emit_2ub(p, 0x0f, 0xb7); 726 emit_modrm(p, dst, src); 727 } 728 729 void x86_cmovcc( struct x86_function *p, 730 struct x86_reg dst, 731 struct x86_reg src, 732 enum x86_cc cc) 733 { 734 DUMP_RRI( dst, src, cc ); 735 emit_2ub( p, 0x0f, 0x40 + cc ); 736 emit_modrm( p, dst, src ); 737 } 738 739 void x86_xor( struct x86_function *p, 740 struct x86_reg dst, 741 struct x86_reg src ) 742 { 743 DUMP_RR( dst, src ); 744 emit_op_modrm( p, 0x33, 0x31, dst, src ); 745 } 746 747 void x86_cmp( struct x86_function *p, 748 struct x86_reg dst, 749 struct x86_reg src ) 750 { 751 DUMP_RR( dst, src ); 752 emit_op_modrm( p, 0x3b, 0x39, dst, src ); 753 } 754 755 void x86_lea( struct x86_function *p, 756 struct x86_reg dst, 757 struct x86_reg src ) 758 { 759 DUMP_RR( dst, src ); 760 emit_1ub(p, 0x8d); 761 emit_modrm( p, dst, src ); 762 } 763 764 void x86_test( struct x86_function *p, 765 struct x86_reg dst, 766 struct x86_reg src ) 767 { 768 DUMP_RR( dst, src ); 769 emit_1ub(p, 0x85); 770 emit_modrm( p, dst, src ); 771 } 772 773 void x86_add( struct x86_function *p, 774 struct x86_reg dst, 775 struct x86_reg src ) 776 { 777 DUMP_RR( dst, src ); 778 emit_op_modrm(p, 0x03, 0x01, dst, src ); 779 } 780 781 /* Calculate EAX * src, results in EDX:EAX. 782 */ 783 void x86_mul( struct x86_function *p, 784 struct x86_reg src ) 785 { 786 DUMP_R( src ); 787 emit_1ub(p, 0xf7); 788 emit_modrm_noreg(p, 4, src ); 789 } 790 791 792 void x86_imul( struct x86_function *p, 793 struct x86_reg dst, 794 struct x86_reg src ) 795 { 796 DUMP_RR( dst, src ); 797 emit_2ub(p, X86_TWOB, 0xAF); 798 emit_modrm(p, dst, src); 799 } 800 801 802 void x86_sub( struct x86_function *p, 803 struct x86_reg dst, 804 struct x86_reg src ) 805 { 806 DUMP_RR( dst, src ); 807 emit_op_modrm(p, 0x2b, 0x29, dst, src ); 808 } 809 810 void x86_or( struct x86_function *p, 811 struct x86_reg dst, 812 struct x86_reg src ) 813 { 814 DUMP_RR( dst, src ); 815 emit_op_modrm( p, 0x0b, 0x09, dst, src ); 816 } 817 818 void x86_and( struct x86_function *p, 819 struct x86_reg dst, 820 struct x86_reg src ) 821 { 822 DUMP_RR( dst, src ); 823 emit_op_modrm( p, 0x23, 0x21, dst, src ); 824 } 825 826 void x86_div( struct x86_function *p, 827 struct x86_reg src ) 828 { 829 assert(src.file == file_REG32 && src.mod == mod_REG); 830 emit_op_modrm(p, 0xf7, 0, x86_make_reg(file_REG32, 6), src); 831 } 832 833 void x86_bswap( struct x86_function *p, struct x86_reg reg ) 834 { 835 DUMP_R(reg); 836 assert(reg.file == file_REG32); 837 assert(reg.mod == mod_REG); 838 emit_2ub(p, 0x0f, 0xc8 + reg.idx); 839 } 840 841 void x86_shr_imm( struct x86_function *p, struct x86_reg reg, unsigned imm ) 842 { 843 DUMP_RI(reg, imm); 844 if(imm == 1) 845 { 846 emit_1ub(p, 0xd1); 847 emit_modrm_noreg(p, 5, reg); 848 } 849 else 850 { 851 emit_1ub(p, 0xc1); 852 emit_modrm_noreg(p, 5, reg); 853 emit_1ub(p, imm); 854 } 855 } 856 857 void x86_sar_imm( struct x86_function *p, struct x86_reg reg, unsigned imm ) 858 { 859 DUMP_RI(reg, imm); 860 if(imm == 1) 861 { 862 emit_1ub(p, 0xd1); 863 emit_modrm_noreg(p, 7, reg); 864 } 865 else 866 { 867 emit_1ub(p, 0xc1); 868 emit_modrm_noreg(p, 7, reg); 869 emit_1ub(p, imm); 870 } 871 } 872 873 void x86_shl_imm( struct x86_function *p, struct x86_reg reg, unsigned imm ) 874 { 875 DUMP_RI(reg, imm); 876 if(imm == 1) 877 { 878 emit_1ub(p, 0xd1); 879 emit_modrm_noreg(p, 4, reg); 880 } 881 else 882 { 883 emit_1ub(p, 0xc1); 884 emit_modrm_noreg(p, 4, reg); 885 emit_1ub(p, imm); 886 } 887 } 888 889 890 /*********************************************************************** 891 * SSE instructions 892 */ 893 894 void sse_prefetchnta( struct x86_function *p, struct x86_reg ptr) 895 { 896 DUMP_R( ptr ); 897 assert(ptr.mod != mod_REG); 898 emit_2ub(p, 0x0f, 0x18); 899 emit_modrm_noreg(p, 0, ptr); 900 } 901 902 void sse_prefetch0( struct x86_function *p, struct x86_reg ptr) 903 { 904 DUMP_R( ptr ); 905 assert(ptr.mod != mod_REG); 906 emit_2ub(p, 0x0f, 0x18); 907 emit_modrm_noreg(p, 1, ptr); 908 } 909 910 void sse_prefetch1( struct x86_function *p, struct x86_reg ptr) 911 { 912 DUMP_R( ptr ); 913 assert(ptr.mod != mod_REG); 914 emit_2ub(p, 0x0f, 0x18); 915 emit_modrm_noreg(p, 2, ptr); 916 } 917 918 void sse_movntps( struct x86_function *p, 919 struct x86_reg dst, 920 struct x86_reg src) 921 { 922 DUMP_RR( dst, src ); 923 924 assert(dst.mod != mod_REG); 925 assert(src.mod == mod_REG); 926 emit_2ub(p, 0x0f, 0x2b); 927 emit_modrm(p, src, dst); 928 } 929 930 931 932 933 void sse_movss( struct x86_function *p, 934 struct x86_reg dst, 935 struct x86_reg src ) 936 { 937 DUMP_RR( dst, src ); 938 emit_2ub(p, 0xF3, X86_TWOB); 939 emit_op_modrm( p, 0x10, 0x11, dst, src ); 940 } 941 942 void sse_movaps( struct x86_function *p, 943 struct x86_reg dst, 944 struct x86_reg src ) 945 { 946 DUMP_RR( dst, src ); 947 emit_1ub(p, X86_TWOB); 948 emit_op_modrm( p, 0x28, 0x29, dst, src ); 949 } 950 951 void sse_movups( struct x86_function *p, 952 struct x86_reg dst, 953 struct x86_reg src ) 954 { 955 DUMP_RR( dst, src ); 956 emit_1ub(p, X86_TWOB); 957 emit_op_modrm( p, 0x10, 0x11, dst, src ); 958 } 959 960 void sse_movhps( struct x86_function *p, 961 struct x86_reg dst, 962 struct x86_reg src ) 963 { 964 DUMP_RR( dst, src ); 965 assert(dst.mod != mod_REG || src.mod != mod_REG); 966 emit_1ub(p, X86_TWOB); 967 emit_op_modrm( p, 0x16, 0x17, dst, src ); /* cf movlhps */ 968 } 969 970 void sse_movlps( struct x86_function *p, 971 struct x86_reg dst, 972 struct x86_reg src ) 973 { 974 DUMP_RR( dst, src ); 975 assert(dst.mod != mod_REG || src.mod != mod_REG); 976 emit_1ub(p, X86_TWOB); 977 emit_op_modrm( p, 0x12, 0x13, dst, src ); /* cf movhlps */ 978 } 979 980 void sse_maxps( struct x86_function *p, 981 struct x86_reg dst, 982 struct x86_reg src ) 983 { 984 DUMP_RR( dst, src ); 985 emit_2ub(p, X86_TWOB, 0x5F); 986 emit_modrm( p, dst, src ); 987 } 988 989 void sse_maxss( struct x86_function *p, 990 struct x86_reg dst, 991 struct x86_reg src ) 992 { 993 DUMP_RR( dst, src ); 994 emit_3ub(p, 0xF3, X86_TWOB, 0x5F); 995 emit_modrm( p, dst, src ); 996 } 997 998 void sse_divss( struct x86_function *p, 999 struct x86_reg dst, 1000 struct x86_reg src ) 1001 { 1002 DUMP_RR( dst, src ); 1003 emit_3ub(p, 0xF3, X86_TWOB, 0x5E); 1004 emit_modrm( p, dst, src ); 1005 } 1006 1007 void sse_minps( struct x86_function *p, 1008 struct x86_reg dst, 1009 struct x86_reg src ) 1010 { 1011 DUMP_RR( dst, src ); 1012 emit_2ub(p, X86_TWOB, 0x5D); 1013 emit_modrm( p, dst, src ); 1014 } 1015 1016 void sse_subps( struct x86_function *p, 1017 struct x86_reg dst, 1018 struct x86_reg src ) 1019 { 1020 DUMP_RR( dst, src ); 1021 emit_2ub(p, X86_TWOB, 0x5C); 1022 emit_modrm( p, dst, src ); 1023 } 1024 1025 void sse_mulps( struct x86_function *p, 1026 struct x86_reg dst, 1027 struct x86_reg src ) 1028 { 1029 DUMP_RR( dst, src ); 1030 emit_2ub(p, X86_TWOB, 0x59); 1031 emit_modrm( p, dst, src ); 1032 } 1033 1034 void sse_mulss( struct x86_function *p, 1035 struct x86_reg dst, 1036 struct x86_reg src ) 1037 { 1038 DUMP_RR( dst, src ); 1039 emit_3ub(p, 0xF3, X86_TWOB, 0x59); 1040 emit_modrm( p, dst, src ); 1041 } 1042 1043 void sse_addps( struct x86_function *p, 1044 struct x86_reg dst, 1045 struct x86_reg src ) 1046 { 1047 DUMP_RR( dst, src ); 1048 emit_2ub(p, X86_TWOB, 0x58); 1049 emit_modrm( p, dst, src ); 1050 } 1051 1052 void sse_addss( struct x86_function *p, 1053 struct x86_reg dst, 1054 struct x86_reg src ) 1055 { 1056 DUMP_RR( dst, src ); 1057 emit_3ub(p, 0xF3, X86_TWOB, 0x58); 1058 emit_modrm( p, dst, src ); 1059 } 1060 1061 void sse_andnps( struct x86_function *p, 1062 struct x86_reg dst, 1063 struct x86_reg src ) 1064 { 1065 DUMP_RR( dst, src ); 1066 emit_2ub(p, X86_TWOB, 0x55); 1067 emit_modrm( p, dst, src ); 1068 } 1069 1070 void sse_andps( struct x86_function *p, 1071 struct x86_reg dst, 1072 struct x86_reg src ) 1073 { 1074 DUMP_RR( dst, src ); 1075 emit_2ub(p, X86_TWOB, 0x54); 1076 emit_modrm( p, dst, src ); 1077 } 1078 1079 void sse_rsqrtps( struct x86_function *p, 1080 struct x86_reg dst, 1081 struct x86_reg src ) 1082 { 1083 DUMP_RR( dst, src ); 1084 emit_2ub(p, X86_TWOB, 0x52); 1085 emit_modrm( p, dst, src ); 1086 } 1087 1088 void sse_rsqrtss( struct x86_function *p, 1089 struct x86_reg dst, 1090 struct x86_reg src ) 1091 { 1092 DUMP_RR( dst, src ); 1093 emit_3ub(p, 0xF3, X86_TWOB, 0x52); 1094 emit_modrm( p, dst, src ); 1095 1096 } 1097 1098 void sse_movhlps( struct x86_function *p, 1099 struct x86_reg dst, 1100 struct x86_reg src ) 1101 { 1102 DUMP_RR( dst, src ); 1103 assert(dst.mod == mod_REG && src.mod == mod_REG); 1104 emit_2ub(p, X86_TWOB, 0x12); 1105 emit_modrm( p, dst, src ); 1106 } 1107 1108 void sse_movlhps( struct x86_function *p, 1109 struct x86_reg dst, 1110 struct x86_reg src ) 1111 { 1112 DUMP_RR( dst, src ); 1113 assert(dst.mod == mod_REG && src.mod == mod_REG); 1114 emit_2ub(p, X86_TWOB, 0x16); 1115 emit_modrm( p, dst, src ); 1116 } 1117 1118 void sse_orps( struct x86_function *p, 1119 struct x86_reg dst, 1120 struct x86_reg src ) 1121 { 1122 DUMP_RR( dst, src ); 1123 emit_2ub(p, X86_TWOB, 0x56); 1124 emit_modrm( p, dst, src ); 1125 } 1126 1127 void sse_xorps( struct x86_function *p, 1128 struct x86_reg dst, 1129 struct x86_reg src ) 1130 { 1131 DUMP_RR( dst, src ); 1132 emit_2ub(p, X86_TWOB, 0x57); 1133 emit_modrm( p, dst, src ); 1134 } 1135 1136 void sse_cvtps2pi( struct x86_function *p, 1137 struct x86_reg dst, 1138 struct x86_reg src ) 1139 { 1140 DUMP_RR( dst, src ); 1141 assert(dst.file == file_MMX && 1142 (src.file == file_XMM || src.mod != mod_REG)); 1143 1144 p->need_emms = 1; 1145 1146 emit_2ub(p, X86_TWOB, 0x2d); 1147 emit_modrm( p, dst, src ); 1148 } 1149 1150 void sse2_cvtdq2ps( struct x86_function *p, 1151 struct x86_reg dst, 1152 struct x86_reg src ) 1153 { 1154 DUMP_RR( dst, src ); 1155 emit_2ub(p, X86_TWOB, 0x5b); 1156 emit_modrm( p, dst, src ); 1157 } 1158 1159 1160 /* Shufps can also be used to implement a reduced swizzle when dest == 1161 * arg0. 1162 */ 1163 void sse_shufps( struct x86_function *p, 1164 struct x86_reg dst, 1165 struct x86_reg src, 1166 unsigned char shuf) 1167 { 1168 DUMP_RRI( dst, src, shuf ); 1169 emit_2ub(p, X86_TWOB, 0xC6); 1170 emit_modrm(p, dst, src); 1171 emit_1ub(p, shuf); 1172 } 1173 1174 void sse_unpckhps( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1175 { 1176 DUMP_RR( dst, src ); 1177 emit_2ub( p, X86_TWOB, 0x15 ); 1178 emit_modrm( p, dst, src ); 1179 } 1180 1181 void sse_unpcklps( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1182 { 1183 DUMP_RR( dst, src ); 1184 emit_2ub( p, X86_TWOB, 0x14 ); 1185 emit_modrm( p, dst, src ); 1186 } 1187 1188 void sse_cmpps( struct x86_function *p, 1189 struct x86_reg dst, 1190 struct x86_reg src, 1191 enum sse_cc cc) 1192 { 1193 DUMP_RRI( dst, src, cc ); 1194 emit_2ub(p, X86_TWOB, 0xC2); 1195 emit_modrm(p, dst, src); 1196 emit_1ub(p, cc); 1197 } 1198 1199 void sse_pmovmskb( struct x86_function *p, 1200 struct x86_reg dst, 1201 struct x86_reg src) 1202 { 1203 DUMP_RR( dst, src ); 1204 emit_3ub(p, 0x66, X86_TWOB, 0xD7); 1205 emit_modrm(p, dst, src); 1206 } 1207 1208 void sse_movmskps( struct x86_function *p, 1209 struct x86_reg dst, 1210 struct x86_reg src) 1211 { 1212 DUMP_RR( dst, src ); 1213 emit_2ub(p, X86_TWOB, 0x50); 1214 emit_modrm(p, dst, src); 1215 } 1216 1217 /*********************************************************************** 1218 * SSE2 instructions 1219 */ 1220 1221 void sse2_movd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1222 { 1223 DUMP_RR(dst, src); 1224 emit_2ub(p, 0x66, 0x0f); 1225 if(dst.mod == mod_REG && dst.file == file_REG32) 1226 { 1227 emit_1ub(p, 0x7e); 1228 emit_modrm(p, src, dst); 1229 } 1230 else 1231 { 1232 emit_op_modrm(p, 0x6e, 0x7e, dst, src); 1233 } 1234 } 1235 1236 void sse2_movq( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1237 { 1238 DUMP_RR(dst, src); 1239 switch (dst.mod) { 1240 case mod_REG: 1241 emit_3ub(p, 0xf3, 0x0f, 0x7e); 1242 emit_modrm(p, dst, src); 1243 break; 1244 case mod_INDIRECT: 1245 case mod_DISP32: 1246 case mod_DISP8: 1247 assert(src.mod == mod_REG); 1248 emit_3ub(p, 0x66, 0x0f, 0xd6); 1249 emit_modrm(p, src, dst); 1250 break; 1251 default: 1252 assert(0); 1253 break; 1254 } 1255 } 1256 1257 void sse2_movdqu( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1258 { 1259 DUMP_RR(dst, src); 1260 emit_2ub(p, 0xf3, 0x0f); 1261 emit_op_modrm(p, 0x6f, 0x7f, dst, src); 1262 } 1263 1264 void sse2_movdqa( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1265 { 1266 DUMP_RR(dst, src); 1267 emit_2ub(p, 0x66, 0x0f); 1268 emit_op_modrm(p, 0x6f, 0x7f, dst, src); 1269 } 1270 1271 void sse2_movsd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1272 { 1273 DUMP_RR(dst, src); 1274 emit_2ub(p, 0xf2, 0x0f); 1275 emit_op_modrm(p, 0x10, 0x11, dst, src); 1276 } 1277 1278 void sse2_movupd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1279 { 1280 DUMP_RR(dst, src); 1281 emit_2ub(p, 0x66, 0x0f); 1282 emit_op_modrm(p, 0x10, 0x11, dst, src); 1283 } 1284 1285 void sse2_movapd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1286 { 1287 DUMP_RR(dst, src); 1288 emit_2ub(p, 0x66, 0x0f); 1289 emit_op_modrm(p, 0x28, 0x29, dst, src); 1290 } 1291 1292 /** 1293 * Perform a reduced swizzle: 1294 */ 1295 void sse2_pshufd( struct x86_function *p, 1296 struct x86_reg dst, 1297 struct x86_reg src, 1298 unsigned char shuf) 1299 { 1300 DUMP_RRI( dst, src, shuf ); 1301 emit_3ub(p, 0x66, X86_TWOB, 0x70); 1302 emit_modrm(p, dst, src); 1303 emit_1ub(p, shuf); 1304 } 1305 1306 void sse2_pshuflw( struct x86_function *p, 1307 struct x86_reg dst, 1308 struct x86_reg src, 1309 unsigned char shuf) 1310 { 1311 DUMP_RRI( dst, src, shuf ); 1312 emit_3ub(p, 0xf2, X86_TWOB, 0x70); 1313 emit_modrm(p, dst, src); 1314 emit_1ub(p, shuf); 1315 } 1316 1317 void sse2_pshufhw( struct x86_function *p, 1318 struct x86_reg dst, 1319 struct x86_reg src, 1320 unsigned char shuf) 1321 { 1322 DUMP_RRI( dst, src, shuf ); 1323 emit_3ub(p, 0xf3, X86_TWOB, 0x70); 1324 emit_modrm(p, dst, src); 1325 emit_1ub(p, shuf); 1326 } 1327 1328 void sse2_cvttps2dq( struct x86_function *p, 1329 struct x86_reg dst, 1330 struct x86_reg src ) 1331 { 1332 DUMP_RR( dst, src ); 1333 emit_3ub( p, 0xF3, X86_TWOB, 0x5B ); 1334 emit_modrm( p, dst, src ); 1335 } 1336 1337 void sse2_cvtps2dq( struct x86_function *p, 1338 struct x86_reg dst, 1339 struct x86_reg src ) 1340 { 1341 DUMP_RR( dst, src ); 1342 emit_3ub(p, 0x66, X86_TWOB, 0x5B); 1343 emit_modrm( p, dst, src ); 1344 } 1345 1346 void sse2_cvtsd2ss( struct x86_function *p, 1347 struct x86_reg dst, 1348 struct x86_reg src ) 1349 { 1350 DUMP_RR( dst, src ); 1351 emit_3ub(p, 0xf2, 0x0f, 0x5a); 1352 emit_modrm( p, dst, src ); 1353 } 1354 1355 void sse2_cvtpd2ps( struct x86_function *p, 1356 struct x86_reg dst, 1357 struct x86_reg src ) 1358 { 1359 DUMP_RR( dst, src ); 1360 emit_3ub(p, 0x66, 0x0f, 0x5a); 1361 emit_modrm( p, dst, src ); 1362 } 1363 1364 void sse2_packssdw( struct x86_function *p, 1365 struct x86_reg dst, 1366 struct x86_reg src ) 1367 { 1368 DUMP_RR( dst, src ); 1369 emit_3ub(p, 0x66, X86_TWOB, 0x6B); 1370 emit_modrm( p, dst, src ); 1371 } 1372 1373 void sse2_packsswb( struct x86_function *p, 1374 struct x86_reg dst, 1375 struct x86_reg src ) 1376 { 1377 DUMP_RR( dst, src ); 1378 emit_3ub(p, 0x66, X86_TWOB, 0x63); 1379 emit_modrm( p, dst, src ); 1380 } 1381 1382 void sse2_packuswb( struct x86_function *p, 1383 struct x86_reg dst, 1384 struct x86_reg src ) 1385 { 1386 DUMP_RR( dst, src ); 1387 emit_3ub(p, 0x66, X86_TWOB, 0x67); 1388 emit_modrm( p, dst, src ); 1389 } 1390 1391 void sse2_punpcklbw( struct x86_function *p, 1392 struct x86_reg dst, 1393 struct x86_reg src ) 1394 { 1395 DUMP_RR( dst, src ); 1396 emit_3ub(p, 0x66, X86_TWOB, 0x60); 1397 emit_modrm( p, dst, src ); 1398 } 1399 1400 void sse2_punpcklwd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1401 { 1402 DUMP_RR( dst, src ); 1403 emit_3ub(p, 0x66, 0x0f, 0x61); 1404 emit_modrm( p, dst, src ); 1405 } 1406 1407 void sse2_punpckldq( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1408 { 1409 DUMP_RR( dst, src ); 1410 emit_3ub(p, 0x66, 0x0f, 0x62); 1411 emit_modrm( p, dst, src ); 1412 } 1413 1414 void sse2_punpcklqdq( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1415 { 1416 DUMP_RR( dst, src ); 1417 emit_3ub(p, 0x66, 0x0f, 0x6c); 1418 emit_modrm( p, dst, src ); 1419 } 1420 1421 void sse2_psllw_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1422 { 1423 DUMP_RI(dst, imm); 1424 emit_3ub(p, 0x66, 0x0f, 0x71); 1425 emit_modrm_noreg(p, 6, dst); 1426 emit_1ub(p, imm); 1427 } 1428 1429 void sse2_pslld_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1430 { 1431 DUMP_RI(dst, imm); 1432 emit_3ub(p, 0x66, 0x0f, 0x72); 1433 emit_modrm_noreg(p, 6, dst); 1434 emit_1ub(p, imm); 1435 } 1436 1437 void sse2_psllq_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1438 { 1439 DUMP_RI(dst, imm); 1440 emit_3ub(p, 0x66, 0x0f, 0x73); 1441 emit_modrm_noreg(p, 6, dst); 1442 emit_1ub(p, imm); 1443 } 1444 1445 void sse2_psrlw_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1446 { 1447 DUMP_RI(dst, imm); 1448 emit_3ub(p, 0x66, 0x0f, 0x71); 1449 emit_modrm_noreg(p, 2, dst); 1450 emit_1ub(p, imm); 1451 } 1452 1453 void sse2_psrld_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1454 { 1455 DUMP_RI(dst, imm); 1456 emit_3ub(p, 0x66, 0x0f, 0x72); 1457 emit_modrm_noreg(p, 2, dst); 1458 emit_1ub(p, imm); 1459 } 1460 1461 void sse2_psrlq_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1462 { 1463 DUMP_RI(dst, imm); 1464 emit_3ub(p, 0x66, 0x0f, 0x73); 1465 emit_modrm_noreg(p, 2, dst); 1466 emit_1ub(p, imm); 1467 } 1468 1469 void sse2_psraw_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1470 { 1471 DUMP_RI(dst, imm); 1472 emit_3ub(p, 0x66, 0x0f, 0x71); 1473 emit_modrm_noreg(p, 4, dst); 1474 emit_1ub(p, imm); 1475 } 1476 1477 void sse2_psrad_imm( struct x86_function *p, struct x86_reg dst, unsigned imm ) 1478 { 1479 DUMP_RI(dst, imm); 1480 emit_3ub(p, 0x66, 0x0f, 0x72); 1481 emit_modrm_noreg(p, 4, dst); 1482 emit_1ub(p, imm); 1483 } 1484 1485 void sse2_por( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1486 { 1487 DUMP_RR(dst, src); 1488 emit_3ub(p, 0x66, 0x0f, 0xeb); 1489 emit_modrm(p, dst, src); 1490 } 1491 1492 void sse2_rcpps( struct x86_function *p, 1493 struct x86_reg dst, 1494 struct x86_reg src ) 1495 { 1496 DUMP_RR( dst, src ); 1497 emit_2ub(p, X86_TWOB, 0x53); 1498 emit_modrm( p, dst, src ); 1499 } 1500 1501 void sse2_rcpss( struct x86_function *p, 1502 struct x86_reg dst, 1503 struct x86_reg src ) 1504 { 1505 DUMP_RR( dst, src ); 1506 emit_3ub(p, 0xF3, X86_TWOB, 0x53); 1507 emit_modrm( p, dst, src ); 1508 } 1509 1510 /*********************************************************************** 1511 * x87 instructions 1512 */ 1513 static void note_x87_pop( struct x86_function *p ) 1514 { 1515 p->x87_stack--; 1516 assert(p->x87_stack >= 0); 1517 } 1518 1519 static void note_x87_push( struct x86_function *p ) 1520 { 1521 p->x87_stack++; 1522 assert(p->x87_stack <= 7); 1523 } 1524 1525 void x87_assert_stack_empty( struct x86_function *p ) 1526 { 1527 assert (p->x87_stack == 0); 1528 } 1529 1530 1531 void x87_fist( struct x86_function *p, struct x86_reg dst ) 1532 { 1533 DUMP_R( dst ); 1534 emit_1ub(p, 0xdb); 1535 emit_modrm_noreg(p, 2, dst); 1536 } 1537 1538 void x87_fistp( struct x86_function *p, struct x86_reg dst ) 1539 { 1540 DUMP_R( dst ); 1541 emit_1ub(p, 0xdb); 1542 emit_modrm_noreg(p, 3, dst); 1543 note_x87_pop(p); 1544 } 1545 1546 void x87_fild( struct x86_function *p, struct x86_reg arg ) 1547 { 1548 DUMP_R( arg ); 1549 emit_1ub(p, 0xdf); 1550 emit_modrm_noreg(p, 0, arg); 1551 note_x87_push(p); 1552 } 1553 1554 void x87_fldz( struct x86_function *p ) 1555 { 1556 DUMP(); 1557 emit_2ub(p, 0xd9, 0xee); 1558 note_x87_push(p); 1559 } 1560 1561 1562 void x87_fldcw( struct x86_function *p, struct x86_reg arg ) 1563 { 1564 DUMP_R( arg ); 1565 assert(arg.file == file_REG32); 1566 assert(arg.mod != mod_REG); 1567 emit_1ub(p, 0xd9); 1568 emit_modrm_noreg(p, 5, arg); 1569 } 1570 1571 void x87_fld1( struct x86_function *p ) 1572 { 1573 DUMP(); 1574 emit_2ub(p, 0xd9, 0xe8); 1575 note_x87_push(p); 1576 } 1577 1578 void x87_fldl2e( struct x86_function *p ) 1579 { 1580 DUMP(); 1581 emit_2ub(p, 0xd9, 0xea); 1582 note_x87_push(p); 1583 } 1584 1585 void x87_fldln2( struct x86_function *p ) 1586 { 1587 DUMP(); 1588 emit_2ub(p, 0xd9, 0xed); 1589 note_x87_push(p); 1590 } 1591 1592 void x87_fwait( struct x86_function *p ) 1593 { 1594 DUMP(); 1595 emit_1ub(p, 0x9b); 1596 } 1597 1598 void x87_fnclex( struct x86_function *p ) 1599 { 1600 DUMP(); 1601 emit_2ub(p, 0xdb, 0xe2); 1602 } 1603 1604 void x87_fclex( struct x86_function *p ) 1605 { 1606 x87_fwait(p); 1607 x87_fnclex(p); 1608 } 1609 1610 void x87_fcmovb( struct x86_function *p, struct x86_reg arg ) 1611 { 1612 DUMP_R( arg ); 1613 assert(arg.file == file_x87); 1614 emit_2ub(p, 0xda, 0xc0+arg.idx); 1615 } 1616 1617 void x87_fcmove( struct x86_function *p, struct x86_reg arg ) 1618 { 1619 DUMP_R( arg ); 1620 assert(arg.file == file_x87); 1621 emit_2ub(p, 0xda, 0xc8+arg.idx); 1622 } 1623 1624 void x87_fcmovbe( struct x86_function *p, struct x86_reg arg ) 1625 { 1626 DUMP_R( arg ); 1627 assert(arg.file == file_x87); 1628 emit_2ub(p, 0xda, 0xd0+arg.idx); 1629 } 1630 1631 void x87_fcmovnb( struct x86_function *p, struct x86_reg arg ) 1632 { 1633 DUMP_R( arg ); 1634 assert(arg.file == file_x87); 1635 emit_2ub(p, 0xdb, 0xc0+arg.idx); 1636 } 1637 1638 void x87_fcmovne( struct x86_function *p, struct x86_reg arg ) 1639 { 1640 DUMP_R( arg ); 1641 assert(arg.file == file_x87); 1642 emit_2ub(p, 0xdb, 0xc8+arg.idx); 1643 } 1644 1645 void x87_fcmovnbe( struct x86_function *p, struct x86_reg arg ) 1646 { 1647 DUMP_R( arg ); 1648 assert(arg.file == file_x87); 1649 emit_2ub(p, 0xdb, 0xd0+arg.idx); 1650 } 1651 1652 1653 1654 static void x87_arith_op( struct x86_function *p, struct x86_reg dst, struct x86_reg arg, 1655 unsigned char dst0ub0, 1656 unsigned char dst0ub1, 1657 unsigned char arg0ub0, 1658 unsigned char arg0ub1, 1659 unsigned char argmem_noreg) 1660 { 1661 assert(dst.file == file_x87); 1662 1663 if (arg.file == file_x87) { 1664 if (dst.idx == 0) 1665 emit_2ub(p, dst0ub0, dst0ub1+arg.idx); 1666 else if (arg.idx == 0) 1667 emit_2ub(p, arg0ub0, arg0ub1+arg.idx); 1668 else 1669 assert(0); 1670 } 1671 else if (dst.idx == 0) { 1672 assert(arg.file == file_REG32); 1673 emit_1ub(p, 0xd8); 1674 emit_modrm_noreg(p, argmem_noreg, arg); 1675 } 1676 else 1677 assert(0); 1678 } 1679 1680 void x87_fmul( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1681 { 1682 DUMP_RR( dst, src ); 1683 x87_arith_op(p, dst, src, 1684 0xd8, 0xc8, 1685 0xdc, 0xc8, 1686 4); 1687 } 1688 1689 void x87_fsub( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1690 { 1691 DUMP_RR( dst, src ); 1692 x87_arith_op(p, dst, src, 1693 0xd8, 0xe0, 1694 0xdc, 0xe8, 1695 4); 1696 } 1697 1698 void x87_fsubr( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1699 { 1700 DUMP_RR( dst, src ); 1701 x87_arith_op(p, dst, src, 1702 0xd8, 0xe8, 1703 0xdc, 0xe0, 1704 5); 1705 } 1706 1707 void x87_fadd( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1708 { 1709 DUMP_RR( dst, src ); 1710 x87_arith_op(p, dst, src, 1711 0xd8, 0xc0, 1712 0xdc, 0xc0, 1713 0); 1714 } 1715 1716 void x87_fdiv( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1717 { 1718 DUMP_RR( dst, src ); 1719 x87_arith_op(p, dst, src, 1720 0xd8, 0xf0, 1721 0xdc, 0xf8, 1722 6); 1723 } 1724 1725 void x87_fdivr( struct x86_function *p, struct x86_reg dst, struct x86_reg src ) 1726 { 1727 DUMP_RR( dst, src ); 1728 x87_arith_op(p, dst, src, 1729 0xd8, 0xf8, 1730 0xdc, 0xf0, 1731 7); 1732 } 1733 1734 void x87_fmulp( struct x86_function *p, struct x86_reg dst ) 1735 { 1736 DUMP_R( dst ); 1737 assert(dst.file == file_x87); 1738 assert(dst.idx >= 1); 1739 emit_2ub(p, 0xde, 0xc8+dst.idx); 1740 note_x87_pop(p); 1741 } 1742 1743 void x87_fsubp( struct x86_function *p, struct x86_reg dst ) 1744 { 1745 DUMP_R( dst ); 1746 assert(dst.file == file_x87); 1747 assert(dst.idx >= 1); 1748 emit_2ub(p, 0xde, 0xe8+dst.idx); 1749 note_x87_pop(p); 1750 } 1751 1752 void x87_fsubrp( struct x86_function *p, struct x86_reg dst ) 1753 { 1754 DUMP_R( dst ); 1755 assert(dst.file == file_x87); 1756 assert(dst.idx >= 1); 1757 emit_2ub(p, 0xde, 0xe0+dst.idx); 1758 note_x87_pop(p); 1759 } 1760 1761 void x87_faddp( struct x86_function *p, struct x86_reg dst ) 1762 { 1763 DUMP_R( dst ); 1764 assert(dst.file == file_x87); 1765 assert(dst.idx >= 1); 1766 emit_2ub(p, 0xde, 0xc0+dst.idx); 1767 note_x87_pop(p); 1768 } 1769 1770 void x87_fdivp( struct x86_function *p, struct x86_reg dst ) 1771 { 1772 DUMP_R( dst ); 1773 assert(dst.file == file_x87); 1774 assert(dst.idx >= 1); 1775 emit_2ub(p, 0xde, 0xf8+dst.idx); 1776 note_x87_pop(p); 1777 } 1778 1779 void x87_fdivrp( struct x86_function *p, struct x86_reg dst ) 1780 { 1781 DUMP_R( dst ); 1782 assert(dst.file == file_x87); 1783 assert(dst.idx >= 1); 1784 emit_2ub(p, 0xde, 0xf0+dst.idx); 1785 note_x87_pop(p); 1786 } 1787 1788 void x87_ftst( struct x86_function *p ) 1789 { 1790 DUMP(); 1791 emit_2ub(p, 0xd9, 0xe4); 1792 } 1793 1794 void x87_fucom( struct x86_function *p, struct x86_reg arg ) 1795 { 1796 DUMP_R( arg ); 1797 assert(arg.file == file_x87); 1798 emit_2ub(p, 0xdd, 0xe0+arg.idx); 1799 } 1800 1801 void x87_fucomp( struct x86_function *p, struct x86_reg arg ) 1802 { 1803 DUMP_R( arg ); 1804 assert(arg.file == file_x87); 1805 emit_2ub(p, 0xdd, 0xe8+arg.idx); 1806 note_x87_pop(p); 1807 } 1808 1809 void x87_fucompp( struct x86_function *p ) 1810 { 1811 DUMP(); 1812 emit_2ub(p, 0xda, 0xe9); 1813 note_x87_pop(p); /* pop twice */ 1814 note_x87_pop(p); /* pop twice */ 1815 } 1816 1817 void x87_fxch( struct x86_function *p, struct x86_reg arg ) 1818 { 1819 DUMP_R( arg ); 1820 assert(arg.file == file_x87); 1821 emit_2ub(p, 0xd9, 0xc8+arg.idx); 1822 } 1823 1824 void x87_fabs( struct x86_function *p ) 1825 { 1826 DUMP(); 1827 emit_2ub(p, 0xd9, 0xe1); 1828 } 1829 1830 void x87_fchs( struct x86_function *p ) 1831 { 1832 DUMP(); 1833 emit_2ub(p, 0xd9, 0xe0); 1834 } 1835 1836 void x87_fcos( struct x86_function *p ) 1837 { 1838 DUMP(); 1839 emit_2ub(p, 0xd9, 0xff); 1840 } 1841 1842 1843 void x87_fprndint( struct x86_function *p ) 1844 { 1845 DUMP(); 1846 emit_2ub(p, 0xd9, 0xfc); 1847 } 1848 1849 void x87_fscale( struct x86_function *p ) 1850 { 1851 DUMP(); 1852 emit_2ub(p, 0xd9, 0xfd); 1853 } 1854 1855 void x87_fsin( struct x86_function *p ) 1856 { 1857 DUMP(); 1858 emit_2ub(p, 0xd9, 0xfe); 1859 } 1860 1861 void x87_fsincos( struct x86_function *p ) 1862 { 1863 DUMP(); 1864 emit_2ub(p, 0xd9, 0xfb); 1865 } 1866 1867 void x87_fsqrt( struct x86_function *p ) 1868 { 1869 DUMP(); 1870 emit_2ub(p, 0xd9, 0xfa); 1871 } 1872 1873 void x87_fxtract( struct x86_function *p ) 1874 { 1875 DUMP(); 1876 emit_2ub(p, 0xd9, 0xf4); 1877 } 1878 1879 /* st0 = (2^st0)-1 1880 * 1881 * Restrictions: -1.0 <= st0 <= 1.0 1882 */ 1883 void x87_f2xm1( struct x86_function *p ) 1884 { 1885 DUMP(); 1886 emit_2ub(p, 0xd9, 0xf0); 1887 } 1888 1889 /* st1 = st1 * log2(st0); 1890 * pop_stack; 1891 */ 1892 void x87_fyl2x( struct x86_function *p ) 1893 { 1894 DUMP(); 1895 emit_2ub(p, 0xd9, 0xf1); 1896 note_x87_pop(p); 1897 } 1898 1899 /* st1 = st1 * log2(st0 + 1.0); 1900 * pop_stack; 1901 * 1902 * A fast operation, with restrictions: -.29 < st0 < .29 1903 */ 1904 void x87_fyl2xp1( struct x86_function *p ) 1905 { 1906 DUMP(); 1907 emit_2ub(p, 0xd9, 0xf9); 1908 note_x87_pop(p); 1909 } 1910 1911 1912 void x87_fld( struct x86_function *p, struct x86_reg arg ) 1913 { 1914 DUMP_R( arg ); 1915 if (arg.file == file_x87) 1916 emit_2ub(p, 0xd9, 0xc0 + arg.idx); 1917 else { 1918 emit_1ub(p, 0xd9); 1919 emit_modrm_noreg(p, 0, arg); 1920 } 1921 note_x87_push(p); 1922 } 1923 1924 void x87_fst( struct x86_function *p, struct x86_reg dst ) 1925 { 1926 DUMP_R( dst ); 1927 if (dst.file == file_x87) 1928 emit_2ub(p, 0xdd, 0xd0 + dst.idx); 1929 else { 1930 emit_1ub(p, 0xd9); 1931 emit_modrm_noreg(p, 2, dst); 1932 } 1933 } 1934 1935 void x87_fstp( struct x86_function *p, struct x86_reg dst ) 1936 { 1937 DUMP_R( dst ); 1938 if (dst.file == file_x87) 1939 emit_2ub(p, 0xdd, 0xd8 + dst.idx); 1940 else { 1941 emit_1ub(p, 0xd9); 1942 emit_modrm_noreg(p, 3, dst); 1943 } 1944 note_x87_pop(p); 1945 } 1946 1947 void x87_fpop( struct x86_function *p ) 1948 { 1949 x87_fstp( p, x86_make_reg( file_x87, 0 )); 1950 } 1951 1952 1953 void x87_fcom( struct x86_function *p, struct x86_reg dst ) 1954 { 1955 DUMP_R( dst ); 1956 if (dst.file == file_x87) 1957 emit_2ub(p, 0xd8, 0xd0 + dst.idx); 1958 else { 1959 emit_1ub(p, 0xd8); 1960 emit_modrm_noreg(p, 2, dst); 1961 } 1962 } 1963 1964 1965 void x87_fcomp( struct x86_function *p, struct x86_reg dst ) 1966 { 1967 DUMP_R( dst ); 1968 if (dst.file == file_x87) 1969 emit_2ub(p, 0xd8, 0xd8 + dst.idx); 1970 else { 1971 emit_1ub(p, 0xd8); 1972 emit_modrm_noreg(p, 3, dst); 1973 } 1974 note_x87_pop(p); 1975 } 1976 1977 void x87_fcomi( struct x86_function *p, struct x86_reg arg ) 1978 { 1979 DUMP_R( arg ); 1980 emit_2ub(p, 0xdb, 0xf0+arg.idx); 1981 } 1982 1983 void x87_fcomip( struct x86_function *p, struct x86_reg arg ) 1984 { 1985 DUMP_R( arg ); 1986 emit_2ub(p, 0xdb, 0xf0+arg.idx); 1987 note_x87_pop(p); 1988 } 1989 1990 1991 void x87_fnstsw( struct x86_function *p, struct x86_reg dst ) 1992 { 1993 DUMP_R( dst ); 1994 assert(dst.file == file_REG32); 1995 1996 if (dst.idx == reg_AX && 1997 dst.mod == mod_REG) 1998 emit_2ub(p, 0xdf, 0xe0); 1999 else { 2000 emit_1ub(p, 0xdd); 2001 emit_modrm_noreg(p, 7, dst); 2002 } 2003 } 2004 2005 2006 void x87_fnstcw( struct x86_function *p, struct x86_reg dst ) 2007 { 2008 DUMP_R( dst ); 2009 assert(dst.file == file_REG32); 2010 2011 emit_1ub(p, 0x9b); /* WAIT -- needed? */ 2012 emit_1ub(p, 0xd9); 2013 emit_modrm_noreg(p, 7, dst); 2014 } 2015 2016 2017 2018 2019 /*********************************************************************** 2020 * MMX instructions 2021 */ 2022 2023 void mmx_emms( struct x86_function *p ) 2024 { 2025 DUMP(); 2026 assert(p->need_emms); 2027 emit_2ub(p, 0x0f, 0x77); 2028 p->need_emms = 0; 2029 } 2030 2031 void mmx_packssdw( struct x86_function *p, 2032 struct x86_reg dst, 2033 struct x86_reg src ) 2034 { 2035 DUMP_RR( dst, src ); 2036 assert(dst.file == file_MMX && 2037 (src.file == file_MMX || src.mod != mod_REG)); 2038 2039 p->need_emms = 1; 2040 2041 emit_2ub(p, X86_TWOB, 0x6b); 2042 emit_modrm( p, dst, src ); 2043 } 2044 2045 void mmx_packuswb( struct x86_function *p, 2046 struct x86_reg dst, 2047 struct x86_reg src ) 2048 { 2049 DUMP_RR( dst, src ); 2050 assert(dst.file == file_MMX && 2051 (src.file == file_MMX || src.mod != mod_REG)); 2052 2053 p->need_emms = 1; 2054 2055 emit_2ub(p, X86_TWOB, 0x67); 2056 emit_modrm( p, dst, src ); 2057 } 2058 2059 void mmx_movd( struct x86_function *p, 2060 struct x86_reg dst, 2061 struct x86_reg src ) 2062 { 2063 DUMP_RR( dst, src ); 2064 p->need_emms = 1; 2065 emit_1ub(p, X86_TWOB); 2066 emit_op_modrm( p, 0x6e, 0x7e, dst, src ); 2067 } 2068 2069 void mmx_movq( struct x86_function *p, 2070 struct x86_reg dst, 2071 struct x86_reg src ) 2072 { 2073 DUMP_RR( dst, src ); 2074 p->need_emms = 1; 2075 emit_1ub(p, X86_TWOB); 2076 emit_op_modrm( p, 0x6f, 0x7f, dst, src ); 2077 } 2078 2079 2080 /*********************************************************************** 2081 * Helper functions 2082 */ 2083 2084 2085 void x86_cdecl_caller_push_regs( struct x86_function *p ) 2086 { 2087 x86_push(p, x86_make_reg(file_REG32, reg_AX)); 2088 x86_push(p, x86_make_reg(file_REG32, reg_CX)); 2089 x86_push(p, x86_make_reg(file_REG32, reg_DX)); 2090 } 2091 2092 void x86_cdecl_caller_pop_regs( struct x86_function *p ) 2093 { 2094 x86_pop(p, x86_make_reg(file_REG32, reg_DX)); 2095 x86_pop(p, x86_make_reg(file_REG32, reg_CX)); 2096 x86_pop(p, x86_make_reg(file_REG32, reg_AX)); 2097 } 2098 2099 2100 struct x86_reg x86_fn_arg( struct x86_function *p, 2101 unsigned arg ) 2102 { 2103 switch(x86_target(p)) 2104 { 2105 case X86_64_WIN64_ABI: 2106 /* Microsoft uses a different calling convention than the rest of the world */ 2107 switch(arg) 2108 { 2109 case 1: 2110 return x86_make_reg(file_REG32, reg_CX); 2111 case 2: 2112 return x86_make_reg(file_REG32, reg_DX); 2113 case 3: 2114 return x86_make_reg(file_REG32, reg_R8); 2115 case 4: 2116 return x86_make_reg(file_REG32, reg_R9); 2117 default: 2118 /* Win64 allocates stack slots as if it pushed the first 4 arguments too */ 2119 return x86_make_disp(x86_make_reg(file_REG32, reg_SP), 2120 p->stack_offset + arg * 8); 2121 } 2122 case X86_64_STD_ABI: 2123 switch(arg) 2124 { 2125 case 1: 2126 return x86_make_reg(file_REG32, reg_DI); 2127 case 2: 2128 return x86_make_reg(file_REG32, reg_SI); 2129 case 3: 2130 return x86_make_reg(file_REG32, reg_DX); 2131 case 4: 2132 return x86_make_reg(file_REG32, reg_CX); 2133 case 5: 2134 return x86_make_reg(file_REG32, reg_R8); 2135 case 6: 2136 return x86_make_reg(file_REG32, reg_R9); 2137 default: 2138 return x86_make_disp(x86_make_reg(file_REG32, reg_SP), 2139 p->stack_offset + (arg - 6) * 8); /* ??? */ 2140 } 2141 case X86_32: 2142 return x86_make_disp(x86_make_reg(file_REG32, reg_SP), 2143 p->stack_offset + arg * 4); /* ??? */ 2144 default: 2145 assert(0 && "Unexpected x86 target ABI in x86_fn_arg"); 2146 return x86_make_reg(file_REG32, reg_CX); /* not used / silence warning */ 2147 } 2148 } 2149 2150 static void x86_init_func_common( struct x86_function *p ) 2151 { 2152 util_cpu_detect(); 2153 p->caps = 0; 2154 if(util_cpu_caps.has_mmx) 2155 p->caps |= X86_MMX; 2156 if(util_cpu_caps.has_mmx2) 2157 p->caps |= X86_MMX2; 2158 if(util_cpu_caps.has_sse) 2159 p->caps |= X86_SSE; 2160 if(util_cpu_caps.has_sse2) 2161 p->caps |= X86_SSE2; 2162 if(util_cpu_caps.has_sse3) 2163 p->caps |= X86_SSE3; 2164 if(util_cpu_caps.has_sse4_1) 2165 p->caps |= X86_SSE4_1; 2166 p->csr = p->store; 2167 DUMP_START(); 2168 } 2169 2170 void x86_init_func( struct x86_function *p ) 2171 { 2172 p->size = 0; 2173 p->store = NULL; 2174 x86_init_func_common(p); 2175 } 2176 2177 void x86_init_func_size( struct x86_function *p, unsigned code_size ) 2178 { 2179 p->size = code_size; 2180 p->store = rtasm_exec_malloc(code_size); 2181 if (p->store == NULL) { 2182 p->store = p->error_overflow; 2183 } 2184 x86_init_func_common(p); 2185 } 2186 2187 void x86_release_func( struct x86_function *p ) 2188 { 2189 if (p->store && p->store != p->error_overflow) 2190 rtasm_exec_free(p->store); 2191 2192 p->store = NULL; 2193 p->csr = NULL; 2194 p->size = 0; 2195 } 2196 2197 2198 static INLINE x86_func 2199 voidptr_to_x86_func(void *v) 2200 { 2201 union { 2202 void *v; 2203 x86_func f; 2204 } u; 2205 assert(sizeof(u.v) == sizeof(u.f)); 2206 u.v = v; 2207 return u.f; 2208 } 2209 2210 2211 x86_func x86_get_func( struct x86_function *p ) 2212 { 2213 DUMP_END(); 2214 if (DISASSEM && p->store) 2215 debug_printf("disassemble %p %p\n", p->store, p->csr); 2216 2217 if (p->store == p->error_overflow) 2218 return voidptr_to_x86_func(NULL); 2219 else 2220 return voidptr_to_x86_func(p->store); 2221 } 2222 2223 #else 2224 2225 void x86sse_dummy( void ); 2226 2227 void x86sse_dummy( void ) 2228 { 2229 } 2230 2231 #endif 2232
