1 #!/usr/bin/env perl 2 3 # ==================================================================== 4 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL 5 # project. The module is, however, dual licensed under OpenSSL and 6 # CRYPTOGAMS licenses depending on where you obtain it. For further 7 # details see http://www.openssl.org/~appro/cryptogams/. 8 # ==================================================================== 9 10 # August 2011. 11 # 12 # Companion to x86_64-mont.pl that optimizes cache-timing attack 13 # countermeasures. The subroutines are produced by replacing bp[i] 14 # references in their x86_64-mont.pl counterparts with cache-neutral 15 # references to powers table computed in BN_mod_exp_mont_consttime. 16 # In addition subroutine that scatters elements of the powers table 17 # is implemented, so that scatter-/gathering can be tuned without 18 # bn_exp.c modifications. 19 20 # August 2013. 21 # 22 # Add MULX/AD*X code paths and additional interfaces to optimize for 23 # branch prediction unit. For input lengths that are multiples of 8 24 # the np argument is not just modulus value, but one interleaved 25 # with 0. This is to optimize post-condition... 26 27 $flavour = shift; 28 $output = shift; 29 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 30 31 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 32 33 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 34 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 35 ( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or 36 die "can't locate x86_64-xlate.pl"; 37 38 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; 39 *STDOUT=*OUT; 40 41 # In upstream, this is controlled by shelling out to the compiler to check 42 # versions, but BoringSSL is intended to be used with pre-generated perlasm 43 # output, so this isn't useful anyway. 44 # 45 # TODO(davidben): Enable this after testing. $addx goes up to 1. 46 $addx = 0; 47 48 # int bn_mul_mont_gather5( 49 $rp="%rdi"; # BN_ULONG *rp, 50 $ap="%rsi"; # const BN_ULONG *ap, 51 $bp="%rdx"; # const BN_ULONG *bp, 52 $np="%rcx"; # const BN_ULONG *np, 53 $n0="%r8"; # const BN_ULONG *n0, 54 $num="%r9"; # int num, 55 # int idx); # 0 to 2^5-1, "index" in $bp holding 56 # pre-computed powers of a', interlaced 57 # in such manner that b[0] is $bp[idx], 58 # b[1] is [2^5+idx], etc. 59 $lo0="%r10"; 60 $hi0="%r11"; 61 $hi1="%r13"; 62 $i="%r14"; 63 $j="%r15"; 64 $m0="%rbx"; 65 $m1="%rbp"; 66 67 $code=<<___; 68 .text 69 70 .extern OPENSSL_ia32cap_P 71 72 .globl bn_mul_mont_gather5 73 .type bn_mul_mont_gather5,\@function,6 74 .align 64 75 bn_mul_mont_gather5: 76 .cfi_startproc 77 mov ${num}d,${num}d 78 mov %rsp,%rax 79 .cfi_def_cfa_register %rax 80 test \$7,${num}d 81 jnz .Lmul_enter 82 ___ 83 $code.=<<___ if ($addx); 84 leaq OPENSSL_ia32cap_P(%rip),%r11 85 mov 8(%r11),%r11d 86 ___ 87 $code.=<<___; 88 jmp .Lmul4x_enter 89 90 .align 16 91 .Lmul_enter: 92 movd `($win64?56:8)`(%rsp),%xmm5 # load 7th argument 93 push %rbx 94 .cfi_push %rbx 95 push %rbp 96 .cfi_push %rbp 97 push %r12 98 .cfi_push %r12 99 push %r13 100 .cfi_push %r13 101 push %r14 102 .cfi_push %r14 103 push %r15 104 .cfi_push %r15 105 106 neg $num 107 mov %rsp,%r11 108 lea -280(%rsp,$num,8),%r10 # future alloca(8*(num+2)+256+8) 109 neg $num # restore $num 110 and \$-1024,%r10 # minimize TLB usage 111 112 # An OS-agnostic version of __chkstk. 113 # 114 # Some OSes (Windows) insist on stack being "wired" to 115 # physical memory in strictly sequential manner, i.e. if stack 116 # allocation spans two pages, then reference to farmost one can 117 # be punishable by SEGV. But page walking can do good even on 118 # other OSes, because it guarantees that villain thread hits 119 # the guard page before it can make damage to innocent one... 120 sub %r10,%r11 121 and \$-4096,%r11 122 lea (%r10,%r11),%rsp 123 mov (%rsp),%r11 124 cmp %r10,%rsp 125 ja .Lmul_page_walk 126 jmp .Lmul_page_walk_done 127 128 .Lmul_page_walk: 129 lea -4096(%rsp),%rsp 130 mov (%rsp),%r11 131 cmp %r10,%rsp 132 ja .Lmul_page_walk 133 .Lmul_page_walk_done: 134 135 lea .Linc(%rip),%r10 136 mov %rax,8(%rsp,$num,8) # tp[num+1]=%rsp 137 .cfi_cfa_expression %rsp+8,$num,8,mul,plus,deref,+8 138 .Lmul_body: 139 140 lea 128($bp),%r12 # reassign $bp (+size optimization) 141 ___ 142 $bp="%r12"; 143 $STRIDE=2**5*8; # 5 is "window size" 144 $N=$STRIDE/4; # should match cache line size 145 $code.=<<___; 146 movdqa 0(%r10),%xmm0 # 00000001000000010000000000000000 147 movdqa 16(%r10),%xmm1 # 00000002000000020000000200000002 148 lea 24-112(%rsp,$num,8),%r10# place the mask after tp[num+3] (+ICache optimization) 149 and \$-16,%r10 150 151 pshufd \$0,%xmm5,%xmm5 # broadcast index 152 movdqa %xmm1,%xmm4 153 movdqa %xmm1,%xmm2 154 ___ 155 ######################################################################## 156 # calculate mask by comparing 0..31 to index and save result to stack 157 # 158 $code.=<<___; 159 paddd %xmm0,%xmm1 160 pcmpeqd %xmm5,%xmm0 # compare to 1,0 161 .byte 0x67 162 movdqa %xmm4,%xmm3 163 ___ 164 for($k=0;$k<$STRIDE/16-4;$k+=4) { 165 $code.=<<___; 166 paddd %xmm1,%xmm2 167 pcmpeqd %xmm5,%xmm1 # compare to 3,2 168 movdqa %xmm0,`16*($k+0)+112`(%r10) 169 movdqa %xmm4,%xmm0 170 171 paddd %xmm2,%xmm3 172 pcmpeqd %xmm5,%xmm2 # compare to 5,4 173 movdqa %xmm1,`16*($k+1)+112`(%r10) 174 movdqa %xmm4,%xmm1 175 176 paddd %xmm3,%xmm0 177 pcmpeqd %xmm5,%xmm3 # compare to 7,6 178 movdqa %xmm2,`16*($k+2)+112`(%r10) 179 movdqa %xmm4,%xmm2 180 181 paddd %xmm0,%xmm1 182 pcmpeqd %xmm5,%xmm0 183 movdqa %xmm3,`16*($k+3)+112`(%r10) 184 movdqa %xmm4,%xmm3 185 ___ 186 } 187 $code.=<<___; # last iteration can be optimized 188 paddd %xmm1,%xmm2 189 pcmpeqd %xmm5,%xmm1 190 movdqa %xmm0,`16*($k+0)+112`(%r10) 191 192 paddd %xmm2,%xmm3 193 .byte 0x67 194 pcmpeqd %xmm5,%xmm2 195 movdqa %xmm1,`16*($k+1)+112`(%r10) 196 197 pcmpeqd %xmm5,%xmm3 198 movdqa %xmm2,`16*($k+2)+112`(%r10) 199 pand `16*($k+0)-128`($bp),%xmm0 # while it's still in register 200 201 pand `16*($k+1)-128`($bp),%xmm1 202 pand `16*($k+2)-128`($bp),%xmm2 203 movdqa %xmm3,`16*($k+3)+112`(%r10) 204 pand `16*($k+3)-128`($bp),%xmm3 205 por %xmm2,%xmm0 206 por %xmm3,%xmm1 207 ___ 208 for($k=0;$k<$STRIDE/16-4;$k+=4) { 209 $code.=<<___; 210 movdqa `16*($k+0)-128`($bp),%xmm4 211 movdqa `16*($k+1)-128`($bp),%xmm5 212 movdqa `16*($k+2)-128`($bp),%xmm2 213 pand `16*($k+0)+112`(%r10),%xmm4 214 movdqa `16*($k+3)-128`($bp),%xmm3 215 pand `16*($k+1)+112`(%r10),%xmm5 216 por %xmm4,%xmm0 217 pand `16*($k+2)+112`(%r10),%xmm2 218 por %xmm5,%xmm1 219 pand `16*($k+3)+112`(%r10),%xmm3 220 por %xmm2,%xmm0 221 por %xmm3,%xmm1 222 ___ 223 } 224 $code.=<<___; 225 por %xmm1,%xmm0 226 pshufd \$0x4e,%xmm0,%xmm1 227 por %xmm1,%xmm0 228 lea $STRIDE($bp),$bp 229 movq %xmm0,$m0 # m0=bp[0] 230 231 mov ($n0),$n0 # pull n0[0] value 232 mov ($ap),%rax 233 234 xor $i,$i # i=0 235 xor $j,$j # j=0 236 237 mov $n0,$m1 238 mulq $m0 # ap[0]*bp[0] 239 mov %rax,$lo0 240 mov ($np),%rax 241 242 imulq $lo0,$m1 # "tp[0]"*n0 243 mov %rdx,$hi0 244 245 mulq $m1 # np[0]*m1 246 add %rax,$lo0 # discarded 247 mov 8($ap),%rax 248 adc \$0,%rdx 249 mov %rdx,$hi1 250 251 lea 1($j),$j # j++ 252 jmp .L1st_enter 253 254 .align 16 255 .L1st: 256 add %rax,$hi1 257 mov ($ap,$j,8),%rax 258 adc \$0,%rdx 259 add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0] 260 mov $lo0,$hi0 261 adc \$0,%rdx 262 mov $hi1,-16(%rsp,$j,8) # tp[j-1] 263 mov %rdx,$hi1 264 265 .L1st_enter: 266 mulq $m0 # ap[j]*bp[0] 267 add %rax,$hi0 268 mov ($np,$j,8),%rax 269 adc \$0,%rdx 270 lea 1($j),$j # j++ 271 mov %rdx,$lo0 272 273 mulq $m1 # np[j]*m1 274 cmp $num,$j 275 jne .L1st # note that upon exit $j==$num, so 276 # they can be used interchangeably 277 278 add %rax,$hi1 279 adc \$0,%rdx 280 add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0] 281 adc \$0,%rdx 282 mov $hi1,-16(%rsp,$num,8) # tp[num-1] 283 mov %rdx,$hi1 284 mov $lo0,$hi0 285 286 xor %rdx,%rdx 287 add $hi0,$hi1 288 adc \$0,%rdx 289 mov $hi1,-8(%rsp,$num,8) 290 mov %rdx,(%rsp,$num,8) # store upmost overflow bit 291 292 lea 1($i),$i # i++ 293 jmp .Louter 294 .align 16 295 .Louter: 296 lea 24+128(%rsp,$num,8),%rdx # where 256-byte mask is (+size optimization) 297 and \$-16,%rdx 298 pxor %xmm4,%xmm4 299 pxor %xmm5,%xmm5 300 ___ 301 for($k=0;$k<$STRIDE/16;$k+=4) { 302 $code.=<<___; 303 movdqa `16*($k+0)-128`($bp),%xmm0 304 movdqa `16*($k+1)-128`($bp),%xmm1 305 movdqa `16*($k+2)-128`($bp),%xmm2 306 movdqa `16*($k+3)-128`($bp),%xmm3 307 pand `16*($k+0)-128`(%rdx),%xmm0 308 pand `16*($k+1)-128`(%rdx),%xmm1 309 por %xmm0,%xmm4 310 pand `16*($k+2)-128`(%rdx),%xmm2 311 por %xmm1,%xmm5 312 pand `16*($k+3)-128`(%rdx),%xmm3 313 por %xmm2,%xmm4 314 por %xmm3,%xmm5 315 ___ 316 } 317 $code.=<<___; 318 por %xmm5,%xmm4 319 pshufd \$0x4e,%xmm4,%xmm0 320 por %xmm4,%xmm0 321 lea $STRIDE($bp),$bp 322 323 mov ($ap),%rax # ap[0] 324 movq %xmm0,$m0 # m0=bp[i] 325 326 xor $j,$j # j=0 327 mov $n0,$m1 328 mov (%rsp),$lo0 329 330 mulq $m0 # ap[0]*bp[i] 331 add %rax,$lo0 # ap[0]*bp[i]+tp[0] 332 mov ($np),%rax 333 adc \$0,%rdx 334 335 imulq $lo0,$m1 # tp[0]*n0 336 mov %rdx,$hi0 337 338 mulq $m1 # np[0]*m1 339 add %rax,$lo0 # discarded 340 mov 8($ap),%rax 341 adc \$0,%rdx 342 mov 8(%rsp),$lo0 # tp[1] 343 mov %rdx,$hi1 344 345 lea 1($j),$j # j++ 346 jmp .Linner_enter 347 348 .align 16 349 .Linner: 350 add %rax,$hi1 351 mov ($ap,$j,8),%rax 352 adc \$0,%rdx 353 add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j] 354 mov (%rsp,$j,8),$lo0 355 adc \$0,%rdx 356 mov $hi1,-16(%rsp,$j,8) # tp[j-1] 357 mov %rdx,$hi1 358 359 .Linner_enter: 360 mulq $m0 # ap[j]*bp[i] 361 add %rax,$hi0 362 mov ($np,$j,8),%rax 363 adc \$0,%rdx 364 add $hi0,$lo0 # ap[j]*bp[i]+tp[j] 365 mov %rdx,$hi0 366 adc \$0,$hi0 367 lea 1($j),$j # j++ 368 369 mulq $m1 # np[j]*m1 370 cmp $num,$j 371 jne .Linner # note that upon exit $j==$num, so 372 # they can be used interchangeably 373 add %rax,$hi1 374 adc \$0,%rdx 375 add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j] 376 mov (%rsp,$num,8),$lo0 377 adc \$0,%rdx 378 mov $hi1,-16(%rsp,$num,8) # tp[num-1] 379 mov %rdx,$hi1 380 381 xor %rdx,%rdx 382 add $hi0,$hi1 383 adc \$0,%rdx 384 add $lo0,$hi1 # pull upmost overflow bit 385 adc \$0,%rdx 386 mov $hi1,-8(%rsp,$num,8) 387 mov %rdx,(%rsp,$num,8) # store upmost overflow bit 388 389 lea 1($i),$i # i++ 390 cmp $num,$i 391 jb .Louter 392 393 xor $i,$i # i=0 and clear CF! 394 mov (%rsp),%rax # tp[0] 395 lea (%rsp),$ap # borrow ap for tp 396 mov $num,$j # j=num 397 jmp .Lsub 398 .align 16 399 .Lsub: 400 sbb ($np,$i,8),%rax 401 mov %rax,($rp,$i,8) # rp[i]=tp[i]-np[i] 402 mov 8($ap,$i,8),%rax # tp[i+1] 403 lea 1($i),$i # i++ 404 dec $j # doesnn't affect CF! 405 jnz .Lsub 406 407 sbb \$0,%rax # handle upmost overflow bit 408 xor $i,$i 409 and %rax,$ap 410 not %rax 411 mov $rp,$np 412 and %rax,$np 413 mov $num,$j # j=num 414 or $np,$ap # ap=borrow?tp:rp 415 .align 16 416 .Lcopy: # copy or in-place refresh 417 mov ($ap,$i,8),%rax 418 mov $i,(%rsp,$i,8) # zap temporary vector 419 mov %rax,($rp,$i,8) # rp[i]=tp[i] 420 lea 1($i),$i 421 sub \$1,$j 422 jnz .Lcopy 423 424 mov 8(%rsp,$num,8),%rsi # restore %rsp 425 .cfi_def_cfa %rsi,8 426 mov \$1,%rax 427 428 mov -48(%rsi),%r15 429 .cfi_restore %r15 430 mov -40(%rsi),%r14 431 .cfi_restore %r14 432 mov -32(%rsi),%r13 433 .cfi_restore %r13 434 mov -24(%rsi),%r12 435 .cfi_restore %r12 436 mov -16(%rsi),%rbp 437 .cfi_restore %rbp 438 mov -8(%rsi),%rbx 439 .cfi_restore %rbx 440 lea (%rsi),%rsp 441 .cfi_def_cfa_register %rsp 442 .Lmul_epilogue: 443 ret 444 .cfi_endproc 445 .size bn_mul_mont_gather5,.-bn_mul_mont_gather5 446 ___ 447 {{{ 448 my @A=("%r10","%r11"); 449 my @N=("%r13","%rdi"); 450 $code.=<<___; 451 .type bn_mul4x_mont_gather5,\@function,6 452 .align 32 453 bn_mul4x_mont_gather5: 454 .cfi_startproc 455 .byte 0x67 456 mov %rsp,%rax 457 .cfi_def_cfa_register %rax 458 .Lmul4x_enter: 459 ___ 460 $code.=<<___ if ($addx); 461 and \$0x80108,%r11d 462 cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1 463 je .Lmulx4x_enter 464 ___ 465 $code.=<<___; 466 push %rbx 467 .cfi_push %rbx 468 push %rbp 469 .cfi_push %rbp 470 push %r12 471 .cfi_push %r12 472 push %r13 473 .cfi_push %r13 474 push %r14 475 .cfi_push %r14 476 push %r15 477 .cfi_push %r15 478 .Lmul4x_prologue: 479 480 .byte 0x67 481 shl \$3,${num}d # convert $num to bytes 482 lea ($num,$num,2),%r10 # 3*$num in bytes 483 neg $num # -$num 484 485 ############################################################## 486 # Ensure that stack frame doesn't alias with $rptr+3*$num 487 # modulo 4096, which covers ret[num], am[num] and n[num] 488 # (see bn_exp.c). This is done to allow memory disambiguation 489 # logic do its magic. [Extra [num] is allocated in order 490 # to align with bn_power5's frame, which is cleansed after 491 # completing exponentiation. Extra 256 bytes is for power mask 492 # calculated from 7th argument, the index.] 493 # 494 lea -320(%rsp,$num,2),%r11 495 mov %rsp,%rbp 496 sub $rp,%r11 497 and \$4095,%r11 498 cmp %r11,%r10 499 jb .Lmul4xsp_alt 500 sub %r11,%rbp # align with $rp 501 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256) 502 jmp .Lmul4xsp_done 503 504 .align 32 505 .Lmul4xsp_alt: 506 lea 4096-320(,$num,2),%r10 507 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256) 508 sub %r10,%r11 509 mov \$0,%r10 510 cmovc %r10,%r11 511 sub %r11,%rbp 512 .Lmul4xsp_done: 513 and \$-64,%rbp 514 mov %rsp,%r11 515 sub %rbp,%r11 516 and \$-4096,%r11 517 lea (%rbp,%r11),%rsp 518 mov (%rsp),%r10 519 cmp %rbp,%rsp 520 ja .Lmul4x_page_walk 521 jmp .Lmul4x_page_walk_done 522 523 .Lmul4x_page_walk: 524 lea -4096(%rsp),%rsp 525 mov (%rsp),%r10 526 cmp %rbp,%rsp 527 ja .Lmul4x_page_walk 528 .Lmul4x_page_walk_done: 529 530 neg $num 531 532 mov %rax,40(%rsp) 533 .cfi_cfa_expression %rsp+40,deref,+8 534 .Lmul4x_body: 535 536 call mul4x_internal 537 538 mov 40(%rsp),%rsi # restore %rsp 539 .cfi_def_cfa %rsi,8 540 mov \$1,%rax 541 542 mov -48(%rsi),%r15 543 .cfi_restore %r15 544 mov -40(%rsi),%r14 545 .cfi_restore %r14 546 mov -32(%rsi),%r13 547 .cfi_restore %r13 548 mov -24(%rsi),%r12 549 .cfi_restore %r12 550 mov -16(%rsi),%rbp 551 .cfi_restore %rbp 552 mov -8(%rsi),%rbx 553 .cfi_restore %rbx 554 lea (%rsi),%rsp 555 .cfi_def_cfa_register %rsp 556 .Lmul4x_epilogue: 557 ret 558 .cfi_endproc 559 .size bn_mul4x_mont_gather5,.-bn_mul4x_mont_gather5 560 561 .type mul4x_internal,\@abi-omnipotent 562 .align 32 563 mul4x_internal: 564 shl \$5,$num # $num was in bytes 565 movd `($win64?56:8)`(%rax),%xmm5 # load 7th argument, index 566 lea .Linc(%rip),%rax 567 lea 128(%rdx,$num),%r13 # end of powers table (+size optimization) 568 shr \$5,$num # restore $num 569 ___ 570 $bp="%r12"; 571 $STRIDE=2**5*8; # 5 is "window size" 572 $N=$STRIDE/4; # should match cache line size 573 $tp=$i; 574 $code.=<<___; 575 movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000 576 movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002 577 lea 88-112(%rsp,$num),%r10 # place the mask after tp[num+1] (+ICache optimization) 578 lea 128(%rdx),$bp # size optimization 579 580 pshufd \$0,%xmm5,%xmm5 # broadcast index 581 movdqa %xmm1,%xmm4 582 .byte 0x67,0x67 583 movdqa %xmm1,%xmm2 584 ___ 585 ######################################################################## 586 # calculate mask by comparing 0..31 to index and save result to stack 587 # 588 $code.=<<___; 589 paddd %xmm0,%xmm1 590 pcmpeqd %xmm5,%xmm0 # compare to 1,0 591 .byte 0x67 592 movdqa %xmm4,%xmm3 593 ___ 594 for($i=0;$i<$STRIDE/16-4;$i+=4) { 595 $code.=<<___; 596 paddd %xmm1,%xmm2 597 pcmpeqd %xmm5,%xmm1 # compare to 3,2 598 movdqa %xmm0,`16*($i+0)+112`(%r10) 599 movdqa %xmm4,%xmm0 600 601 paddd %xmm2,%xmm3 602 pcmpeqd %xmm5,%xmm2 # compare to 5,4 603 movdqa %xmm1,`16*($i+1)+112`(%r10) 604 movdqa %xmm4,%xmm1 605 606 paddd %xmm3,%xmm0 607 pcmpeqd %xmm5,%xmm3 # compare to 7,6 608 movdqa %xmm2,`16*($i+2)+112`(%r10) 609 movdqa %xmm4,%xmm2 610 611 paddd %xmm0,%xmm1 612 pcmpeqd %xmm5,%xmm0 613 movdqa %xmm3,`16*($i+3)+112`(%r10) 614 movdqa %xmm4,%xmm3 615 ___ 616 } 617 $code.=<<___; # last iteration can be optimized 618 paddd %xmm1,%xmm2 619 pcmpeqd %xmm5,%xmm1 620 movdqa %xmm0,`16*($i+0)+112`(%r10) 621 622 paddd %xmm2,%xmm3 623 .byte 0x67 624 pcmpeqd %xmm5,%xmm2 625 movdqa %xmm1,`16*($i+1)+112`(%r10) 626 627 pcmpeqd %xmm5,%xmm3 628 movdqa %xmm2,`16*($i+2)+112`(%r10) 629 pand `16*($i+0)-128`($bp),%xmm0 # while it's still in register 630 631 pand `16*($i+1)-128`($bp),%xmm1 632 pand `16*($i+2)-128`($bp),%xmm2 633 movdqa %xmm3,`16*($i+3)+112`(%r10) 634 pand `16*($i+3)-128`($bp),%xmm3 635 por %xmm2,%xmm0 636 por %xmm3,%xmm1 637 ___ 638 for($i=0;$i<$STRIDE/16-4;$i+=4) { 639 $code.=<<___; 640 movdqa `16*($i+0)-128`($bp),%xmm4 641 movdqa `16*($i+1)-128`($bp),%xmm5 642 movdqa `16*($i+2)-128`($bp),%xmm2 643 pand `16*($i+0)+112`(%r10),%xmm4 644 movdqa `16*($i+3)-128`($bp),%xmm3 645 pand `16*($i+1)+112`(%r10),%xmm5 646 por %xmm4,%xmm0 647 pand `16*($i+2)+112`(%r10),%xmm2 648 por %xmm5,%xmm1 649 pand `16*($i+3)+112`(%r10),%xmm3 650 por %xmm2,%xmm0 651 por %xmm3,%xmm1 652 ___ 653 } 654 $code.=<<___; 655 por %xmm1,%xmm0 656 pshufd \$0x4e,%xmm0,%xmm1 657 por %xmm1,%xmm0 658 lea $STRIDE($bp),$bp 659 movq %xmm0,$m0 # m0=bp[0] 660 661 mov %r13,16+8(%rsp) # save end of b[num] 662 mov $rp, 56+8(%rsp) # save $rp 663 664 mov ($n0),$n0 # pull n0[0] value 665 mov ($ap),%rax 666 lea ($ap,$num),$ap # end of a[num] 667 neg $num 668 669 mov $n0,$m1 670 mulq $m0 # ap[0]*bp[0] 671 mov %rax,$A[0] 672 mov ($np),%rax 673 674 imulq $A[0],$m1 # "tp[0]"*n0 675 lea 64+8(%rsp),$tp 676 mov %rdx,$A[1] 677 678 mulq $m1 # np[0]*m1 679 add %rax,$A[0] # discarded 680 mov 8($ap,$num),%rax 681 adc \$0,%rdx 682 mov %rdx,$N[1] 683 684 mulq $m0 685 add %rax,$A[1] 686 mov 8*1($np),%rax 687 adc \$0,%rdx 688 mov %rdx,$A[0] 689 690 mulq $m1 691 add %rax,$N[1] 692 mov 16($ap,$num),%rax 693 adc \$0,%rdx 694 add $A[1],$N[1] 695 lea 4*8($num),$j # j=4 696 lea 8*4($np),$np 697 adc \$0,%rdx 698 mov $N[1],($tp) 699 mov %rdx,$N[0] 700 jmp .L1st4x 701 702 .align 32 703 .L1st4x: 704 mulq $m0 # ap[j]*bp[0] 705 add %rax,$A[0] 706 mov -8*2($np),%rax 707 lea 32($tp),$tp 708 adc \$0,%rdx 709 mov %rdx,$A[1] 710 711 mulq $m1 # np[j]*m1 712 add %rax,$N[0] 713 mov -8($ap,$j),%rax 714 adc \$0,%rdx 715 add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] 716 adc \$0,%rdx 717 mov $N[0],-24($tp) # tp[j-1] 718 mov %rdx,$N[1] 719 720 mulq $m0 # ap[j]*bp[0] 721 add %rax,$A[1] 722 mov -8*1($np),%rax 723 adc \$0,%rdx 724 mov %rdx,$A[0] 725 726 mulq $m1 # np[j]*m1 727 add %rax,$N[1] 728 mov ($ap,$j),%rax 729 adc \$0,%rdx 730 add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] 731 adc \$0,%rdx 732 mov $N[1],-16($tp) # tp[j-1] 733 mov %rdx,$N[0] 734 735 mulq $m0 # ap[j]*bp[0] 736 add %rax,$A[0] 737 mov 8*0($np),%rax 738 adc \$0,%rdx 739 mov %rdx,$A[1] 740 741 mulq $m1 # np[j]*m1 742 add %rax,$N[0] 743 mov 8($ap,$j),%rax 744 adc \$0,%rdx 745 add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] 746 adc \$0,%rdx 747 mov $N[0],-8($tp) # tp[j-1] 748 mov %rdx,$N[1] 749 750 mulq $m0 # ap[j]*bp[0] 751 add %rax,$A[1] 752 mov 8*1($np),%rax 753 adc \$0,%rdx 754 mov %rdx,$A[0] 755 756 mulq $m1 # np[j]*m1 757 add %rax,$N[1] 758 mov 16($ap,$j),%rax 759 adc \$0,%rdx 760 add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] 761 lea 8*4($np),$np 762 adc \$0,%rdx 763 mov $N[1],($tp) # tp[j-1] 764 mov %rdx,$N[0] 765 766 add \$32,$j # j+=4 767 jnz .L1st4x 768 769 mulq $m0 # ap[j]*bp[0] 770 add %rax,$A[0] 771 mov -8*2($np),%rax 772 lea 32($tp),$tp 773 adc \$0,%rdx 774 mov %rdx,$A[1] 775 776 mulq $m1 # np[j]*m1 777 add %rax,$N[0] 778 mov -8($ap),%rax 779 adc \$0,%rdx 780 add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] 781 adc \$0,%rdx 782 mov $N[0],-24($tp) # tp[j-1] 783 mov %rdx,$N[1] 784 785 mulq $m0 # ap[j]*bp[0] 786 add %rax,$A[1] 787 mov -8*1($np),%rax 788 adc \$0,%rdx 789 mov %rdx,$A[0] 790 791 mulq $m1 # np[j]*m1 792 add %rax,$N[1] 793 mov ($ap,$num),%rax # ap[0] 794 adc \$0,%rdx 795 add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] 796 adc \$0,%rdx 797 mov $N[1],-16($tp) # tp[j-1] 798 mov %rdx,$N[0] 799 800 lea ($np,$num),$np # rewind $np 801 802 xor $N[1],$N[1] 803 add $A[0],$N[0] 804 adc \$0,$N[1] 805 mov $N[0],-8($tp) 806 807 jmp .Louter4x 808 809 .align 32 810 .Louter4x: 811 lea 16+128($tp),%rdx # where 256-byte mask is (+size optimization) 812 pxor %xmm4,%xmm4 813 pxor %xmm5,%xmm5 814 ___ 815 for($i=0;$i<$STRIDE/16;$i+=4) { 816 $code.=<<___; 817 movdqa `16*($i+0)-128`($bp),%xmm0 818 movdqa `16*($i+1)-128`($bp),%xmm1 819 movdqa `16*($i+2)-128`($bp),%xmm2 820 movdqa `16*($i+3)-128`($bp),%xmm3 821 pand `16*($i+0)-128`(%rdx),%xmm0 822 pand `16*($i+1)-128`(%rdx),%xmm1 823 por %xmm0,%xmm4 824 pand `16*($i+2)-128`(%rdx),%xmm2 825 por %xmm1,%xmm5 826 pand `16*($i+3)-128`(%rdx),%xmm3 827 por %xmm2,%xmm4 828 por %xmm3,%xmm5 829 ___ 830 } 831 $code.=<<___; 832 por %xmm5,%xmm4 833 pshufd \$0x4e,%xmm4,%xmm0 834 por %xmm4,%xmm0 835 lea $STRIDE($bp),$bp 836 movq %xmm0,$m0 # m0=bp[i] 837 838 mov ($tp,$num),$A[0] 839 mov $n0,$m1 840 mulq $m0 # ap[0]*bp[i] 841 add %rax,$A[0] # ap[0]*bp[i]+tp[0] 842 mov ($np),%rax 843 adc \$0,%rdx 844 845 imulq $A[0],$m1 # tp[0]*n0 846 mov %rdx,$A[1] 847 mov $N[1],($tp) # store upmost overflow bit 848 849 lea ($tp,$num),$tp # rewind $tp 850 851 mulq $m1 # np[0]*m1 852 add %rax,$A[0] # "$N[0]", discarded 853 mov 8($ap,$num),%rax 854 adc \$0,%rdx 855 mov %rdx,$N[1] 856 857 mulq $m0 # ap[j]*bp[i] 858 add %rax,$A[1] 859 mov 8*1($np),%rax 860 adc \$0,%rdx 861 add 8($tp),$A[1] # +tp[1] 862 adc \$0,%rdx 863 mov %rdx,$A[0] 864 865 mulq $m1 # np[j]*m1 866 add %rax,$N[1] 867 mov 16($ap,$num),%rax 868 adc \$0,%rdx 869 add $A[1],$N[1] # np[j]*m1+ap[j]*bp[i]+tp[j] 870 lea 4*8($num),$j # j=4 871 lea 8*4($np),$np 872 adc \$0,%rdx 873 mov %rdx,$N[0] 874 jmp .Linner4x 875 876 .align 32 877 .Linner4x: 878 mulq $m0 # ap[j]*bp[i] 879 add %rax,$A[0] 880 mov -8*2($np),%rax 881 adc \$0,%rdx 882 add 16($tp),$A[0] # ap[j]*bp[i]+tp[j] 883 lea 32($tp),$tp 884 adc \$0,%rdx 885 mov %rdx,$A[1] 886 887 mulq $m1 # np[j]*m1 888 add %rax,$N[0] 889 mov -8($ap,$j),%rax 890 adc \$0,%rdx 891 add $A[0],$N[0] 892 adc \$0,%rdx 893 mov $N[1],-32($tp) # tp[j-1] 894 mov %rdx,$N[1] 895 896 mulq $m0 # ap[j]*bp[i] 897 add %rax,$A[1] 898 mov -8*1($np),%rax 899 adc \$0,%rdx 900 add -8($tp),$A[1] 901 adc \$0,%rdx 902 mov %rdx,$A[0] 903 904 mulq $m1 # np[j]*m1 905 add %rax,$N[1] 906 mov ($ap,$j),%rax 907 adc \$0,%rdx 908 add $A[1],$N[1] 909 adc \$0,%rdx 910 mov $N[0],-24($tp) # tp[j-1] 911 mov %rdx,$N[0] 912 913 mulq $m0 # ap[j]*bp[i] 914 add %rax,$A[0] 915 mov 8*0($np),%rax 916 adc \$0,%rdx 917 add ($tp),$A[0] # ap[j]*bp[i]+tp[j] 918 adc \$0,%rdx 919 mov %rdx,$A[1] 920 921 mulq $m1 # np[j]*m1 922 add %rax,$N[0] 923 mov 8($ap,$j),%rax 924 adc \$0,%rdx 925 add $A[0],$N[0] 926 adc \$0,%rdx 927 mov $N[1],-16($tp) # tp[j-1] 928 mov %rdx,$N[1] 929 930 mulq $m0 # ap[j]*bp[i] 931 add %rax,$A[1] 932 mov 8*1($np),%rax 933 adc \$0,%rdx 934 add 8($tp),$A[1] 935 adc \$0,%rdx 936 mov %rdx,$A[0] 937 938 mulq $m1 # np[j]*m1 939 add %rax,$N[1] 940 mov 16($ap,$j),%rax 941 adc \$0,%rdx 942 add $A[1],$N[1] 943 lea 8*4($np),$np 944 adc \$0,%rdx 945 mov $N[0],-8($tp) # tp[j-1] 946 mov %rdx,$N[0] 947 948 add \$32,$j # j+=4 949 jnz .Linner4x 950 951 mulq $m0 # ap[j]*bp[i] 952 add %rax,$A[0] 953 mov -8*2($np),%rax 954 adc \$0,%rdx 955 add 16($tp),$A[0] # ap[j]*bp[i]+tp[j] 956 lea 32($tp),$tp 957 adc \$0,%rdx 958 mov %rdx,$A[1] 959 960 mulq $m1 # np[j]*m1 961 add %rax,$N[0] 962 mov -8($ap),%rax 963 adc \$0,%rdx 964 add $A[0],$N[0] 965 adc \$0,%rdx 966 mov $N[1],-32($tp) # tp[j-1] 967 mov %rdx,$N[1] 968 969 mulq $m0 # ap[j]*bp[i] 970 add %rax,$A[1] 971 mov $m1,%rax 972 mov -8*1($np),$m1 973 adc \$0,%rdx 974 add -8($tp),$A[1] 975 adc \$0,%rdx 976 mov %rdx,$A[0] 977 978 mulq $m1 # np[j]*m1 979 add %rax,$N[1] 980 mov ($ap,$num),%rax # ap[0] 981 adc \$0,%rdx 982 add $A[1],$N[1] 983 adc \$0,%rdx 984 mov $N[0],-24($tp) # tp[j-1] 985 mov %rdx,$N[0] 986 987 mov $N[1],-16($tp) # tp[j-1] 988 lea ($np,$num),$np # rewind $np 989 990 xor $N[1],$N[1] 991 add $A[0],$N[0] 992 adc \$0,$N[1] 993 add ($tp),$N[0] # pull upmost overflow bit 994 adc \$0,$N[1] # upmost overflow bit 995 mov $N[0],-8($tp) 996 997 cmp 16+8(%rsp),$bp 998 jb .Louter4x 999 ___ 1000 if (1) { 1001 $code.=<<___; 1002 xor %rax,%rax 1003 sub $N[0],$m1 # compare top-most words 1004 adc $j,$j # $j is zero 1005 or $j,$N[1] 1006 sub $N[1],%rax # %rax=-$N[1] 1007 lea ($tp,$num),%rbx # tptr in .sqr4x_sub 1008 mov ($np),%r12 1009 lea ($np),%rbp # nptr in .sqr4x_sub 1010 mov %r9,%rcx 1011 sar \$3+2,%rcx 1012 mov 56+8(%rsp),%rdi # rptr in .sqr4x_sub 1013 dec %r12 # so that after 'not' we get -n[0] 1014 xor %r10,%r10 1015 mov 8*1(%rbp),%r13 1016 mov 8*2(%rbp),%r14 1017 mov 8*3(%rbp),%r15 1018 jmp .Lsqr4x_sub_entry 1019 ___ 1020 } else { 1021 my @ri=("%rax",$bp,$m0,$m1); 1022 my $rp="%rdx"; 1023 $code.=<<___ 1024 xor \$1,$N[1] 1025 lea ($tp,$num),$tp # rewind $tp 1026 sar \$5,$num # cf=0 1027 lea ($np,$N[1],8),$np 1028 mov 56+8(%rsp),$rp # restore $rp 1029 jmp .Lsub4x 1030 1031 .align 32 1032 .Lsub4x: 1033 .byte 0x66 1034 mov 8*0($tp),@ri[0] 1035 mov 8*1($tp),@ri[1] 1036 .byte 0x66 1037 sbb 16*0($np),@ri[0] 1038 mov 8*2($tp),@ri[2] 1039 sbb 16*1($np),@ri[1] 1040 mov 3*8($tp),@ri[3] 1041 lea 4*8($tp),$tp 1042 sbb 16*2($np),@ri[2] 1043 mov @ri[0],8*0($rp) 1044 sbb 16*3($np),@ri[3] 1045 lea 16*4($np),$np 1046 mov @ri[1],8*1($rp) 1047 mov @ri[2],8*2($rp) 1048 mov @ri[3],8*3($rp) 1049 lea 8*4($rp),$rp 1050 1051 inc $num 1052 jnz .Lsub4x 1053 1054 ret 1055 ___ 1056 } 1057 $code.=<<___; 1058 .size mul4x_internal,.-mul4x_internal 1059 ___ 1060 }}} 1061 {{{ 1063 ###################################################################### 1064 # void bn_power5( 1065 my $rptr="%rdi"; # BN_ULONG *rptr, 1066 my $aptr="%rsi"; # const BN_ULONG *aptr, 1067 my $bptr="%rdx"; # const void *table, 1068 my $nptr="%rcx"; # const BN_ULONG *nptr, 1069 my $n0 ="%r8"; # const BN_ULONG *n0); 1070 my $num ="%r9"; # int num, has to be divisible by 8 1071 # int pwr 1072 1073 my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); 1074 my @A0=("%r10","%r11"); 1075 my @A1=("%r12","%r13"); 1076 my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); 1077 1078 $code.=<<___; 1079 .globl bn_power5 1080 .type bn_power5,\@function,6 1081 .align 32 1082 bn_power5: 1083 .cfi_startproc 1084 mov %rsp,%rax 1085 .cfi_def_cfa_register %rax 1086 ___ 1087 $code.=<<___ if ($addx); 1088 leaq OPENSSL_ia32cap_P(%rip),%r11 1089 mov 8(%r11),%r11d 1090 and \$0x80108,%r11d 1091 cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1 1092 je .Lpowerx5_enter 1093 ___ 1094 $code.=<<___; 1095 push %rbx 1096 .cfi_push %rbx 1097 push %rbp 1098 .cfi_push %rbp 1099 push %r12 1100 .cfi_push %r12 1101 push %r13 1102 .cfi_push %r13 1103 push %r14 1104 .cfi_push %r14 1105 push %r15 1106 .cfi_push %r15 1107 .Lpower5_prologue: 1108 1109 shl \$3,${num}d # convert $num to bytes 1110 lea ($num,$num,2),%r10d # 3*$num 1111 neg $num 1112 mov ($n0),$n0 # *n0 1113 1114 ############################################################## 1115 # Ensure that stack frame doesn't alias with $rptr+3*$num 1116 # modulo 4096, which covers ret[num], am[num] and n[num] 1117 # (see bn_exp.c). This is done to allow memory disambiguation 1118 # logic do its magic. [Extra 256 bytes is for power mask 1119 # calculated from 7th argument, the index.] 1120 # 1121 lea -320(%rsp,$num,2),%r11 1122 mov %rsp,%rbp 1123 sub $rptr,%r11 1124 and \$4095,%r11 1125 cmp %r11,%r10 1126 jb .Lpwr_sp_alt 1127 sub %r11,%rbp # align with $aptr 1128 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256) 1129 jmp .Lpwr_sp_done 1130 1131 .align 32 1132 .Lpwr_sp_alt: 1133 lea 4096-320(,$num,2),%r10 1134 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*num*8+256) 1135 sub %r10,%r11 1136 mov \$0,%r10 1137 cmovc %r10,%r11 1138 sub %r11,%rbp 1139 .Lpwr_sp_done: 1140 and \$-64,%rbp 1141 mov %rsp,%r11 1142 sub %rbp,%r11 1143 and \$-4096,%r11 1144 lea (%rbp,%r11),%rsp 1145 mov (%rsp),%r10 1146 cmp %rbp,%rsp 1147 ja .Lpwr_page_walk 1148 jmp .Lpwr_page_walk_done 1149 1150 .Lpwr_page_walk: 1151 lea -4096(%rsp),%rsp 1152 mov (%rsp),%r10 1153 cmp %rbp,%rsp 1154 ja .Lpwr_page_walk 1155 .Lpwr_page_walk_done: 1156 1157 mov $num,%r10 1158 neg $num 1159 1160 ############################################################## 1161 # Stack layout 1162 # 1163 # +0 saved $num, used in reduction section 1164 # +8 &t[2*$num], used in reduction section 1165 # +32 saved *n0 1166 # +40 saved %rsp 1167 # +48 t[2*$num] 1168 # 1169 mov $n0, 32(%rsp) 1170 mov %rax, 40(%rsp) # save original %rsp 1171 .cfi_cfa_expression %rsp+40,deref,+8 1172 .Lpower5_body: 1173 movq $rptr,%xmm1 # save $rptr, used in sqr8x 1174 movq $nptr,%xmm2 # save $nptr 1175 movq %r10, %xmm3 # -$num, used in sqr8x 1176 movq $bptr,%xmm4 1177 1178 call __bn_sqr8x_internal 1179 call __bn_post4x_internal 1180 call __bn_sqr8x_internal 1181 call __bn_post4x_internal 1182 call __bn_sqr8x_internal 1183 call __bn_post4x_internal 1184 call __bn_sqr8x_internal 1185 call __bn_post4x_internal 1186 call __bn_sqr8x_internal 1187 call __bn_post4x_internal 1188 1189 movq %xmm2,$nptr 1190 movq %xmm4,$bptr 1191 mov $aptr,$rptr 1192 mov 40(%rsp),%rax 1193 lea 32(%rsp),$n0 1194 1195 call mul4x_internal 1196 1197 mov 40(%rsp),%rsi # restore %rsp 1198 .cfi_def_cfa %rsi,8 1199 mov \$1,%rax 1200 mov -48(%rsi),%r15 1201 .cfi_restore %r15 1202 mov -40(%rsi),%r14 1203 .cfi_restore %r14 1204 mov -32(%rsi),%r13 1205 .cfi_restore %r13 1206 mov -24(%rsi),%r12 1207 .cfi_restore %r12 1208 mov -16(%rsi),%rbp 1209 .cfi_restore %rbp 1210 mov -8(%rsi),%rbx 1211 .cfi_restore %rbx 1212 lea (%rsi),%rsp 1213 .cfi_def_cfa_register %rsp 1214 .Lpower5_epilogue: 1215 ret 1216 .cfi_endproc 1217 .size bn_power5,.-bn_power5 1218 1219 .globl bn_sqr8x_internal 1220 .hidden bn_sqr8x_internal 1221 .type bn_sqr8x_internal,\@abi-omnipotent 1222 .align 32 1223 bn_sqr8x_internal: 1224 __bn_sqr8x_internal: 1225 ############################################################## 1226 # Squaring part: 1227 # 1228 # a) multiply-n-add everything but a[i]*a[i]; 1229 # b) shift result of a) by 1 to the left and accumulate 1230 # a[i]*a[i] products; 1231 # 1232 ############################################################## 1233 # a[1]a[0] 1234 # a[2]a[0] 1235 # a[3]a[0] 1236 # a[2]a[1] 1237 # a[4]a[0] 1238 # a[3]a[1] 1239 # a[5]a[0] 1240 # a[4]a[1] 1241 # a[3]a[2] 1242 # a[6]a[0] 1243 # a[5]a[1] 1244 # a[4]a[2] 1245 # a[7]a[0] 1246 # a[6]a[1] 1247 # a[5]a[2] 1248 # a[4]a[3] 1249 # a[7]a[1] 1250 # a[6]a[2] 1251 # a[5]a[3] 1252 # a[7]a[2] 1253 # a[6]a[3] 1254 # a[5]a[4] 1255 # a[7]a[3] 1256 # a[6]a[4] 1257 # a[7]a[4] 1258 # a[6]a[5] 1259 # a[7]a[5] 1260 # a[7]a[6] 1261 # a[1]a[0] 1262 # a[2]a[0] 1263 # a[3]a[0] 1264 # a[4]a[0] 1265 # a[5]a[0] 1266 # a[6]a[0] 1267 # a[7]a[0] 1268 # a[2]a[1] 1269 # a[3]a[1] 1270 # a[4]a[1] 1271 # a[5]a[1] 1272 # a[6]a[1] 1273 # a[7]a[1] 1274 # a[3]a[2] 1275 # a[4]a[2] 1276 # a[5]a[2] 1277 # a[6]a[2] 1278 # a[7]a[2] 1279 # a[4]a[3] 1280 # a[5]a[3] 1281 # a[6]a[3] 1282 # a[7]a[3] 1283 # a[5]a[4] 1284 # a[6]a[4] 1285 # a[7]a[4] 1286 # a[6]a[5] 1287 # a[7]a[5] 1288 # a[7]a[6] 1289 # a[0]a[0] 1290 # a[1]a[1] 1291 # a[2]a[2] 1292 # a[3]a[3] 1293 # a[4]a[4] 1294 # a[5]a[5] 1295 # a[6]a[6] 1296 # a[7]a[7] 1297 1298 lea 32(%r10),$i # $i=-($num-32) 1299 lea ($aptr,$num),$aptr # end of a[] buffer, ($aptr,$i)=&ap[2] 1300 1301 mov $num,$j # $j=$num 1302 1303 # comments apply to $num==8 case 1304 mov -32($aptr,$i),$a0 # a[0] 1305 lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] 1306 mov -24($aptr,$i),%rax # a[1] 1307 lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] 1308 mov -16($aptr,$i),$ai # a[2] 1309 mov %rax,$a1 1310 1311 mul $a0 # a[1]*a[0] 1312 mov %rax,$A0[0] # a[1]*a[0] 1313 mov $ai,%rax # a[2] 1314 mov %rdx,$A0[1] 1315 mov $A0[0],-24($tptr,$i) # t[1] 1316 1317 mul $a0 # a[2]*a[0] 1318 add %rax,$A0[1] 1319 mov $ai,%rax 1320 adc \$0,%rdx 1321 mov $A0[1],-16($tptr,$i) # t[2] 1322 mov %rdx,$A0[0] 1323 1324 1325 mov -8($aptr,$i),$ai # a[3] 1326 mul $a1 # a[2]*a[1] 1327 mov %rax,$A1[0] # a[2]*a[1]+t[3] 1328 mov $ai,%rax 1329 mov %rdx,$A1[1] 1330 1331 lea ($i),$j 1332 mul $a0 # a[3]*a[0] 1333 add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] 1334 mov $ai,%rax 1335 mov %rdx,$A0[1] 1336 adc \$0,$A0[1] 1337 add $A1[0],$A0[0] 1338 adc \$0,$A0[1] 1339 mov $A0[0],-8($tptr,$j) # t[3] 1340 jmp .Lsqr4x_1st 1341 1342 .align 32 1343 .Lsqr4x_1st: 1344 mov ($aptr,$j),$ai # a[4] 1345 mul $a1 # a[3]*a[1] 1346 add %rax,$A1[1] # a[3]*a[1]+t[4] 1347 mov $ai,%rax 1348 mov %rdx,$A1[0] 1349 adc \$0,$A1[0] 1350 1351 mul $a0 # a[4]*a[0] 1352 add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] 1353 mov $ai,%rax # a[3] 1354 mov 8($aptr,$j),$ai # a[5] 1355 mov %rdx,$A0[0] 1356 adc \$0,$A0[0] 1357 add $A1[1],$A0[1] 1358 adc \$0,$A0[0] 1359 1360 1361 mul $a1 # a[4]*a[3] 1362 add %rax,$A1[0] # a[4]*a[3]+t[5] 1363 mov $ai,%rax 1364 mov $A0[1],($tptr,$j) # t[4] 1365 mov %rdx,$A1[1] 1366 adc \$0,$A1[1] 1367 1368 mul $a0 # a[5]*a[2] 1369 add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] 1370 mov $ai,%rax 1371 mov 16($aptr,$j),$ai # a[6] 1372 mov %rdx,$A0[1] 1373 adc \$0,$A0[1] 1374 add $A1[0],$A0[0] 1375 adc \$0,$A0[1] 1376 1377 mul $a1 # a[5]*a[3] 1378 add %rax,$A1[1] # a[5]*a[3]+t[6] 1379 mov $ai,%rax 1380 mov $A0[0],8($tptr,$j) # t[5] 1381 mov %rdx,$A1[0] 1382 adc \$0,$A1[0] 1383 1384 mul $a0 # a[6]*a[2] 1385 add %rax,$A0[1] # a[6]*a[2]+a[5]*a[3]+t[6] 1386 mov $ai,%rax # a[3] 1387 mov 24($aptr,$j),$ai # a[7] 1388 mov %rdx,$A0[0] 1389 adc \$0,$A0[0] 1390 add $A1[1],$A0[1] 1391 adc \$0,$A0[0] 1392 1393 1394 mul $a1 # a[6]*a[5] 1395 add %rax,$A1[0] # a[6]*a[5]+t[7] 1396 mov $ai,%rax 1397 mov $A0[1],16($tptr,$j) # t[6] 1398 mov %rdx,$A1[1] 1399 adc \$0,$A1[1] 1400 lea 32($j),$j 1401 1402 mul $a0 # a[7]*a[4] 1403 add %rax,$A0[0] # a[7]*a[4]+a[6]*a[5]+t[6] 1404 mov $ai,%rax 1405 mov %rdx,$A0[1] 1406 adc \$0,$A0[1] 1407 add $A1[0],$A0[0] 1408 adc \$0,$A0[1] 1409 mov $A0[0],-8($tptr,$j) # t[7] 1410 1411 cmp \$0,$j 1412 jne .Lsqr4x_1st 1413 1414 mul $a1 # a[7]*a[5] 1415 add %rax,$A1[1] 1416 lea 16($i),$i 1417 adc \$0,%rdx 1418 add $A0[1],$A1[1] 1419 adc \$0,%rdx 1420 1421 mov $A1[1],($tptr) # t[8] 1422 mov %rdx,$A1[0] 1423 mov %rdx,8($tptr) # t[9] 1424 jmp .Lsqr4x_outer 1425 1426 .align 32 1427 .Lsqr4x_outer: # comments apply to $num==6 case 1428 mov -32($aptr,$i),$a0 # a[0] 1429 lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] 1430 mov -24($aptr,$i),%rax # a[1] 1431 lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] 1432 mov -16($aptr,$i),$ai # a[2] 1433 mov %rax,$a1 1434 1435 mul $a0 # a[1]*a[0] 1436 mov -24($tptr,$i),$A0[0] # t[1] 1437 add %rax,$A0[0] # a[1]*a[0]+t[1] 1438 mov $ai,%rax # a[2] 1439 adc \$0,%rdx 1440 mov $A0[0],-24($tptr,$i) # t[1] 1441 mov %rdx,$A0[1] 1442 1443 mul $a0 # a[2]*a[0] 1444 add %rax,$A0[1] 1445 mov $ai,%rax 1446 adc \$0,%rdx 1447 add -16($tptr,$i),$A0[1] # a[2]*a[0]+t[2] 1448 mov %rdx,$A0[0] 1449 adc \$0,$A0[0] 1450 mov $A0[1],-16($tptr,$i) # t[2] 1451 1452 xor $A1[0],$A1[0] 1453 1454 mov -8($aptr,$i),$ai # a[3] 1455 mul $a1 # a[2]*a[1] 1456 add %rax,$A1[0] # a[2]*a[1]+t[3] 1457 mov $ai,%rax 1458 adc \$0,%rdx 1459 add -8($tptr,$i),$A1[0] 1460 mov %rdx,$A1[1] 1461 adc \$0,$A1[1] 1462 1463 mul $a0 # a[3]*a[0] 1464 add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] 1465 mov $ai,%rax 1466 adc \$0,%rdx 1467 add $A1[0],$A0[0] 1468 mov %rdx,$A0[1] 1469 adc \$0,$A0[1] 1470 mov $A0[0],-8($tptr,$i) # t[3] 1471 1472 lea ($i),$j 1473 jmp .Lsqr4x_inner 1474 1475 .align 32 1476 .Lsqr4x_inner: 1477 mov ($aptr,$j),$ai # a[4] 1478 mul $a1 # a[3]*a[1] 1479 add %rax,$A1[1] # a[3]*a[1]+t[4] 1480 mov $ai,%rax 1481 mov %rdx,$A1[0] 1482 adc \$0,$A1[0] 1483 add ($tptr,$j),$A1[1] 1484 adc \$0,$A1[0] 1485 1486 .byte 0x67 1487 mul $a0 # a[4]*a[0] 1488 add %rax,$A0[1] # a[4]*a[0]+a[3]*a[1]+t[4] 1489 mov $ai,%rax # a[3] 1490 mov 8($aptr,$j),$ai # a[5] 1491 mov %rdx,$A0[0] 1492 adc \$0,$A0[0] 1493 add $A1[1],$A0[1] 1494 adc \$0,$A0[0] 1495 1496 mul $a1 # a[4]*a[3] 1497 add %rax,$A1[0] # a[4]*a[3]+t[5] 1498 mov $A0[1],($tptr,$j) # t[4] 1499 mov $ai,%rax 1500 mov %rdx,$A1[1] 1501 adc \$0,$A1[1] 1502 add 8($tptr,$j),$A1[0] 1503 lea 16($j),$j # j++ 1504 adc \$0,$A1[1] 1505 1506 mul $a0 # a[5]*a[2] 1507 add %rax,$A0[0] # a[5]*a[2]+a[4]*a[3]+t[5] 1508 mov $ai,%rax 1509 adc \$0,%rdx 1510 add $A1[0],$A0[0] 1511 mov %rdx,$A0[1] 1512 adc \$0,$A0[1] 1513 mov $A0[0],-8($tptr,$j) # t[5], "preloaded t[1]" below 1514 1515 cmp \$0,$j 1516 jne .Lsqr4x_inner 1517 1518 .byte 0x67 1519 mul $a1 # a[5]*a[3] 1520 add %rax,$A1[1] 1521 adc \$0,%rdx 1522 add $A0[1],$A1[1] 1523 adc \$0,%rdx 1524 1525 mov $A1[1],($tptr) # t[6], "preloaded t[2]" below 1526 mov %rdx,$A1[0] 1527 mov %rdx,8($tptr) # t[7], "preloaded t[3]" below 1528 1529 add \$16,$i 1530 jnz .Lsqr4x_outer 1531 1532 # comments apply to $num==4 case 1533 mov -32($aptr),$a0 # a[0] 1534 lea 48+8(%rsp,$num,2),$tptr # end of tp[] buffer, &tp[2*$num] 1535 mov -24($aptr),%rax # a[1] 1536 lea -32($tptr,$i),$tptr # end of tp[] window, &tp[2*$num-"$i"] 1537 mov -16($aptr),$ai # a[2] 1538 mov %rax,$a1 1539 1540 mul $a0 # a[1]*a[0] 1541 add %rax,$A0[0] # a[1]*a[0]+t[1], preloaded t[1] 1542 mov $ai,%rax # a[2] 1543 mov %rdx,$A0[1] 1544 adc \$0,$A0[1] 1545 1546 mul $a0 # a[2]*a[0] 1547 add %rax,$A0[1] 1548 mov $ai,%rax 1549 mov $A0[0],-24($tptr) # t[1] 1550 mov %rdx,$A0[0] 1551 adc \$0,$A0[0] 1552 add $A1[1],$A0[1] # a[2]*a[0]+t[2], preloaded t[2] 1553 mov -8($aptr),$ai # a[3] 1554 adc \$0,$A0[0] 1555 1556 mul $a1 # a[2]*a[1] 1557 add %rax,$A1[0] # a[2]*a[1]+t[3], preloaded t[3] 1558 mov $ai,%rax 1559 mov $A0[1],-16($tptr) # t[2] 1560 mov %rdx,$A1[1] 1561 adc \$0,$A1[1] 1562 1563 mul $a0 # a[3]*a[0] 1564 add %rax,$A0[0] # a[3]*a[0]+a[2]*a[1]+t[3] 1565 mov $ai,%rax 1566 mov %rdx,$A0[1] 1567 adc \$0,$A0[1] 1568 add $A1[0],$A0[0] 1569 adc \$0,$A0[1] 1570 mov $A0[0],-8($tptr) # t[3] 1571 1572 mul $a1 # a[3]*a[1] 1573 add %rax,$A1[1] 1574 mov -16($aptr),%rax # a[2] 1575 adc \$0,%rdx 1576 add $A0[1],$A1[1] 1577 adc \$0,%rdx 1578 1579 mov $A1[1],($tptr) # t[4] 1580 mov %rdx,$A1[0] 1581 mov %rdx,8($tptr) # t[5] 1582 1583 mul $ai # a[2]*a[3] 1584 ___ 1585 { 1586 my ($shift,$carry)=($a0,$a1); 1587 my @S=(@A1,$ai,$n0); 1588 $code.=<<___; 1589 add \$16,$i 1590 xor $shift,$shift 1591 sub $num,$i # $i=16-$num 1592 xor $carry,$carry 1593 1594 add $A1[0],%rax # t[5] 1595 adc \$0,%rdx 1596 mov %rax,8($tptr) # t[5] 1597 mov %rdx,16($tptr) # t[6] 1598 mov $carry,24($tptr) # t[7] 1599 1600 mov -16($aptr,$i),%rax # a[0] 1601 lea 48+8(%rsp),$tptr 1602 xor $A0[0],$A0[0] # t[0] 1603 mov 8($tptr),$A0[1] # t[1] 1604 1605 lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift 1606 shr \$63,$A0[0] 1607 lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | 1608 shr \$63,$A0[1] 1609 or $A0[0],$S[1] # | t[2*i]>>63 1610 mov 16($tptr),$A0[0] # t[2*i+2] # prefetch 1611 mov $A0[1],$shift # shift=t[2*i+1]>>63 1612 mul %rax # a[i]*a[i] 1613 neg $carry # mov $carry,cf 1614 mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch 1615 adc %rax,$S[0] 1616 mov -8($aptr,$i),%rax # a[i+1] # prefetch 1617 mov $S[0],($tptr) 1618 adc %rdx,$S[1] 1619 1620 lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift 1621 mov $S[1],8($tptr) 1622 sbb $carry,$carry # mov cf,$carry 1623 shr \$63,$A0[0] 1624 lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | 1625 shr \$63,$A0[1] 1626 or $A0[0],$S[3] # | t[2*i]>>63 1627 mov 32($tptr),$A0[0] # t[2*i+2] # prefetch 1628 mov $A0[1],$shift # shift=t[2*i+1]>>63 1629 mul %rax # a[i]*a[i] 1630 neg $carry # mov $carry,cf 1631 mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch 1632 adc %rax,$S[2] 1633 mov 0($aptr,$i),%rax # a[i+1] # prefetch 1634 mov $S[2],16($tptr) 1635 adc %rdx,$S[3] 1636 lea 16($i),$i 1637 mov $S[3],24($tptr) 1638 sbb $carry,$carry # mov cf,$carry 1639 lea 64($tptr),$tptr 1640 jmp .Lsqr4x_shift_n_add 1641 1642 .align 32 1643 .Lsqr4x_shift_n_add: 1644 lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift 1645 shr \$63,$A0[0] 1646 lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | 1647 shr \$63,$A0[1] 1648 or $A0[0],$S[1] # | t[2*i]>>63 1649 mov -16($tptr),$A0[0] # t[2*i+2] # prefetch 1650 mov $A0[1],$shift # shift=t[2*i+1]>>63 1651 mul %rax # a[i]*a[i] 1652 neg $carry # mov $carry,cf 1653 mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch 1654 adc %rax,$S[0] 1655 mov -8($aptr,$i),%rax # a[i+1] # prefetch 1656 mov $S[0],-32($tptr) 1657 adc %rdx,$S[1] 1658 1659 lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift 1660 mov $S[1],-24($tptr) 1661 sbb $carry,$carry # mov cf,$carry 1662 shr \$63,$A0[0] 1663 lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | 1664 shr \$63,$A0[1] 1665 or $A0[0],$S[3] # | t[2*i]>>63 1666 mov 0($tptr),$A0[0] # t[2*i+2] # prefetch 1667 mov $A0[1],$shift # shift=t[2*i+1]>>63 1668 mul %rax # a[i]*a[i] 1669 neg $carry # mov $carry,cf 1670 mov 8($tptr),$A0[1] # t[2*i+2+1] # prefetch 1671 adc %rax,$S[2] 1672 mov 0($aptr,$i),%rax # a[i+1] # prefetch 1673 mov $S[2],-16($tptr) 1674 adc %rdx,$S[3] 1675 1676 lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift 1677 mov $S[3],-8($tptr) 1678 sbb $carry,$carry # mov cf,$carry 1679 shr \$63,$A0[0] 1680 lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | 1681 shr \$63,$A0[1] 1682 or $A0[0],$S[1] # | t[2*i]>>63 1683 mov 16($tptr),$A0[0] # t[2*i+2] # prefetch 1684 mov $A0[1],$shift # shift=t[2*i+1]>>63 1685 mul %rax # a[i]*a[i] 1686 neg $carry # mov $carry,cf 1687 mov 24($tptr),$A0[1] # t[2*i+2+1] # prefetch 1688 adc %rax,$S[0] 1689 mov 8($aptr,$i),%rax # a[i+1] # prefetch 1690 mov $S[0],0($tptr) 1691 adc %rdx,$S[1] 1692 1693 lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1 | shift 1694 mov $S[1],8($tptr) 1695 sbb $carry,$carry # mov cf,$carry 1696 shr \$63,$A0[0] 1697 lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | 1698 shr \$63,$A0[1] 1699 or $A0[0],$S[3] # | t[2*i]>>63 1700 mov 32($tptr),$A0[0] # t[2*i+2] # prefetch 1701 mov $A0[1],$shift # shift=t[2*i+1]>>63 1702 mul %rax # a[i]*a[i] 1703 neg $carry # mov $carry,cf 1704 mov 40($tptr),$A0[1] # t[2*i+2+1] # prefetch 1705 adc %rax,$S[2] 1706 mov 16($aptr,$i),%rax # a[i+1] # prefetch 1707 mov $S[2],16($tptr) 1708 adc %rdx,$S[3] 1709 mov $S[3],24($tptr) 1710 sbb $carry,$carry # mov cf,$carry 1711 lea 64($tptr),$tptr 1712 add \$32,$i 1713 jnz .Lsqr4x_shift_n_add 1714 1715 lea ($shift,$A0[0],2),$S[0] # t[2*i]<<1 | shift 1716 .byte 0x67 1717 shr \$63,$A0[0] 1718 lea ($j,$A0[1],2),$S[1] # t[2*i+1]<<1 | 1719 shr \$63,$A0[1] 1720 or $A0[0],$S[1] # | t[2*i]>>63 1721 mov -16($tptr),$A0[0] # t[2*i+2] # prefetch 1722 mov $A0[1],$shift # shift=t[2*i+1]>>63 1723 mul %rax # a[i]*a[i] 1724 neg $carry # mov $carry,cf 1725 mov -8($tptr),$A0[1] # t[2*i+2+1] # prefetch 1726 adc %rax,$S[0] 1727 mov -8($aptr),%rax # a[i+1] # prefetch 1728 mov $S[0],-32($tptr) 1729 adc %rdx,$S[1] 1730 1731 lea ($shift,$A0[0],2),$S[2] # t[2*i]<<1|shift 1732 mov $S[1],-24($tptr) 1733 sbb $carry,$carry # mov cf,$carry 1734 shr \$63,$A0[0] 1735 lea ($j,$A0[1],2),$S[3] # t[2*i+1]<<1 | 1736 shr \$63,$A0[1] 1737 or $A0[0],$S[3] # | t[2*i]>>63 1738 mul %rax # a[i]*a[i] 1739 neg $carry # mov $carry,cf 1740 adc %rax,$S[2] 1741 adc %rdx,$S[3] 1742 mov $S[2],-16($tptr) 1743 mov $S[3],-8($tptr) 1744 ___ 1745 } 1747 ###################################################################### 1748 # Montgomery reduction part, "word-by-word" algorithm. 1749 # 1750 # This new path is inspired by multiple submissions from Intel, by 1751 # Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford, 1752 # Vinodh Gopal... 1753 { 1754 my ($nptr,$tptr,$carry,$m0)=("%rbp","%rdi","%rsi","%rbx"); 1755 1756 $code.=<<___; 1757 movq %xmm2,$nptr 1758 __bn_sqr8x_reduction: 1759 xor %rax,%rax 1760 lea ($nptr,$num),%rcx # end of n[] 1761 lea 48+8(%rsp,$num,2),%rdx # end of t[] buffer 1762 mov %rcx,0+8(%rsp) 1763 lea 48+8(%rsp,$num),$tptr # end of initial t[] window 1764 mov %rdx,8+8(%rsp) 1765 neg $num 1766 jmp .L8x_reduction_loop 1767 1768 .align 32 1769 .L8x_reduction_loop: 1770 lea ($tptr,$num),$tptr # start of current t[] window 1771 .byte 0x66 1772 mov 8*0($tptr),$m0 1773 mov 8*1($tptr),%r9 1774 mov 8*2($tptr),%r10 1775 mov 8*3($tptr),%r11 1776 mov 8*4($tptr),%r12 1777 mov 8*5($tptr),%r13 1778 mov 8*6($tptr),%r14 1779 mov 8*7($tptr),%r15 1780 mov %rax,(%rdx) # store top-most carry bit 1781 lea 8*8($tptr),$tptr 1782 1783 .byte 0x67 1784 mov $m0,%r8 1785 imulq 32+8(%rsp),$m0 # n0*a[0] 1786 mov 8*0($nptr),%rax # n[0] 1787 mov \$8,%ecx 1788 jmp .L8x_reduce 1789 1790 .align 32 1791 .L8x_reduce: 1792 mulq $m0 1793 mov 8*1($nptr),%rax # n[1] 1794 neg %r8 1795 mov %rdx,%r8 1796 adc \$0,%r8 1797 1798 mulq $m0 1799 add %rax,%r9 1800 mov 8*2($nptr),%rax 1801 adc \$0,%rdx 1802 add %r9,%r8 1803 mov $m0,48-8+8(%rsp,%rcx,8) # put aside n0*a[i] 1804 mov %rdx,%r9 1805 adc \$0,%r9 1806 1807 mulq $m0 1808 add %rax,%r10 1809 mov 8*3($nptr),%rax 1810 adc \$0,%rdx 1811 add %r10,%r9 1812 mov 32+8(%rsp),$carry # pull n0, borrow $carry 1813 mov %rdx,%r10 1814 adc \$0,%r10 1815 1816 mulq $m0 1817 add %rax,%r11 1818 mov 8*4($nptr),%rax 1819 adc \$0,%rdx 1820 imulq %r8,$carry # modulo-scheduled 1821 add %r11,%r10 1822 mov %rdx,%r11 1823 adc \$0,%r11 1824 1825 mulq $m0 1826 add %rax,%r12 1827 mov 8*5($nptr),%rax 1828 adc \$0,%rdx 1829 add %r12,%r11 1830 mov %rdx,%r12 1831 adc \$0,%r12 1832 1833 mulq $m0 1834 add %rax,%r13 1835 mov 8*6($nptr),%rax 1836 adc \$0,%rdx 1837 add %r13,%r12 1838 mov %rdx,%r13 1839 adc \$0,%r13 1840 1841 mulq $m0 1842 add %rax,%r14 1843 mov 8*7($nptr),%rax 1844 adc \$0,%rdx 1845 add %r14,%r13 1846 mov %rdx,%r14 1847 adc \$0,%r14 1848 1849 mulq $m0 1850 mov $carry,$m0 # n0*a[i] 1851 add %rax,%r15 1852 mov 8*0($nptr),%rax # n[0] 1853 adc \$0,%rdx 1854 add %r15,%r14 1855 mov %rdx,%r15 1856 adc \$0,%r15 1857 1858 dec %ecx 1859 jnz .L8x_reduce 1860 1861 lea 8*8($nptr),$nptr 1862 xor %rax,%rax 1863 mov 8+8(%rsp),%rdx # pull end of t[] 1864 cmp 0+8(%rsp),$nptr # end of n[]? 1865 jae .L8x_no_tail 1866 1867 .byte 0x66 1868 add 8*0($tptr),%r8 1869 adc 8*1($tptr),%r9 1870 adc 8*2($tptr),%r10 1871 adc 8*3($tptr),%r11 1872 adc 8*4($tptr),%r12 1873 adc 8*5($tptr),%r13 1874 adc 8*6($tptr),%r14 1875 adc 8*7($tptr),%r15 1876 sbb $carry,$carry # top carry 1877 1878 mov 48+56+8(%rsp),$m0 # pull n0*a[0] 1879 mov \$8,%ecx 1880 mov 8*0($nptr),%rax 1881 jmp .L8x_tail 1882 1883 .align 32 1884 .L8x_tail: 1885 mulq $m0 1886 add %rax,%r8 1887 mov 8*1($nptr),%rax 1888 mov %r8,($tptr) # save result 1889 mov %rdx,%r8 1890 adc \$0,%r8 1891 1892 mulq $m0 1893 add %rax,%r9 1894 mov 8*2($nptr),%rax 1895 adc \$0,%rdx 1896 add %r9,%r8 1897 lea 8($tptr),$tptr # $tptr++ 1898 mov %rdx,%r9 1899 adc \$0,%r9 1900 1901 mulq $m0 1902 add %rax,%r10 1903 mov 8*3($nptr),%rax 1904 adc \$0,%rdx 1905 add %r10,%r9 1906 mov %rdx,%r10 1907 adc \$0,%r10 1908 1909 mulq $m0 1910 add %rax,%r11 1911 mov 8*4($nptr),%rax 1912 adc \$0,%rdx 1913 add %r11,%r10 1914 mov %rdx,%r11 1915 adc \$0,%r11 1916 1917 mulq $m0 1918 add %rax,%r12 1919 mov 8*5($nptr),%rax 1920 adc \$0,%rdx 1921 add %r12,%r11 1922 mov %rdx,%r12 1923 adc \$0,%r12 1924 1925 mulq $m0 1926 add %rax,%r13 1927 mov 8*6($nptr),%rax 1928 adc \$0,%rdx 1929 add %r13,%r12 1930 mov %rdx,%r13 1931 adc \$0,%r13 1932 1933 mulq $m0 1934 add %rax,%r14 1935 mov 8*7($nptr),%rax 1936 adc \$0,%rdx 1937 add %r14,%r13 1938 mov %rdx,%r14 1939 adc \$0,%r14 1940 1941 mulq $m0 1942 mov 48-16+8(%rsp,%rcx,8),$m0# pull n0*a[i] 1943 add %rax,%r15 1944 adc \$0,%rdx 1945 add %r15,%r14 1946 mov 8*0($nptr),%rax # pull n[0] 1947 mov %rdx,%r15 1948 adc \$0,%r15 1949 1950 dec %ecx 1951 jnz .L8x_tail 1952 1953 lea 8*8($nptr),$nptr 1954 mov 8+8(%rsp),%rdx # pull end of t[] 1955 cmp 0+8(%rsp),$nptr # end of n[]? 1956 jae .L8x_tail_done # break out of loop 1957 1958 mov 48+56+8(%rsp),$m0 # pull n0*a[0] 1959 neg $carry 1960 mov 8*0($nptr),%rax # pull n[0] 1961 adc 8*0($tptr),%r8 1962 adc 8*1($tptr),%r9 1963 adc 8*2($tptr),%r10 1964 adc 8*3($tptr),%r11 1965 adc 8*4($tptr),%r12 1966 adc 8*5($tptr),%r13 1967 adc 8*6($tptr),%r14 1968 adc 8*7($tptr),%r15 1969 sbb $carry,$carry # top carry 1970 1971 mov \$8,%ecx 1972 jmp .L8x_tail 1973 1974 .align 32 1975 .L8x_tail_done: 1976 xor %rax,%rax 1977 add (%rdx),%r8 # can this overflow? 1978 adc \$0,%r9 1979 adc \$0,%r10 1980 adc \$0,%r11 1981 adc \$0,%r12 1982 adc \$0,%r13 1983 adc \$0,%r14 1984 adc \$0,%r15 1985 adc \$0,%rax 1986 1987 neg $carry 1988 .L8x_no_tail: 1989 adc 8*0($tptr),%r8 1990 adc 8*1($tptr),%r9 1991 adc 8*2($tptr),%r10 1992 adc 8*3($tptr),%r11 1993 adc 8*4($tptr),%r12 1994 adc 8*5($tptr),%r13 1995 adc 8*6($tptr),%r14 1996 adc 8*7($tptr),%r15 1997 adc \$0,%rax # top-most carry 1998 mov -8($nptr),%rcx # np[num-1] 1999 xor $carry,$carry 2000 2001 movq %xmm2,$nptr # restore $nptr 2002 2003 mov %r8,8*0($tptr) # store top 512 bits 2004 mov %r9,8*1($tptr) 2005 movq %xmm3,$num # $num is %r9, can't be moved upwards 2006 mov %r10,8*2($tptr) 2007 mov %r11,8*3($tptr) 2008 mov %r12,8*4($tptr) 2009 mov %r13,8*5($tptr) 2010 mov %r14,8*6($tptr) 2011 mov %r15,8*7($tptr) 2012 lea 8*8($tptr),$tptr 2013 2014 cmp %rdx,$tptr # end of t[]? 2015 jb .L8x_reduction_loop 2016 ret 2017 .size bn_sqr8x_internal,.-bn_sqr8x_internal 2018 ___ 2019 } 2021 ############################################################## 2022 # Post-condition, 4x unrolled 2023 # 2024 { 2025 my ($tptr,$nptr)=("%rbx","%rbp"); 2026 $code.=<<___; 2027 .type __bn_post4x_internal,\@abi-omnipotent 2028 .align 32 2029 __bn_post4x_internal: 2030 mov 8*0($nptr),%r12 2031 lea (%rdi,$num),$tptr # %rdi was $tptr above 2032 mov $num,%rcx 2033 movq %xmm1,$rptr # restore $rptr 2034 neg %rax 2035 movq %xmm1,$aptr # prepare for back-to-back call 2036 sar \$3+2,%rcx 2037 dec %r12 # so that after 'not' we get -n[0] 2038 xor %r10,%r10 2039 mov 8*1($nptr),%r13 2040 mov 8*2($nptr),%r14 2041 mov 8*3($nptr),%r15 2042 jmp .Lsqr4x_sub_entry 2043 2044 .align 16 2045 .Lsqr4x_sub: 2046 mov 8*0($nptr),%r12 2047 mov 8*1($nptr),%r13 2048 mov 8*2($nptr),%r14 2049 mov 8*3($nptr),%r15 2050 .Lsqr4x_sub_entry: 2051 lea 8*4($nptr),$nptr 2052 not %r12 2053 not %r13 2054 not %r14 2055 not %r15 2056 and %rax,%r12 2057 and %rax,%r13 2058 and %rax,%r14 2059 and %rax,%r15 2060 2061 neg %r10 # mov %r10,%cf 2062 adc 8*0($tptr),%r12 2063 adc 8*1($tptr),%r13 2064 adc 8*2($tptr),%r14 2065 adc 8*3($tptr),%r15 2066 mov %r12,8*0($rptr) 2067 lea 8*4($tptr),$tptr 2068 mov %r13,8*1($rptr) 2069 sbb %r10,%r10 # mov %cf,%r10 2070 mov %r14,8*2($rptr) 2071 mov %r15,8*3($rptr) 2072 lea 8*4($rptr),$rptr 2073 2074 inc %rcx # pass %cf 2075 jnz .Lsqr4x_sub 2076 2077 mov $num,%r10 # prepare for back-to-back call 2078 neg $num # restore $num 2079 ret 2080 .size __bn_post4x_internal,.-__bn_post4x_internal 2081 ___ 2082 } 2083 { 2084 $code.=<<___; 2085 .globl bn_from_montgomery 2086 .type bn_from_montgomery,\@abi-omnipotent 2087 .align 32 2088 bn_from_montgomery: 2089 testl \$7,`($win64?"48(%rsp)":"%r9d")` 2090 jz bn_from_mont8x 2091 xor %eax,%eax 2092 ret 2093 .size bn_from_montgomery,.-bn_from_montgomery 2094 2095 .type bn_from_mont8x,\@function,6 2096 .align 32 2097 bn_from_mont8x: 2098 .cfi_startproc 2099 .byte 0x67 2100 mov %rsp,%rax 2101 .cfi_def_cfa_register %rax 2102 push %rbx 2103 .cfi_push %rbx 2104 push %rbp 2105 .cfi_push %rbp 2106 push %r12 2107 .cfi_push %r12 2108 push %r13 2109 .cfi_push %r13 2110 push %r14 2111 .cfi_push %r14 2112 push %r15 2113 .cfi_push %r15 2114 .Lfrom_prologue: 2115 2116 shl \$3,${num}d # convert $num to bytes 2117 lea ($num,$num,2),%r10 # 3*$num in bytes 2118 neg $num 2119 mov ($n0),$n0 # *n0 2120 2121 ############################################################## 2122 # Ensure that stack frame doesn't alias with $rptr+3*$num 2123 # modulo 4096, which covers ret[num], am[num] and n[num] 2124 # (see bn_exp.c). The stack is allocated to aligned with 2125 # bn_power5's frame, and as bn_from_montgomery happens to be 2126 # last operation, we use the opportunity to cleanse it. 2127 # 2128 lea -320(%rsp,$num,2),%r11 2129 mov %rsp,%rbp 2130 sub $rptr,%r11 2131 and \$4095,%r11 2132 cmp %r11,%r10 2133 jb .Lfrom_sp_alt 2134 sub %r11,%rbp # align with $aptr 2135 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256) 2136 jmp .Lfrom_sp_done 2137 2138 .align 32 2139 .Lfrom_sp_alt: 2140 lea 4096-320(,$num,2),%r10 2141 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256) 2142 sub %r10,%r11 2143 mov \$0,%r10 2144 cmovc %r10,%r11 2145 sub %r11,%rbp 2146 .Lfrom_sp_done: 2147 and \$-64,%rbp 2148 mov %rsp,%r11 2149 sub %rbp,%r11 2150 and \$-4096,%r11 2151 lea (%rbp,%r11),%rsp 2152 mov (%rsp),%r10 2153 cmp %rbp,%rsp 2154 ja .Lfrom_page_walk 2155 jmp .Lfrom_page_walk_done 2156 2157 .Lfrom_page_walk: 2158 lea -4096(%rsp),%rsp 2159 mov (%rsp),%r10 2160 cmp %rbp,%rsp 2161 ja .Lfrom_page_walk 2162 .Lfrom_page_walk_done: 2163 2164 mov $num,%r10 2165 neg $num 2166 2167 ############################################################## 2168 # Stack layout 2169 # 2170 # +0 saved $num, used in reduction section 2171 # +8 &t[2*$num], used in reduction section 2172 # +32 saved *n0 2173 # +40 saved %rsp 2174 # +48 t[2*$num] 2175 # 2176 mov $n0, 32(%rsp) 2177 mov %rax, 40(%rsp) # save original %rsp 2178 .cfi_cfa_expression %rsp+40,deref,+8 2179 .Lfrom_body: 2180 mov $num,%r11 2181 lea 48(%rsp),%rax 2182 pxor %xmm0,%xmm0 2183 jmp .Lmul_by_1 2184 2185 .align 32 2186 .Lmul_by_1: 2187 movdqu ($aptr),%xmm1 2188 movdqu 16($aptr),%xmm2 2189 movdqu 32($aptr),%xmm3 2190 movdqa %xmm0,(%rax,$num) 2191 movdqu 48($aptr),%xmm4 2192 movdqa %xmm0,16(%rax,$num) 2193 .byte 0x48,0x8d,0xb6,0x40,0x00,0x00,0x00 # lea 64($aptr),$aptr 2194 movdqa %xmm1,(%rax) 2195 movdqa %xmm0,32(%rax,$num) 2196 movdqa %xmm2,16(%rax) 2197 movdqa %xmm0,48(%rax,$num) 2198 movdqa %xmm3,32(%rax) 2199 movdqa %xmm4,48(%rax) 2200 lea 64(%rax),%rax 2201 sub \$64,%r11 2202 jnz .Lmul_by_1 2203 2204 movq $rptr,%xmm1 2205 movq $nptr,%xmm2 2206 .byte 0x67 2207 mov $nptr,%rbp 2208 movq %r10, %xmm3 # -num 2209 ___ 2210 $code.=<<___ if ($addx); 2211 leaq OPENSSL_ia32cap_P(%rip),%r11 2212 mov 8(%r11),%r11d 2213 and \$0x80108,%r11d 2214 cmp \$0x80108,%r11d # check for AD*X+BMI2+BMI1 2215 jne .Lfrom_mont_nox 2216 2217 lea (%rax,$num),$rptr 2218 call __bn_sqrx8x_reduction 2219 call __bn_postx4x_internal 2220 2221 pxor %xmm0,%xmm0 2222 lea 48(%rsp),%rax 2223 jmp .Lfrom_mont_zero 2224 2225 .align 32 2226 .Lfrom_mont_nox: 2227 ___ 2228 $code.=<<___; 2229 call __bn_sqr8x_reduction 2230 call __bn_post4x_internal 2231 2232 pxor %xmm0,%xmm0 2233 lea 48(%rsp),%rax 2234 jmp .Lfrom_mont_zero 2235 2236 .align 32 2237 .Lfrom_mont_zero: 2238 mov 40(%rsp),%rsi # restore %rsp 2239 .cfi_def_cfa %rsi,8 2240 movdqa %xmm0,16*0(%rax) 2241 movdqa %xmm0,16*1(%rax) 2242 movdqa %xmm0,16*2(%rax) 2243 movdqa %xmm0,16*3(%rax) 2244 lea 16*4(%rax),%rax 2245 sub \$32,$num 2246 jnz .Lfrom_mont_zero 2247 2248 mov \$1,%rax 2249 mov -48(%rsi),%r15 2250 .cfi_restore %r15 2251 mov -40(%rsi),%r14 2252 .cfi_restore %r14 2253 mov -32(%rsi),%r13 2254 .cfi_restore %r13 2255 mov -24(%rsi),%r12 2256 .cfi_restore %r12 2257 mov -16(%rsi),%rbp 2258 .cfi_restore %rbp 2259 mov -8(%rsi),%rbx 2260 .cfi_restore %rbx 2261 lea (%rsi),%rsp 2262 .cfi_def_cfa_register %rsp 2263 .Lfrom_epilogue: 2264 ret 2265 .cfi_endproc 2266 .size bn_from_mont8x,.-bn_from_mont8x 2267 ___ 2268 } 2269 }}} 2270 2272 if ($addx) {{{ 2273 my $bp="%rdx"; # restore original value 2274 2275 $code.=<<___; 2276 .type bn_mulx4x_mont_gather5,\@function,6 2277 .align 32 2278 bn_mulx4x_mont_gather5: 2279 .cfi_startproc 2280 mov %rsp,%rax 2281 .cfi_def_cfa_register %rax 2282 .Lmulx4x_enter: 2283 push %rbx 2284 .cfi_push %rbx 2285 push %rbp 2286 .cfi_push %rbp 2287 push %r12 2288 .cfi_push %r12 2289 push %r13 2290 .cfi_push %r13 2291 push %r14 2292 .cfi_push %r14 2293 push %r15 2294 .cfi_push %r15 2295 .Lmulx4x_prologue: 2296 2297 shl \$3,${num}d # convert $num to bytes 2298 lea ($num,$num,2),%r10 # 3*$num in bytes 2299 neg $num # -$num 2300 mov ($n0),$n0 # *n0 2301 2302 ############################################################## 2303 # Ensure that stack frame doesn't alias with $rptr+3*$num 2304 # modulo 4096, which covers ret[num], am[num] and n[num] 2305 # (see bn_exp.c). This is done to allow memory disambiguation 2306 # logic do its magic. [Extra [num] is allocated in order 2307 # to align with bn_power5's frame, which is cleansed after 2308 # completing exponentiation. Extra 256 bytes is for power mask 2309 # calculated from 7th argument, the index.] 2310 # 2311 lea -320(%rsp,$num,2),%r11 2312 mov %rsp,%rbp 2313 sub $rp,%r11 2314 and \$4095,%r11 2315 cmp %r11,%r10 2316 jb .Lmulx4xsp_alt 2317 sub %r11,%rbp # align with $aptr 2318 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256) 2319 jmp .Lmulx4xsp_done 2320 2321 .Lmulx4xsp_alt: 2322 lea 4096-320(,$num,2),%r10 2323 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256) 2324 sub %r10,%r11 2325 mov \$0,%r10 2326 cmovc %r10,%r11 2327 sub %r11,%rbp 2328 .Lmulx4xsp_done: 2329 and \$-64,%rbp # ensure alignment 2330 mov %rsp,%r11 2331 sub %rbp,%r11 2332 and \$-4096,%r11 2333 lea (%rbp,%r11),%rsp 2334 mov (%rsp),%r10 2335 cmp %rbp,%rsp 2336 ja .Lmulx4x_page_walk 2337 jmp .Lmulx4x_page_walk_done 2338 2339 .Lmulx4x_page_walk: 2340 lea -4096(%rsp),%rsp 2341 mov (%rsp),%r10 2342 cmp %rbp,%rsp 2343 ja .Lmulx4x_page_walk 2344 .Lmulx4x_page_walk_done: 2345 2346 ############################################################## 2347 # Stack layout 2348 # +0 -num 2349 # +8 off-loaded &b[i] 2350 # +16 end of b[num] 2351 # +24 inner counter 2352 # +32 saved n0 2353 # +40 saved %rsp 2354 # +48 2355 # +56 saved rp 2356 # +64 tmp[num+1] 2357 # 2358 mov $n0, 32(%rsp) # save *n0 2359 mov %rax,40(%rsp) # save original %rsp 2360 .cfi_cfa_expression %rsp+40,deref,+8 2361 .Lmulx4x_body: 2362 call mulx4x_internal 2363 2364 mov 40(%rsp),%rsi # restore %rsp 2365 .cfi_def_cfa %rsi,8 2366 mov \$1,%rax 2367 2368 mov -48(%rsi),%r15 2369 .cfi_restore %r15 2370 mov -40(%rsi),%r14 2371 .cfi_restore %r14 2372 mov -32(%rsi),%r13 2373 .cfi_restore %r13 2374 mov -24(%rsi),%r12 2375 .cfi_restore %r12 2376 mov -16(%rsi),%rbp 2377 .cfi_restore %rbp 2378 mov -8(%rsi),%rbx 2379 .cfi_restore %rbx 2380 lea (%rsi),%rsp 2381 .cfi_def_cfa_register %rsp 2382 .Lmulx4x_epilogue: 2383 ret 2384 .cfi_endproc 2385 .size bn_mulx4x_mont_gather5,.-bn_mulx4x_mont_gather5 2386 2387 .type mulx4x_internal,\@abi-omnipotent 2388 .align 32 2389 mulx4x_internal: 2390 mov $num,8(%rsp) # save -$num (it was in bytes) 2391 mov $num,%r10 2392 neg $num # restore $num 2393 shl \$5,$num 2394 neg %r10 # restore $num 2395 lea 128($bp,$num),%r13 # end of powers table (+size optimization) 2396 shr \$5+5,$num 2397 movd `($win64?56:8)`(%rax),%xmm5 # load 7th argument 2398 sub \$1,$num 2399 lea .Linc(%rip),%rax 2400 mov %r13,16+8(%rsp) # end of b[num] 2401 mov $num,24+8(%rsp) # inner counter 2402 mov $rp, 56+8(%rsp) # save $rp 2403 ___ 2404 my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)= 2405 ("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax"); 2406 my $rptr=$bptr; 2407 my $STRIDE=2**5*8; # 5 is "window size" 2408 my $N=$STRIDE/4; # should match cache line size 2409 $code.=<<___; 2410 movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000 2411 movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002 2412 lea 88-112(%rsp,%r10),%r10 # place the mask after tp[num+1] (+ICache optimizaton) 2413 lea 128($bp),$bptr # size optimization 2414 2415 pshufd \$0,%xmm5,%xmm5 # broadcast index 2416 movdqa %xmm1,%xmm4 2417 .byte 0x67 2418 movdqa %xmm1,%xmm2 2419 ___ 2420 ######################################################################## 2421 # calculate mask by comparing 0..31 to index and save result to stack 2422 # 2423 $code.=<<___; 2424 .byte 0x67 2425 paddd %xmm0,%xmm1 2426 pcmpeqd %xmm5,%xmm0 # compare to 1,0 2427 movdqa %xmm4,%xmm3 2428 ___ 2429 for($i=0;$i<$STRIDE/16-4;$i+=4) { 2430 $code.=<<___; 2431 paddd %xmm1,%xmm2 2432 pcmpeqd %xmm5,%xmm1 # compare to 3,2 2433 movdqa %xmm0,`16*($i+0)+112`(%r10) 2434 movdqa %xmm4,%xmm0 2435 2436 paddd %xmm2,%xmm3 2437 pcmpeqd %xmm5,%xmm2 # compare to 5,4 2438 movdqa %xmm1,`16*($i+1)+112`(%r10) 2439 movdqa %xmm4,%xmm1 2440 2441 paddd %xmm3,%xmm0 2442 pcmpeqd %xmm5,%xmm3 # compare to 7,6 2443 movdqa %xmm2,`16*($i+2)+112`(%r10) 2444 movdqa %xmm4,%xmm2 2445 2446 paddd %xmm0,%xmm1 2447 pcmpeqd %xmm5,%xmm0 2448 movdqa %xmm3,`16*($i+3)+112`(%r10) 2449 movdqa %xmm4,%xmm3 2450 ___ 2451 } 2452 $code.=<<___; # last iteration can be optimized 2453 .byte 0x67 2454 paddd %xmm1,%xmm2 2455 pcmpeqd %xmm5,%xmm1 2456 movdqa %xmm0,`16*($i+0)+112`(%r10) 2457 2458 paddd %xmm2,%xmm3 2459 pcmpeqd %xmm5,%xmm2 2460 movdqa %xmm1,`16*($i+1)+112`(%r10) 2461 2462 pcmpeqd %xmm5,%xmm3 2463 movdqa %xmm2,`16*($i+2)+112`(%r10) 2464 2465 pand `16*($i+0)-128`($bptr),%xmm0 # while it's still in register 2466 pand `16*($i+1)-128`($bptr),%xmm1 2467 pand `16*($i+2)-128`($bptr),%xmm2 2468 movdqa %xmm3,`16*($i+3)+112`(%r10) 2469 pand `16*($i+3)-128`($bptr),%xmm3 2470 por %xmm2,%xmm0 2471 por %xmm3,%xmm1 2472 ___ 2473 for($i=0;$i<$STRIDE/16-4;$i+=4) { 2474 $code.=<<___; 2475 movdqa `16*($i+0)-128`($bptr),%xmm4 2476 movdqa `16*($i+1)-128`($bptr),%xmm5 2477 movdqa `16*($i+2)-128`($bptr),%xmm2 2478 pand `16*($i+0)+112`(%r10),%xmm4 2479 movdqa `16*($i+3)-128`($bptr),%xmm3 2480 pand `16*($i+1)+112`(%r10),%xmm5 2481 por %xmm4,%xmm0 2482 pand `16*($i+2)+112`(%r10),%xmm2 2483 por %xmm5,%xmm1 2484 pand `16*($i+3)+112`(%r10),%xmm3 2485 por %xmm2,%xmm0 2486 por %xmm3,%xmm1 2487 ___ 2488 } 2489 $code.=<<___; 2490 pxor %xmm1,%xmm0 2491 pshufd \$0x4e,%xmm0,%xmm1 2492 por %xmm1,%xmm0 2493 lea $STRIDE($bptr),$bptr 2494 movq %xmm0,%rdx # bp[0] 2495 lea 64+8*4+8(%rsp),$tptr 2496 2497 mov %rdx,$bi 2498 mulx 0*8($aptr),$mi,%rax # a[0]*b[0] 2499 mulx 1*8($aptr),%r11,%r12 # a[1]*b[0] 2500 add %rax,%r11 2501 mulx 2*8($aptr),%rax,%r13 # ... 2502 adc %rax,%r12 2503 adc \$0,%r13 2504 mulx 3*8($aptr),%rax,%r14 2505 2506 mov $mi,%r15 2507 imulq 32+8(%rsp),$mi # "t[0]"*n0 2508 xor $zero,$zero # cf=0, of=0 2509 mov $mi,%rdx 2510 2511 mov $bptr,8+8(%rsp) # off-load &b[i] 2512 2513 lea 4*8($aptr),$aptr 2514 adcx %rax,%r13 2515 adcx $zero,%r14 # cf=0 2516 2517 mulx 0*8($nptr),%rax,%r10 2518 adcx %rax,%r15 # discarded 2519 adox %r11,%r10 2520 mulx 1*8($nptr),%rax,%r11 2521 adcx %rax,%r10 2522 adox %r12,%r11 2523 mulx 2*8($nptr),%rax,%r12 2524 mov 24+8(%rsp),$bptr # counter value 2525 mov %r10,-8*4($tptr) 2526 adcx %rax,%r11 2527 adox %r13,%r12 2528 mulx 3*8($nptr),%rax,%r15 2529 mov $bi,%rdx 2530 mov %r11,-8*3($tptr) 2531 adcx %rax,%r12 2532 adox $zero,%r15 # of=0 2533 lea 4*8($nptr),$nptr 2534 mov %r12,-8*2($tptr) 2535 jmp .Lmulx4x_1st 2536 2537 .align 32 2538 .Lmulx4x_1st: 2539 adcx $zero,%r15 # cf=0, modulo-scheduled 2540 mulx 0*8($aptr),%r10,%rax # a[4]*b[0] 2541 adcx %r14,%r10 2542 mulx 1*8($aptr),%r11,%r14 # a[5]*b[0] 2543 adcx %rax,%r11 2544 mulx 2*8($aptr),%r12,%rax # ... 2545 adcx %r14,%r12 2546 mulx 3*8($aptr),%r13,%r14 2547 .byte 0x67,0x67 2548 mov $mi,%rdx 2549 adcx %rax,%r13 2550 adcx $zero,%r14 # cf=0 2551 lea 4*8($aptr),$aptr 2552 lea 4*8($tptr),$tptr 2553 2554 adox %r15,%r10 2555 mulx 0*8($nptr),%rax,%r15 2556 adcx %rax,%r10 2557 adox %r15,%r11 2558 mulx 1*8($nptr),%rax,%r15 2559 adcx %rax,%r11 2560 adox %r15,%r12 2561 mulx 2*8($nptr),%rax,%r15 2562 mov %r10,-5*8($tptr) 2563 adcx %rax,%r12 2564 mov %r11,-4*8($tptr) 2565 adox %r15,%r13 2566 mulx 3*8($nptr),%rax,%r15 2567 mov $bi,%rdx 2568 mov %r12,-3*8($tptr) 2569 adcx %rax,%r13 2570 adox $zero,%r15 2571 lea 4*8($nptr),$nptr 2572 mov %r13,-2*8($tptr) 2573 2574 dec $bptr # of=0, pass cf 2575 jnz .Lmulx4x_1st 2576 2577 mov 8(%rsp),$num # load -num 2578 adc $zero,%r15 # modulo-scheduled 2579 lea ($aptr,$num),$aptr # rewind $aptr 2580 add %r15,%r14 2581 mov 8+8(%rsp),$bptr # re-load &b[i] 2582 adc $zero,$zero # top-most carry 2583 mov %r14,-1*8($tptr) 2584 jmp .Lmulx4x_outer 2585 2586 .align 32 2587 .Lmulx4x_outer: 2588 lea 16-256($tptr),%r10 # where 256-byte mask is (+density control) 2589 pxor %xmm4,%xmm4 2590 .byte 0x67,0x67 2591 pxor %xmm5,%xmm5 2592 ___ 2593 for($i=0;$i<$STRIDE/16;$i+=4) { 2594 $code.=<<___; 2595 movdqa `16*($i+0)-128`($bptr),%xmm0 2596 movdqa `16*($i+1)-128`($bptr),%xmm1 2597 movdqa `16*($i+2)-128`($bptr),%xmm2 2598 pand `16*($i+0)+256`(%r10),%xmm0 2599 movdqa `16*($i+3)-128`($bptr),%xmm3 2600 pand `16*($i+1)+256`(%r10),%xmm1 2601 por %xmm0,%xmm4 2602 pand `16*($i+2)+256`(%r10),%xmm2 2603 por %xmm1,%xmm5 2604 pand `16*($i+3)+256`(%r10),%xmm3 2605 por %xmm2,%xmm4 2606 por %xmm3,%xmm5 2607 ___ 2608 } 2609 $code.=<<___; 2610 por %xmm5,%xmm4 2611 pshufd \$0x4e,%xmm4,%xmm0 2612 por %xmm4,%xmm0 2613 lea $STRIDE($bptr),$bptr 2614 movq %xmm0,%rdx # m0=bp[i] 2615 2616 mov $zero,($tptr) # save top-most carry 2617 lea 4*8($tptr,$num),$tptr # rewind $tptr 2618 mulx 0*8($aptr),$mi,%r11 # a[0]*b[i] 2619 xor $zero,$zero # cf=0, of=0 2620 mov %rdx,$bi 2621 mulx 1*8($aptr),%r14,%r12 # a[1]*b[i] 2622 adox -4*8($tptr),$mi # +t[0] 2623 adcx %r14,%r11 2624 mulx 2*8($aptr),%r15,%r13 # ... 2625 adox -3*8($tptr),%r11 2626 adcx %r15,%r12 2627 mulx 3*8($aptr),%rdx,%r14 2628 adox -2*8($tptr),%r12 2629 adcx %rdx,%r13 2630 lea ($nptr,$num),$nptr # rewind $nptr 2631 lea 4*8($aptr),$aptr 2632 adox -1*8($tptr),%r13 2633 adcx $zero,%r14 2634 adox $zero,%r14 2635 2636 mov $mi,%r15 2637 imulq 32+8(%rsp),$mi # "t[0]"*n0 2638 2639 mov $mi,%rdx 2640 xor $zero,$zero # cf=0, of=0 2641 mov $bptr,8+8(%rsp) # off-load &b[i] 2642 2643 mulx 0*8($nptr),%rax,%r10 2644 adcx %rax,%r15 # discarded 2645 adox %r11,%r10 2646 mulx 1*8($nptr),%rax,%r11 2647 adcx %rax,%r10 2648 adox %r12,%r11 2649 mulx 2*8($nptr),%rax,%r12 2650 adcx %rax,%r11 2651 adox %r13,%r12 2652 mulx 3*8($nptr),%rax,%r15 2653 mov $bi,%rdx 2654 mov 24+8(%rsp),$bptr # counter value 2655 mov %r10,-8*4($tptr) 2656 adcx %rax,%r12 2657 mov %r11,-8*3($tptr) 2658 adox $zero,%r15 # of=0 2659 mov %r12,-8*2($tptr) 2660 lea 4*8($nptr),$nptr 2661 jmp .Lmulx4x_inner 2662 2663 .align 32 2664 .Lmulx4x_inner: 2665 mulx 0*8($aptr),%r10,%rax # a[4]*b[i] 2666 adcx $zero,%r15 # cf=0, modulo-scheduled 2667 adox %r14,%r10 2668 mulx 1*8($aptr),%r11,%r14 # a[5]*b[i] 2669 adcx 0*8($tptr),%r10 2670 adox %rax,%r11 2671 mulx 2*8($aptr),%r12,%rax # ... 2672 adcx 1*8($tptr),%r11 2673 adox %r14,%r12 2674 mulx 3*8($aptr),%r13,%r14 2675 mov $mi,%rdx 2676 adcx 2*8($tptr),%r12 2677 adox %rax,%r13 2678 adcx 3*8($tptr),%r13 2679 adox $zero,%r14 # of=0 2680 lea 4*8($aptr),$aptr 2681 lea 4*8($tptr),$tptr 2682 adcx $zero,%r14 # cf=0 2683 2684 adox %r15,%r10 2685 mulx 0*8($nptr),%rax,%r15 2686 adcx %rax,%r10 2687 adox %r15,%r11 2688 mulx 1*8($nptr),%rax,%r15 2689 adcx %rax,%r11 2690 adox %r15,%r12 2691 mulx 2*8($nptr),%rax,%r15 2692 mov %r10,-5*8($tptr) 2693 adcx %rax,%r12 2694 adox %r15,%r13 2695 mov %r11,-4*8($tptr) 2696 mulx 3*8($nptr),%rax,%r15 2697 mov $bi,%rdx 2698 lea 4*8($nptr),$nptr 2699 mov %r12,-3*8($tptr) 2700 adcx %rax,%r13 2701 adox $zero,%r15 2702 mov %r13,-2*8($tptr) 2703 2704 dec $bptr # of=0, pass cf 2705 jnz .Lmulx4x_inner 2706 2707 mov 0+8(%rsp),$num # load -num 2708 adc $zero,%r15 # modulo-scheduled 2709 sub 0*8($tptr),$bptr # pull top-most carry to %cf 2710 mov 8+8(%rsp),$bptr # re-load &b[i] 2711 mov 16+8(%rsp),%r10 2712 adc %r15,%r14 2713 lea ($aptr,$num),$aptr # rewind $aptr 2714 adc $zero,$zero # top-most carry 2715 mov %r14,-1*8($tptr) 2716 2717 cmp %r10,$bptr 2718 jb .Lmulx4x_outer 2719 2720 mov -8($nptr),%r10 2721 mov $zero,%r8 2722 mov ($nptr,$num),%r12 2723 lea ($nptr,$num),%rbp # rewind $nptr 2724 mov $num,%rcx 2725 lea ($tptr,$num),%rdi # rewind $tptr 2726 xor %eax,%eax 2727 xor %r15,%r15 2728 sub %r14,%r10 # compare top-most words 2729 adc %r15,%r15 2730 or %r15,%r8 2731 sar \$3+2,%rcx 2732 sub %r8,%rax # %rax=-%r8 2733 mov 56+8(%rsp),%rdx # restore rp 2734 dec %r12 # so that after 'not' we get -n[0] 2735 mov 8*1(%rbp),%r13 2736 xor %r8,%r8 2737 mov 8*2(%rbp),%r14 2738 mov 8*3(%rbp),%r15 2739 jmp .Lsqrx4x_sub_entry # common post-condition 2740 .size mulx4x_internal,.-mulx4x_internal 2741 ___ 2742 }{ 2744 ###################################################################### 2745 # void bn_power5( 2746 my $rptr="%rdi"; # BN_ULONG *rptr, 2747 my $aptr="%rsi"; # const BN_ULONG *aptr, 2748 my $bptr="%rdx"; # const void *table, 2749 my $nptr="%rcx"; # const BN_ULONG *nptr, 2750 my $n0 ="%r8"; # const BN_ULONG *n0); 2751 my $num ="%r9"; # int num, has to be divisible by 8 2752 # int pwr); 2753 2754 my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); 2755 my @A0=("%r10","%r11"); 2756 my @A1=("%r12","%r13"); 2757 my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); 2758 2759 $code.=<<___; 2760 .type bn_powerx5,\@function,6 2761 .align 32 2762 bn_powerx5: 2763 .cfi_startproc 2764 mov %rsp,%rax 2765 .cfi_def_cfa_register %rax 2766 .Lpowerx5_enter: 2767 push %rbx 2768 .cfi_push %rbx 2769 push %rbp 2770 .cfi_push %rbp 2771 push %r12 2772 .cfi_push %r12 2773 push %r13 2774 .cfi_push %r13 2775 push %r14 2776 .cfi_push %r14 2777 push %r15 2778 .cfi_push %r15 2779 .Lpowerx5_prologue: 2780 2781 shl \$3,${num}d # convert $num to bytes 2782 lea ($num,$num,2),%r10 # 3*$num in bytes 2783 neg $num 2784 mov ($n0),$n0 # *n0 2785 2786 ############################################################## 2787 # Ensure that stack frame doesn't alias with $rptr+3*$num 2788 # modulo 4096, which covers ret[num], am[num] and n[num] 2789 # (see bn_exp.c). This is done to allow memory disambiguation 2790 # logic do its magic. [Extra 256 bytes is for power mask 2791 # calculated from 7th argument, the index.] 2792 # 2793 lea -320(%rsp,$num,2),%r11 2794 mov %rsp,%rbp 2795 sub $rptr,%r11 2796 and \$4095,%r11 2797 cmp %r11,%r10 2798 jb .Lpwrx_sp_alt 2799 sub %r11,%rbp # align with $aptr 2800 lea -320(%rbp,$num,2),%rbp # future alloca(frame+2*$num*8+256) 2801 jmp .Lpwrx_sp_done 2802 2803 .align 32 2804 .Lpwrx_sp_alt: 2805 lea 4096-320(,$num,2),%r10 2806 lea -320(%rbp,$num,2),%rbp # alloca(frame+2*$num*8+256) 2807 sub %r10,%r11 2808 mov \$0,%r10 2809 cmovc %r10,%r11 2810 sub %r11,%rbp 2811 .Lpwrx_sp_done: 2812 and \$-64,%rbp 2813 mov %rsp,%r11 2814 sub %rbp,%r11 2815 and \$-4096,%r11 2816 lea (%rbp,%r11),%rsp 2817 mov (%rsp),%r10 2818 cmp %rbp,%rsp 2819 ja .Lpwrx_page_walk 2820 jmp .Lpwrx_page_walk_done 2821 2822 .Lpwrx_page_walk: 2823 lea -4096(%rsp),%rsp 2824 mov (%rsp),%r10 2825 cmp %rbp,%rsp 2826 ja .Lpwrx_page_walk 2827 .Lpwrx_page_walk_done: 2828 2829 mov $num,%r10 2830 neg $num 2831 2832 ############################################################## 2833 # Stack layout 2834 # 2835 # +0 saved $num, used in reduction section 2836 # +8 &t[2*$num], used in reduction section 2837 # +16 intermediate carry bit 2838 # +24 top-most carry bit, used in reduction section 2839 # +32 saved *n0 2840 # +40 saved %rsp 2841 # +48 t[2*$num] 2842 # 2843 pxor %xmm0,%xmm0 2844 movq $rptr,%xmm1 # save $rptr 2845 movq $nptr,%xmm2 # save $nptr 2846 movq %r10, %xmm3 # -$num 2847 movq $bptr,%xmm4 2848 mov $n0, 32(%rsp) 2849 mov %rax, 40(%rsp) # save original %rsp 2850 .cfi_cfa_expression %rsp+40,deref,+8 2851 .Lpowerx5_body: 2852 2853 call __bn_sqrx8x_internal 2854 call __bn_postx4x_internal 2855 call __bn_sqrx8x_internal 2856 call __bn_postx4x_internal 2857 call __bn_sqrx8x_internal 2858 call __bn_postx4x_internal 2859 call __bn_sqrx8x_internal 2860 call __bn_postx4x_internal 2861 call __bn_sqrx8x_internal 2862 call __bn_postx4x_internal 2863 2864 mov %r10,$num # -num 2865 mov $aptr,$rptr 2866 movq %xmm2,$nptr 2867 movq %xmm4,$bptr 2868 mov 40(%rsp),%rax 2869 2870 call mulx4x_internal 2871 2872 mov 40(%rsp),%rsi # restore %rsp 2873 .cfi_def_cfa %rsi,8 2874 mov \$1,%rax 2875 2876 mov -48(%rsi),%r15 2877 .cfi_restore %r15 2878 mov -40(%rsi),%r14 2879 .cfi_restore %r14 2880 mov -32(%rsi),%r13 2881 .cfi_restore %r13 2882 mov -24(%rsi),%r12 2883 .cfi_restore %r12 2884 mov -16(%rsi),%rbp 2885 .cfi_restore %rbp 2886 mov -8(%rsi),%rbx 2887 .cfi_restore %rbx 2888 lea (%rsi),%rsp 2889 .cfi_def_cfa_register %rsp 2890 .Lpowerx5_epilogue: 2891 ret 2892 .cfi_endproc 2893 .size bn_powerx5,.-bn_powerx5 2894 2895 .globl bn_sqrx8x_internal 2896 .hidden bn_sqrx8x_internal 2897 .type bn_sqrx8x_internal,\@abi-omnipotent 2898 .align 32 2899 bn_sqrx8x_internal: 2900 __bn_sqrx8x_internal: 2901 ################################################################## 2902 # Squaring part: 2903 # 2904 # a) multiply-n-add everything but a[i]*a[i]; 2905 # b) shift result of a) by 1 to the left and accumulate 2906 # a[i]*a[i] products; 2907 # 2908 ################################################################## 2909 # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] 2910 # a[1]a[0] 2911 # a[2]a[0] 2912 # a[3]a[0] 2913 # a[2]a[1] 2914 # a[3]a[1] 2915 # a[3]a[2] 2916 # 2917 # a[4]a[0] 2918 # a[5]a[0] 2919 # a[6]a[0] 2920 # a[7]a[0] 2921 # a[4]a[1] 2922 # a[5]a[1] 2923 # a[6]a[1] 2924 # a[7]a[1] 2925 # a[4]a[2] 2926 # a[5]a[2] 2927 # a[6]a[2] 2928 # a[7]a[2] 2929 # a[4]a[3] 2930 # a[5]a[3] 2931 # a[6]a[3] 2932 # a[7]a[3] 2933 # 2934 # a[5]a[4] 2935 # a[6]a[4] 2936 # a[7]a[4] 2937 # a[6]a[5] 2938 # a[7]a[5] 2939 # a[7]a[6] 2940 # a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] 2941 ___ 2942 { 2943 my ($zero,$carry)=("%rbp","%rcx"); 2944 my $aaptr=$zero; 2945 $code.=<<___; 2946 lea 48+8(%rsp),$tptr 2947 lea ($aptr,$num),$aaptr 2948 mov $num,0+8(%rsp) # save $num 2949 mov $aaptr,8+8(%rsp) # save end of $aptr 2950 jmp .Lsqr8x_zero_start 2951 2952 .align 32 2953 .byte 0x66,0x66,0x66,0x2e,0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00 2954 .Lsqrx8x_zero: 2955 .byte 0x3e 2956 movdqa %xmm0,0*8($tptr) 2957 movdqa %xmm0,2*8($tptr) 2958 movdqa %xmm0,4*8($tptr) 2959 movdqa %xmm0,6*8($tptr) 2960 .Lsqr8x_zero_start: # aligned at 32 2961 movdqa %xmm0,8*8($tptr) 2962 movdqa %xmm0,10*8($tptr) 2963 movdqa %xmm0,12*8($tptr) 2964 movdqa %xmm0,14*8($tptr) 2965 lea 16*8($tptr),$tptr 2966 sub \$64,$num 2967 jnz .Lsqrx8x_zero 2968 2969 mov 0*8($aptr),%rdx # a[0], modulo-scheduled 2970 #xor %r9,%r9 # t[1], ex-$num, zero already 2971 xor %r10,%r10 2972 xor %r11,%r11 2973 xor %r12,%r12 2974 xor %r13,%r13 2975 xor %r14,%r14 2976 xor %r15,%r15 2977 lea 48+8(%rsp),$tptr 2978 xor $zero,$zero # cf=0, cf=0 2979 jmp .Lsqrx8x_outer_loop 2980 2981 .align 32 2982 .Lsqrx8x_outer_loop: 2983 mulx 1*8($aptr),%r8,%rax # a[1]*a[0] 2984 adcx %r9,%r8 # a[1]*a[0]+=t[1] 2985 adox %rax,%r10 2986 mulx 2*8($aptr),%r9,%rax # a[2]*a[0] 2987 adcx %r10,%r9 2988 adox %rax,%r11 2989 .byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 # mulx 3*8($aptr),%r10,%rax # ... 2990 adcx %r11,%r10 2991 adox %rax,%r12 2992 .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 # mulx 4*8($aptr),%r11,%rax 2993 adcx %r12,%r11 2994 adox %rax,%r13 2995 mulx 5*8($aptr),%r12,%rax 2996 adcx %r13,%r12 2997 adox %rax,%r14 2998 mulx 6*8($aptr),%r13,%rax 2999 adcx %r14,%r13 3000 adox %r15,%rax 3001 mulx 7*8($aptr),%r14,%r15 3002 mov 1*8($aptr),%rdx # a[1] 3003 adcx %rax,%r14 3004 adox $zero,%r15 3005 adc 8*8($tptr),%r15 3006 mov %r8,1*8($tptr) # t[1] 3007 mov %r9,2*8($tptr) # t[2] 3008 sbb $carry,$carry # mov %cf,$carry 3009 xor $zero,$zero # cf=0, of=0 3010 3011 3012 mulx 2*8($aptr),%r8,%rbx # a[2]*a[1] 3013 mulx 3*8($aptr),%r9,%rax # a[3]*a[1] 3014 adcx %r10,%r8 3015 adox %rbx,%r9 3016 mulx 4*8($aptr),%r10,%rbx # ... 3017 adcx %r11,%r9 3018 adox %rax,%r10 3019 .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 # mulx 5*8($aptr),%r11,%rax 3020 adcx %r12,%r10 3021 adox %rbx,%r11 3022 .byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r12,%rbx 3023 adcx %r13,%r11 3024 adox %r14,%r12 3025 .byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r13,%r14 3026 mov 2*8($aptr),%rdx # a[2] 3027 adcx %rax,%r12 3028 adox %rbx,%r13 3029 adcx %r15,%r13 3030 adox $zero,%r14 # of=0 3031 adcx $zero,%r14 # cf=0 3032 3033 mov %r8,3*8($tptr) # t[3] 3034 mov %r9,4*8($tptr) # t[4] 3035 3036 mulx 3*8($aptr),%r8,%rbx # a[3]*a[2] 3037 mulx 4*8($aptr),%r9,%rax # a[4]*a[2] 3038 adcx %r10,%r8 3039 adox %rbx,%r9 3040 mulx 5*8($aptr),%r10,%rbx # ... 3041 adcx %r11,%r9 3042 adox %rax,%r10 3043 .byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 # mulx 6*8($aptr),%r11,%rax 3044 adcx %r12,%r10 3045 adox %r13,%r11 3046 .byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 7*8($aptr),%r12,%r13 3047 .byte 0x3e 3048 mov 3*8($aptr),%rdx # a[3] 3049 adcx %rbx,%r11 3050 adox %rax,%r12 3051 adcx %r14,%r12 3052 mov %r8,5*8($tptr) # t[5] 3053 mov %r9,6*8($tptr) # t[6] 3054 mulx 4*8($aptr),%r8,%rax # a[4]*a[3] 3055 adox $zero,%r13 # of=0 3056 adcx $zero,%r13 # cf=0 3057 3058 mulx 5*8($aptr),%r9,%rbx # a[5]*a[3] 3059 adcx %r10,%r8 3060 adox %rax,%r9 3061 mulx 6*8($aptr),%r10,%rax # ... 3062 adcx %r11,%r9 3063 adox %r12,%r10 3064 mulx 7*8($aptr),%r11,%r12 3065 mov 4*8($aptr),%rdx # a[4] 3066 mov 5*8($aptr),%r14 # a[5] 3067 adcx %rbx,%r10 3068 adox %rax,%r11 3069 mov 6*8($aptr),%r15 # a[6] 3070 adcx %r13,%r11 3071 adox $zero,%r12 # of=0 3072 adcx $zero,%r12 # cf=0 3073 3074 mov %r8,7*8($tptr) # t[7] 3075 mov %r9,8*8($tptr) # t[8] 3076 3077 mulx %r14,%r9,%rax # a[5]*a[4] 3078 mov 7*8($aptr),%r8 # a[7] 3079 adcx %r10,%r9 3080 mulx %r15,%r10,%rbx # a[6]*a[4] 3081 adox %rax,%r10 3082 adcx %r11,%r10 3083 mulx %r8,%r11,%rax # a[7]*a[4] 3084 mov %r14,%rdx # a[5] 3085 adox %rbx,%r11 3086 adcx %r12,%r11 3087 #adox $zero,%rax # of=0 3088 adcx $zero,%rax # cf=0 3089 3090 mulx %r15,%r14,%rbx # a[6]*a[5] 3091 mulx %r8,%r12,%r13 # a[7]*a[5] 3092 mov %r15,%rdx # a[6] 3093 lea 8*8($aptr),$aptr 3094 adcx %r14,%r11 3095 adox %rbx,%r12 3096 adcx %rax,%r12 3097 adox $zero,%r13 3098 3099 .byte 0x67,0x67 3100 mulx %r8,%r8,%r14 # a[7]*a[6] 3101 adcx %r8,%r13 3102 adcx $zero,%r14 3103 3104 cmp 8+8(%rsp),$aptr 3105 je .Lsqrx8x_outer_break 3106 3107 neg $carry # mov $carry,%cf 3108 mov \$-8,%rcx 3109 mov $zero,%r15 3110 mov 8*8($tptr),%r8 3111 adcx 9*8($tptr),%r9 # +=t[9] 3112 adcx 10*8($tptr),%r10 # ... 3113 adcx 11*8($tptr),%r11 3114 adc 12*8($tptr),%r12 3115 adc 13*8($tptr),%r13 3116 adc 14*8($tptr),%r14 3117 adc 15*8($tptr),%r15 3118 lea ($aptr),$aaptr 3119 lea 2*64($tptr),$tptr 3120 sbb %rax,%rax # mov %cf,$carry 3121 3122 mov -64($aptr),%rdx # a[0] 3123 mov %rax,16+8(%rsp) # offload $carry 3124 mov $tptr,24+8(%rsp) 3125 3126 #lea 8*8($tptr),$tptr # see 2*8*8($tptr) above 3127 xor %eax,%eax # cf=0, of=0 3128 jmp .Lsqrx8x_loop 3129 3130 .align 32 3131 .Lsqrx8x_loop: 3132 mov %r8,%rbx 3133 mulx 0*8($aaptr),%rax,%r8 # a[8]*a[i] 3134 adcx %rax,%rbx # +=t[8] 3135 adox %r9,%r8 3136 3137 mulx 1*8($aaptr),%rax,%r9 # ... 3138 adcx %rax,%r8 3139 adox %r10,%r9 3140 3141 mulx 2*8($aaptr),%rax,%r10 3142 adcx %rax,%r9 3143 adox %r11,%r10 3144 3145 mulx 3*8($aaptr),%rax,%r11 3146 adcx %rax,%r10 3147 adox %r12,%r11 3148 3149 .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 4*8($aaptr),%rax,%r12 3150 adcx %rax,%r11 3151 adox %r13,%r12 3152 3153 mulx 5*8($aaptr),%rax,%r13 3154 adcx %rax,%r12 3155 adox %r14,%r13 3156 3157 mulx 6*8($aaptr),%rax,%r14 3158 mov %rbx,($tptr,%rcx,8) # store t[8+i] 3159 mov \$0,%ebx 3160 adcx %rax,%r13 3161 adox %r15,%r14 3162 3163 .byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 # mulx 7*8($aaptr),%rax,%r15 3164 mov 8($aptr,%rcx,8),%rdx # a[i] 3165 adcx %rax,%r14 3166 adox %rbx,%r15 # %rbx is 0, of=0 3167 adcx %rbx,%r15 # cf=0 3168 3169 .byte 0x67 3170 inc %rcx # of=0 3171 jnz .Lsqrx8x_loop 3172 3173 lea 8*8($aaptr),$aaptr 3174 mov \$-8,%rcx 3175 cmp 8+8(%rsp),$aaptr # done? 3176 je .Lsqrx8x_break 3177 3178 sub 16+8(%rsp),%rbx # mov 16(%rsp),%cf 3179 .byte 0x66 3180 mov -64($aptr),%rdx 3181 adcx 0*8($tptr),%r8 3182 adcx 1*8($tptr),%r9 3183 adc 2*8($tptr),%r10 3184 adc 3*8($tptr),%r11 3185 adc 4*8($tptr),%r12 3186 adc 5*8($tptr),%r13 3187 adc 6*8($tptr),%r14 3188 adc 7*8($tptr),%r15 3189 lea 8*8($tptr),$tptr 3190 .byte 0x67 3191 sbb %rax,%rax # mov %cf,%rax 3192 xor %ebx,%ebx # cf=0, of=0 3193 mov %rax,16+8(%rsp) # offload carry 3194 jmp .Lsqrx8x_loop 3195 3196 .align 32 3197 .Lsqrx8x_break: 3198 sub 16+8(%rsp),%r8 # consume last carry 3199 mov 24+8(%rsp),$carry # initial $tptr, borrow $carry 3200 mov 0*8($aptr),%rdx # a[8], modulo-scheduled 3201 xor %ebp,%ebp # xor $zero,$zero 3202 mov %r8,0*8($tptr) 3203 cmp $carry,$tptr # cf=0, of=0 3204 je .Lsqrx8x_outer_loop 3205 3206 mov %r9,1*8($tptr) 3207 mov 1*8($carry),%r9 3208 mov %r10,2*8($tptr) 3209 mov 2*8($carry),%r10 3210 mov %r11,3*8($tptr) 3211 mov 3*8($carry),%r11 3212 mov %r12,4*8($tptr) 3213 mov 4*8($carry),%r12 3214 mov %r13,5*8($tptr) 3215 mov 5*8($carry),%r13 3216 mov %r14,6*8($tptr) 3217 mov 6*8($carry),%r14 3218 mov %r15,7*8($tptr) 3219 mov 7*8($carry),%r15 3220 mov $carry,$tptr 3221 jmp .Lsqrx8x_outer_loop 3222 3223 .align 32 3224 .Lsqrx8x_outer_break: 3225 mov %r9,9*8($tptr) # t[9] 3226 movq %xmm3,%rcx # -$num 3227 mov %r10,10*8($tptr) # ... 3228 mov %r11,11*8($tptr) 3229 mov %r12,12*8($tptr) 3230 mov %r13,13*8($tptr) 3231 mov %r14,14*8($tptr) 3232 ___ 3233 }{ 3235 my $i="%rcx"; 3236 $code.=<<___; 3237 lea 48+8(%rsp),$tptr 3238 mov ($aptr,$i),%rdx # a[0] 3239 3240 mov 8($tptr),$A0[1] # t[1] 3241 xor $A0[0],$A0[0] # t[0], of=0, cf=0 3242 mov 0+8(%rsp),$num # restore $num 3243 adox $A0[1],$A0[1] 3244 mov 16($tptr),$A1[0] # t[2] # prefetch 3245 mov 24($tptr),$A1[1] # t[3] # prefetch 3246 #jmp .Lsqrx4x_shift_n_add # happens to be aligned 3247 3248 .align 32 3249 .Lsqrx4x_shift_n_add: 3250 mulx %rdx,%rax,%rbx 3251 adox $A1[0],$A1[0] 3252 adcx $A0[0],%rax 3253 .byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 # mov 8($aptr,$i),%rdx # a[i+1] # prefetch 3254 .byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 # mov 32($tptr),$A0[0] # t[2*i+4] # prefetch 3255 adox $A1[1],$A1[1] 3256 adcx $A0[1],%rbx 3257 mov 40($tptr),$A0[1] # t[2*i+4+1] # prefetch 3258 mov %rax,0($tptr) 3259 mov %rbx,8($tptr) 3260 3261 mulx %rdx,%rax,%rbx 3262 adox $A0[0],$A0[0] 3263 adcx $A1[0],%rax 3264 mov 16($aptr,$i),%rdx # a[i+2] # prefetch 3265 mov 48($tptr),$A1[0] # t[2*i+6] # prefetch 3266 adox $A0[1],$A0[1] 3267 adcx $A1[1],%rbx 3268 mov 56($tptr),$A1[1] # t[2*i+6+1] # prefetch 3269 mov %rax,16($tptr) 3270 mov %rbx,24($tptr) 3271 3272 mulx %rdx,%rax,%rbx 3273 adox $A1[0],$A1[0] 3274 adcx $A0[0],%rax 3275 mov 24($aptr,$i),%rdx # a[i+3] # prefetch 3276 lea 32($i),$i 3277 mov 64($tptr),$A0[0] # t[2*i+8] # prefetch 3278 adox $A1[1],$A1[1] 3279 adcx $A0[1],%rbx 3280 mov 72($tptr),$A0[1] # t[2*i+8+1] # prefetch 3281 mov %rax,32($tptr) 3282 mov %rbx,40($tptr) 3283 3284 mulx %rdx,%rax,%rbx 3285 adox $A0[0],$A0[0] 3286 adcx $A1[0],%rax 3287 jrcxz .Lsqrx4x_shift_n_add_break 3288 .byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 # mov 0($aptr,$i),%rdx # a[i+4] # prefetch 3289 adox $A0[1],$A0[1] 3290 adcx $A1[1],%rbx 3291 mov 80($tptr),$A1[0] # t[2*i+10] # prefetch 3292 mov 88($tptr),$A1[1] # t[2*i+10+1] # prefetch 3293 mov %rax,48($tptr) 3294 mov %rbx,56($tptr) 3295 lea 64($tptr),$tptr 3296 nop 3297 jmp .Lsqrx4x_shift_n_add 3298 3299 .align 32 3300 .Lsqrx4x_shift_n_add_break: 3301 adcx $A1[1],%rbx 3302 mov %rax,48($tptr) 3303 mov %rbx,56($tptr) 3304 lea 64($tptr),$tptr # end of t[] buffer 3305 ___ 3306 } 3308 ###################################################################### 3309 # Montgomery reduction part, "word-by-word" algorithm. 3310 # 3311 # This new path is inspired by multiple submissions from Intel, by 3312 # Shay Gueron, Vlad Krasnov, Erdinc Ozturk, James Guilford, 3313 # Vinodh Gopal... 3314 { 3315 my ($nptr,$carry,$m0)=("%rbp","%rsi","%rdx"); 3316 3317 $code.=<<___; 3318 movq %xmm2,$nptr 3319 __bn_sqrx8x_reduction: 3320 xor %eax,%eax # initial top-most carry bit 3321 mov 32+8(%rsp),%rbx # n0 3322 mov 48+8(%rsp),%rdx # "%r8", 8*0($tptr) 3323 lea -8*8($nptr,$num),%rcx # end of n[] 3324 #lea 48+8(%rsp,$num,2),$tptr # end of t[] buffer 3325 mov %rcx, 0+8(%rsp) # save end of n[] 3326 mov $tptr,8+8(%rsp) # save end of t[] 3327 3328 lea 48+8(%rsp),$tptr # initial t[] window 3329 jmp .Lsqrx8x_reduction_loop 3330 3331 .align 32 3332 .Lsqrx8x_reduction_loop: 3333 mov 8*1($tptr),%r9 3334 mov 8*2($tptr),%r10 3335 mov 8*3($tptr),%r11 3336 mov 8*4($tptr),%r12 3337 mov %rdx,%r8 3338 imulq %rbx,%rdx # n0*a[i] 3339 mov 8*5($tptr),%r13 3340 mov 8*6($tptr),%r14 3341 mov 8*7($tptr),%r15 3342 mov %rax,24+8(%rsp) # store top-most carry bit 3343 3344 lea 8*8($tptr),$tptr 3345 xor $carry,$carry # cf=0,of=0 3346 mov \$-8,%rcx 3347 jmp .Lsqrx8x_reduce 3348 3349 .align 32 3350 .Lsqrx8x_reduce: 3351 mov %r8, %rbx 3352 mulx 8*0($nptr),%rax,%r8 # n[0] 3353 adcx %rbx,%rax # discarded 3354 adox %r9,%r8 3355 3356 mulx 8*1($nptr),%rbx,%r9 # n[1] 3357 adcx %rbx,%r8 3358 adox %r10,%r9 3359 3360 mulx 8*2($nptr),%rbx,%r10 3361 adcx %rbx,%r9 3362 adox %r11,%r10 3363 3364 mulx 8*3($nptr),%rbx,%r11 3365 adcx %rbx,%r10 3366 adox %r12,%r11 3367 3368 .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 8*4($nptr),%rbx,%r12 3369 mov %rdx,%rax 3370 mov %r8,%rdx 3371 adcx %rbx,%r11 3372 adox %r13,%r12 3373 3374 mulx 32+8(%rsp),%rbx,%rdx # %rdx discarded 3375 mov %rax,%rdx 3376 mov %rax,64+48+8(%rsp,%rcx,8) # put aside n0*a[i] 3377 3378 mulx 8*5($nptr),%rax,%r13 3379 adcx %rax,%r12 3380 adox %r14,%r13 3381 3382 mulx 8*6($nptr),%rax,%r14 3383 adcx %rax,%r13 3384 adox %r15,%r14 3385 3386 mulx 8*7($nptr),%rax,%r15 3387 mov %rbx,%rdx 3388 adcx %rax,%r14 3389 adox $carry,%r15 # $carry is 0 3390 adcx $carry,%r15 # cf=0 3391 3392 .byte 0x67,0x67,0x67 3393 inc %rcx # of=0 3394 jnz .Lsqrx8x_reduce 3395 3396 mov $carry,%rax # xor %rax,%rax 3397 cmp 0+8(%rsp),$nptr # end of n[]? 3398 jae .Lsqrx8x_no_tail 3399 3400 mov 48+8(%rsp),%rdx # pull n0*a[0] 3401 add 8*0($tptr),%r8 3402 lea 8*8($nptr),$nptr 3403 mov \$-8,%rcx 3404 adcx 8*1($tptr),%r9 3405 adcx 8*2($tptr),%r10 3406 adc 8*3($tptr),%r11 3407 adc 8*4($tptr),%r12 3408 adc 8*5($tptr),%r13 3409 adc 8*6($tptr),%r14 3410 adc 8*7($tptr),%r15 3411 lea 8*8($tptr),$tptr 3412 sbb %rax,%rax # top carry 3413 3414 xor $carry,$carry # of=0, cf=0 3415 mov %rax,16+8(%rsp) 3416 jmp .Lsqrx8x_tail 3417 3418 .align 32 3419 .Lsqrx8x_tail: 3420 mov %r8,%rbx 3421 mulx 8*0($nptr),%rax,%r8 3422 adcx %rax,%rbx 3423 adox %r9,%r8 3424 3425 mulx 8*1($nptr),%rax,%r9 3426 adcx %rax,%r8 3427 adox %r10,%r9 3428 3429 mulx 8*2($nptr),%rax,%r10 3430 adcx %rax,%r9 3431 adox %r11,%r10 3432 3433 mulx 8*3($nptr),%rax,%r11 3434 adcx %rax,%r10 3435 adox %r12,%r11 3436 3437 .byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 8*4($nptr),%rax,%r12 3438 adcx %rax,%r11 3439 adox %r13,%r12 3440 3441 mulx 8*5($nptr),%rax,%r13 3442 adcx %rax,%r12 3443 adox %r14,%r13 3444 3445 mulx 8*6($nptr),%rax,%r14 3446 adcx %rax,%r13 3447 adox %r15,%r14 3448 3449 mulx 8*7($nptr),%rax,%r15 3450 mov 72+48+8(%rsp,%rcx,8),%rdx # pull n0*a[i] 3451 adcx %rax,%r14 3452 adox $carry,%r15 3453 mov %rbx,($tptr,%rcx,8) # save result 3454 mov %r8,%rbx 3455 adcx $carry,%r15 # cf=0 3456 3457 inc %rcx # of=0 3458 jnz .Lsqrx8x_tail 3459 3460 cmp 0+8(%rsp),$nptr # end of n[]? 3461 jae .Lsqrx8x_tail_done # break out of loop 3462 3463 sub 16+8(%rsp),$carry # mov 16(%rsp),%cf 3464 mov 48+8(%rsp),%rdx # pull n0*a[0] 3465 lea 8*8($nptr),$nptr 3466 adc 8*0($tptr),%r8 3467 adc 8*1($tptr),%r9 3468 adc 8*2($tptr),%r10 3469 adc 8*3($tptr),%r11 3470 adc 8*4($tptr),%r12 3471 adc 8*5($tptr),%r13 3472 adc 8*6($tptr),%r14 3473 adc 8*7($tptr),%r15 3474 lea 8*8($tptr),$tptr 3475 sbb %rax,%rax 3476 sub \$8,%rcx # mov \$-8,%rcx 3477 3478 xor $carry,$carry # of=0, cf=0 3479 mov %rax,16+8(%rsp) 3480 jmp .Lsqrx8x_tail 3481 3482 .align 32 3483 .Lsqrx8x_tail_done: 3484 xor %rax,%rax 3485 add 24+8(%rsp),%r8 # can this overflow? 3486 adc \$0,%r9 3487 adc \$0,%r10 3488 adc \$0,%r11 3489 adc \$0,%r12 3490 adc \$0,%r13 3491 adc \$0,%r14 3492 adc \$0,%r15 3493 adc \$0,%rax 3494 3495 sub 16+8(%rsp),$carry # mov 16(%rsp),%cf 3496 .Lsqrx8x_no_tail: # %cf is 0 if jumped here 3497 adc 8*0($tptr),%r8 3498 movq %xmm3,%rcx 3499 adc 8*1($tptr),%r9 3500 mov 8*7($nptr),$carry 3501 movq %xmm2,$nptr # restore $nptr 3502 adc 8*2($tptr),%r10 3503 adc 8*3($tptr),%r11 3504 adc 8*4($tptr),%r12 3505 adc 8*5($tptr),%r13 3506 adc 8*6($tptr),%r14 3507 adc 8*7($tptr),%r15 3508 adc \$0,%rax # top-most carry 3509 3510 mov 32+8(%rsp),%rbx # n0 3511 mov 8*8($tptr,%rcx),%rdx # modulo-scheduled "%r8" 3512 3513 mov %r8,8*0($tptr) # store top 512 bits 3514 lea 8*8($tptr),%r8 # borrow %r8 3515 mov %r9,8*1($tptr) 3516 mov %r10,8*2($tptr) 3517 mov %r11,8*3($tptr) 3518 mov %r12,8*4($tptr) 3519 mov %r13,8*5($tptr) 3520 mov %r14,8*6($tptr) 3521 mov %r15,8*7($tptr) 3522 3523 lea 8*8($tptr,%rcx),$tptr # start of current t[] window 3524 cmp 8+8(%rsp),%r8 # end of t[]? 3525 jb .Lsqrx8x_reduction_loop 3526 ret 3527 .size bn_sqrx8x_internal,.-bn_sqrx8x_internal 3528 ___ 3529 } 3531 ############################################################## 3532 # Post-condition, 4x unrolled 3533 # 3534 { 3535 my ($rptr,$nptr)=("%rdx","%rbp"); 3536 $code.=<<___; 3537 .align 32 3538 __bn_postx4x_internal: 3539 mov 8*0($nptr),%r12 3540 mov %rcx,%r10 # -$num 3541 mov %rcx,%r9 # -$num 3542 neg %rax 3543 sar \$3+2,%rcx 3544 #lea 48+8(%rsp,%r9),$tptr 3545 movq %xmm1,$rptr # restore $rptr 3546 movq %xmm1,$aptr # prepare for back-to-back call 3547 dec %r12 # so that after 'not' we get -n[0] 3548 mov 8*1($nptr),%r13 3549 xor %r8,%r8 3550 mov 8*2($nptr),%r14 3551 mov 8*3($nptr),%r15 3552 jmp .Lsqrx4x_sub_entry 3553 3554 .align 16 3555 .Lsqrx4x_sub: 3556 mov 8*0($nptr),%r12 3557 mov 8*1($nptr),%r13 3558 mov 8*2($nptr),%r14 3559 mov 8*3($nptr),%r15 3560 .Lsqrx4x_sub_entry: 3561 andn %rax,%r12,%r12 3562 lea 8*4($nptr),$nptr 3563 andn %rax,%r13,%r13 3564 andn %rax,%r14,%r14 3565 andn %rax,%r15,%r15 3566 3567 neg %r8 # mov %r8,%cf 3568 adc 8*0($tptr),%r12 3569 adc 8*1($tptr),%r13 3570 adc 8*2($tptr),%r14 3571 adc 8*3($tptr),%r15 3572 mov %r12,8*0($rptr) 3573 lea 8*4($tptr),$tptr 3574 mov %r13,8*1($rptr) 3575 sbb %r8,%r8 # mov %cf,%r8 3576 mov %r14,8*2($rptr) 3577 mov %r15,8*3($rptr) 3578 lea 8*4($rptr),$rptr 3579 3580 inc %rcx 3581 jnz .Lsqrx4x_sub 3582 3583 neg %r9 # restore $num 3584 3585 ret 3586 .size __bn_postx4x_internal,.-__bn_postx4x_internal 3587 ___ 3588 } 3589 }}} 3590 { 3591 my ($inp,$num,$tbl,$idx)=$win64?("%rcx","%edx","%r8", "%r9d") : # Win64 order 3592 ("%rdi","%esi","%rdx","%ecx"); # Unix order 3593 my $out=$inp; 3594 my $STRIDE=2**5*8; 3595 my $N=$STRIDE/4; 3596 3597 $code.=<<___; 3598 .globl bn_scatter5 3599 .type bn_scatter5,\@abi-omnipotent 3600 .align 16 3601 bn_scatter5: 3602 cmp \$0, $num 3603 jz .Lscatter_epilogue 3604 lea ($tbl,$idx,8),$tbl 3605 .Lscatter: 3606 mov ($inp),%rax 3607 lea 8($inp),$inp 3608 mov %rax,($tbl) 3609 lea 32*8($tbl),$tbl 3610 sub \$1,$num 3611 jnz .Lscatter 3612 .Lscatter_epilogue: 3613 ret 3614 .size bn_scatter5,.-bn_scatter5 3615 3616 .globl bn_gather5 3617 .type bn_gather5,\@abi-omnipotent 3618 .align 32 3619 bn_gather5: 3620 .LSEH_begin_bn_gather5: # Win64 thing, but harmless in other cases 3621 # I can't trust assembler to use specific encoding:-( 3622 .byte 0x4c,0x8d,0x14,0x24 #lea (%rsp),%r10 3623 .byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 #sub $0x108,%rsp 3624 lea .Linc(%rip),%rax 3625 and \$-16,%rsp # shouldn't be formally required 3626 3627 movd $idx,%xmm5 3628 movdqa 0(%rax),%xmm0 # 00000001000000010000000000000000 3629 movdqa 16(%rax),%xmm1 # 00000002000000020000000200000002 3630 lea 128($tbl),%r11 # size optimization 3631 lea 128(%rsp),%rax # size optimization 3632 3633 pshufd \$0,%xmm5,%xmm5 # broadcast $idx 3634 movdqa %xmm1,%xmm4 3635 movdqa %xmm1,%xmm2 3636 ___ 3637 ######################################################################## 3638 # calculate mask by comparing 0..31 to $idx and save result to stack 3639 # 3640 for($i=0;$i<$STRIDE/16;$i+=4) { 3641 $code.=<<___; 3642 paddd %xmm0,%xmm1 3643 pcmpeqd %xmm5,%xmm0 # compare to 1,0 3644 ___ 3645 $code.=<<___ if ($i); 3646 movdqa %xmm3,`16*($i-1)-128`(%rax) 3647 ___ 3648 $code.=<<___; 3649 movdqa %xmm4,%xmm3 3650 3651 paddd %xmm1,%xmm2 3652 pcmpeqd %xmm5,%xmm1 # compare to 3,2 3653 movdqa %xmm0,`16*($i+0)-128`(%rax) 3654 movdqa %xmm4,%xmm0 3655 3656 paddd %xmm2,%xmm3 3657 pcmpeqd %xmm5,%xmm2 # compare to 5,4 3658 movdqa %xmm1,`16*($i+1)-128`(%rax) 3659 movdqa %xmm4,%xmm1 3660 3661 paddd %xmm3,%xmm0 3662 pcmpeqd %xmm5,%xmm3 # compare to 7,6 3663 movdqa %xmm2,`16*($i+2)-128`(%rax) 3664 movdqa %xmm4,%xmm2 3665 ___ 3666 } 3667 $code.=<<___; 3668 movdqa %xmm3,`16*($i-1)-128`(%rax) 3669 jmp .Lgather 3670 3671 .align 32 3672 .Lgather: 3673 pxor %xmm4,%xmm4 3674 pxor %xmm5,%xmm5 3675 ___ 3676 for($i=0;$i<$STRIDE/16;$i+=4) { 3677 $code.=<<___; 3678 movdqa `16*($i+0)-128`(%r11),%xmm0 3679 movdqa `16*($i+1)-128`(%r11),%xmm1 3680 movdqa `16*($i+2)-128`(%r11),%xmm2 3681 pand `16*($i+0)-128`(%rax),%xmm0 3682 movdqa `16*($i+3)-128`(%r11),%xmm3 3683 pand `16*($i+1)-128`(%rax),%xmm1 3684 por %xmm0,%xmm4 3685 pand `16*($i+2)-128`(%rax),%xmm2 3686 por %xmm1,%xmm5 3687 pand `16*($i+3)-128`(%rax),%xmm3 3688 por %xmm2,%xmm4 3689 por %xmm3,%xmm5 3690 ___ 3691 } 3692 $code.=<<___; 3693 por %xmm5,%xmm4 3694 lea $STRIDE(%r11),%r11 3695 pshufd \$0x4e,%xmm4,%xmm0 3696 por %xmm4,%xmm0 3697 movq %xmm0,($out) # m0=bp[0] 3698 lea 8($out),$out 3699 sub \$1,$num 3700 jnz .Lgather 3701 3702 lea (%r10),%rsp 3703 ret 3704 .LSEH_end_bn_gather5: 3705 .size bn_gather5,.-bn_gather5 3706 ___ 3707 } 3708 $code.=<<___; 3709 .align 64 3710 .Linc: 3711 .long 0,0, 1,1 3712 .long 2,2, 2,2 3713 .asciz "Montgomery Multiplication with scatter/gather for x86_64, CRYPTOGAMS by <appro\@openssl.org>" 3714 ___ 3715 3716 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 3717 # CONTEXT *context,DISPATCHER_CONTEXT *disp) 3718 if ($win64) { 3719 $rec="%rcx"; 3720 $frame="%rdx"; 3721 $context="%r8"; 3722 $disp="%r9"; 3723 3724 $code.=<<___; 3725 .extern __imp_RtlVirtualUnwind 3726 .type mul_handler,\@abi-omnipotent 3727 .align 16 3728 mul_handler: 3729 push %rsi 3730 push %rdi 3731 push %rbx 3732 push %rbp 3733 push %r12 3734 push %r13 3735 push %r14 3736 push %r15 3737 pushfq 3738 sub \$64,%rsp 3739 3740 mov 120($context),%rax # pull context->Rax 3741 mov 248($context),%rbx # pull context->Rip 3742 3743 mov 8($disp),%rsi # disp->ImageBase 3744 mov 56($disp),%r11 # disp->HandlerData 3745 3746 mov 0(%r11),%r10d # HandlerData[0] 3747 lea (%rsi,%r10),%r10 # end of prologue label 3748 cmp %r10,%rbx # context->Rip<end of prologue label 3749 jb .Lcommon_seh_tail 3750 3751 mov 4(%r11),%r10d # HandlerData[1] 3752 lea (%rsi,%r10),%r10 # beginning of body label 3753 cmp %r10,%rbx # context->Rip<body label 3754 jb .Lcommon_pop_regs 3755 3756 mov 152($context),%rax # pull context->Rsp 3757 3758 mov 8(%r11),%r10d # HandlerData[2] 3759 lea (%rsi,%r10),%r10 # epilogue label 3760 cmp %r10,%rbx # context->Rip>=epilogue label 3761 jae .Lcommon_seh_tail 3762 3763 lea .Lmul_epilogue(%rip),%r10 3764 cmp %r10,%rbx 3765 ja .Lbody_40 3766 3767 mov 192($context),%r10 # pull $num 3768 mov 8(%rax,%r10,8),%rax # pull saved stack pointer 3769 3770 jmp .Lcommon_pop_regs 3771 3772 .Lbody_40: 3773 mov 40(%rax),%rax # pull saved stack pointer 3774 .Lcommon_pop_regs: 3775 mov -8(%rax),%rbx 3776 mov -16(%rax),%rbp 3777 mov -24(%rax),%r12 3778 mov -32(%rax),%r13 3779 mov -40(%rax),%r14 3780 mov -48(%rax),%r15 3781 mov %rbx,144($context) # restore context->Rbx 3782 mov %rbp,160($context) # restore context->Rbp 3783 mov %r12,216($context) # restore context->R12 3784 mov %r13,224($context) # restore context->R13 3785 mov %r14,232($context) # restore context->R14 3786 mov %r15,240($context) # restore context->R15 3787 3788 .Lcommon_seh_tail: 3789 mov 8(%rax),%rdi 3790 mov 16(%rax),%rsi 3791 mov %rax,152($context) # restore context->Rsp 3792 mov %rsi,168($context) # restore context->Rsi 3793 mov %rdi,176($context) # restore context->Rdi 3794 3795 mov 40($disp),%rdi # disp->ContextRecord 3796 mov $context,%rsi # context 3797 mov \$154,%ecx # sizeof(CONTEXT) 3798 .long 0xa548f3fc # cld; rep movsq 3799 3800 mov $disp,%rsi 3801 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 3802 mov 8(%rsi),%rdx # arg2, disp->ImageBase 3803 mov 0(%rsi),%r8 # arg3, disp->ControlPc 3804 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 3805 mov 40(%rsi),%r10 # disp->ContextRecord 3806 lea 56(%rsi),%r11 # &disp->HandlerData 3807 lea 24(%rsi),%r12 # &disp->EstablisherFrame 3808 mov %r10,32(%rsp) # arg5 3809 mov %r11,40(%rsp) # arg6 3810 mov %r12,48(%rsp) # arg7 3811 mov %rcx,56(%rsp) # arg8, (NULL) 3812 call *__imp_RtlVirtualUnwind(%rip) 3813 3814 mov \$1,%eax # ExceptionContinueSearch 3815 add \$64,%rsp 3816 popfq 3817 pop %r15 3818 pop %r14 3819 pop %r13 3820 pop %r12 3821 pop %rbp 3822 pop %rbx 3823 pop %rdi 3824 pop %rsi 3825 ret 3826 .size mul_handler,.-mul_handler 3827 3828 .section .pdata 3829 .align 4 3830 .rva .LSEH_begin_bn_mul_mont_gather5 3831 .rva .LSEH_end_bn_mul_mont_gather5 3832 .rva .LSEH_info_bn_mul_mont_gather5 3833 3834 .rva .LSEH_begin_bn_mul4x_mont_gather5 3835 .rva .LSEH_end_bn_mul4x_mont_gather5 3836 .rva .LSEH_info_bn_mul4x_mont_gather5 3837 3838 .rva .LSEH_begin_bn_power5 3839 .rva .LSEH_end_bn_power5 3840 .rva .LSEH_info_bn_power5 3841 3842 .rva .LSEH_begin_bn_from_mont8x 3843 .rva .LSEH_end_bn_from_mont8x 3844 .rva .LSEH_info_bn_from_mont8x 3845 ___ 3846 $code.=<<___ if ($addx); 3847 .rva .LSEH_begin_bn_mulx4x_mont_gather5 3848 .rva .LSEH_end_bn_mulx4x_mont_gather5 3849 .rva .LSEH_info_bn_mulx4x_mont_gather5 3850 3851 .rva .LSEH_begin_bn_powerx5 3852 .rva .LSEH_end_bn_powerx5 3853 .rva .LSEH_info_bn_powerx5 3854 ___ 3855 $code.=<<___; 3856 .rva .LSEH_begin_bn_gather5 3857 .rva .LSEH_end_bn_gather5 3858 .rva .LSEH_info_bn_gather5 3859 3860 .section .xdata 3861 .align 8 3862 .LSEH_info_bn_mul_mont_gather5: 3863 .byte 9,0,0,0 3864 .rva mul_handler 3865 .rva .Lmul_body,.Lmul_body,.Lmul_epilogue # HandlerData[] 3866 .align 8 3867 .LSEH_info_bn_mul4x_mont_gather5: 3868 .byte 9,0,0,0 3869 .rva mul_handler 3870 .rva .Lmul4x_prologue,.Lmul4x_body,.Lmul4x_epilogue # HandlerData[] 3871 .align 8 3872 .LSEH_info_bn_power5: 3873 .byte 9,0,0,0 3874 .rva mul_handler 3875 .rva .Lpower5_prologue,.Lpower5_body,.Lpower5_epilogue # HandlerData[] 3876 .align 8 3877 .LSEH_info_bn_from_mont8x: 3878 .byte 9,0,0,0 3879 .rva mul_handler 3880 .rva .Lfrom_prologue,.Lfrom_body,.Lfrom_epilogue # HandlerData[] 3881 ___ 3882 $code.=<<___ if ($addx); 3883 .align 8 3884 .LSEH_info_bn_mulx4x_mont_gather5: 3885 .byte 9,0,0,0 3886 .rva mul_handler 3887 .rva .Lmulx4x_prologue,.Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[] 3888 .align 8 3889 .LSEH_info_bn_powerx5: 3890 .byte 9,0,0,0 3891 .rva mul_handler 3892 .rva .Lpowerx5_prologue,.Lpowerx5_body,.Lpowerx5_epilogue # HandlerData[] 3893 ___ 3894 $code.=<<___; 3895 .align 8 3896 .LSEH_info_bn_gather5: 3897 .byte 0x01,0x0b,0x03,0x0a 3898 .byte 0x0b,0x01,0x21,0x00 # sub rsp,0x108 3899 .byte 0x04,0xa3,0x00,0x00 # lea r10,(rsp) 3900 .align 8 3901 ___ 3902 } 3903 3904 $code =~ s/\`([^\`]*)\`/eval($1)/gem; 3905 3906 print $code; 3907 close STDOUT; 3908