1 #!/usr/bin/env perl 2 # 3 # Copyright (c) 2010-2011 Intel Corp. 4 # Author: Vinodh.Gopal (at] intel.com 5 # Jim Guilford 6 # Erdinc.Ozturk (at] intel.com 7 # Maxim.Perminov (at] intel.com 8 # 9 # More information about algorithm used can be found at: 10 # http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf 11 # 12 # ==================================================================== 13 # Copyright (c) 2011 The OpenSSL Project. All rights reserved. 14 # 15 # Redistribution and use in source and binary forms, with or without 16 # modification, are permitted provided that the following conditions 17 # are met: 18 # 19 # 1. Redistributions of source code must retain the above copyright 20 # notice, this list of conditions and the following disclaimer. 21 # 22 # 2. Redistributions in binary form must reproduce the above copyright 23 # notice, this list of conditions and the following disclaimer in 24 # the documentation and/or other materials provided with the 25 # distribution. 26 # 27 # 3. All advertising materials mentioning features or use of this 28 # software must display the following acknowledgment: 29 # "This product includes software developed by the OpenSSL Project 30 # for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" 31 # 32 # 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 33 # endorse or promote products derived from this software without 34 # prior written permission. For written permission, please contact 35 # licensing (at] OpenSSL.org. 36 # 37 # 5. Products derived from this software may not be called "OpenSSL" 38 # nor may "OpenSSL" appear in their names without prior written 39 # permission of the OpenSSL Project. 40 # 41 # 6. Redistributions of any form whatsoever must retain the following 42 # acknowledgment: 43 # "This product includes software developed by the OpenSSL Project 44 # for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" 45 # 46 # THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 47 # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 49 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 50 # ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 51 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 52 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 53 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 55 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 56 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 57 # OF THE POSSIBILITY OF SUCH DAMAGE. 58 # ==================================================================== 59 60 $flavour = shift; 61 $output = shift; 62 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 63 64 my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 65 66 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 67 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 68 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 69 die "can't locate x86_64-xlate.pl"; 70 71 open OUT,"| \"$^X\" $xlate $flavour $output"; 72 *STDOUT=*OUT; 73 74 use strict; 75 my $code=".text\n\n"; 76 my $m=0; 77 78 # 79 # Define x512 macros 80 # 81 82 #MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2 83 # 84 # uses rax, rdx, and args 85 sub MULSTEP_512_ADD 86 { 87 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_; 88 my @X=@$x; # make a copy 89 $code.=<<___; 90 mov (+8*0)($SRC2), %rax 91 mul $OP # rdx:rax = %OP * [0] 92 mov ($ASRC), $X[0] 93 add %rax, $X[0] 94 adc \$0, %rdx 95 mov $X[0], $DST 96 ___ 97 for(my $i=1;$i<8;$i++) { 98 $code.=<<___; 99 mov %rdx, $TMP 100 101 mov (+8*$i)($SRC2), %rax 102 mul $OP # rdx:rax = %OP * [$i] 103 mov (+8*$i)($ASRC), $X[$i] 104 add %rax, $X[$i] 105 adc \$0, %rdx 106 add $TMP, $X[$i] 107 adc \$0, %rdx 108 ___ 109 } 110 $code.=<<___; 111 mov %rdx, $X[0] 112 ___ 113 } 114 115 #MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp 116 # 117 # uses rax, rdx, and args 118 sub MULSTEP_512 119 { 120 my ($x, $DST, $SRC2, $OP, $TMP)=@_; 121 my @X=@$x; # make a copy 122 $code.=<<___; 123 mov (+8*0)($SRC2), %rax 124 mul $OP # rdx:rax = %OP * [0] 125 add %rax, $X[0] 126 adc \$0, %rdx 127 mov $X[0], $DST 128 ___ 129 for(my $i=1;$i<8;$i++) { 130 $code.=<<___; 131 mov %rdx, $TMP 132 133 mov (+8*$i)($SRC2), %rax 134 mul $OP # rdx:rax = %OP * [$i] 135 add %rax, $X[$i] 136 adc \$0, %rdx 137 add $TMP, $X[$i] 138 adc \$0, %rdx 139 ___ 140 } 141 $code.=<<___; 142 mov %rdx, $X[0] 143 ___ 144 } 145 146 # 147 # Swizzle Macros 148 # 149 150 # macro to copy data from flat space to swizzled table 151 #MACRO swizzle pDst, pSrc, tmp1, tmp2 152 # pDst and pSrc are modified 153 sub swizzle 154 { 155 my ($pDst, $pSrc, $cnt, $d0)=@_; 156 $code.=<<___; 157 mov \$8, $cnt 158 loop_$m: 159 mov ($pSrc), $d0 160 mov $d0#w, ($pDst) 161 shr \$16, $d0 162 mov $d0#w, (+64*1)($pDst) 163 shr \$16, $d0 164 mov $d0#w, (+64*2)($pDst) 165 shr \$16, $d0 166 mov $d0#w, (+64*3)($pDst) 167 lea 8($pSrc), $pSrc 168 lea 64*4($pDst), $pDst 169 dec $cnt 170 jnz loop_$m 171 ___ 172 173 $m++; 174 } 175 176 # macro to copy data from swizzled table to flat space 177 #MACRO unswizzle pDst, pSrc, tmp*3 178 sub unswizzle 179 { 180 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_; 181 $code.=<<___; 182 mov \$4, $cnt 183 loop_$m: 184 movzxw (+64*3+256*0)($pSrc), $d0 185 movzxw (+64*3+256*1)($pSrc), $d1 186 shl \$16, $d0 187 shl \$16, $d1 188 mov (+64*2+256*0)($pSrc), $d0#w 189 mov (+64*2+256*1)($pSrc), $d1#w 190 shl \$16, $d0 191 shl \$16, $d1 192 mov (+64*1+256*0)($pSrc), $d0#w 193 mov (+64*1+256*1)($pSrc), $d1#w 194 shl \$16, $d0 195 shl \$16, $d1 196 mov (+64*0+256*0)($pSrc), $d0#w 197 mov (+64*0+256*1)($pSrc), $d1#w 198 mov $d0, (+8*0)($pDst) 199 mov $d1, (+8*1)($pDst) 200 lea 256*2($pSrc), $pSrc 201 lea 8*2($pDst), $pDst 202 sub \$1, $cnt 203 jnz loop_$m 204 ___ 205 206 $m++; 207 } 208 209 # 210 # Data Structures 211 # 212 213 # Reduce Data 214 # 215 # 216 # Offset Value 217 # 0C0 Carries 218 # 0B8 X2[10] 219 # 0B0 X2[9] 220 # 0A8 X2[8] 221 # 0A0 X2[7] 222 # 098 X2[6] 223 # 090 X2[5] 224 # 088 X2[4] 225 # 080 X2[3] 226 # 078 X2[2] 227 # 070 X2[1] 228 # 068 X2[0] 229 # 060 X1[12] P[10] 230 # 058 X1[11] P[9] Z[8] 231 # 050 X1[10] P[8] Z[7] 232 # 048 X1[9] P[7] Z[6] 233 # 040 X1[8] P[6] Z[5] 234 # 038 X1[7] P[5] Z[4] 235 # 030 X1[6] P[4] Z[3] 236 # 028 X1[5] P[3] Z[2] 237 # 020 X1[4] P[2] Z[1] 238 # 018 X1[3] P[1] Z[0] 239 # 010 X1[2] P[0] Y[2] 240 # 008 X1[1] Q[1] Y[1] 241 # 000 X1[0] Q[0] Y[0] 242 243 my $X1_offset = 0; # 13 qwords 244 my $X2_offset = $X1_offset + 13*8; # 11 qwords 245 my $Carries_offset = $X2_offset + 11*8; # 1 qword 246 my $Q_offset = 0; # 2 qwords 247 my $P_offset = $Q_offset + 2*8; # 11 qwords 248 my $Y_offset = 0; # 3 qwords 249 my $Z_offset = $Y_offset + 3*8; # 9 qwords 250 251 my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords) 252 253 # 254 # Stack Frame 255 # 256 # 257 # offset value 258 # ... <old stack contents> 259 # ... 260 # 280 Garray 261 262 # 278 tmp16[15] 263 # ... ... 264 # 200 tmp16[0] 265 266 # 1F8 tmp[7] 267 # ... ... 268 # 1C0 tmp[0] 269 270 # 1B8 GT[7] 271 # ... ... 272 # 180 GT[0] 273 274 # 178 Reduce Data 275 # ... ... 276 # 0B8 Reduce Data 277 # 0B0 reserved 278 # 0A8 reserved 279 # 0A0 reserved 280 # 098 reserved 281 # 090 reserved 282 # 088 reduce result addr 283 # 080 exp[8] 284 285 # ... 286 # 048 exp[1] 287 # 040 exp[0] 288 289 # 038 reserved 290 # 030 loop_idx 291 # 028 pg 292 # 020 i 293 # 018 pData ; arg 4 294 # 010 pG ; arg 2 295 # 008 pResult ; arg 1 296 # 000 rsp ; stack pointer before subtract 297 298 my $rsp_offset = 0; 299 my $pResult_offset = 8*1 + $rsp_offset; 300 my $pG_offset = 8*1 + $pResult_offset; 301 my $pData_offset = 8*1 + $pG_offset; 302 my $i_offset = 8*1 + $pData_offset; 303 my $pg_offset = 8*1 + $i_offset; 304 my $loop_idx_offset = 8*1 + $pg_offset; 305 my $reserved1_offset = 8*1 + $loop_idx_offset; 306 my $exp_offset = 8*1 + $reserved1_offset; 307 my $red_result_addr_offset= 8*9 + $exp_offset; 308 my $reserved2_offset = 8*1 + $red_result_addr_offset; 309 my $Reduce_Data_offset = 8*5 + $reserved2_offset; 310 my $GT_offset = $Red_Data_Size + $Reduce_Data_offset; 311 my $tmp_offset = 8*8 + $GT_offset; 312 my $tmp16_offset = 8*8 + $tmp_offset; 313 my $garray_offset = 8*16 + $tmp16_offset; 314 my $mem_size = 8*8*32 + $garray_offset; 315 316 # 317 # Offsets within Reduce Data 318 # 319 # 320 # struct MODF_2FOLD_MONT_512_C1_DATA { 321 # UINT64 t[8][8]; 322 # UINT64 m[8]; 323 # UINT64 m1[8]; /* 2^768 % m */ 324 # UINT64 m2[8]; /* 2^640 % m */ 325 # UINT64 k1[2]; /* (- 1/m) % 2^128 */ 326 # }; 327 328 my $T = 0; 329 my $M = 512; # = 8 * 8 * 8 330 my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */ 331 my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */ 332 my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */ 333 334 # 335 # FUNCTIONS 336 # 337 338 {{{ 339 # 340 # MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords) 341 # and add 512-bits (8 qwords) 342 # to get 640 bits (10 qwords) 343 # Input: 128-bit mul source: [rdi+8*1], rbp 344 # 512-bit mul source: [rsi+8*n] 345 # 512-bit add source: r15, r14, ..., r9, r8 346 # Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0] 347 # Clobbers all regs except: rcx, rsi, rdi 348 $code.=<<___; 349 .type MULADD_128x512,\@abi-omnipotent 350 .align 16 351 MULADD_128x512: 352 ___ 353 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx"); 354 $code.=<<___; 355 mov (+8*1)(%rdi), %rbp 356 ___ 357 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx"); 358 $code.=<<___; 359 ret 360 .size MULADD_128x512,.-MULADD_128x512 361 ___ 362 }}} 363 364 {{{ 365 #MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0 366 # 367 # Inputs: pDst: Destination (768 bits, 12 qwords) 368 # pA: Multiplicand (1024 bits, 16 qwords) 369 # pB: Multiplicand (512 bits, 8 qwords) 370 # Dst = Ah * B + Al 371 # where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits) 372 # Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0] 373 # Uses registers: arguments, RAX, RDX 374 sub MULADD_256x512 375 { 376 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_; 377 $code.=<<___; 378 mov (+8*12)($pA), $OP 379 ___ 380 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP); 381 push(@$X,shift(@$X)); 382 383 $code.=<<___; 384 mov (+8*13)($pA), $OP 385 ___ 386 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP); 387 push(@$X,shift(@$X)); 388 389 $code.=<<___; 390 mov (+8*14)($pA), $OP 391 ___ 392 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP); 393 push(@$X,shift(@$X)); 394 395 $code.=<<___; 396 mov (+8*15)($pA), $OP 397 ___ 398 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP); 399 push(@$X,shift(@$X)); 400 } 401 402 # 403 # mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */ 404 # UINT64 *m, /* 512 bits, 8 qwords */ 405 # MODF_2FOLD_MONT_512_C1_DATA *data, 406 # UINT64 *r) /* 512 bits, 8 qwords */ 407 # Input: x (number to be reduced): tmp16 (Implicit) 408 # m (modulus): [pM] (Implicit) 409 # data (reduce data): [pData] (Implicit) 410 # Output: r (result): Address in [red_res_addr] 411 # result also in: r9, r8, r15, r14, r13, r12, r11, r10 412 413 my @X=map("%r$_",(8..15)); 414 415 $code.=<<___; 416 .type mont_reduce,\@abi-omnipotent 417 .align 16 418 mont_reduce: 419 ___ 420 421 my $STACK_DEPTH = 8; 422 # 423 # X1 = Xh * M1 + Xl 424 $code.=<<___; 425 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords 426 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords 427 add \$$M1, %rsi 428 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords 429 430 ___ 431 432 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times 433 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0] 434 435 $code.=<<___; 436 xor %rax, %rax 437 # X1 += xl 438 add (+8*8)(%rcx), $X[4] 439 adc (+8*9)(%rcx), $X[5] 440 adc (+8*10)(%rcx), $X[6] 441 adc (+8*11)(%rcx), $X[7] 442 adc \$0, %rax 443 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0] 444 445 # 446 # check for carry ;; carry stored in rax 447 mov $X[4], (+8*8)(%rdi) # rdi points to X1 448 mov $X[5], (+8*9)(%rdi) 449 mov $X[6], %rbp 450 mov $X[7], (+8*11)(%rdi) 451 452 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 453 454 mov (+8*0)(%rdi), $X[4] 455 mov (+8*1)(%rdi), $X[5] 456 mov (+8*2)(%rdi), $X[6] 457 mov (+8*3)(%rdi), $X[7] 458 459 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8 460 # rdi -> X1 461 # rsi -> M1 462 463 # 464 # X2 = Xh * M2 + Xl 465 # do first part (X2 = Xh * M2) 466 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords 467 # Xh is actually { [rdi+8*1], rbp } 468 add \$`$M2-$M1`, %rsi # rsi -> M2 469 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords 470 ___ 471 unshift(@X,pop(@X)); unshift(@X,pop(@X)); 472 $code.=<<___; 473 474 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 475 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 476 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax 477 478 # X2 += Xl 479 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl 480 adc (+8*9-8*10)(%rdi), $X[7] 481 mov $X[6], (+8*8)(%rcx) 482 mov $X[7], (+8*9)(%rcx) 483 484 adc %rax, %rax 485 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 486 487 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords 488 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords 489 490 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half) 491 # B1:B0 = rsi[1:0] = K1[1:0] 492 # A1:A0 = rcx[1:0] = X2[1:0] 493 # Result = rdi[1],rbp = Q[1],rbp 494 mov (%rsi), %r8 # B0 495 mov (+8*1)(%rsi), %rbx # B1 496 497 mov (%rcx), %rax # A0 498 mul %r8 # B0 499 mov %rax, %rbp 500 mov %rdx, %r9 501 502 mov (+8*1)(%rcx), %rax # A1 503 mul %r8 # B0 504 add %rax, %r9 505 506 mov (%rcx), %rax # A0 507 mul %rbx # B1 508 add %rax, %r9 509 510 mov %r9, (+8*1)(%rdi) 511 # end MUL_128x128t128 512 513 sub \$`$K1-$M`, %rsi 514 515 mov (%rcx), $X[6] 516 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0] 517 518 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 519 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 520 521 # load first half of m to rdx, rdi, rbx, rax 522 # moved this here for efficiency 523 mov (+8*0)(%rsi), %rax 524 mov (+8*1)(%rsi), %rbx 525 mov (+8*2)(%rsi), %rdi 526 mov (+8*3)(%rsi), %rdx 527 528 # continue with reduction 529 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp 530 531 add (+8*8)(%rcx), $X[6] 532 adc (+8*9)(%rcx), $X[7] 533 534 #accumulate the final carry to rbp 535 adc %rbp, %rbp 536 537 # Add in overflow corrections: R = (X2>>128) += T[overflow] 538 # R = {r9, r8, r15, r14, ..., r10} 539 shl \$3, %rbp 540 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T) 541 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out 542 543 # rsi will be used to generate a mask after the addition 544 xor %rsi, %rsi 545 546 add (+8*8*0)(%rbp), $X[0] 547 adc (+8*8*1)(%rbp), $X[1] 548 adc (+8*8*2)(%rbp), $X[2] 549 adc (+8*8*3)(%rbp), $X[3] 550 adc (+8*8*4)(%rbp), $X[4] 551 adc (+8*8*5)(%rbp), $X[5] 552 adc (+8*8*6)(%rbp), $X[6] 553 adc (+8*8*7)(%rbp), $X[7] 554 555 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF 556 # if carry is clear: rsi = 0x0000000000000000 557 sbb \$0, %rsi 558 559 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 560 and %rsi, %rax 561 and %rsi, %rbx 562 and %rsi, %rdi 563 and %rsi, %rdx 564 565 mov \$1, %rbp 566 sub %rax, $X[0] 567 sbb %rbx, $X[1] 568 sbb %rdi, $X[2] 569 sbb %rdx, $X[3] 570 571 # if there is a borrow: rbp = 0 572 # if there is no borrow: rbp = 1 573 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m 574 sbb \$0, %rbp 575 576 #load second half of m to rdx, rdi, rbx, rax 577 578 add \$$M, %rcx 579 mov (+8*4)(%rcx), %rax 580 mov (+8*5)(%rcx), %rbx 581 mov (+8*6)(%rcx), %rdi 582 mov (+8*7)(%rcx), %rdx 583 584 # use the rsi mask as before 585 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 586 and %rsi, %rax 587 and %rsi, %rbx 588 and %rsi, %rdi 589 and %rsi, %rdx 590 591 # if rbp = 0, there was a borrow before, it is moved to the carry flag 592 # if rbp = 1, there was not a borrow before, carry flag is cleared 593 sub \$1, %rbp 594 595 sbb %rax, $X[4] 596 sbb %rbx, $X[5] 597 sbb %rdi, $X[6] 598 sbb %rdx, $X[7] 599 600 # write R back to memory 601 602 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi 603 mov $X[0], (+8*0)(%rsi) 604 mov $X[1], (+8*1)(%rsi) 605 mov $X[2], (+8*2)(%rsi) 606 mov $X[3], (+8*3)(%rsi) 607 mov $X[4], (+8*4)(%rsi) 608 mov $X[5], (+8*5)(%rsi) 609 mov $X[6], (+8*6)(%rsi) 610 mov $X[7], (+8*7)(%rsi) 611 612 ret 613 .size mont_reduce,.-mont_reduce 614 ___ 615 }}} 616 617 {{{ 618 #MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2 619 # 620 # Inputs: pDst: Destination (1024 bits, 16 qwords) 621 # pA: Multiplicand (512 bits, 8 qwords) 622 # pB: Multiplicand (512 bits, 8 qwords) 623 # Uses registers rax, rdx, args 624 # B operand in [pB] and also in x7...x0 625 sub MUL_512x512 626 { 627 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_; 628 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 629 my @X=@$x; # make a copy 630 631 $code.=<<___; 632 mov (+8*0)($pA), $OP 633 634 mov $X[0], %rax 635 mul $OP # rdx:rax = %OP * [0] 636 mov %rax, (+$pDst_o+8*0)($pDst) 637 mov %rdx, $X[0] 638 ___ 639 for(my $i=1;$i<8;$i++) { 640 $code.=<<___; 641 mov $X[$i], %rax 642 mul $OP # rdx:rax = %OP * [$i] 643 add %rax, $X[$i-1] 644 adc \$0, %rdx 645 mov %rdx, $X[$i] 646 ___ 647 } 648 649 for(my $i=1;$i<8;$i++) { 650 $code.=<<___; 651 mov (+8*$i)($pA), $OP 652 ___ 653 654 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP); 655 push(@X,shift(@X)); 656 } 657 658 $code.=<<___; 659 mov $X[0], (+$pDst_o+8*8)($pDst) 660 mov $X[1], (+$pDst_o+8*9)($pDst) 661 mov $X[2], (+$pDst_o+8*10)($pDst) 662 mov $X[3], (+$pDst_o+8*11)($pDst) 663 mov $X[4], (+$pDst_o+8*12)($pDst) 664 mov $X[5], (+$pDst_o+8*13)($pDst) 665 mov $X[6], (+$pDst_o+8*14)($pDst) 666 mov $X[7], (+$pDst_o+8*15)($pDst) 667 ___ 668 } 669 670 # 671 # mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits) 672 # Input: src1: Address of source 1: rdi 673 # src2: Address of source 2: rsi 674 # Output: dst: Address of destination: [red_res_addr] 675 # src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10 676 # Temp: Clobbers [tmp16], all registers 677 $code.=<<___; 678 .type mont_mul_a3b,\@abi-omnipotent 679 .align 16 680 mont_mul_a3b: 681 # 682 # multiply tmp = src1 * src2 683 # For multiply: dst = rcx, src1 = rdi, src2 = rsi 684 # stack depth is extra 8 from call 685 ___ 686 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx"); 687 $code.=<<___; 688 # 689 # Dst = tmp % m 690 # Call reduce(tmp, m, data, dst) 691 692 # tail recursion optimization: jmp to mont_reduce and return from there 693 jmp mont_reduce 694 # call mont_reduce 695 # ret 696 .size mont_mul_a3b,.-mont_mul_a3b 697 ___ 698 }}} 699 700 {{{ 701 #SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4 702 # 703 # Input in memory [pA] and also in x7...x0 704 # Uses all argument registers plus rax and rdx 705 # 706 # This version computes all of the off-diagonal terms into memory, 707 # and then it adds in the diagonal terms 708 709 sub SQR_512 710 { 711 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_; 712 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 713 my @X=@$x; # make a copy 714 $code.=<<___; 715 # ------------------ 716 # first pass 01...07 717 # ------------------ 718 mov $X[0], $A 719 720 mov $X[1],%rax 721 mul $A 722 mov %rax, (+$pDst_o+8*1)($pDst) 723 ___ 724 for(my $i=2;$i<8;$i++) { 725 $code.=<<___; 726 mov %rdx, $X[$i-2] 727 mov $X[$i],%rax 728 mul $A 729 add %rax, $X[$i-2] 730 adc \$0, %rdx 731 ___ 732 } 733 $code.=<<___; 734 mov %rdx, $x7 735 736 mov $X[0], (+$pDst_o+8*2)($pDst) 737 738 # ------------------ 739 # second pass 12...17 740 # ------------------ 741 742 mov (+8*1)($pA), $A 743 744 mov (+8*2)($pA),%rax 745 mul $A 746 add %rax, $X[1] 747 adc \$0, %rdx 748 mov $X[1], (+$pDst_o+8*3)($pDst) 749 750 mov %rdx, $X[0] 751 mov (+8*3)($pA),%rax 752 mul $A 753 add %rax, $X[2] 754 adc \$0, %rdx 755 add $X[0], $X[2] 756 adc \$0, %rdx 757 mov $X[2], (+$pDst_o+8*4)($pDst) 758 759 mov %rdx, $X[0] 760 mov (+8*4)($pA),%rax 761 mul $A 762 add %rax, $X[3] 763 adc \$0, %rdx 764 add $X[0], $X[3] 765 adc \$0, %rdx 766 767 mov %rdx, $X[0] 768 mov (+8*5)($pA),%rax 769 mul $A 770 add %rax, $X[4] 771 adc \$0, %rdx 772 add $X[0], $X[4] 773 adc \$0, %rdx 774 775 mov %rdx, $X[0] 776 mov $X[6],%rax 777 mul $A 778 add %rax, $X[5] 779 adc \$0, %rdx 780 add $X[0], $X[5] 781 adc \$0, %rdx 782 783 mov %rdx, $X[0] 784 mov $X[7],%rax 785 mul $A 786 add %rax, $x7 787 adc \$0, %rdx 788 add $X[0], $x7 789 adc \$0, %rdx 790 791 mov %rdx, $X[1] 792 793 # ------------------ 794 # third pass 23...27 795 # ------------------ 796 mov (+8*2)($pA), $A 797 798 mov (+8*3)($pA),%rax 799 mul $A 800 add %rax, $X[3] 801 adc \$0, %rdx 802 mov $X[3], (+$pDst_o+8*5)($pDst) 803 804 mov %rdx, $X[0] 805 mov (+8*4)($pA),%rax 806 mul $A 807 add %rax, $X[4] 808 adc \$0, %rdx 809 add $X[0], $X[4] 810 adc \$0, %rdx 811 mov $X[4], (+$pDst_o+8*6)($pDst) 812 813 mov %rdx, $X[0] 814 mov (+8*5)($pA),%rax 815 mul $A 816 add %rax, $X[5] 817 adc \$0, %rdx 818 add $X[0], $X[5] 819 adc \$0, %rdx 820 821 mov %rdx, $X[0] 822 mov $X[6],%rax 823 mul $A 824 add %rax, $x7 825 adc \$0, %rdx 826 add $X[0], $x7 827 adc \$0, %rdx 828 829 mov %rdx, $X[0] 830 mov $X[7],%rax 831 mul $A 832 add %rax, $X[1] 833 adc \$0, %rdx 834 add $X[0], $X[1] 835 adc \$0, %rdx 836 837 mov %rdx, $X[2] 838 839 # ------------------ 840 # fourth pass 34...37 841 # ------------------ 842 843 mov (+8*3)($pA), $A 844 845 mov (+8*4)($pA),%rax 846 mul $A 847 add %rax, $X[5] 848 adc \$0, %rdx 849 mov $X[5], (+$pDst_o+8*7)($pDst) 850 851 mov %rdx, $X[0] 852 mov (+8*5)($pA),%rax 853 mul $A 854 add %rax, $x7 855 adc \$0, %rdx 856 add $X[0], $x7 857 adc \$0, %rdx 858 mov $x7, (+$pDst_o+8*8)($pDst) 859 860 mov %rdx, $X[0] 861 mov $X[6],%rax 862 mul $A 863 add %rax, $X[1] 864 adc \$0, %rdx 865 add $X[0], $X[1] 866 adc \$0, %rdx 867 868 mov %rdx, $X[0] 869 mov $X[7],%rax 870 mul $A 871 add %rax, $X[2] 872 adc \$0, %rdx 873 add $X[0], $X[2] 874 adc \$0, %rdx 875 876 mov %rdx, $X[5] 877 878 # ------------------ 879 # fifth pass 45...47 880 # ------------------ 881 mov (+8*4)($pA), $A 882 883 mov (+8*5)($pA),%rax 884 mul $A 885 add %rax, $X[1] 886 adc \$0, %rdx 887 mov $X[1], (+$pDst_o+8*9)($pDst) 888 889 mov %rdx, $X[0] 890 mov $X[6],%rax 891 mul $A 892 add %rax, $X[2] 893 adc \$0, %rdx 894 add $X[0], $X[2] 895 adc \$0, %rdx 896 mov $X[2], (+$pDst_o+8*10)($pDst) 897 898 mov %rdx, $X[0] 899 mov $X[7],%rax 900 mul $A 901 add %rax, $X[5] 902 adc \$0, %rdx 903 add $X[0], $X[5] 904 adc \$0, %rdx 905 906 mov %rdx, $X[1] 907 908 # ------------------ 909 # sixth pass 56...57 910 # ------------------ 911 mov (+8*5)($pA), $A 912 913 mov $X[6],%rax 914 mul $A 915 add %rax, $X[5] 916 adc \$0, %rdx 917 mov $X[5], (+$pDst_o+8*11)($pDst) 918 919 mov %rdx, $X[0] 920 mov $X[7],%rax 921 mul $A 922 add %rax, $X[1] 923 adc \$0, %rdx 924 add $X[0], $X[1] 925 adc \$0, %rdx 926 mov $X[1], (+$pDst_o+8*12)($pDst) 927 928 mov %rdx, $X[2] 929 930 # ------------------ 931 # seventh pass 67 932 # ------------------ 933 mov $X[6], $A 934 935 mov $X[7],%rax 936 mul $A 937 add %rax, $X[2] 938 adc \$0, %rdx 939 mov $X[2], (+$pDst_o+8*13)($pDst) 940 941 mov %rdx, (+$pDst_o+8*14)($pDst) 942 943 # start finalize (add in squares, and double off-terms) 944 mov (+$pDst_o+8*1)($pDst), $X[0] 945 mov (+$pDst_o+8*2)($pDst), $X[1] 946 mov (+$pDst_o+8*3)($pDst), $X[2] 947 mov (+$pDst_o+8*4)($pDst), $X[3] 948 mov (+$pDst_o+8*5)($pDst), $X[4] 949 mov (+$pDst_o+8*6)($pDst), $X[5] 950 951 mov (+8*3)($pA), %rax 952 mul %rax 953 mov %rax, $x6 954 mov %rdx, $X[6] 955 956 add $X[0], $X[0] 957 adc $X[1], $X[1] 958 adc $X[2], $X[2] 959 adc $X[3], $X[3] 960 adc $X[4], $X[4] 961 adc $X[5], $X[5] 962 adc \$0, $X[6] 963 964 mov (+8*0)($pA), %rax 965 mul %rax 966 mov %rax, (+$pDst_o+8*0)($pDst) 967 mov %rdx, $A 968 969 mov (+8*1)($pA), %rax 970 mul %rax 971 972 add $A, $X[0] 973 adc %rax, $X[1] 974 adc \$0, %rdx 975 976 mov %rdx, $A 977 mov $X[0], (+$pDst_o+8*1)($pDst) 978 mov $X[1], (+$pDst_o+8*2)($pDst) 979 980 mov (+8*2)($pA), %rax 981 mul %rax 982 983 add $A, $X[2] 984 adc %rax, $X[3] 985 adc \$0, %rdx 986 987 mov %rdx, $A 988 989 mov $X[2], (+$pDst_o+8*3)($pDst) 990 mov $X[3], (+$pDst_o+8*4)($pDst) 991 992 xor $tmp, $tmp 993 add $A, $X[4] 994 adc $x6, $X[5] 995 adc \$0, $tmp 996 997 mov $X[4], (+$pDst_o+8*5)($pDst) 998 mov $X[5], (+$pDst_o+8*6)($pDst) 999 1000 # %%tmp has 0/1 in column 7 1001 # %%A6 has a full value in column 7 1002 1003 mov (+$pDst_o+8*7)($pDst), $X[0] 1004 mov (+$pDst_o+8*8)($pDst), $X[1] 1005 mov (+$pDst_o+8*9)($pDst), $X[2] 1006 mov (+$pDst_o+8*10)($pDst), $X[3] 1007 mov (+$pDst_o+8*11)($pDst), $X[4] 1008 mov (+$pDst_o+8*12)($pDst), $X[5] 1009 mov (+$pDst_o+8*13)($pDst), $x6 1010 mov (+$pDst_o+8*14)($pDst), $x7 1011 1012 mov $X[7], %rax 1013 mul %rax 1014 mov %rax, $X[7] 1015 mov %rdx, $A 1016 1017 add $X[0], $X[0] 1018 adc $X[1], $X[1] 1019 adc $X[2], $X[2] 1020 adc $X[3], $X[3] 1021 adc $X[4], $X[4] 1022 adc $X[5], $X[5] 1023 adc $x6, $x6 1024 adc $x7, $x7 1025 adc \$0, $A 1026 1027 add $tmp, $X[0] 1028 1029 mov (+8*4)($pA), %rax 1030 mul %rax 1031 1032 add $X[6], $X[0] 1033 adc %rax, $X[1] 1034 adc \$0, %rdx 1035 1036 mov %rdx, $tmp 1037 1038 mov $X[0], (+$pDst_o+8*7)($pDst) 1039 mov $X[1], (+$pDst_o+8*8)($pDst) 1040 1041 mov (+8*5)($pA), %rax 1042 mul %rax 1043 1044 add $tmp, $X[2] 1045 adc %rax, $X[3] 1046 adc \$0, %rdx 1047 1048 mov %rdx, $tmp 1049 1050 mov $X[2], (+$pDst_o+8*9)($pDst) 1051 mov $X[3], (+$pDst_o+8*10)($pDst) 1052 1053 mov (+8*6)($pA), %rax 1054 mul %rax 1055 1056 add $tmp, $X[4] 1057 adc %rax, $X[5] 1058 adc \$0, %rdx 1059 1060 mov $X[4], (+$pDst_o+8*11)($pDst) 1061 mov $X[5], (+$pDst_o+8*12)($pDst) 1062 1063 add %rdx, $x6 1064 adc $X[7], $x7 1065 adc \$0, $A 1066 1067 mov $x6, (+$pDst_o+8*13)($pDst) 1068 mov $x7, (+$pDst_o+8*14)($pDst) 1069 mov $A, (+$pDst_o+8*15)($pDst) 1070 ___ 1071 } 1072 1073 # 1074 # sqr_reduce: subroutine to compute Result = reduce(Result * Result) 1075 # 1076 # input and result also in: r9, r8, r15, r14, r13, r12, r11, r10 1077 # 1078 $code.=<<___; 1079 .type sqr_reduce,\@abi-omnipotent 1080 .align 16 1081 sqr_reduce: 1082 mov (+$pResult_offset+8)(%rsp), %rcx 1083 ___ 1084 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi"); 1085 $code.=<<___; 1086 # tail recursion optimization: jmp to mont_reduce and return from there 1087 jmp mont_reduce 1088 # call mont_reduce 1089 # ret 1090 .size sqr_reduce,.-sqr_reduce 1091 ___ 1092 }}} 1093 1094 # 1095 # MAIN FUNCTION 1096 # 1097 1098 #mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */ 1099 # UINT64 *g, /* 512 bits, 8 qwords */ 1100 # UINT64 *exp, /* 512 bits, 8 qwords */ 1101 # struct mod_ctx_512 *data) 1102 1103 # window size = 5 1104 # table size = 2^5 = 32 1105 #table_entries equ 32 1106 #table_size equ table_entries * 8 1107 $code.=<<___; 1108 .globl mod_exp_512 1109 .type mod_exp_512,\@function,4 1110 mod_exp_512: 1111 push %rbp 1112 push %rbx 1113 push %r12 1114 push %r13 1115 push %r14 1116 push %r15 1117 1118 # adjust stack down and then align it with cache boundary 1119 mov %rsp, %r8 1120 sub \$$mem_size, %rsp 1121 and \$-64, %rsp 1122 1123 # store previous stack pointer and arguments 1124 mov %r8, (+$rsp_offset)(%rsp) 1125 mov %rdi, (+$pResult_offset)(%rsp) 1126 mov %rsi, (+$pG_offset)(%rsp) 1127 mov %rcx, (+$pData_offset)(%rsp) 1128 .Lbody: 1129 # transform g into montgomery space 1130 # GT = reduce(g * C2) = reduce(g * (2^256)) 1131 # reduce expects to have the input in [tmp16] 1132 pxor %xmm4, %xmm4 1133 movdqu (+16*0)(%rsi), %xmm0 1134 movdqu (+16*1)(%rsi), %xmm1 1135 movdqu (+16*2)(%rsi), %xmm2 1136 movdqu (+16*3)(%rsi), %xmm3 1137 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp) 1138 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp) 1139 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1140 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1141 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp) 1142 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp) 1143 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp) 1144 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp) 1145 1146 # load pExp before rdx gets blown away 1147 movdqu (+16*0)(%rdx), %xmm0 1148 movdqu (+16*1)(%rdx), %xmm1 1149 movdqu (+16*2)(%rdx), %xmm2 1150 movdqu (+16*3)(%rdx), %xmm3 1151 1152 lea (+$GT_offset)(%rsp), %rbx 1153 mov %rbx, (+$red_result_addr_offset)(%rsp) 1154 call mont_reduce 1155 1156 # Initialize tmp = C 1157 lea (+$tmp_offset)(%rsp), %rcx 1158 xor %rax, %rax 1159 mov %rax, (+8*0)(%rcx) 1160 mov %rax, (+8*1)(%rcx) 1161 mov %rax, (+8*3)(%rcx) 1162 mov %rax, (+8*4)(%rcx) 1163 mov %rax, (+8*5)(%rcx) 1164 mov %rax, (+8*6)(%rcx) 1165 mov %rax, (+8*7)(%rcx) 1166 mov %rax, (+$exp_offset+8*8)(%rsp) 1167 movq \$1, (+8*2)(%rcx) 1168 1169 lea (+$garray_offset)(%rsp), %rbp 1170 mov %rcx, %rsi # pTmp 1171 mov %rbp, %rdi # Garray[][0] 1172 ___ 1173 1174 &swizzle("%rdi", "%rcx", "%rax", "%rbx"); 1175 1176 # for (rax = 31; rax != 0; rax--) { 1177 # tmp = reduce(tmp * G) 1178 # swizzle(pg, tmp); 1179 # pg += 2; } 1180 $code.=<<___; 1181 mov \$31, %rax 1182 mov %rax, (+$i_offset)(%rsp) 1183 mov %rbp, (+$pg_offset)(%rsp) 1184 # rsi -> pTmp 1185 mov %rsi, (+$red_result_addr_offset)(%rsp) 1186 mov (+8*0)(%rsi), %r10 1187 mov (+8*1)(%rsi), %r11 1188 mov (+8*2)(%rsi), %r12 1189 mov (+8*3)(%rsi), %r13 1190 mov (+8*4)(%rsi), %r14 1191 mov (+8*5)(%rsi), %r15 1192 mov (+8*6)(%rsi), %r8 1193 mov (+8*7)(%rsi), %r9 1194 init_loop: 1195 lea (+$GT_offset)(%rsp), %rdi 1196 call mont_mul_a3b 1197 lea (+$tmp_offset)(%rsp), %rsi 1198 mov (+$pg_offset)(%rsp), %rbp 1199 add \$2, %rbp 1200 mov %rbp, (+$pg_offset)(%rsp) 1201 mov %rsi, %rcx # rcx = rsi = addr of tmp 1202 ___ 1203 1204 &swizzle("%rbp", "%rcx", "%rax", "%rbx"); 1205 $code.=<<___; 1206 mov (+$i_offset)(%rsp), %rax 1207 sub \$1, %rax 1208 mov %rax, (+$i_offset)(%rsp) 1209 jne init_loop 1210 1211 # 1212 # Copy exponent onto stack 1213 movdqa %xmm0, (+$exp_offset+16*0)(%rsp) 1214 movdqa %xmm1, (+$exp_offset+16*1)(%rsp) 1215 movdqa %xmm2, (+$exp_offset+16*2)(%rsp) 1216 movdqa %xmm3, (+$exp_offset+16*3)(%rsp) 1217 1218 1219 # 1220 # Do exponentiation 1221 # Initialize result to G[exp{511:507}] 1222 mov (+$exp_offset+62)(%rsp), %eax 1223 mov %rax, %rdx 1224 shr \$11, %rax 1225 and \$0x07FF, %edx 1226 mov %edx, (+$exp_offset+62)(%rsp) 1227 lea (+$garray_offset)(%rsp,%rax,2), %rsi 1228 mov (+$pResult_offset)(%rsp), %rdx 1229 ___ 1230 1231 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1232 1233 # 1234 # Loop variables 1235 # rcx = [loop_idx] = index: 510-5 to 0 by 5 1236 $code.=<<___; 1237 movq \$505, (+$loop_idx_offset)(%rsp) 1238 1239 mov (+$pResult_offset)(%rsp), %rcx 1240 mov %rcx, (+$red_result_addr_offset)(%rsp) 1241 mov (+8*0)(%rcx), %r10 1242 mov (+8*1)(%rcx), %r11 1243 mov (+8*2)(%rcx), %r12 1244 mov (+8*3)(%rcx), %r13 1245 mov (+8*4)(%rcx), %r14 1246 mov (+8*5)(%rcx), %r15 1247 mov (+8*6)(%rcx), %r8 1248 mov (+8*7)(%rcx), %r9 1249 jmp sqr_2 1250 1251 main_loop_a3b: 1252 call sqr_reduce 1253 call sqr_reduce 1254 call sqr_reduce 1255 sqr_2: 1256 call sqr_reduce 1257 call sqr_reduce 1258 1259 # 1260 # Do multiply, first look up proper value in Garray 1261 mov (+$loop_idx_offset)(%rsp), %rcx # bit index 1262 mov %rcx, %rax 1263 shr \$4, %rax # rax is word pointer 1264 mov (+$exp_offset)(%rsp,%rax,2), %edx 1265 and \$15, %rcx 1266 shrq %cl, %rdx 1267 and \$0x1F, %rdx 1268 1269 lea (+$garray_offset)(%rsp,%rdx,2), %rsi 1270 lea (+$tmp_offset)(%rsp), %rdx 1271 mov %rdx, %rdi 1272 ___ 1273 1274 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1275 # rdi = tmp = pG 1276 1277 # 1278 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData) 1279 # result result pG M Data 1280 $code.=<<___; 1281 mov (+$pResult_offset)(%rsp), %rsi 1282 call mont_mul_a3b 1283 1284 # 1285 # finish loop 1286 mov (+$loop_idx_offset)(%rsp), %rcx 1287 sub \$5, %rcx 1288 mov %rcx, (+$loop_idx_offset)(%rsp) 1289 jge main_loop_a3b 1290 1291 # 1292 1293 end_main_loop_a3b: 1294 # transform result out of Montgomery space 1295 # result = reduce(result) 1296 mov (+$pResult_offset)(%rsp), %rdx 1297 pxor %xmm4, %xmm4 1298 movdqu (+16*0)(%rdx), %xmm0 1299 movdqu (+16*1)(%rdx), %xmm1 1300 movdqu (+16*2)(%rdx), %xmm2 1301 movdqu (+16*3)(%rdx), %xmm3 1302 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp) 1303 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp) 1304 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1305 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1306 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp) 1307 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp) 1308 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp) 1309 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp) 1310 call mont_reduce 1311 1312 # If result > m, subract m 1313 # load result into r15:r8 1314 mov (+$pResult_offset)(%rsp), %rax 1315 mov (+8*0)(%rax), %r8 1316 mov (+8*1)(%rax), %r9 1317 mov (+8*2)(%rax), %r10 1318 mov (+8*3)(%rax), %r11 1319 mov (+8*4)(%rax), %r12 1320 mov (+8*5)(%rax), %r13 1321 mov (+8*6)(%rax), %r14 1322 mov (+8*7)(%rax), %r15 1323 1324 # subtract m 1325 mov (+$pData_offset)(%rsp), %rbx 1326 add \$$M, %rbx 1327 1328 sub (+8*0)(%rbx), %r8 1329 sbb (+8*1)(%rbx), %r9 1330 sbb (+8*2)(%rbx), %r10 1331 sbb (+8*3)(%rbx), %r11 1332 sbb (+8*4)(%rbx), %r12 1333 sbb (+8*5)(%rbx), %r13 1334 sbb (+8*6)(%rbx), %r14 1335 sbb (+8*7)(%rbx), %r15 1336 1337 # if Carry is clear, replace result with difference 1338 mov (+8*0)(%rax), %rsi 1339 mov (+8*1)(%rax), %rdi 1340 mov (+8*2)(%rax), %rcx 1341 mov (+8*3)(%rax), %rdx 1342 cmovnc %r8, %rsi 1343 cmovnc %r9, %rdi 1344 cmovnc %r10, %rcx 1345 cmovnc %r11, %rdx 1346 mov %rsi, (+8*0)(%rax) 1347 mov %rdi, (+8*1)(%rax) 1348 mov %rcx, (+8*2)(%rax) 1349 mov %rdx, (+8*3)(%rax) 1350 1351 mov (+8*4)(%rax), %rsi 1352 mov (+8*5)(%rax), %rdi 1353 mov (+8*6)(%rax), %rcx 1354 mov (+8*7)(%rax), %rdx 1355 cmovnc %r12, %rsi 1356 cmovnc %r13, %rdi 1357 cmovnc %r14, %rcx 1358 cmovnc %r15, %rdx 1359 mov %rsi, (+8*4)(%rax) 1360 mov %rdi, (+8*5)(%rax) 1361 mov %rcx, (+8*6)(%rax) 1362 mov %rdx, (+8*7)(%rax) 1363 1364 mov (+$rsp_offset)(%rsp), %rsi 1365 mov 0(%rsi),%r15 1366 mov 8(%rsi),%r14 1367 mov 16(%rsi),%r13 1368 mov 24(%rsi),%r12 1369 mov 32(%rsi),%rbx 1370 mov 40(%rsi),%rbp 1371 lea 48(%rsi),%rsp 1372 .Lepilogue: 1373 ret 1374 .size mod_exp_512, . - mod_exp_512 1375 ___ 1376 1377 if ($win64) { 1378 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 1379 # CONTEXT *context,DISPATCHER_CONTEXT *disp) 1380 my $rec="%rcx"; 1381 my $frame="%rdx"; 1382 my $context="%r8"; 1383 my $disp="%r9"; 1384 1385 $code.=<<___; 1386 .extern __imp_RtlVirtualUnwind 1387 .type mod_exp_512_se_handler,\@abi-omnipotent 1388 .align 16 1389 mod_exp_512_se_handler: 1390 push %rsi 1391 push %rdi 1392 push %rbx 1393 push %rbp 1394 push %r12 1395 push %r13 1396 push %r14 1397 push %r15 1398 pushfq 1399 sub \$64,%rsp 1400 1401 mov 120($context),%rax # pull context->Rax 1402 mov 248($context),%rbx # pull context->Rip 1403 1404 lea .Lbody(%rip),%r10 1405 cmp %r10,%rbx # context->Rip<prologue label 1406 jb .Lin_prologue 1407 1408 mov 152($context),%rax # pull context->Rsp 1409 1410 lea .Lepilogue(%rip),%r10 1411 cmp %r10,%rbx # context->Rip>=epilogue label 1412 jae .Lin_prologue 1413 1414 mov $rsp_offset(%rax),%rax # pull saved Rsp 1415 1416 mov 32(%rax),%rbx 1417 mov 40(%rax),%rbp 1418 mov 24(%rax),%r12 1419 mov 16(%rax),%r13 1420 mov 8(%rax),%r14 1421 mov 0(%rax),%r15 1422 lea 48(%rax),%rax 1423 mov %rbx,144($context) # restore context->Rbx 1424 mov %rbp,160($context) # restore context->Rbp 1425 mov %r12,216($context) # restore context->R12 1426 mov %r13,224($context) # restore context->R13 1427 mov %r14,232($context) # restore context->R14 1428 mov %r15,240($context) # restore context->R15 1429 1430 .Lin_prologue: 1431 mov 8(%rax),%rdi 1432 mov 16(%rax),%rsi 1433 mov %rax,152($context) # restore context->Rsp 1434 mov %rsi,168($context) # restore context->Rsi 1435 mov %rdi,176($context) # restore context->Rdi 1436 1437 mov 40($disp),%rdi # disp->ContextRecord 1438 mov $context,%rsi # context 1439 mov \$154,%ecx # sizeof(CONTEXT) 1440 .long 0xa548f3fc # cld; rep movsq 1441 1442 mov $disp,%rsi 1443 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1444 mov 8(%rsi),%rdx # arg2, disp->ImageBase 1445 mov 0(%rsi),%r8 # arg3, disp->ControlPc 1446 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1447 mov 40(%rsi),%r10 # disp->ContextRecord 1448 lea 56(%rsi),%r11 # &disp->HandlerData 1449 lea 24(%rsi),%r12 # &disp->EstablisherFrame 1450 mov %r10,32(%rsp) # arg5 1451 mov %r11,40(%rsp) # arg6 1452 mov %r12,48(%rsp) # arg7 1453 mov %rcx,56(%rsp) # arg8, (NULL) 1454 call *__imp_RtlVirtualUnwind(%rip) 1455 1456 mov \$1,%eax # ExceptionContinueSearch 1457 add \$64,%rsp 1458 popfq 1459 pop %r15 1460 pop %r14 1461 pop %r13 1462 pop %r12 1463 pop %rbp 1464 pop %rbx 1465 pop %rdi 1466 pop %rsi 1467 ret 1468 .size mod_exp_512_se_handler,.-mod_exp_512_se_handler 1469 1470 .section .pdata 1471 .align 4 1472 .rva .LSEH_begin_mod_exp_512 1473 .rva .LSEH_end_mod_exp_512 1474 .rva .LSEH_info_mod_exp_512 1475 1476 .section .xdata 1477 .align 8 1478 .LSEH_info_mod_exp_512: 1479 .byte 9,0,0,0 1480 .rva mod_exp_512_se_handler 1481 ___ 1482 } 1483 1484 sub reg_part { 1485 my ($reg,$conv)=@_; 1486 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } 1487 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } 1488 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } 1489 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } 1490 return $reg; 1491 } 1492 1493 $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; 1494 $code =~ s/\`([^\`]*)\`/eval $1/gem; 1495 $code =~ s/(\(\+[^)]+\))/eval $1/gem; 1496 print $code; 1497 close STDOUT; 1498
