1 #!/usr/bin/env perl 2 # 3 # Copyright (c) 2010-2011 Intel Corp. 4 # Author: Vinodh.Gopal (at] intel.com 5 # Jim Guilford 6 # Erdinc.Ozturk (at] intel.com 7 # Maxim.Perminov (at] intel.com 8 # 9 # More information about algorithm used can be found at: 10 # http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf 11 # 12 # ==================================================================== 13 # Copyright (c) 2011 The OpenSSL Project. All rights reserved. 14 # 15 # Redistribution and use in source and binary forms, with or without 16 # modification, are permitted provided that the following conditions 17 # are met: 18 # 19 # 1. Redistributions of source code must retain the above copyright 20 # notice, this list of conditions and the following disclaimer. 21 # 22 # 2. Redistributions in binary form must reproduce the above copyright 23 # notice, this list of conditions and the following disclaimer in 24 # the documentation and/or other materials provided with the 25 # distribution. 26 # 27 # 3. All advertising materials mentioning features or use of this 28 # software must display the following acknowledgment: 29 # "This product includes software developed by the OpenSSL Project 30 # for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" 31 # 32 # 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 33 # endorse or promote products derived from this software without 34 # prior written permission. For written permission, please contact 35 # licensing (at] OpenSSL.org. 36 # 37 # 5. Products derived from this software may not be called "OpenSSL" 38 # nor may "OpenSSL" appear in their names without prior written 39 # permission of the OpenSSL Project. 40 # 41 # 6. Redistributions of any form whatsoever must retain the following 42 # acknowledgment: 43 # "This product includes software developed by the OpenSSL Project 44 # for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" 45 # 46 # THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 47 # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 49 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 50 # ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 51 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 52 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 53 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 55 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 56 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 57 # OF THE POSSIBILITY OF SUCH DAMAGE. 58 # ==================================================================== 59 60 $flavour = shift; 61 $output = shift; 62 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 63 64 my $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 65 66 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 67 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 68 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 69 die "can't locate x86_64-xlate.pl"; 70 71 open STDOUT,"| $^X $xlate $flavour $output"; 72 73 use strict; 74 my $code=".text\n\n"; 75 my $m=0; 76 77 # 78 # Define x512 macros 79 # 80 81 #MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2 82 # 83 # uses rax, rdx, and args 84 sub MULSTEP_512_ADD 85 { 86 my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_; 87 my @X=@$x; # make a copy 88 $code.=<<___; 89 mov (+8*0)($SRC2), %rax 90 mul $OP # rdx:rax = %OP * [0] 91 mov ($ASRC), $X[0] 92 add %rax, $X[0] 93 adc \$0, %rdx 94 mov $X[0], $DST 95 ___ 96 for(my $i=1;$i<8;$i++) { 97 $code.=<<___; 98 mov %rdx, $TMP 99 100 mov (+8*$i)($SRC2), %rax 101 mul $OP # rdx:rax = %OP * [$i] 102 mov (+8*$i)($ASRC), $X[$i] 103 add %rax, $X[$i] 104 adc \$0, %rdx 105 add $TMP, $X[$i] 106 adc \$0, %rdx 107 ___ 108 } 109 $code.=<<___; 110 mov %rdx, $X[0] 111 ___ 112 } 113 114 #MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp 115 # 116 # uses rax, rdx, and args 117 sub MULSTEP_512 118 { 119 my ($x, $DST, $SRC2, $OP, $TMP)=@_; 120 my @X=@$x; # make a copy 121 $code.=<<___; 122 mov (+8*0)($SRC2), %rax 123 mul $OP # rdx:rax = %OP * [0] 124 add %rax, $X[0] 125 adc \$0, %rdx 126 mov $X[0], $DST 127 ___ 128 for(my $i=1;$i<8;$i++) { 129 $code.=<<___; 130 mov %rdx, $TMP 131 132 mov (+8*$i)($SRC2), %rax 133 mul $OP # rdx:rax = %OP * [$i] 134 add %rax, $X[$i] 135 adc \$0, %rdx 136 add $TMP, $X[$i] 137 adc \$0, %rdx 138 ___ 139 } 140 $code.=<<___; 141 mov %rdx, $X[0] 142 ___ 143 } 144 145 # 146 # Swizzle Macros 147 # 148 149 # macro to copy data from flat space to swizzled table 150 #MACRO swizzle pDst, pSrc, tmp1, tmp2 151 # pDst and pSrc are modified 152 sub swizzle 153 { 154 my ($pDst, $pSrc, $cnt, $d0)=@_; 155 $code.=<<___; 156 mov \$8, $cnt 157 loop_$m: 158 mov ($pSrc), $d0 159 mov $d0#w, ($pDst) 160 shr \$16, $d0 161 mov $d0#w, (+64*1)($pDst) 162 shr \$16, $d0 163 mov $d0#w, (+64*2)($pDst) 164 shr \$16, $d0 165 mov $d0#w, (+64*3)($pDst) 166 lea 8($pSrc), $pSrc 167 lea 64*4($pDst), $pDst 168 dec $cnt 169 jnz loop_$m 170 ___ 171 172 $m++; 173 } 174 175 # macro to copy data from swizzled table to flat space 176 #MACRO unswizzle pDst, pSrc, tmp*3 177 sub unswizzle 178 { 179 my ($pDst, $pSrc, $cnt, $d0, $d1)=@_; 180 $code.=<<___; 181 mov \$4, $cnt 182 loop_$m: 183 movzxw (+64*3+256*0)($pSrc), $d0 184 movzxw (+64*3+256*1)($pSrc), $d1 185 shl \$16, $d0 186 shl \$16, $d1 187 mov (+64*2+256*0)($pSrc), $d0#w 188 mov (+64*2+256*1)($pSrc), $d1#w 189 shl \$16, $d0 190 shl \$16, $d1 191 mov (+64*1+256*0)($pSrc), $d0#w 192 mov (+64*1+256*1)($pSrc), $d1#w 193 shl \$16, $d0 194 shl \$16, $d1 195 mov (+64*0+256*0)($pSrc), $d0#w 196 mov (+64*0+256*1)($pSrc), $d1#w 197 mov $d0, (+8*0)($pDst) 198 mov $d1, (+8*1)($pDst) 199 lea 256*2($pSrc), $pSrc 200 lea 8*2($pDst), $pDst 201 sub \$1, $cnt 202 jnz loop_$m 203 ___ 204 205 $m++; 206 } 207 208 # 209 # Data Structures 210 # 211 212 # Reduce Data 213 # 214 # 215 # Offset Value 216 # 0C0 Carries 217 # 0B8 X2[10] 218 # 0B0 X2[9] 219 # 0A8 X2[8] 220 # 0A0 X2[7] 221 # 098 X2[6] 222 # 090 X2[5] 223 # 088 X2[4] 224 # 080 X2[3] 225 # 078 X2[2] 226 # 070 X2[1] 227 # 068 X2[0] 228 # 060 X1[12] P[10] 229 # 058 X1[11] P[9] Z[8] 230 # 050 X1[10] P[8] Z[7] 231 # 048 X1[9] P[7] Z[6] 232 # 040 X1[8] P[6] Z[5] 233 # 038 X1[7] P[5] Z[4] 234 # 030 X1[6] P[4] Z[3] 235 # 028 X1[5] P[3] Z[2] 236 # 020 X1[4] P[2] Z[1] 237 # 018 X1[3] P[1] Z[0] 238 # 010 X1[2] P[0] Y[2] 239 # 008 X1[1] Q[1] Y[1] 240 # 000 X1[0] Q[0] Y[0] 241 242 my $X1_offset = 0; # 13 qwords 243 my $X2_offset = $X1_offset + 13*8; # 11 qwords 244 my $Carries_offset = $X2_offset + 11*8; # 1 qword 245 my $Q_offset = 0; # 2 qwords 246 my $P_offset = $Q_offset + 2*8; # 11 qwords 247 my $Y_offset = 0; # 3 qwords 248 my $Z_offset = $Y_offset + 3*8; # 9 qwords 249 250 my $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords) 251 252 # 253 # Stack Frame 254 # 255 # 256 # offset value 257 # ... <old stack contents> 258 # ... 259 # 280 Garray 260 261 # 278 tmp16[15] 262 # ... ... 263 # 200 tmp16[0] 264 265 # 1F8 tmp[7] 266 # ... ... 267 # 1C0 tmp[0] 268 269 # 1B8 GT[7] 270 # ... ... 271 # 180 GT[0] 272 273 # 178 Reduce Data 274 # ... ... 275 # 0B8 Reduce Data 276 # 0B0 reserved 277 # 0A8 reserved 278 # 0A0 reserved 279 # 098 reserved 280 # 090 reserved 281 # 088 reduce result addr 282 # 080 exp[8] 283 284 # ... 285 # 048 exp[1] 286 # 040 exp[0] 287 288 # 038 reserved 289 # 030 loop_idx 290 # 028 pg 291 # 020 i 292 # 018 pData ; arg 4 293 # 010 pG ; arg 2 294 # 008 pResult ; arg 1 295 # 000 rsp ; stack pointer before subtract 296 297 my $rsp_offset = 0; 298 my $pResult_offset = 8*1 + $rsp_offset; 299 my $pG_offset = 8*1 + $pResult_offset; 300 my $pData_offset = 8*1 + $pG_offset; 301 my $i_offset = 8*1 + $pData_offset; 302 my $pg_offset = 8*1 + $i_offset; 303 my $loop_idx_offset = 8*1 + $pg_offset; 304 my $reserved1_offset = 8*1 + $loop_idx_offset; 305 my $exp_offset = 8*1 + $reserved1_offset; 306 my $red_result_addr_offset= 8*9 + $exp_offset; 307 my $reserved2_offset = 8*1 + $red_result_addr_offset; 308 my $Reduce_Data_offset = 8*5 + $reserved2_offset; 309 my $GT_offset = $Red_Data_Size + $Reduce_Data_offset; 310 my $tmp_offset = 8*8 + $GT_offset; 311 my $tmp16_offset = 8*8 + $tmp_offset; 312 my $garray_offset = 8*16 + $tmp16_offset; 313 my $mem_size = 8*8*32 + $garray_offset; 314 315 # 316 # Offsets within Reduce Data 317 # 318 # 319 # struct MODF_2FOLD_MONT_512_C1_DATA { 320 # UINT64 t[8][8]; 321 # UINT64 m[8]; 322 # UINT64 m1[8]; /* 2^768 % m */ 323 # UINT64 m2[8]; /* 2^640 % m */ 324 # UINT64 k1[2]; /* (- 1/m) % 2^128 */ 325 # }; 326 327 my $T = 0; 328 my $M = 512; # = 8 * 8 * 8 329 my $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */ 330 my $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */ 331 my $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */ 332 333 # 334 # FUNCTIONS 335 # 336 337 {{{ 338 # 339 # MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords) 340 # and add 512-bits (8 qwords) 341 # to get 640 bits (10 qwords) 342 # Input: 128-bit mul source: [rdi+8*1], rbp 343 # 512-bit mul source: [rsi+8*n] 344 # 512-bit add source: r15, r14, ..., r9, r8 345 # Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0] 346 # Clobbers all regs except: rcx, rsi, rdi 347 $code.=<<___; 348 .type MULADD_128x512,\@abi-omnipotent 349 .align 16 350 MULADD_128x512: 351 ___ 352 &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx"); 353 $code.=<<___; 354 mov (+8*1)(%rdi), %rbp 355 ___ 356 &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx"); 357 $code.=<<___; 358 ret 359 .size MULADD_128x512,.-MULADD_128x512 360 ___ 361 }}} 362 363 {{{ 364 #MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0 365 # 366 # Inputs: pDst: Destination (768 bits, 12 qwords) 367 # pA: Multiplicand (1024 bits, 16 qwords) 368 # pB: Multiplicand (512 bits, 8 qwords) 369 # Dst = Ah * B + Al 370 # where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits) 371 # Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0] 372 # Uses registers: arguments, RAX, RDX 373 sub MULADD_256x512 374 { 375 my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_; 376 $code.=<<___; 377 mov (+8*12)($pA), $OP 378 ___ 379 &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP); 380 push(@$X,shift(@$X)); 381 382 $code.=<<___; 383 mov (+8*13)($pA), $OP 384 ___ 385 &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP); 386 push(@$X,shift(@$X)); 387 388 $code.=<<___; 389 mov (+8*14)($pA), $OP 390 ___ 391 &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP); 392 push(@$X,shift(@$X)); 393 394 $code.=<<___; 395 mov (+8*15)($pA), $OP 396 ___ 397 &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP); 398 push(@$X,shift(@$X)); 399 } 400 401 # 402 # mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */ 403 # UINT64 *m, /* 512 bits, 8 qwords */ 404 # MODF_2FOLD_MONT_512_C1_DATA *data, 405 # UINT64 *r) /* 512 bits, 8 qwords */ 406 # Input: x (number to be reduced): tmp16 (Implicit) 407 # m (modulus): [pM] (Implicit) 408 # data (reduce data): [pData] (Implicit) 409 # Output: r (result): Address in [red_res_addr] 410 # result also in: r9, r8, r15, r14, r13, r12, r11, r10 411 412 my @X=map("%r$_",(8..15)); 413 414 $code.=<<___; 415 .type mont_reduce,\@abi-omnipotent 416 .align 16 417 mont_reduce: 418 ___ 419 420 my $STACK_DEPTH = 8; 421 # 422 # X1 = Xh * M1 + Xl 423 $code.=<<___; 424 lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords 425 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords 426 add \$$M1, %rsi 427 lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords 428 429 ___ 430 431 &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times 432 # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0] 433 434 $code.=<<___; 435 xor %rax, %rax 436 # X1 += xl 437 add (+8*8)(%rcx), $X[4] 438 adc (+8*9)(%rcx), $X[5] 439 adc (+8*10)(%rcx), $X[6] 440 adc (+8*11)(%rcx), $X[7] 441 adc \$0, %rax 442 # X1 is now rax, r11-r8, r15-r12, tmp16[3:0] 443 444 # 445 # check for carry ;; carry stored in rax 446 mov $X[4], (+8*8)(%rdi) # rdi points to X1 447 mov $X[5], (+8*9)(%rdi) 448 mov $X[6], %rbp 449 mov $X[7], (+8*11)(%rdi) 450 451 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 452 453 mov (+8*0)(%rdi), $X[4] 454 mov (+8*1)(%rdi), $X[5] 455 mov (+8*2)(%rdi), $X[6] 456 mov (+8*3)(%rdi), $X[7] 457 458 # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8 459 # rdi -> X1 460 # rsi -> M1 461 462 # 463 # X2 = Xh * M2 + Xl 464 # do first part (X2 = Xh * M2) 465 add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords 466 # Xh is actually { [rdi+8*1], rbp } 467 add \$`$M2-$M1`, %rsi # rsi -> M2 468 lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords 469 ___ 470 unshift(@X,pop(@X)); unshift(@X,pop(@X)); 471 $code.=<<___; 472 473 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 474 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 475 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax 476 477 # X2 += Xl 478 add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl 479 adc (+8*9-8*10)(%rdi), $X[7] 480 mov $X[6], (+8*8)(%rcx) 481 mov $X[7], (+8*9)(%rcx) 482 483 adc %rax, %rax 484 mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 485 486 lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords 487 add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords 488 489 # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half) 490 # B1:B0 = rsi[1:0] = K1[1:0] 491 # A1:A0 = rcx[1:0] = X2[1:0] 492 # Result = rdi[1],rbp = Q[1],rbp 493 mov (%rsi), %r8 # B0 494 mov (+8*1)(%rsi), %rbx # B1 495 496 mov (%rcx), %rax # A0 497 mul %r8 # B0 498 mov %rax, %rbp 499 mov %rdx, %r9 500 501 mov (+8*1)(%rcx), %rax # A1 502 mul %r8 # B0 503 add %rax, %r9 504 505 mov (%rcx), %rax # A0 506 mul %rbx # B1 507 add %rax, %r9 508 509 mov %r9, (+8*1)(%rdi) 510 # end MUL_128x128t128 511 512 sub \$`$K1-$M`, %rsi 513 514 mov (%rcx), $X[6] 515 mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0] 516 517 call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 518 # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 519 520 # load first half of m to rdx, rdi, rbx, rax 521 # moved this here for efficiency 522 mov (+8*0)(%rsi), %rax 523 mov (+8*1)(%rsi), %rbx 524 mov (+8*2)(%rsi), %rdi 525 mov (+8*3)(%rsi), %rdx 526 527 # continue with reduction 528 mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp 529 530 add (+8*8)(%rcx), $X[6] 531 adc (+8*9)(%rcx), $X[7] 532 533 #accumulate the final carry to rbp 534 adc %rbp, %rbp 535 536 # Add in overflow corrections: R = (X2>>128) += T[overflow] 537 # R = {r9, r8, r15, r14, ..., r10} 538 shl \$3, %rbp 539 mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T) 540 add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out 541 542 # rsi will be used to generate a mask after the addition 543 xor %rsi, %rsi 544 545 add (+8*8*0)(%rbp), $X[0] 546 adc (+8*8*1)(%rbp), $X[1] 547 adc (+8*8*2)(%rbp), $X[2] 548 adc (+8*8*3)(%rbp), $X[3] 549 adc (+8*8*4)(%rbp), $X[4] 550 adc (+8*8*5)(%rbp), $X[5] 551 adc (+8*8*6)(%rbp), $X[6] 552 adc (+8*8*7)(%rbp), $X[7] 553 554 # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF 555 # if carry is clear: rsi = 0x0000000000000000 556 sbb \$0, %rsi 557 558 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 559 and %rsi, %rax 560 and %rsi, %rbx 561 and %rsi, %rdi 562 and %rsi, %rdx 563 564 mov \$1, %rbp 565 sub %rax, $X[0] 566 sbb %rbx, $X[1] 567 sbb %rdi, $X[2] 568 sbb %rdx, $X[3] 569 570 # if there is a borrow: rbp = 0 571 # if there is no borrow: rbp = 1 572 # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m 573 sbb \$0, %rbp 574 575 #load second half of m to rdx, rdi, rbx, rax 576 577 add \$$M, %rcx 578 mov (+8*4)(%rcx), %rax 579 mov (+8*5)(%rcx), %rbx 580 mov (+8*6)(%rcx), %rdi 581 mov (+8*7)(%rcx), %rdx 582 583 # use the rsi mask as before 584 # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 585 and %rsi, %rax 586 and %rsi, %rbx 587 and %rsi, %rdi 588 and %rsi, %rdx 589 590 # if rbp = 0, there was a borrow before, it is moved to the carry flag 591 # if rbp = 1, there was not a borrow before, carry flag is cleared 592 sub \$1, %rbp 593 594 sbb %rax, $X[4] 595 sbb %rbx, $X[5] 596 sbb %rdi, $X[6] 597 sbb %rdx, $X[7] 598 599 # write R back to memory 600 601 mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi 602 mov $X[0], (+8*0)(%rsi) 603 mov $X[1], (+8*1)(%rsi) 604 mov $X[2], (+8*2)(%rsi) 605 mov $X[3], (+8*3)(%rsi) 606 mov $X[4], (+8*4)(%rsi) 607 mov $X[5], (+8*5)(%rsi) 608 mov $X[6], (+8*6)(%rsi) 609 mov $X[7], (+8*7)(%rsi) 610 611 ret 612 .size mont_reduce,.-mont_reduce 613 ___ 614 }}} 615 616 {{{ 617 #MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2 618 # 619 # Inputs: pDst: Destination (1024 bits, 16 qwords) 620 # pA: Multiplicand (512 bits, 8 qwords) 621 # pB: Multiplicand (512 bits, 8 qwords) 622 # Uses registers rax, rdx, args 623 # B operand in [pB] and also in x7...x0 624 sub MUL_512x512 625 { 626 my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_; 627 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 628 my @X=@$x; # make a copy 629 630 $code.=<<___; 631 mov (+8*0)($pA), $OP 632 633 mov $X[0], %rax 634 mul $OP # rdx:rax = %OP * [0] 635 mov %rax, (+$pDst_o+8*0)($pDst) 636 mov %rdx, $X[0] 637 ___ 638 for(my $i=1;$i<8;$i++) { 639 $code.=<<___; 640 mov $X[$i], %rax 641 mul $OP # rdx:rax = %OP * [$i] 642 add %rax, $X[$i-1] 643 adc \$0, %rdx 644 mov %rdx, $X[$i] 645 ___ 646 } 647 648 for(my $i=1;$i<8;$i++) { 649 $code.=<<___; 650 mov (+8*$i)($pA), $OP 651 ___ 652 653 &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP); 654 push(@X,shift(@X)); 655 } 656 657 $code.=<<___; 658 mov $X[0], (+$pDst_o+8*8)($pDst) 659 mov $X[1], (+$pDst_o+8*9)($pDst) 660 mov $X[2], (+$pDst_o+8*10)($pDst) 661 mov $X[3], (+$pDst_o+8*11)($pDst) 662 mov $X[4], (+$pDst_o+8*12)($pDst) 663 mov $X[5], (+$pDst_o+8*13)($pDst) 664 mov $X[6], (+$pDst_o+8*14)($pDst) 665 mov $X[7], (+$pDst_o+8*15)($pDst) 666 ___ 667 } 668 669 # 670 # mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits) 671 # Input: src1: Address of source 1: rdi 672 # src2: Address of source 2: rsi 673 # Output: dst: Address of destination: [red_res_addr] 674 # src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10 675 # Temp: Clobbers [tmp16], all registers 676 $code.=<<___; 677 .type mont_mul_a3b,\@abi-omnipotent 678 .align 16 679 mont_mul_a3b: 680 # 681 # multiply tmp = src1 * src2 682 # For multiply: dst = rcx, src1 = rdi, src2 = rsi 683 # stack depth is extra 8 from call 684 ___ 685 &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx"); 686 $code.=<<___; 687 # 688 # Dst = tmp % m 689 # Call reduce(tmp, m, data, dst) 690 691 # tail recursion optimization: jmp to mont_reduce and return from there 692 jmp mont_reduce 693 # call mont_reduce 694 # ret 695 .size mont_mul_a3b,.-mont_mul_a3b 696 ___ 697 }}} 698 699 {{{ 700 #SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4 701 # 702 # Input in memory [pA] and also in x7...x0 703 # Uses all argument registers plus rax and rdx 704 # 705 # This version computes all of the off-diagonal terms into memory, 706 # and then it adds in the diagonal terms 707 708 sub SQR_512 709 { 710 my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_; 711 my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 712 my @X=@$x; # make a copy 713 $code.=<<___; 714 # ------------------ 715 # first pass 01...07 716 # ------------------ 717 mov $X[0], $A 718 719 mov $X[1],%rax 720 mul $A 721 mov %rax, (+$pDst_o+8*1)($pDst) 722 ___ 723 for(my $i=2;$i<8;$i++) { 724 $code.=<<___; 725 mov %rdx, $X[$i-2] 726 mov $X[$i],%rax 727 mul $A 728 add %rax, $X[$i-2] 729 adc \$0, %rdx 730 ___ 731 } 732 $code.=<<___; 733 mov %rdx, $x7 734 735 mov $X[0], (+$pDst_o+8*2)($pDst) 736 737 # ------------------ 738 # second pass 12...17 739 # ------------------ 740 741 mov (+8*1)($pA), $A 742 743 mov (+8*2)($pA),%rax 744 mul $A 745 add %rax, $X[1] 746 adc \$0, %rdx 747 mov $X[1], (+$pDst_o+8*3)($pDst) 748 749 mov %rdx, $X[0] 750 mov (+8*3)($pA),%rax 751 mul $A 752 add %rax, $X[2] 753 adc \$0, %rdx 754 add $X[0], $X[2] 755 adc \$0, %rdx 756 mov $X[2], (+$pDst_o+8*4)($pDst) 757 758 mov %rdx, $X[0] 759 mov (+8*4)($pA),%rax 760 mul $A 761 add %rax, $X[3] 762 adc \$0, %rdx 763 add $X[0], $X[3] 764 adc \$0, %rdx 765 766 mov %rdx, $X[0] 767 mov (+8*5)($pA),%rax 768 mul $A 769 add %rax, $X[4] 770 adc \$0, %rdx 771 add $X[0], $X[4] 772 adc \$0, %rdx 773 774 mov %rdx, $X[0] 775 mov $X[6],%rax 776 mul $A 777 add %rax, $X[5] 778 adc \$0, %rdx 779 add $X[0], $X[5] 780 adc \$0, %rdx 781 782 mov %rdx, $X[0] 783 mov $X[7],%rax 784 mul $A 785 add %rax, $x7 786 adc \$0, %rdx 787 add $X[0], $x7 788 adc \$0, %rdx 789 790 mov %rdx, $X[1] 791 792 # ------------------ 793 # third pass 23...27 794 # ------------------ 795 mov (+8*2)($pA), $A 796 797 mov (+8*3)($pA),%rax 798 mul $A 799 add %rax, $X[3] 800 adc \$0, %rdx 801 mov $X[3], (+$pDst_o+8*5)($pDst) 802 803 mov %rdx, $X[0] 804 mov (+8*4)($pA),%rax 805 mul $A 806 add %rax, $X[4] 807 adc \$0, %rdx 808 add $X[0], $X[4] 809 adc \$0, %rdx 810 mov $X[4], (+$pDst_o+8*6)($pDst) 811 812 mov %rdx, $X[0] 813 mov (+8*5)($pA),%rax 814 mul $A 815 add %rax, $X[5] 816 adc \$0, %rdx 817 add $X[0], $X[5] 818 adc \$0, %rdx 819 820 mov %rdx, $X[0] 821 mov $X[6],%rax 822 mul $A 823 add %rax, $x7 824 adc \$0, %rdx 825 add $X[0], $x7 826 adc \$0, %rdx 827 828 mov %rdx, $X[0] 829 mov $X[7],%rax 830 mul $A 831 add %rax, $X[1] 832 adc \$0, %rdx 833 add $X[0], $X[1] 834 adc \$0, %rdx 835 836 mov %rdx, $X[2] 837 838 # ------------------ 839 # fourth pass 34...37 840 # ------------------ 841 842 mov (+8*3)($pA), $A 843 844 mov (+8*4)($pA),%rax 845 mul $A 846 add %rax, $X[5] 847 adc \$0, %rdx 848 mov $X[5], (+$pDst_o+8*7)($pDst) 849 850 mov %rdx, $X[0] 851 mov (+8*5)($pA),%rax 852 mul $A 853 add %rax, $x7 854 adc \$0, %rdx 855 add $X[0], $x7 856 adc \$0, %rdx 857 mov $x7, (+$pDst_o+8*8)($pDst) 858 859 mov %rdx, $X[0] 860 mov $X[6],%rax 861 mul $A 862 add %rax, $X[1] 863 adc \$0, %rdx 864 add $X[0], $X[1] 865 adc \$0, %rdx 866 867 mov %rdx, $X[0] 868 mov $X[7],%rax 869 mul $A 870 add %rax, $X[2] 871 adc \$0, %rdx 872 add $X[0], $X[2] 873 adc \$0, %rdx 874 875 mov %rdx, $X[5] 876 877 # ------------------ 878 # fifth pass 45...47 879 # ------------------ 880 mov (+8*4)($pA), $A 881 882 mov (+8*5)($pA),%rax 883 mul $A 884 add %rax, $X[1] 885 adc \$0, %rdx 886 mov $X[1], (+$pDst_o+8*9)($pDst) 887 888 mov %rdx, $X[0] 889 mov $X[6],%rax 890 mul $A 891 add %rax, $X[2] 892 adc \$0, %rdx 893 add $X[0], $X[2] 894 adc \$0, %rdx 895 mov $X[2], (+$pDst_o+8*10)($pDst) 896 897 mov %rdx, $X[0] 898 mov $X[7],%rax 899 mul $A 900 add %rax, $X[5] 901 adc \$0, %rdx 902 add $X[0], $X[5] 903 adc \$0, %rdx 904 905 mov %rdx, $X[1] 906 907 # ------------------ 908 # sixth pass 56...57 909 # ------------------ 910 mov (+8*5)($pA), $A 911 912 mov $X[6],%rax 913 mul $A 914 add %rax, $X[5] 915 adc \$0, %rdx 916 mov $X[5], (+$pDst_o+8*11)($pDst) 917 918 mov %rdx, $X[0] 919 mov $X[7],%rax 920 mul $A 921 add %rax, $X[1] 922 adc \$0, %rdx 923 add $X[0], $X[1] 924 adc \$0, %rdx 925 mov $X[1], (+$pDst_o+8*12)($pDst) 926 927 mov %rdx, $X[2] 928 929 # ------------------ 930 # seventh pass 67 931 # ------------------ 932 mov $X[6], $A 933 934 mov $X[7],%rax 935 mul $A 936 add %rax, $X[2] 937 adc \$0, %rdx 938 mov $X[2], (+$pDst_o+8*13)($pDst) 939 940 mov %rdx, (+$pDst_o+8*14)($pDst) 941 942 # start finalize (add in squares, and double off-terms) 943 mov (+$pDst_o+8*1)($pDst), $X[0] 944 mov (+$pDst_o+8*2)($pDst), $X[1] 945 mov (+$pDst_o+8*3)($pDst), $X[2] 946 mov (+$pDst_o+8*4)($pDst), $X[3] 947 mov (+$pDst_o+8*5)($pDst), $X[4] 948 mov (+$pDst_o+8*6)($pDst), $X[5] 949 950 mov (+8*3)($pA), %rax 951 mul %rax 952 mov %rax, $x6 953 mov %rdx, $X[6] 954 955 add $X[0], $X[0] 956 adc $X[1], $X[1] 957 adc $X[2], $X[2] 958 adc $X[3], $X[3] 959 adc $X[4], $X[4] 960 adc $X[5], $X[5] 961 adc \$0, $X[6] 962 963 mov (+8*0)($pA), %rax 964 mul %rax 965 mov %rax, (+$pDst_o+8*0)($pDst) 966 mov %rdx, $A 967 968 mov (+8*1)($pA), %rax 969 mul %rax 970 971 add $A, $X[0] 972 adc %rax, $X[1] 973 adc \$0, %rdx 974 975 mov %rdx, $A 976 mov $X[0], (+$pDst_o+8*1)($pDst) 977 mov $X[1], (+$pDst_o+8*2)($pDst) 978 979 mov (+8*2)($pA), %rax 980 mul %rax 981 982 add $A, $X[2] 983 adc %rax, $X[3] 984 adc \$0, %rdx 985 986 mov %rdx, $A 987 988 mov $X[2], (+$pDst_o+8*3)($pDst) 989 mov $X[3], (+$pDst_o+8*4)($pDst) 990 991 xor $tmp, $tmp 992 add $A, $X[4] 993 adc $x6, $X[5] 994 adc \$0, $tmp 995 996 mov $X[4], (+$pDst_o+8*5)($pDst) 997 mov $X[5], (+$pDst_o+8*6)($pDst) 998 999 # %%tmp has 0/1 in column 7 1000 # %%A6 has a full value in column 7 1001 1002 mov (+$pDst_o+8*7)($pDst), $X[0] 1003 mov (+$pDst_o+8*8)($pDst), $X[1] 1004 mov (+$pDst_o+8*9)($pDst), $X[2] 1005 mov (+$pDst_o+8*10)($pDst), $X[3] 1006 mov (+$pDst_o+8*11)($pDst), $X[4] 1007 mov (+$pDst_o+8*12)($pDst), $X[5] 1008 mov (+$pDst_o+8*13)($pDst), $x6 1009 mov (+$pDst_o+8*14)($pDst), $x7 1010 1011 mov $X[7], %rax 1012 mul %rax 1013 mov %rax, $X[7] 1014 mov %rdx, $A 1015 1016 add $X[0], $X[0] 1017 adc $X[1], $X[1] 1018 adc $X[2], $X[2] 1019 adc $X[3], $X[3] 1020 adc $X[4], $X[4] 1021 adc $X[5], $X[5] 1022 adc $x6, $x6 1023 adc $x7, $x7 1024 adc \$0, $A 1025 1026 add $tmp, $X[0] 1027 1028 mov (+8*4)($pA), %rax 1029 mul %rax 1030 1031 add $X[6], $X[0] 1032 adc %rax, $X[1] 1033 adc \$0, %rdx 1034 1035 mov %rdx, $tmp 1036 1037 mov $X[0], (+$pDst_o+8*7)($pDst) 1038 mov $X[1], (+$pDst_o+8*8)($pDst) 1039 1040 mov (+8*5)($pA), %rax 1041 mul %rax 1042 1043 add $tmp, $X[2] 1044 adc %rax, $X[3] 1045 adc \$0, %rdx 1046 1047 mov %rdx, $tmp 1048 1049 mov $X[2], (+$pDst_o+8*9)($pDst) 1050 mov $X[3], (+$pDst_o+8*10)($pDst) 1051 1052 mov (+8*6)($pA), %rax 1053 mul %rax 1054 1055 add $tmp, $X[4] 1056 adc %rax, $X[5] 1057 adc \$0, %rdx 1058 1059 mov $X[4], (+$pDst_o+8*11)($pDst) 1060 mov $X[5], (+$pDst_o+8*12)($pDst) 1061 1062 add %rdx, $x6 1063 adc $X[7], $x7 1064 adc \$0, $A 1065 1066 mov $x6, (+$pDst_o+8*13)($pDst) 1067 mov $x7, (+$pDst_o+8*14)($pDst) 1068 mov $A, (+$pDst_o+8*15)($pDst) 1069 ___ 1070 } 1071 1072 # 1073 # sqr_reduce: subroutine to compute Result = reduce(Result * Result) 1074 # 1075 # input and result also in: r9, r8, r15, r14, r13, r12, r11, r10 1076 # 1077 $code.=<<___; 1078 .type sqr_reduce,\@abi-omnipotent 1079 .align 16 1080 sqr_reduce: 1081 mov (+$pResult_offset+8)(%rsp), %rcx 1082 ___ 1083 &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi"); 1084 $code.=<<___; 1085 # tail recursion optimization: jmp to mont_reduce and return from there 1086 jmp mont_reduce 1087 # call mont_reduce 1088 # ret 1089 .size sqr_reduce,.-sqr_reduce 1090 ___ 1091 }}} 1092 1093 # 1094 # MAIN FUNCTION 1095 # 1096 1097 #mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */ 1098 # UINT64 *g, /* 512 bits, 8 qwords */ 1099 # UINT64 *exp, /* 512 bits, 8 qwords */ 1100 # struct mod_ctx_512 *data) 1101 1102 # window size = 5 1103 # table size = 2^5 = 32 1104 #table_entries equ 32 1105 #table_size equ table_entries * 8 1106 $code.=<<___; 1107 .globl mod_exp_512 1108 .type mod_exp_512,\@function,4 1109 mod_exp_512: 1110 push %rbp 1111 push %rbx 1112 push %r12 1113 push %r13 1114 push %r14 1115 push %r15 1116 1117 # adjust stack down and then align it with cache boundary 1118 mov %rsp, %r8 1119 sub \$$mem_size, %rsp 1120 and \$-64, %rsp 1121 1122 # store previous stack pointer and arguments 1123 mov %r8, (+$rsp_offset)(%rsp) 1124 mov %rdi, (+$pResult_offset)(%rsp) 1125 mov %rsi, (+$pG_offset)(%rsp) 1126 mov %rcx, (+$pData_offset)(%rsp) 1127 .Lbody: 1128 # transform g into montgomery space 1129 # GT = reduce(g * C2) = reduce(g * (2^256)) 1130 # reduce expects to have the input in [tmp16] 1131 pxor %xmm4, %xmm4 1132 movdqu (+16*0)(%rsi), %xmm0 1133 movdqu (+16*1)(%rsi), %xmm1 1134 movdqu (+16*2)(%rsi), %xmm2 1135 movdqu (+16*3)(%rsi), %xmm3 1136 movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp) 1137 movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp) 1138 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1139 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1140 movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp) 1141 movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp) 1142 movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp) 1143 movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp) 1144 1145 # load pExp before rdx gets blown away 1146 movdqu (+16*0)(%rdx), %xmm0 1147 movdqu (+16*1)(%rdx), %xmm1 1148 movdqu (+16*2)(%rdx), %xmm2 1149 movdqu (+16*3)(%rdx), %xmm3 1150 1151 lea (+$GT_offset)(%rsp), %rbx 1152 mov %rbx, (+$red_result_addr_offset)(%rsp) 1153 call mont_reduce 1154 1155 # Initialize tmp = C 1156 lea (+$tmp_offset)(%rsp), %rcx 1157 xor %rax, %rax 1158 mov %rax, (+8*0)(%rcx) 1159 mov %rax, (+8*1)(%rcx) 1160 mov %rax, (+8*3)(%rcx) 1161 mov %rax, (+8*4)(%rcx) 1162 mov %rax, (+8*5)(%rcx) 1163 mov %rax, (+8*6)(%rcx) 1164 mov %rax, (+8*7)(%rcx) 1165 mov %rax, (+$exp_offset+8*8)(%rsp) 1166 movq \$1, (+8*2)(%rcx) 1167 1168 lea (+$garray_offset)(%rsp), %rbp 1169 mov %rcx, %rsi # pTmp 1170 mov %rbp, %rdi # Garray[][0] 1171 ___ 1172 1173 &swizzle("%rdi", "%rcx", "%rax", "%rbx"); 1174 1175 # for (rax = 31; rax != 0; rax--) { 1176 # tmp = reduce(tmp * G) 1177 # swizzle(pg, tmp); 1178 # pg += 2; } 1179 $code.=<<___; 1180 mov \$31, %rax 1181 mov %rax, (+$i_offset)(%rsp) 1182 mov %rbp, (+$pg_offset)(%rsp) 1183 # rsi -> pTmp 1184 mov %rsi, (+$red_result_addr_offset)(%rsp) 1185 mov (+8*0)(%rsi), %r10 1186 mov (+8*1)(%rsi), %r11 1187 mov (+8*2)(%rsi), %r12 1188 mov (+8*3)(%rsi), %r13 1189 mov (+8*4)(%rsi), %r14 1190 mov (+8*5)(%rsi), %r15 1191 mov (+8*6)(%rsi), %r8 1192 mov (+8*7)(%rsi), %r9 1193 init_loop: 1194 lea (+$GT_offset)(%rsp), %rdi 1195 call mont_mul_a3b 1196 lea (+$tmp_offset)(%rsp), %rsi 1197 mov (+$pg_offset)(%rsp), %rbp 1198 add \$2, %rbp 1199 mov %rbp, (+$pg_offset)(%rsp) 1200 mov %rsi, %rcx # rcx = rsi = addr of tmp 1201 ___ 1202 1203 &swizzle("%rbp", "%rcx", "%rax", "%rbx"); 1204 $code.=<<___; 1205 mov (+$i_offset)(%rsp), %rax 1206 sub \$1, %rax 1207 mov %rax, (+$i_offset)(%rsp) 1208 jne init_loop 1209 1210 # 1211 # Copy exponent onto stack 1212 movdqa %xmm0, (+$exp_offset+16*0)(%rsp) 1213 movdqa %xmm1, (+$exp_offset+16*1)(%rsp) 1214 movdqa %xmm2, (+$exp_offset+16*2)(%rsp) 1215 movdqa %xmm3, (+$exp_offset+16*3)(%rsp) 1216 1217 1218 # 1219 # Do exponentiation 1220 # Initialize result to G[exp{511:507}] 1221 mov (+$exp_offset+62)(%rsp), %eax 1222 mov %rax, %rdx 1223 shr \$11, %rax 1224 and \$0x07FF, %edx 1225 mov %edx, (+$exp_offset+62)(%rsp) 1226 lea (+$garray_offset)(%rsp,%rax,2), %rsi 1227 mov (+$pResult_offset)(%rsp), %rdx 1228 ___ 1229 1230 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1231 1232 # 1233 # Loop variables 1234 # rcx = [loop_idx] = index: 510-5 to 0 by 5 1235 $code.=<<___; 1236 movq \$505, (+$loop_idx_offset)(%rsp) 1237 1238 mov (+$pResult_offset)(%rsp), %rcx 1239 mov %rcx, (+$red_result_addr_offset)(%rsp) 1240 mov (+8*0)(%rcx), %r10 1241 mov (+8*1)(%rcx), %r11 1242 mov (+8*2)(%rcx), %r12 1243 mov (+8*3)(%rcx), %r13 1244 mov (+8*4)(%rcx), %r14 1245 mov (+8*5)(%rcx), %r15 1246 mov (+8*6)(%rcx), %r8 1247 mov (+8*7)(%rcx), %r9 1248 jmp sqr_2 1249 1250 main_loop_a3b: 1251 call sqr_reduce 1252 call sqr_reduce 1253 call sqr_reduce 1254 sqr_2: 1255 call sqr_reduce 1256 call sqr_reduce 1257 1258 # 1259 # Do multiply, first look up proper value in Garray 1260 mov (+$loop_idx_offset)(%rsp), %rcx # bit index 1261 mov %rcx, %rax 1262 shr \$4, %rax # rax is word pointer 1263 mov (+$exp_offset)(%rsp,%rax,2), %edx 1264 and \$15, %rcx 1265 shrq %cl, %rdx 1266 and \$0x1F, %rdx 1267 1268 lea (+$garray_offset)(%rsp,%rdx,2), %rsi 1269 lea (+$tmp_offset)(%rsp), %rdx 1270 mov %rdx, %rdi 1271 ___ 1272 1273 &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1274 # rdi = tmp = pG 1275 1276 # 1277 # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData) 1278 # result result pG M Data 1279 $code.=<<___; 1280 mov (+$pResult_offset)(%rsp), %rsi 1281 call mont_mul_a3b 1282 1283 # 1284 # finish loop 1285 mov (+$loop_idx_offset)(%rsp), %rcx 1286 sub \$5, %rcx 1287 mov %rcx, (+$loop_idx_offset)(%rsp) 1288 jge main_loop_a3b 1289 1290 # 1291 1292 end_main_loop_a3b: 1293 # transform result out of Montgomery space 1294 # result = reduce(result) 1295 mov (+$pResult_offset)(%rsp), %rdx 1296 pxor %xmm4, %xmm4 1297 movdqu (+16*0)(%rdx), %xmm0 1298 movdqu (+16*1)(%rdx), %xmm1 1299 movdqu (+16*2)(%rdx), %xmm2 1300 movdqu (+16*3)(%rdx), %xmm3 1301 movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp) 1302 movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp) 1303 movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1304 movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1305 movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp) 1306 movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp) 1307 movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp) 1308 movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp) 1309 call mont_reduce 1310 1311 # If result > m, subract m 1312 # load result into r15:r8 1313 mov (+$pResult_offset)(%rsp), %rax 1314 mov (+8*0)(%rax), %r8 1315 mov (+8*1)(%rax), %r9 1316 mov (+8*2)(%rax), %r10 1317 mov (+8*3)(%rax), %r11 1318 mov (+8*4)(%rax), %r12 1319 mov (+8*5)(%rax), %r13 1320 mov (+8*6)(%rax), %r14 1321 mov (+8*7)(%rax), %r15 1322 1323 # subtract m 1324 mov (+$pData_offset)(%rsp), %rbx 1325 add \$$M, %rbx 1326 1327 sub (+8*0)(%rbx), %r8 1328 sbb (+8*1)(%rbx), %r9 1329 sbb (+8*2)(%rbx), %r10 1330 sbb (+8*3)(%rbx), %r11 1331 sbb (+8*4)(%rbx), %r12 1332 sbb (+8*5)(%rbx), %r13 1333 sbb (+8*6)(%rbx), %r14 1334 sbb (+8*7)(%rbx), %r15 1335 1336 # if Carry is clear, replace result with difference 1337 mov (+8*0)(%rax), %rsi 1338 mov (+8*1)(%rax), %rdi 1339 mov (+8*2)(%rax), %rcx 1340 mov (+8*3)(%rax), %rdx 1341 cmovnc %r8, %rsi 1342 cmovnc %r9, %rdi 1343 cmovnc %r10, %rcx 1344 cmovnc %r11, %rdx 1345 mov %rsi, (+8*0)(%rax) 1346 mov %rdi, (+8*1)(%rax) 1347 mov %rcx, (+8*2)(%rax) 1348 mov %rdx, (+8*3)(%rax) 1349 1350 mov (+8*4)(%rax), %rsi 1351 mov (+8*5)(%rax), %rdi 1352 mov (+8*6)(%rax), %rcx 1353 mov (+8*7)(%rax), %rdx 1354 cmovnc %r12, %rsi 1355 cmovnc %r13, %rdi 1356 cmovnc %r14, %rcx 1357 cmovnc %r15, %rdx 1358 mov %rsi, (+8*4)(%rax) 1359 mov %rdi, (+8*5)(%rax) 1360 mov %rcx, (+8*6)(%rax) 1361 mov %rdx, (+8*7)(%rax) 1362 1363 mov (+$rsp_offset)(%rsp), %rsi 1364 mov 0(%rsi),%r15 1365 mov 8(%rsi),%r14 1366 mov 16(%rsi),%r13 1367 mov 24(%rsi),%r12 1368 mov 32(%rsi),%rbx 1369 mov 40(%rsi),%rbp 1370 lea 48(%rsi),%rsp 1371 .Lepilogue: 1372 ret 1373 .size mod_exp_512, . - mod_exp_512 1374 ___ 1375 1376 if ($win64) { 1377 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 1378 # CONTEXT *context,DISPATCHER_CONTEXT *disp) 1379 my $rec="%rcx"; 1380 my $frame="%rdx"; 1381 my $context="%r8"; 1382 my $disp="%r9"; 1383 1384 $code.=<<___; 1385 .extern __imp_RtlVirtualUnwind 1386 .type mod_exp_512_se_handler,\@abi-omnipotent 1387 .align 16 1388 mod_exp_512_se_handler: 1389 push %rsi 1390 push %rdi 1391 push %rbx 1392 push %rbp 1393 push %r12 1394 push %r13 1395 push %r14 1396 push %r15 1397 pushfq 1398 sub \$64,%rsp 1399 1400 mov 120($context),%rax # pull context->Rax 1401 mov 248($context),%rbx # pull context->Rip 1402 1403 lea .Lbody(%rip),%r10 1404 cmp %r10,%rbx # context->Rip<prologue label 1405 jb .Lin_prologue 1406 1407 mov 152($context),%rax # pull context->Rsp 1408 1409 lea .Lepilogue(%rip),%r10 1410 cmp %r10,%rbx # context->Rip>=epilogue label 1411 jae .Lin_prologue 1412 1413 mov $rsp_offset(%rax),%rax # pull saved Rsp 1414 1415 mov 32(%rax),%rbx 1416 mov 40(%rax),%rbp 1417 mov 24(%rax),%r12 1418 mov 16(%rax),%r13 1419 mov 8(%rax),%r14 1420 mov 0(%rax),%r15 1421 lea 48(%rax),%rax 1422 mov %rbx,144($context) # restore context->Rbx 1423 mov %rbp,160($context) # restore context->Rbp 1424 mov %r12,216($context) # restore context->R12 1425 mov %r13,224($context) # restore context->R13 1426 mov %r14,232($context) # restore context->R14 1427 mov %r15,240($context) # restore context->R15 1428 1429 .Lin_prologue: 1430 mov 8(%rax),%rdi 1431 mov 16(%rax),%rsi 1432 mov %rax,152($context) # restore context->Rsp 1433 mov %rsi,168($context) # restore context->Rsi 1434 mov %rdi,176($context) # restore context->Rdi 1435 1436 mov 40($disp),%rdi # disp->ContextRecord 1437 mov $context,%rsi # context 1438 mov \$154,%ecx # sizeof(CONTEXT) 1439 .long 0xa548f3fc # cld; rep movsq 1440 1441 mov $disp,%rsi 1442 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1443 mov 8(%rsi),%rdx # arg2, disp->ImageBase 1444 mov 0(%rsi),%r8 # arg3, disp->ControlPc 1445 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1446 mov 40(%rsi),%r10 # disp->ContextRecord 1447 lea 56(%rsi),%r11 # &disp->HandlerData 1448 lea 24(%rsi),%r12 # &disp->EstablisherFrame 1449 mov %r10,32(%rsp) # arg5 1450 mov %r11,40(%rsp) # arg6 1451 mov %r12,48(%rsp) # arg7 1452 mov %rcx,56(%rsp) # arg8, (NULL) 1453 call *__imp_RtlVirtualUnwind(%rip) 1454 1455 mov \$1,%eax # ExceptionContinueSearch 1456 add \$64,%rsp 1457 popfq 1458 pop %r15 1459 pop %r14 1460 pop %r13 1461 pop %r12 1462 pop %rbp 1463 pop %rbx 1464 pop %rdi 1465 pop %rsi 1466 ret 1467 .size mod_exp_512_se_handler,.-mod_exp_512_se_handler 1468 1469 .section .pdata 1470 .align 4 1471 .rva .LSEH_begin_mod_exp_512 1472 .rva .LSEH_end_mod_exp_512 1473 .rva .LSEH_info_mod_exp_512 1474 1475 .section .xdata 1476 .align 8 1477 .LSEH_info_mod_exp_512: 1478 .byte 9,0,0,0 1479 .rva mod_exp_512_se_handler 1480 ___ 1481 } 1482 1483 sub reg_part { 1484 my ($reg,$conv)=@_; 1485 if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } 1486 elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } 1487 elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } 1488 elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } 1489 return $reg; 1490 } 1491 1492 $code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; 1493 $code =~ s/\`([^\`]*)\`/eval $1/gem; 1494 $code =~ s/(\(\+[^)]+\))/eval $1/gem; 1495 print $code; 1496 close STDOUT; 1497
