1 #!/usr/bin/env perl 2 # 3 # ==================================================================== 4 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL 5 # project. The module is, however, dual licensed under OpenSSL and 6 # CRYPTOGAMS licenses depending on where you obtain it. For further 7 # details see http://www.openssl.org/~appro/cryptogams/. 8 # ==================================================================== 9 # 10 # March, June 2010 11 # 12 # The module implements "4-bit" GCM GHASH function and underlying 13 # single multiplication operation in GF(2^128). "4-bit" means that 14 # it uses 256 bytes per-key table [+128 bytes shared table]. GHASH 15 # function features so called "528B" variant utilizing additional 16 # 256+16 bytes of per-key storage [+512 bytes shared table]. 17 # Performance results are for this streamed GHASH subroutine and are 18 # expressed in cycles per processed byte, less is better: 19 # 20 # gcc 3.4.x(*) assembler 21 # 22 # P4 28.6 14.0 +100% 23 # Opteron 19.3 7.7 +150% 24 # Core2 17.8 8.1(**) +120% 25 # 26 # (*) comparison is not completely fair, because C results are 27 # for vanilla "256B" implementation, while assembler results 28 # are for "528B";-) 29 # (**) it's mystery [to me] why Core2 result is not same as for 30 # Opteron; 31 32 # May 2010 33 # 34 # Add PCLMULQDQ version performing at 2.02 cycles per processed byte. 35 # See ghash-x86.pl for background information and details about coding 36 # techniques. 37 # 38 # Special thanks to David Woodhouse <dwmw2 (at] infradead.org> for 39 # providing access to a Westmere-based system on behalf of Intel 40 # Open Source Technology Centre. 41 42 $flavour = shift; 43 $output = shift; 44 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 45 46 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 47 48 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 49 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 50 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 51 die "can't locate x86_64-xlate.pl"; 52 53 open OUT,"| \"$^X\" $xlate $flavour $output"; 54 *STDOUT=*OUT; 55 56 # common register layout 57 $nlo="%rax"; 58 $nhi="%rbx"; 59 $Zlo="%r8"; 60 $Zhi="%r9"; 61 $tmp="%r10"; 62 $rem_4bit = "%r11"; 63 64 $Xi="%rdi"; 65 $Htbl="%rsi"; 66 67 # per-function register layout 68 $cnt="%rcx"; 69 $rem="%rdx"; 70 71 sub LB() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/ or 72 $r =~ s/%[er]([sd]i)/%\1l/ or 73 $r =~ s/%[er](bp)/%\1l/ or 74 $r =~ s/%(r[0-9]+)[d]?/%\1b/; $r; } 75 76 sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm 77 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; 78 my $arg = pop; 79 $arg = "\$$arg" if ($arg*1 eq $arg); 80 $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; 81 } 82 84 { my $N; 85 sub loop() { 86 my $inp = shift; 87 88 $N++; 89 $code.=<<___; 90 xor $nlo,$nlo 91 xor $nhi,$nhi 92 mov `&LB("$Zlo")`,`&LB("$nlo")` 93 mov `&LB("$Zlo")`,`&LB("$nhi")` 94 shl \$4,`&LB("$nlo")` 95 mov \$14,$cnt 96 mov 8($Htbl,$nlo),$Zlo 97 mov ($Htbl,$nlo),$Zhi 98 and \$0xf0,`&LB("$nhi")` 99 mov $Zlo,$rem 100 jmp .Loop$N 101 102 .align 16 103 .Loop$N: 104 shr \$4,$Zlo 105 and \$0xf,$rem 106 mov $Zhi,$tmp 107 mov ($inp,$cnt),`&LB("$nlo")` 108 shr \$4,$Zhi 109 xor 8($Htbl,$nhi),$Zlo 110 shl \$60,$tmp 111 xor ($Htbl,$nhi),$Zhi 112 mov `&LB("$nlo")`,`&LB("$nhi")` 113 xor ($rem_4bit,$rem,8),$Zhi 114 mov $Zlo,$rem 115 shl \$4,`&LB("$nlo")` 116 xor $tmp,$Zlo 117 dec $cnt 118 js .Lbreak$N 119 120 shr \$4,$Zlo 121 and \$0xf,$rem 122 mov $Zhi,$tmp 123 shr \$4,$Zhi 124 xor 8($Htbl,$nlo),$Zlo 125 shl \$60,$tmp 126 xor ($Htbl,$nlo),$Zhi 127 and \$0xf0,`&LB("$nhi")` 128 xor ($rem_4bit,$rem,8),$Zhi 129 mov $Zlo,$rem 130 xor $tmp,$Zlo 131 jmp .Loop$N 132 133 .align 16 134 .Lbreak$N: 135 shr \$4,$Zlo 136 and \$0xf,$rem 137 mov $Zhi,$tmp 138 shr \$4,$Zhi 139 xor 8($Htbl,$nlo),$Zlo 140 shl \$60,$tmp 141 xor ($Htbl,$nlo),$Zhi 142 and \$0xf0,`&LB("$nhi")` 143 xor ($rem_4bit,$rem,8),$Zhi 144 mov $Zlo,$rem 145 xor $tmp,$Zlo 146 147 shr \$4,$Zlo 148 and \$0xf,$rem 149 mov $Zhi,$tmp 150 shr \$4,$Zhi 151 xor 8($Htbl,$nhi),$Zlo 152 shl \$60,$tmp 153 xor ($Htbl,$nhi),$Zhi 154 xor $tmp,$Zlo 155 xor ($rem_4bit,$rem,8),$Zhi 156 157 bswap $Zlo 158 bswap $Zhi 159 ___ 160 }} 161 162 $code=<<___; 163 .text 164 165 .globl gcm_gmult_4bit 166 .type gcm_gmult_4bit,\@function,2 167 .align 16 168 gcm_gmult_4bit: 169 push %rbx 170 push %rbp # %rbp and %r12 are pushed exclusively in 171 push %r12 # order to reuse Win64 exception handler... 172 .Lgmult_prologue: 173 174 movzb 15($Xi),$Zlo 175 lea .Lrem_4bit(%rip),$rem_4bit 176 ___ 177 &loop ($Xi); 178 $code.=<<___; 179 mov $Zlo,8($Xi) 180 mov $Zhi,($Xi) 181 182 mov 16(%rsp),%rbx 183 lea 24(%rsp),%rsp 184 .Lgmult_epilogue: 185 ret 186 .size gcm_gmult_4bit,.-gcm_gmult_4bit 187 ___ 188 190 # per-function register layout 191 $inp="%rdx"; 192 $len="%rcx"; 193 $rem_8bit=$rem_4bit; 194 195 $code.=<<___; 196 .globl gcm_ghash_4bit 197 .type gcm_ghash_4bit,\@function,4 198 .align 16 199 gcm_ghash_4bit: 200 push %rbx 201 push %rbp 202 push %r12 203 push %r13 204 push %r14 205 push %r15 206 sub \$280,%rsp 207 .Lghash_prologue: 208 mov $inp,%r14 # reassign couple of args 209 mov $len,%r15 210 ___ 211 { my $inp="%r14"; 212 my $dat="%edx"; 213 my $len="%r15"; 214 my @nhi=("%ebx","%ecx"); 215 my @rem=("%r12","%r13"); 216 my $Hshr4="%rbp"; 217 218 &sub ($Htbl,-128); # size optimization 219 &lea ($Hshr4,"16+128(%rsp)"); 220 { my @lo =($nlo,$nhi); 221 my @hi =($Zlo,$Zhi); 222 223 &xor ($dat,$dat); 224 for ($i=0,$j=-2;$i<18;$i++,$j++) { 225 &mov ("$j(%rsp)",&LB($dat)) if ($i>1); 226 &or ($lo[0],$tmp) if ($i>1); 227 &mov (&LB($dat),&LB($lo[1])) if ($i>0 && $i<17); 228 &shr ($lo[1],4) if ($i>0 && $i<17); 229 &mov ($tmp,$hi[1]) if ($i>0 && $i<17); 230 &shr ($hi[1],4) if ($i>0 && $i<17); 231 &mov ("8*$j($Hshr4)",$hi[0]) if ($i>1); 232 &mov ($hi[0],"16*$i+0-128($Htbl)") if ($i<16); 233 &shl (&LB($dat),4) if ($i>0 && $i<17); 234 &mov ("8*$j-128($Hshr4)",$lo[0]) if ($i>1); 235 &mov ($lo[0],"16*$i+8-128($Htbl)") if ($i<16); 236 &shl ($tmp,60) if ($i>0 && $i<17); 237 238 push (@lo,shift(@lo)); 239 push (@hi,shift(@hi)); 240 } 241 } 242 &add ($Htbl,-128); 243 &mov ($Zlo,"8($Xi)"); 244 &mov ($Zhi,"0($Xi)"); 245 &add ($len,$inp); # pointer to the end of data 246 &lea ($rem_8bit,".Lrem_8bit(%rip)"); 247 &jmp (".Louter_loop"); 248 249 $code.=".align 16\n.Louter_loop:\n"; 250 &xor ($Zhi,"($inp)"); 251 &mov ("%rdx","8($inp)"); 252 &lea ($inp,"16($inp)"); 253 &xor ("%rdx",$Zlo); 254 &mov ("($Xi)",$Zhi); 255 &mov ("8($Xi)","%rdx"); 256 &shr ("%rdx",32); 257 258 &xor ($nlo,$nlo); 259 &rol ($dat,8); 260 &mov (&LB($nlo),&LB($dat)); 261 &movz ($nhi[0],&LB($dat)); 262 &shl (&LB($nlo),4); 263 &shr ($nhi[0],4); 264 265 for ($j=11,$i=0;$i<15;$i++) { 266 &rol ($dat,8); 267 &xor ($Zlo,"8($Htbl,$nlo)") if ($i>0); 268 &xor ($Zhi,"($Htbl,$nlo)") if ($i>0); 269 &mov ($Zlo,"8($Htbl,$nlo)") if ($i==0); 270 &mov ($Zhi,"($Htbl,$nlo)") if ($i==0); 271 272 &mov (&LB($nlo),&LB($dat)); 273 &xor ($Zlo,$tmp) if ($i>0); 274 &movzw ($rem[1],"($rem_8bit,$rem[1],2)") if ($i>0); 275 276 &movz ($nhi[1],&LB($dat)); 277 &shl (&LB($nlo),4); 278 &movzb ($rem[0],"(%rsp,$nhi[0])"); 279 280 &shr ($nhi[1],4) if ($i<14); 281 &and ($nhi[1],0xf0) if ($i==14); 282 &shl ($rem[1],48) if ($i>0); 283 &xor ($rem[0],$Zlo); 284 285 &mov ($tmp,$Zhi); 286 &xor ($Zhi,$rem[1]) if ($i>0); 287 &shr ($Zlo,8); 288 289 &movz ($rem[0],&LB($rem[0])); 290 &mov ($dat,"$j($Xi)") if (--$j%4==0); 291 &shr ($Zhi,8); 292 293 &xor ($Zlo,"-128($Hshr4,$nhi[0],8)"); 294 &shl ($tmp,56); 295 &xor ($Zhi,"($Hshr4,$nhi[0],8)"); 296 297 unshift (@nhi,pop(@nhi)); # "rotate" registers 298 unshift (@rem,pop(@rem)); 299 } 300 &movzw ($rem[1],"($rem_8bit,$rem[1],2)"); 301 &xor ($Zlo,"8($Htbl,$nlo)"); 302 &xor ($Zhi,"($Htbl,$nlo)"); 303 304 &shl ($rem[1],48); 305 &xor ($Zlo,$tmp); 306 307 &xor ($Zhi,$rem[1]); 308 &movz ($rem[0],&LB($Zlo)); 309 &shr ($Zlo,4); 310 311 &mov ($tmp,$Zhi); 312 &shl (&LB($rem[0]),4); 313 &shr ($Zhi,4); 314 315 &xor ($Zlo,"8($Htbl,$nhi[0])"); 316 &movzw ($rem[0],"($rem_8bit,$rem[0],2)"); 317 &shl ($tmp,60); 318 319 &xor ($Zhi,"($Htbl,$nhi[0])"); 320 &xor ($Zlo,$tmp); 321 &shl ($rem[0],48); 322 323 &bswap ($Zlo); 324 &xor ($Zhi,$rem[0]); 325 326 &bswap ($Zhi); 327 &cmp ($inp,$len); 328 &jb (".Louter_loop"); 329 } 330 $code.=<<___; 331 mov $Zlo,8($Xi) 332 mov $Zhi,($Xi) 333 334 lea 280(%rsp),%rsi 335 mov 0(%rsi),%r15 336 mov 8(%rsi),%r14 337 mov 16(%rsi),%r13 338 mov 24(%rsi),%r12 339 mov 32(%rsi),%rbp 340 mov 40(%rsi),%rbx 341 lea 48(%rsi),%rsp 342 .Lghash_epilogue: 343 ret 344 .size gcm_ghash_4bit,.-gcm_ghash_4bit 345 ___ 346 348 ###################################################################### 349 # PCLMULQDQ version. 350 351 @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order 352 ("%rdi","%rsi","%rdx","%rcx"); # Unix order 353 354 ($Xi,$Xhi)=("%xmm0","%xmm1"); $Hkey="%xmm2"; 355 ($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5"); 356 357 sub clmul64x64_T2 { # minimal register pressure 358 my ($Xhi,$Xi,$Hkey,$modulo)=@_; 359 360 $code.=<<___ if (!defined($modulo)); 361 movdqa $Xi,$Xhi # 362 pshufd \$0b01001110,$Xi,$T1 363 pshufd \$0b01001110,$Hkey,$T2 364 pxor $Xi,$T1 # 365 pxor $Hkey,$T2 366 ___ 367 $code.=<<___; 368 pclmulqdq \$0x00,$Hkey,$Xi ####### 369 pclmulqdq \$0x11,$Hkey,$Xhi ####### 370 pclmulqdq \$0x00,$T2,$T1 ####### 371 pxor $Xi,$T1 # 372 pxor $Xhi,$T1 # 373 374 movdqa $T1,$T2 # 375 psrldq \$8,$T1 376 pslldq \$8,$T2 # 377 pxor $T1,$Xhi 378 pxor $T2,$Xi # 379 ___ 380 } 381 382 sub reduction_alg9 { # 17/13 times faster than Intel version 383 my ($Xhi,$Xi) = @_; 384 385 $code.=<<___; 386 # 1st phase 387 movdqa $Xi,$T1 # 388 psllq \$1,$Xi 389 pxor $T1,$Xi # 390 psllq \$5,$Xi # 391 pxor $T1,$Xi # 392 psllq \$57,$Xi # 393 movdqa $Xi,$T2 # 394 pslldq \$8,$Xi 395 psrldq \$8,$T2 # 396 pxor $T1,$Xi 397 pxor $T2,$Xhi # 398 399 # 2nd phase 400 movdqa $Xi,$T2 401 psrlq \$5,$Xi 402 pxor $T2,$Xi # 403 psrlq \$1,$Xi # 404 pxor $T2,$Xi # 405 pxor $Xhi,$T2 406 psrlq \$1,$Xi # 407 pxor $T2,$Xi # 408 ___ 409 } 410 412 { my ($Htbl,$Xip)=@_4args; 413 414 $code.=<<___; 415 .globl gcm_init_clmul 416 .type gcm_init_clmul,\@abi-omnipotent 417 .align 16 418 gcm_init_clmul: 419 movdqu ($Xip),$Hkey 420 pshufd \$0b01001110,$Hkey,$Hkey # dword swap 421 422 # <<1 twist 423 pshufd \$0b11111111,$Hkey,$T2 # broadcast uppermost dword 424 movdqa $Hkey,$T1 425 psllq \$1,$Hkey 426 pxor $T3,$T3 # 427 psrlq \$63,$T1 428 pcmpgtd $T2,$T3 # broadcast carry bit 429 pslldq \$8,$T1 430 por $T1,$Hkey # H<<=1 431 432 # magic reduction 433 pand .L0x1c2_polynomial(%rip),$T3 434 pxor $T3,$Hkey # if(carry) H^=0x1c2_polynomial 435 436 # calculate H^2 437 movdqa $Hkey,$Xi 438 ___ 439 &clmul64x64_T2 ($Xhi,$Xi,$Hkey); 440 &reduction_alg9 ($Xhi,$Xi); 441 $code.=<<___; 442 movdqu $Hkey,($Htbl) # save H 443 movdqu $Xi,16($Htbl) # save H^2 444 ret 445 .size gcm_init_clmul,.-gcm_init_clmul 446 ___ 447 } 448 449 { my ($Xip,$Htbl)=@_4args; 450 451 $code.=<<___; 452 .globl gcm_gmult_clmul 453 .type gcm_gmult_clmul,\@abi-omnipotent 454 .align 16 455 gcm_gmult_clmul: 456 movdqu ($Xip),$Xi 457 movdqa .Lbswap_mask(%rip),$T3 458 movdqu ($Htbl),$Hkey 459 pshufb $T3,$Xi 460 ___ 461 &clmul64x64_T2 ($Xhi,$Xi,$Hkey); 462 &reduction_alg9 ($Xhi,$Xi); 463 $code.=<<___; 464 pshufb $T3,$Xi 465 movdqu $Xi,($Xip) 466 ret 467 .size gcm_gmult_clmul,.-gcm_gmult_clmul 468 ___ 469 } 470 472 { my ($Xip,$Htbl,$inp,$len)=@_4args; 473 my $Xn="%xmm6"; 474 my $Xhn="%xmm7"; 475 my $Hkey2="%xmm8"; 476 my $T1n="%xmm9"; 477 my $T2n="%xmm10"; 478 479 $code.=<<___; 480 .globl gcm_ghash_clmul 481 .type gcm_ghash_clmul,\@abi-omnipotent 482 .align 16 483 gcm_ghash_clmul: 484 ___ 485 $code.=<<___ if ($win64); 486 .LSEH_begin_gcm_ghash_clmul: 487 # I can't trust assembler to use specific encoding:-( 488 .byte 0x48,0x83,0xec,0x58 #sub \$0x58,%rsp 489 .byte 0x0f,0x29,0x34,0x24 #movaps %xmm6,(%rsp) 490 .byte 0x0f,0x29,0x7c,0x24,0x10 #movdqa %xmm7,0x10(%rsp) 491 .byte 0x44,0x0f,0x29,0x44,0x24,0x20 #movaps %xmm8,0x20(%rsp) 492 .byte 0x44,0x0f,0x29,0x4c,0x24,0x30 #movaps %xmm9,0x30(%rsp) 493 .byte 0x44,0x0f,0x29,0x54,0x24,0x40 #movaps %xmm10,0x40(%rsp) 494 ___ 495 $code.=<<___; 496 movdqa .Lbswap_mask(%rip),$T3 497 498 movdqu ($Xip),$Xi 499 movdqu ($Htbl),$Hkey 500 pshufb $T3,$Xi 501 502 sub \$0x10,$len 503 jz .Lodd_tail 504 505 movdqu 16($Htbl),$Hkey2 506 ####### 507 # Xi+2 =[H*(Ii+1 + Xi+1)] mod P = 508 # [(H*Ii+1) + (H*Xi+1)] mod P = 509 # [(H*Ii+1) + H^2*(Ii+Xi)] mod P 510 # 511 movdqu ($inp),$T1 # Ii 512 movdqu 16($inp),$Xn # Ii+1 513 pshufb $T3,$T1 514 pshufb $T3,$Xn 515 pxor $T1,$Xi # Ii+Xi 516 ___ 517 &clmul64x64_T2 ($Xhn,$Xn,$Hkey); # H*Ii+1 518 $code.=<<___; 519 movdqa $Xi,$Xhi # 520 pshufd \$0b01001110,$Xi,$T1 521 pshufd \$0b01001110,$Hkey2,$T2 522 pxor $Xi,$T1 # 523 pxor $Hkey2,$T2 524 525 lea 32($inp),$inp # i+=2 526 sub \$0x20,$len 527 jbe .Leven_tail 528 529 .Lmod_loop: 530 ___ 531 &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) 532 $code.=<<___; 533 movdqu ($inp),$T1 # Ii 534 pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 535 pxor $Xhn,$Xhi 536 537 movdqu 16($inp),$Xn # Ii+1 538 pshufb $T3,$T1 539 pshufb $T3,$Xn 540 541 movdqa $Xn,$Xhn # 542 pshufd \$0b01001110,$Xn,$T1n 543 pshufd \$0b01001110,$Hkey,$T2n 544 pxor $Xn,$T1n # 545 pxor $Hkey,$T2n 546 pxor $T1,$Xhi # "Ii+Xi", consume early 547 548 movdqa $Xi,$T1 # 1st phase 549 psllq \$1,$Xi 550 pxor $T1,$Xi # 551 psllq \$5,$Xi # 552 pxor $T1,$Xi # 553 pclmulqdq \$0x00,$Hkey,$Xn ####### 554 psllq \$57,$Xi # 555 movdqa $Xi,$T2 # 556 pslldq \$8,$Xi 557 psrldq \$8,$T2 # 558 pxor $T1,$Xi 559 pxor $T2,$Xhi # 560 561 pclmulqdq \$0x11,$Hkey,$Xhn ####### 562 movdqa $Xi,$T2 # 2nd phase 563 psrlq \$5,$Xi 564 pxor $T2,$Xi # 565 psrlq \$1,$Xi # 566 pxor $T2,$Xi # 567 pxor $Xhi,$T2 568 psrlq \$1,$Xi # 569 pxor $T2,$Xi # 570 571 pclmulqdq \$0x00,$T2n,$T1n ####### 572 movdqa $Xi,$Xhi # 573 pshufd \$0b01001110,$Xi,$T1 574 pshufd \$0b01001110,$Hkey2,$T2 575 pxor $Xi,$T1 # 576 pxor $Hkey2,$T2 577 578 pxor $Xn,$T1n # 579 pxor $Xhn,$T1n # 580 movdqa $T1n,$T2n # 581 psrldq \$8,$T1n 582 pslldq \$8,$T2n # 583 pxor $T1n,$Xhn 584 pxor $T2n,$Xn # 585 586 lea 32($inp),$inp 587 sub \$0x20,$len 588 ja .Lmod_loop 589 590 .Leven_tail: 591 ___ 592 &clmul64x64_T2 ($Xhi,$Xi,$Hkey2,1); # H^2*(Ii+Xi) 593 $code.=<<___; 594 pxor $Xn,$Xi # (H*Ii+1) + H^2*(Ii+Xi) 595 pxor $Xhn,$Xhi 596 ___ 597 &reduction_alg9 ($Xhi,$Xi); 598 $code.=<<___; 599 test $len,$len 600 jnz .Ldone 601 602 .Lodd_tail: 603 movdqu ($inp),$T1 # Ii 604 pshufb $T3,$T1 605 pxor $T1,$Xi # Ii+Xi 606 ___ 607 &clmul64x64_T2 ($Xhi,$Xi,$Hkey); # H*(Ii+Xi) 608 &reduction_alg9 ($Xhi,$Xi); 609 $code.=<<___; 610 .Ldone: 611 pshufb $T3,$Xi 612 movdqu $Xi,($Xip) 613 ___ 614 $code.=<<___ if ($win64); 615 movaps (%rsp),%xmm6 616 movaps 0x10(%rsp),%xmm7 617 movaps 0x20(%rsp),%xmm8 618 movaps 0x30(%rsp),%xmm9 619 movaps 0x40(%rsp),%xmm10 620 add \$0x58,%rsp 621 ___ 622 $code.=<<___; 623 ret 624 .LSEH_end_gcm_ghash_clmul: 625 .size gcm_ghash_clmul,.-gcm_ghash_clmul 626 ___ 627 } 628 629 $code.=<<___; 630 .align 64 631 .Lbswap_mask: 632 .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 633 .L0x1c2_polynomial: 634 .byte 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 635 .align 64 636 .type .Lrem_4bit,\@object 637 .Lrem_4bit: 638 .long 0,`0x0000<<16`,0,`0x1C20<<16`,0,`0x3840<<16`,0,`0x2460<<16` 639 .long 0,`0x7080<<16`,0,`0x6CA0<<16`,0,`0x48C0<<16`,0,`0x54E0<<16` 640 .long 0,`0xE100<<16`,0,`0xFD20<<16`,0,`0xD940<<16`,0,`0xC560<<16` 641 .long 0,`0x9180<<16`,0,`0x8DA0<<16`,0,`0xA9C0<<16`,0,`0xB5E0<<16` 642 .type .Lrem_8bit,\@object 643 .Lrem_8bit: 644 .value 0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E 645 .value 0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E 646 .value 0x1C20,0x1DE2,0x1FA4,0x1E66,0x1B28,0x1AEA,0x18AC,0x196E 647 .value 0x1230,0x13F2,0x11B4,0x1076,0x1538,0x14FA,0x16BC,0x177E 648 .value 0x3840,0x3982,0x3BC4,0x3A06,0x3F48,0x3E8A,0x3CCC,0x3D0E 649 .value 0x3650,0x3792,0x35D4,0x3416,0x3158,0x309A,0x32DC,0x331E 650 .value 0x2460,0x25A2,0x27E4,0x2626,0x2368,0x22AA,0x20EC,0x212E 651 .value 0x2A70,0x2BB2,0x29F4,0x2836,0x2D78,0x2CBA,0x2EFC,0x2F3E 652 .value 0x7080,0x7142,0x7304,0x72C6,0x7788,0x764A,0x740C,0x75CE 653 .value 0x7E90,0x7F52,0x7D14,0x7CD6,0x7998,0x785A,0x7A1C,0x7BDE 654 .value 0x6CA0,0x6D62,0x6F24,0x6EE6,0x6BA8,0x6A6A,0x682C,0x69EE 655 .value 0x62B0,0x6372,0x6134,0x60F6,0x65B8,0x647A,0x663C,0x67FE 656 .value 0x48C0,0x4902,0x4B44,0x4A86,0x4FC8,0x4E0A,0x4C4C,0x4D8E 657 .value 0x46D0,0x4712,0x4554,0x4496,0x41D8,0x401A,0x425C,0x439E 658 .value 0x54E0,0x5522,0x5764,0x56A6,0x53E8,0x522A,0x506C,0x51AE 659 .value 0x5AF0,0x5B32,0x5974,0x58B6,0x5DF8,0x5C3A,0x5E7C,0x5FBE 660 .value 0xE100,0xE0C2,0xE284,0xE346,0xE608,0xE7CA,0xE58C,0xE44E 661 .value 0xEF10,0xEED2,0xEC94,0xED56,0xE818,0xE9DA,0xEB9C,0xEA5E 662 .value 0xFD20,0xFCE2,0xFEA4,0xFF66,0xFA28,0xFBEA,0xF9AC,0xF86E 663 .value 0xF330,0xF2F2,0xF0B4,0xF176,0xF438,0xF5FA,0xF7BC,0xF67E 664 .value 0xD940,0xD882,0xDAC4,0xDB06,0xDE48,0xDF8A,0xDDCC,0xDC0E 665 .value 0xD750,0xD692,0xD4D4,0xD516,0xD058,0xD19A,0xD3DC,0xD21E 666 .value 0xC560,0xC4A2,0xC6E4,0xC726,0xC268,0xC3AA,0xC1EC,0xC02E 667 .value 0xCB70,0xCAB2,0xC8F4,0xC936,0xCC78,0xCDBA,0xCFFC,0xCE3E 668 .value 0x9180,0x9042,0x9204,0x93C6,0x9688,0x974A,0x950C,0x94CE 669 .value 0x9F90,0x9E52,0x9C14,0x9DD6,0x9898,0x995A,0x9B1C,0x9ADE 670 .value 0x8DA0,0x8C62,0x8E24,0x8FE6,0x8AA8,0x8B6A,0x892C,0x88EE 671 .value 0x83B0,0x8272,0x8034,0x81F6,0x84B8,0x857A,0x873C,0x86FE 672 .value 0xA9C0,0xA802,0xAA44,0xAB86,0xAEC8,0xAF0A,0xAD4C,0xAC8E 673 .value 0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E 674 .value 0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE 675 .value 0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE 676 677 .asciz "GHASH for x86_64, CRYPTOGAMS by <appro\@openssl.org>" 678 .align 64 679 ___ 680 682 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 683 # CONTEXT *context,DISPATCHER_CONTEXT *disp) 684 if ($win64) { 685 $rec="%rcx"; 686 $frame="%rdx"; 687 $context="%r8"; 688 $disp="%r9"; 689 690 $code.=<<___; 691 .extern __imp_RtlVirtualUnwind 692 .type se_handler,\@abi-omnipotent 693 .align 16 694 se_handler: 695 push %rsi 696 push %rdi 697 push %rbx 698 push %rbp 699 push %r12 700 push %r13 701 push %r14 702 push %r15 703 pushfq 704 sub \$64,%rsp 705 706 mov 120($context),%rax # pull context->Rax 707 mov 248($context),%rbx # pull context->Rip 708 709 mov 8($disp),%rsi # disp->ImageBase 710 mov 56($disp),%r11 # disp->HandlerData 711 712 mov 0(%r11),%r10d # HandlerData[0] 713 lea (%rsi,%r10),%r10 # prologue label 714 cmp %r10,%rbx # context->Rip<prologue label 715 jb .Lin_prologue 716 717 mov 152($context),%rax # pull context->Rsp 718 719 mov 4(%r11),%r10d # HandlerData[1] 720 lea (%rsi,%r10),%r10 # epilogue label 721 cmp %r10,%rbx # context->Rip>=epilogue label 722 jae .Lin_prologue 723 724 lea 24(%rax),%rax # adjust "rsp" 725 726 mov -8(%rax),%rbx 727 mov -16(%rax),%rbp 728 mov -24(%rax),%r12 729 mov %rbx,144($context) # restore context->Rbx 730 mov %rbp,160($context) # restore context->Rbp 731 mov %r12,216($context) # restore context->R12 732 733 .Lin_prologue: 734 mov 8(%rax),%rdi 735 mov 16(%rax),%rsi 736 mov %rax,152($context) # restore context->Rsp 737 mov %rsi,168($context) # restore context->Rsi 738 mov %rdi,176($context) # restore context->Rdi 739 740 mov 40($disp),%rdi # disp->ContextRecord 741 mov $context,%rsi # context 742 mov \$`1232/8`,%ecx # sizeof(CONTEXT) 743 .long 0xa548f3fc # cld; rep movsq 744 745 mov $disp,%rsi 746 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 747 mov 8(%rsi),%rdx # arg2, disp->ImageBase 748 mov 0(%rsi),%r8 # arg3, disp->ControlPc 749 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 750 mov 40(%rsi),%r10 # disp->ContextRecord 751 lea 56(%rsi),%r11 # &disp->HandlerData 752 lea 24(%rsi),%r12 # &disp->EstablisherFrame 753 mov %r10,32(%rsp) # arg5 754 mov %r11,40(%rsp) # arg6 755 mov %r12,48(%rsp) # arg7 756 mov %rcx,56(%rsp) # arg8, (NULL) 757 call *__imp_RtlVirtualUnwind(%rip) 758 759 mov \$1,%eax # ExceptionContinueSearch 760 add \$64,%rsp 761 popfq 762 pop %r15 763 pop %r14 764 pop %r13 765 pop %r12 766 pop %rbp 767 pop %rbx 768 pop %rdi 769 pop %rsi 770 ret 771 .size se_handler,.-se_handler 772 773 .section .pdata 774 .align 4 775 .rva .LSEH_begin_gcm_gmult_4bit 776 .rva .LSEH_end_gcm_gmult_4bit 777 .rva .LSEH_info_gcm_gmult_4bit 778 779 .rva .LSEH_begin_gcm_ghash_4bit 780 .rva .LSEH_end_gcm_ghash_4bit 781 .rva .LSEH_info_gcm_ghash_4bit 782 783 .rva .LSEH_begin_gcm_ghash_clmul 784 .rva .LSEH_end_gcm_ghash_clmul 785 .rva .LSEH_info_gcm_ghash_clmul 786 787 .section .xdata 788 .align 8 789 .LSEH_info_gcm_gmult_4bit: 790 .byte 9,0,0,0 791 .rva se_handler 792 .rva .Lgmult_prologue,.Lgmult_epilogue # HandlerData 793 .LSEH_info_gcm_ghash_4bit: 794 .byte 9,0,0,0 795 .rva se_handler 796 .rva .Lghash_prologue,.Lghash_epilogue # HandlerData 797 .LSEH_info_gcm_ghash_clmul: 798 .byte 0x01,0x1f,0x0b,0x00 799 .byte 0x1f,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 800 .byte 0x19,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 801 .byte 0x13,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 802 .byte 0x0d,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 803 .byte 0x08,0x68,0x00,0x00 #movaps (rsp),xmm6 804 .byte 0x04,0xa2,0x00,0x00 #sub rsp,0x58 805 ___ 806 } 807 809 $code =~ s/\`([^\`]*)\`/eval($1)/gem; 810 811 print $code; 812 813 close STDOUT; 814
