Home | History | Annotate | Download | only in asm 
      1 #!/usr/bin/env perl
      2 #
      3 # ====================================================================
      4 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL
      5 # project. The module is, however, dual licensed under OpenSSL and
      6 # CRYPTOGAMS licenses depending on where you obtain it. For further
      7 # details see http://www.openssl.org/~appro/cryptogams/.
      8 # ====================================================================
      9 #
     10 # March, June 2010
     11 #
     12 # The module implements "4-bit" GCM GHASH function and underlying
     13 # single multiplication operation in GF(2^128). "4-bit" means that
     14 # it uses 256 bytes per-key table [+128 bytes shared table]. GHASH
     15 # function features so called "528B" variant utilizing additional
     16 # 256+16 bytes of per-key storage [+512 bytes shared table].
     17 # Performance results are for this streamed GHASH subroutine and are
     18 # expressed in cycles per processed byte, less is better:
     19 #
     20 #		gcc 3.4.x(*)	assembler
     21 #
     22 # P4		28.6		14.0		+100%
     23 # Opteron	19.3		7.7		+150%
     24 # Core2		17.8		8.1(**)		+120%
     25 #
     26 # (*)	comparison is not completely fair, because C results are
     27 #	for vanilla "256B" implementation, while assembler results
     28 #	are for "528B";-)
     29 # (**)	it's mystery [to me] why Core2 result is not same as for
     30 #	Opteron;
     31 
     32 # May 2010
     33 #
     34 # Add PCLMULQDQ version performing at 2.02 cycles per processed byte.
     35 # See ghash-x86.pl for background information and details about coding
     36 # techniques.
     37 #
     38 # Special thanks to David Woodhouse <dwmw2 (at] infradead.org> for
     39 # providing access to a Westmere-based system on behalf of Intel
     40 # Open Source Technology Centre.
     41 
     42 $flavour = shift;
     43 $output  = shift;
     44 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
     45 
     46 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
     47 
     48 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
     49 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
     50 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
     51 die "can't locate x86_64-xlate.pl";
     52 
     53 open STDOUT,"| $^X $xlate $flavour $output";
     54 
     55 # common register layout
     56 $nlo="%rax";
     57 $nhi="%rbx";
     58 $Zlo="%r8";
     59 $Zhi="%r9";
     60 $tmp="%r10";
     61 $rem_4bit = "%r11";
     62 
     63 $Xi="%rdi";
     64 $Htbl="%rsi";
     65 
     66 # per-function register layout
     67 $cnt="%rcx";
     68 $rem="%rdx";
     69 
     70 sub LB() { my $r=shift; $r =~ s/%[er]([a-d])x/%\1l/	or
     71 			$r =~ s/%[er]([sd]i)/%\1l/	or
     72 			$r =~ s/%[er](bp)/%\1l/		or
     73 			$r =~ s/%(r[0-9]+)[d]?/%\1b/;   $r; }
     74 
     75 sub AUTOLOAD()		# thunk [simplified] 32-bit style perlasm
     76 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
     77   my $arg = pop;
     78     $arg = "\$$arg" if ($arg*1 eq $arg);
     79     $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
     80 }
     81 
     83 { my $N;
     84   sub loop() {
     85   my $inp = shift;
     86 
     87 	$N++;
     88 $code.=<<___;
     89 	xor	$nlo,$nlo
     90 	xor	$nhi,$nhi
     91 	mov	`&LB("$Zlo")`,`&LB("$nlo")`
     92 	mov	`&LB("$Zlo")`,`&LB("$nhi")`
     93 	shl	\$4,`&LB("$nlo")`
     94 	mov	\$14,$cnt
     95 	mov	8($Htbl,$nlo),$Zlo
     96 	mov	($Htbl,$nlo),$Zhi
     97 	and	\$0xf0,`&LB("$nhi")`
     98 	mov	$Zlo,$rem
     99 	jmp	.Loop$N
    100 
    101 .align	16
    102 .Loop$N:
    103 	shr	\$4,$Zlo
    104 	and	\$0xf,$rem
    105 	mov	$Zhi,$tmp
    106 	mov	($inp,$cnt),`&LB("$nlo")`
    107 	shr	\$4,$Zhi
    108 	xor	8($Htbl,$nhi),$Zlo
    109 	shl	\$60,$tmp
    110 	xor	($Htbl,$nhi),$Zhi
    111 	mov	`&LB("$nlo")`,`&LB("$nhi")`
    112 	xor	($rem_4bit,$rem,8),$Zhi
    113 	mov	$Zlo,$rem
    114 	shl	\$4,`&LB("$nlo")`
    115 	xor	$tmp,$Zlo
    116 	dec	$cnt
    117 	js	.Lbreak$N
    118 
    119 	shr	\$4,$Zlo
    120 	and	\$0xf,$rem
    121 	mov	$Zhi,$tmp
    122 	shr	\$4,$Zhi
    123 	xor	8($Htbl,$nlo),$Zlo
    124 	shl	\$60,$tmp
    125 	xor	($Htbl,$nlo),$Zhi
    126 	and	\$0xf0,`&LB("$nhi")`
    127 	xor	($rem_4bit,$rem,8),$Zhi
    128 	mov	$Zlo,$rem
    129 	xor	$tmp,$Zlo
    130 	jmp	.Loop$N
    131 
    132 .align	16
    133 .Lbreak$N:
    134 	shr	\$4,$Zlo
    135 	and	\$0xf,$rem
    136 	mov	$Zhi,$tmp
    137 	shr	\$4,$Zhi
    138 	xor	8($Htbl,$nlo),$Zlo
    139 	shl	\$60,$tmp
    140 	xor	($Htbl,$nlo),$Zhi
    141 	and	\$0xf0,`&LB("$nhi")`
    142 	xor	($rem_4bit,$rem,8),$Zhi
    143 	mov	$Zlo,$rem
    144 	xor	$tmp,$Zlo
    145 
    146 	shr	\$4,$Zlo
    147 	and	\$0xf,$rem
    148 	mov	$Zhi,$tmp
    149 	shr	\$4,$Zhi
    150 	xor	8($Htbl,$nhi),$Zlo
    151 	shl	\$60,$tmp
    152 	xor	($Htbl,$nhi),$Zhi
    153 	xor	$tmp,$Zlo
    154 	xor	($rem_4bit,$rem,8),$Zhi
    155 
    156 	bswap	$Zlo
    157 	bswap	$Zhi
    158 ___
    159 }}
    160 
    161 $code=<<___;
    162 .text
    163 
    164 .globl	gcm_gmult_4bit
    165 .type	gcm_gmult_4bit,\@function,2
    166 .align	16
    167 gcm_gmult_4bit:
    168 	push	%rbx
    169 	push	%rbp		# %rbp and %r12 are pushed exclusively in
    170 	push	%r12		# order to reuse Win64 exception handler...
    171 .Lgmult_prologue:
    172 
    173 	movzb	15($Xi),$Zlo
    174 	lea	.Lrem_4bit(%rip),$rem_4bit
    175 ___
    176 	&loop	($Xi);
    177 $code.=<<___;
    178 	mov	$Zlo,8($Xi)
    179 	mov	$Zhi,($Xi)
    180 
    181 	mov	16(%rsp),%rbx
    182 	lea	24(%rsp),%rsp
    183 .Lgmult_epilogue:
    184 	ret
    185 .size	gcm_gmult_4bit,.-gcm_gmult_4bit
    186 ___
    187 
    189 # per-function register layout
    190 $inp="%rdx";
    191 $len="%rcx";
    192 $rem_8bit=$rem_4bit;
    193 
    194 $code.=<<___;
    195 .globl	gcm_ghash_4bit
    196 .type	gcm_ghash_4bit,\@function,4
    197 .align	16
    198 gcm_ghash_4bit:
    199 	push	%rbx
    200 	push	%rbp
    201 	push	%r12
    202 	push	%r13
    203 	push	%r14
    204 	push	%r15
    205 	sub	\$280,%rsp
    206 .Lghash_prologue:
    207 	mov	$inp,%r14		# reassign couple of args
    208 	mov	$len,%r15
    209 ___
    210 { my $inp="%r14";
    211   my $dat="%edx";
    212   my $len="%r15";
    213   my @nhi=("%ebx","%ecx");
    214   my @rem=("%r12","%r13");
    215   my $Hshr4="%rbp";
    216 
    217 	&sub	($Htbl,-128);		# size optimization
    218 	&lea	($Hshr4,"16+128(%rsp)");
    219 	{ my @lo =($nlo,$nhi);
    220           my @hi =($Zlo,$Zhi);
    221 
    222 	  &xor	($dat,$dat);
    223 	  for ($i=0,$j=-2;$i<18;$i++,$j++) {
    224 	    &mov	("$j(%rsp)",&LB($dat))		if ($i>1);
    225 	    &or		($lo[0],$tmp)			if ($i>1);
    226 	    &mov	(&LB($dat),&LB($lo[1]))		if ($i>0 && $i<17);
    227 	    &shr	($lo[1],4)			if ($i>0 && $i<17);
    228 	    &mov	($tmp,$hi[1])			if ($i>0 && $i<17);
    229 	    &shr	($hi[1],4)			if ($i>0 && $i<17);
    230 	    &mov	("8*$j($Hshr4)",$hi[0])		if ($i>1);
    231 	    &mov	($hi[0],"16*$i+0-128($Htbl)")	if ($i<16);
    232 	    &shl	(&LB($dat),4)			if ($i>0 && $i<17);
    233 	    &mov	("8*$j-128($Hshr4)",$lo[0])	if ($i>1);
    234 	    &mov	($lo[0],"16*$i+8-128($Htbl)")	if ($i<16);
    235 	    &shl	($tmp,60)			if ($i>0 && $i<17);
    236 
    237 	    push	(@lo,shift(@lo));
    238 	    push	(@hi,shift(@hi));
    239 	  }
    240 	}
    241 	&add	($Htbl,-128);
    242 	&mov	($Zlo,"8($Xi)");
    243 	&mov	($Zhi,"0($Xi)");
    244 	&add	($len,$inp);		# pointer to the end of data
    245 	&lea	($rem_8bit,".Lrem_8bit(%rip)");
    246 	&jmp	(".Louter_loop");
    247 
    248 $code.=".align	16\n.Louter_loop:\n";
    249 	&xor	($Zhi,"($inp)");
    250 	&mov	("%rdx","8($inp)");
    251 	&lea	($inp,"16($inp)");
    252 	&xor	("%rdx",$Zlo);
    253 	&mov	("($Xi)",$Zhi);
    254 	&mov	("8($Xi)","%rdx");
    255 	&shr	("%rdx",32);
    256 
    257 	&xor	($nlo,$nlo);
    258 	&rol	($dat,8);
    259 	&mov	(&LB($nlo),&LB($dat));
    260 	&movz	($nhi[0],&LB($dat));
    261 	&shl	(&LB($nlo),4);
    262 	&shr	($nhi[0],4);
    263 
    264 	for ($j=11,$i=0;$i<15;$i++) {
    265 	    &rol	($dat,8);
    266 	    &xor	($Zlo,"8($Htbl,$nlo)")			if ($i>0);
    267 	    &xor	($Zhi,"($Htbl,$nlo)")			if ($i>0);
    268 	    &mov	($Zlo,"8($Htbl,$nlo)")			if ($i==0);
    269 	    &mov	($Zhi,"($Htbl,$nlo)")			if ($i==0);
    270 
    271 	    &mov	(&LB($nlo),&LB($dat));
    272 	    &xor	($Zlo,$tmp)				if ($i>0);
    273 	    &movzw	($rem[1],"($rem_8bit,$rem[1],2)")	if ($i>0);
    274 
    275 	    &movz	($nhi[1],&LB($dat));
    276 	    &shl	(&LB($nlo),4);
    277 	    &movzb	($rem[0],"(%rsp,$nhi[0])");
    278 
    279 	    &shr	($nhi[1],4)				if ($i<14);
    280 	    &and	($nhi[1],0xf0)				if ($i==14);
    281 	    &shl	($rem[1],48)				if ($i>0);
    282 	    &xor	($rem[0],$Zlo);
    283 
    284 	    &mov	($tmp,$Zhi);
    285 	    &xor	($Zhi,$rem[1])				if ($i>0);
    286 	    &shr	($Zlo,8);
    287 
    288 	    &movz	($rem[0],&LB($rem[0]));
    289 	    &mov	($dat,"$j($Xi)")			if (--$j%4==0);
    290 	    &shr	($Zhi,8);
    291 
    292 	    &xor	($Zlo,"-128($Hshr4,$nhi[0],8)");
    293 	    &shl	($tmp,56);
    294 	    &xor	($Zhi,"($Hshr4,$nhi[0],8)");
    295 
    296 	    unshift	(@nhi,pop(@nhi));		# "rotate" registers
    297 	    unshift	(@rem,pop(@rem));
    298 	}
    299 	&movzw	($rem[1],"($rem_8bit,$rem[1],2)");
    300 	&xor	($Zlo,"8($Htbl,$nlo)");
    301 	&xor	($Zhi,"($Htbl,$nlo)");
    302 
    303 	&shl	($rem[1],48);
    304 	&xor	($Zlo,$tmp);
    305 
    306 	&xor	($Zhi,$rem[1]);
    307 	&movz	($rem[0],&LB($Zlo));
    308 	&shr	($Zlo,4);
    309 
    310 	&mov	($tmp,$Zhi);
    311 	&shl	(&LB($rem[0]),4);
    312 	&shr	($Zhi,4);
    313 
    314 	&xor	($Zlo,"8($Htbl,$nhi[0])");
    315 	&movzw	($rem[0],"($rem_8bit,$rem[0],2)");
    316 	&shl	($tmp,60);
    317 
    318 	&xor	($Zhi,"($Htbl,$nhi[0])");
    319 	&xor	($Zlo,$tmp);
    320 	&shl	($rem[0],48);
    321 
    322 	&bswap	($Zlo);
    323 	&xor	($Zhi,$rem[0]);
    324 
    325 	&bswap	($Zhi);
    326 	&cmp	($inp,$len);
    327 	&jb	(".Louter_loop");
    328 }
    329 $code.=<<___;
    330 	mov	$Zlo,8($Xi)
    331 	mov	$Zhi,($Xi)
    332 
    333 	lea	280(%rsp),%rsi
    334 	mov	0(%rsi),%r15
    335 	mov	8(%rsi),%r14
    336 	mov	16(%rsi),%r13
    337 	mov	24(%rsi),%r12
    338 	mov	32(%rsi),%rbp
    339 	mov	40(%rsi),%rbx
    340 	lea	48(%rsi),%rsp
    341 .Lghash_epilogue:
    342 	ret
    343 .size	gcm_ghash_4bit,.-gcm_ghash_4bit
    344 ___
    345 
    347 ######################################################################
    348 # PCLMULQDQ version.
    349 
    350 @_4args=$win64?	("%rcx","%rdx","%r8", "%r9") :	# Win64 order
    351 		("%rdi","%rsi","%rdx","%rcx");	# Unix order
    352 
    353 ($Xi,$Xhi)=("%xmm0","%xmm1");	$Hkey="%xmm2";
    354 ($T1,$T2,$T3)=("%xmm3","%xmm4","%xmm5");
    355 
    356 sub clmul64x64_T2 {	# minimal register pressure
    357 my ($Xhi,$Xi,$Hkey,$modulo)=@_;
    358 
    359 $code.=<<___ if (!defined($modulo));
    360 	movdqa		$Xi,$Xhi		#
    361 	pshufd		\$0b01001110,$Xi,$T1
    362 	pshufd		\$0b01001110,$Hkey,$T2
    363 	pxor		$Xi,$T1			#
    364 	pxor		$Hkey,$T2
    365 ___
    366 $code.=<<___;
    367 	pclmulqdq	\$0x00,$Hkey,$Xi	#######
    368 	pclmulqdq	\$0x11,$Hkey,$Xhi	#######
    369 	pclmulqdq	\$0x00,$T2,$T1		#######
    370 	pxor		$Xi,$T1			#
    371 	pxor		$Xhi,$T1		#
    372 
    373 	movdqa		$T1,$T2			#
    374 	psrldq		\$8,$T1
    375 	pslldq		\$8,$T2			#
    376 	pxor		$T1,$Xhi
    377 	pxor		$T2,$Xi			#
    378 ___
    379 }
    380 
    381 sub reduction_alg9 {	# 17/13 times faster than Intel version
    382 my ($Xhi,$Xi) = @_;
    383 
    384 $code.=<<___;
    385 	# 1st phase
    386 	movdqa		$Xi,$T1			#
    387 	psllq		\$1,$Xi
    388 	pxor		$T1,$Xi			#
    389 	psllq		\$5,$Xi			#
    390 	pxor		$T1,$Xi			#
    391 	psllq		\$57,$Xi		#
    392 	movdqa		$Xi,$T2			#
    393 	pslldq		\$8,$Xi
    394 	psrldq		\$8,$T2			#	
    395 	pxor		$T1,$Xi
    396 	pxor		$T2,$Xhi		#
    397 
    398 	# 2nd phase
    399 	movdqa		$Xi,$T2
    400 	psrlq		\$5,$Xi
    401 	pxor		$T2,$Xi			#
    402 	psrlq		\$1,$Xi			#
    403 	pxor		$T2,$Xi			#
    404 	pxor		$Xhi,$T2
    405 	psrlq		\$1,$Xi			#
    406 	pxor		$T2,$Xi			#
    407 ___
    408 }
    409 
    411 { my ($Htbl,$Xip)=@_4args;
    412 
    413 $code.=<<___;
    414 .globl	gcm_init_clmul
    415 .type	gcm_init_clmul,\@abi-omnipotent
    416 .align	16
    417 gcm_init_clmul:
    418 	movdqu		($Xip),$Hkey
    419 	pshufd		\$0b01001110,$Hkey,$Hkey	# dword swap
    420 
    421 	# <<1 twist
    422 	pshufd		\$0b11111111,$Hkey,$T2	# broadcast uppermost dword
    423 	movdqa		$Hkey,$T1
    424 	psllq		\$1,$Hkey
    425 	pxor		$T3,$T3			#
    426 	psrlq		\$63,$T1
    427 	pcmpgtd		$T2,$T3			# broadcast carry bit
    428 	pslldq		\$8,$T1
    429 	por		$T1,$Hkey		# H<<=1
    430 
    431 	# magic reduction
    432 	pand		.L0x1c2_polynomial(%rip),$T3
    433 	pxor		$T3,$Hkey		# if(carry) H^=0x1c2_polynomial
    434 
    435 	# calculate H^2
    436 	movdqa		$Hkey,$Xi
    437 ___
    438 	&clmul64x64_T2	($Xhi,$Xi,$Hkey);
    439 	&reduction_alg9	($Xhi,$Xi);
    440 $code.=<<___;
    441 	movdqu		$Hkey,($Htbl)		# save H
    442 	movdqu		$Xi,16($Htbl)		# save H^2
    443 	ret
    444 .size	gcm_init_clmul,.-gcm_init_clmul
    445 ___
    446 }
    447 
    448 { my ($Xip,$Htbl)=@_4args;
    449 
    450 $code.=<<___;
    451 .globl	gcm_gmult_clmul
    452 .type	gcm_gmult_clmul,\@abi-omnipotent
    453 .align	16
    454 gcm_gmult_clmul:
    455 	movdqu		($Xip),$Xi
    456 	movdqa		.Lbswap_mask(%rip),$T3
    457 	movdqu		($Htbl),$Hkey
    458 	pshufb		$T3,$Xi
    459 ___
    460 	&clmul64x64_T2	($Xhi,$Xi,$Hkey);
    461 	&reduction_alg9	($Xhi,$Xi);
    462 $code.=<<___;
    463 	pshufb		$T3,$Xi
    464 	movdqu		$Xi,($Xip)
    465 	ret
    466 .size	gcm_gmult_clmul,.-gcm_gmult_clmul
    467 ___
    468 }
    469 
    471 { my ($Xip,$Htbl,$inp,$len)=@_4args;
    472   my $Xn="%xmm6";
    473   my $Xhn="%xmm7";
    474   my $Hkey2="%xmm8";
    475   my $T1n="%xmm9";
    476   my $T2n="%xmm10";
    477 
    478 $code.=<<___;
    479 .globl	gcm_ghash_clmul
    480 .type	gcm_ghash_clmul,\@abi-omnipotent
    481 .align	16
    482 gcm_ghash_clmul:
    483 ___
    484 $code.=<<___ if ($win64);
    485 .LSEH_begin_gcm_ghash_clmul:
    486 	# I can't trust assembler to use specific encoding:-(
    487 	.byte	0x48,0x83,0xec,0x58		#sub	\$0x58,%rsp
    488 	.byte	0x0f,0x29,0x34,0x24		#movaps	%xmm6,(%rsp)
    489 	.byte	0x0f,0x29,0x7c,0x24,0x10	#movdqa	%xmm7,0x10(%rsp)
    490 	.byte	0x44,0x0f,0x29,0x44,0x24,0x20	#movaps	%xmm8,0x20(%rsp)
    491 	.byte	0x44,0x0f,0x29,0x4c,0x24,0x30	#movaps	%xmm9,0x30(%rsp)
    492 	.byte	0x44,0x0f,0x29,0x54,0x24,0x40	#movaps	%xmm10,0x40(%rsp)
    493 ___
    494 $code.=<<___;
    495 	movdqa		.Lbswap_mask(%rip),$T3
    496 
    497 	movdqu		($Xip),$Xi
    498 	movdqu		($Htbl),$Hkey
    499 	pshufb		$T3,$Xi
    500 
    501 	sub		\$0x10,$len
    502 	jz		.Lodd_tail
    503 
    504 	movdqu		16($Htbl),$Hkey2
    505 	#######
    506 	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
    507 	#	[(H*Ii+1) + (H*Xi+1)] mod P =
    508 	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
    509 	#
    510 	movdqu		($inp),$T1		# Ii
    511 	movdqu		16($inp),$Xn		# Ii+1
    512 	pshufb		$T3,$T1
    513 	pshufb		$T3,$Xn
    514 	pxor		$T1,$Xi			# Ii+Xi
    515 ___
    516 	&clmul64x64_T2	($Xhn,$Xn,$Hkey);	# H*Ii+1
    517 $code.=<<___;
    518 	movdqa		$Xi,$Xhi		#
    519 	pshufd		\$0b01001110,$Xi,$T1
    520 	pshufd		\$0b01001110,$Hkey2,$T2
    521 	pxor		$Xi,$T1			#
    522 	pxor		$Hkey2,$T2
    523 
    524 	lea		32($inp),$inp		# i+=2
    525 	sub		\$0x20,$len
    526 	jbe		.Leven_tail
    527 
    528 .Lmod_loop:
    529 ___
    530 	&clmul64x64_T2	($Xhi,$Xi,$Hkey2,1);	# H^2*(Ii+Xi)
    531 $code.=<<___;
    532 	movdqu		($inp),$T1		# Ii
    533 	pxor		$Xn,$Xi			# (H*Ii+1) + H^2*(Ii+Xi)
    534 	pxor		$Xhn,$Xhi
    535 
    536 	movdqu		16($inp),$Xn		# Ii+1
    537 	pshufb		$T3,$T1
    538 	pshufb		$T3,$Xn
    539 
    540 	movdqa		$Xn,$Xhn		#
    541 	pshufd		\$0b01001110,$Xn,$T1n
    542 	pshufd		\$0b01001110,$Hkey,$T2n
    543 	pxor		$Xn,$T1n		#
    544 	pxor		$Hkey,$T2n
    545 	 pxor		$T1,$Xhi		# "Ii+Xi", consume early
    546 
    547 	  movdqa	$Xi,$T1			# 1st phase
    548 	  psllq		\$1,$Xi
    549 	  pxor		$T1,$Xi			#
    550 	  psllq		\$5,$Xi			#
    551 	  pxor		$T1,$Xi			#
    552 	pclmulqdq	\$0x00,$Hkey,$Xn	#######
    553 	  psllq		\$57,$Xi		#
    554 	  movdqa	$Xi,$T2			#
    555 	  pslldq	\$8,$Xi
    556 	  psrldq	\$8,$T2			#	
    557 	  pxor		$T1,$Xi
    558 	  pxor		$T2,$Xhi		#
    559 
    560 	pclmulqdq	\$0x11,$Hkey,$Xhn	#######
    561 	  movdqa	$Xi,$T2			# 2nd phase
    562 	  psrlq		\$5,$Xi
    563 	  pxor		$T2,$Xi			#
    564 	  psrlq		\$1,$Xi			#
    565 	  pxor		$T2,$Xi			#
    566 	  pxor		$Xhi,$T2
    567 	  psrlq		\$1,$Xi			#
    568 	  pxor		$T2,$Xi			#
    569 
    570 	pclmulqdq	\$0x00,$T2n,$T1n	#######
    571 	 movdqa		$Xi,$Xhi		#
    572 	 pshufd		\$0b01001110,$Xi,$T1
    573 	 pshufd		\$0b01001110,$Hkey2,$T2
    574 	 pxor		$Xi,$T1			#
    575 	 pxor		$Hkey2,$T2
    576 
    577 	pxor		$Xn,$T1n		#
    578 	pxor		$Xhn,$T1n		#
    579 	movdqa		$T1n,$T2n		#
    580 	psrldq		\$8,$T1n
    581 	pslldq		\$8,$T2n		#
    582 	pxor		$T1n,$Xhn
    583 	pxor		$T2n,$Xn		#
    584 
    585 	lea		32($inp),$inp
    586 	sub		\$0x20,$len
    587 	ja		.Lmod_loop
    588 
    589 .Leven_tail:
    590 ___
    591 	&clmul64x64_T2	($Xhi,$Xi,$Hkey2,1);	# H^2*(Ii+Xi)
    592 $code.=<<___;
    593 	pxor		$Xn,$Xi			# (H*Ii+1) + H^2*(Ii+Xi)
    594 	pxor		$Xhn,$Xhi
    595 ___
    596 	&reduction_alg9	($Xhi,$Xi);
    597 $code.=<<___;
    598 	test		$len,$len
    599 	jnz		.Ldone
    600 
    601 .Lodd_tail:
    602 	movdqu		($inp),$T1		# Ii
    603 	pshufb		$T3,$T1
    604 	pxor		$T1,$Xi			# Ii+Xi
    605 ___
    606 	&clmul64x64_T2	($Xhi,$Xi,$Hkey);	# H*(Ii+Xi)
    607 	&reduction_alg9	($Xhi,$Xi);
    608 $code.=<<___;
    609 .Ldone:
    610 	pshufb		$T3,$Xi
    611 	movdqu		$Xi,($Xip)
    612 ___
    613 $code.=<<___ if ($win64);
    614 	movaps	(%rsp),%xmm6
    615 	movaps	0x10(%rsp),%xmm7
    616 	movaps	0x20(%rsp),%xmm8
    617 	movaps	0x30(%rsp),%xmm9
    618 	movaps	0x40(%rsp),%xmm10
    619 	add	\$0x58,%rsp
    620 ___
    621 $code.=<<___;
    622 	ret
    623 .LSEH_end_gcm_ghash_clmul:
    624 .size	gcm_ghash_clmul,.-gcm_ghash_clmul
    625 ___
    626 }
    627 
    628 $code.=<<___;
    629 .align	64
    630 .Lbswap_mask:
    631 	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
    632 .L0x1c2_polynomial:
    633 	.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
    634 .align	64
    635 .type	.Lrem_4bit,\@object
    636 .Lrem_4bit:
    637 	.long	0,`0x0000<<16`,0,`0x1C20<<16`,0,`0x3840<<16`,0,`0x2460<<16`
    638 	.long	0,`0x7080<<16`,0,`0x6CA0<<16`,0,`0x48C0<<16`,0,`0x54E0<<16`
    639 	.long	0,`0xE100<<16`,0,`0xFD20<<16`,0,`0xD940<<16`,0,`0xC560<<16`
    640 	.long	0,`0x9180<<16`,0,`0x8DA0<<16`,0,`0xA9C0<<16`,0,`0xB5E0<<16`
    641 .type	.Lrem_8bit,\@object
    642 .Lrem_8bit:
    643 	.value	0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E
    644 	.value	0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E
    645 	.value	0x1C20,0x1DE2,0x1FA4,0x1E66,0x1B28,0x1AEA,0x18AC,0x196E
    646 	.value	0x1230,0x13F2,0x11B4,0x1076,0x1538,0x14FA,0x16BC,0x177E
    647 	.value	0x3840,0x3982,0x3BC4,0x3A06,0x3F48,0x3E8A,0x3CCC,0x3D0E
    648 	.value	0x3650,0x3792,0x35D4,0x3416,0x3158,0x309A,0x32DC,0x331E
    649 	.value	0x2460,0x25A2,0x27E4,0x2626,0x2368,0x22AA,0x20EC,0x212E
    650 	.value	0x2A70,0x2BB2,0x29F4,0x2836,0x2D78,0x2CBA,0x2EFC,0x2F3E
    651 	.value	0x7080,0x7142,0x7304,0x72C6,0x7788,0x764A,0x740C,0x75CE
    652 	.value	0x7E90,0x7F52,0x7D14,0x7CD6,0x7998,0x785A,0x7A1C,0x7BDE
    653 	.value	0x6CA0,0x6D62,0x6F24,0x6EE6,0x6BA8,0x6A6A,0x682C,0x69EE
    654 	.value	0x62B0,0x6372,0x6134,0x60F6,0x65B8,0x647A,0x663C,0x67FE
    655 	.value	0x48C0,0x4902,0x4B44,0x4A86,0x4FC8,0x4E0A,0x4C4C,0x4D8E
    656 	.value	0x46D0,0x4712,0x4554,0x4496,0x41D8,0x401A,0x425C,0x439E
    657 	.value	0x54E0,0x5522,0x5764,0x56A6,0x53E8,0x522A,0x506C,0x51AE
    658 	.value	0x5AF0,0x5B32,0x5974,0x58B6,0x5DF8,0x5C3A,0x5E7C,0x5FBE
    659 	.value	0xE100,0xE0C2,0xE284,0xE346,0xE608,0xE7CA,0xE58C,0xE44E
    660 	.value	0xEF10,0xEED2,0xEC94,0xED56,0xE818,0xE9DA,0xEB9C,0xEA5E
    661 	.value	0xFD20,0xFCE2,0xFEA4,0xFF66,0xFA28,0xFBEA,0xF9AC,0xF86E
    662 	.value	0xF330,0xF2F2,0xF0B4,0xF176,0xF438,0xF5FA,0xF7BC,0xF67E
    663 	.value	0xD940,0xD882,0xDAC4,0xDB06,0xDE48,0xDF8A,0xDDCC,0xDC0E
    664 	.value	0xD750,0xD692,0xD4D4,0xD516,0xD058,0xD19A,0xD3DC,0xD21E
    665 	.value	0xC560,0xC4A2,0xC6E4,0xC726,0xC268,0xC3AA,0xC1EC,0xC02E
    666 	.value	0xCB70,0xCAB2,0xC8F4,0xC936,0xCC78,0xCDBA,0xCFFC,0xCE3E
    667 	.value	0x9180,0x9042,0x9204,0x93C6,0x9688,0x974A,0x950C,0x94CE
    668 	.value	0x9F90,0x9E52,0x9C14,0x9DD6,0x9898,0x995A,0x9B1C,0x9ADE
    669 	.value	0x8DA0,0x8C62,0x8E24,0x8FE6,0x8AA8,0x8B6A,0x892C,0x88EE
    670 	.value	0x83B0,0x8272,0x8034,0x81F6,0x84B8,0x857A,0x873C,0x86FE
    671 	.value	0xA9C0,0xA802,0xAA44,0xAB86,0xAEC8,0xAF0A,0xAD4C,0xAC8E
    672 	.value	0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E
    673 	.value	0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE
    674 	.value	0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE
    675 
    676 .asciz	"GHASH for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
    677 .align	64
    678 ___
    679 
    681 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
    682 #		CONTEXT *context,DISPATCHER_CONTEXT *disp)
    683 if ($win64) {
    684 $rec="%rcx";
    685 $frame="%rdx";
    686 $context="%r8";
    687 $disp="%r9";
    688 
    689 $code.=<<___;
    690 .extern	__imp_RtlVirtualUnwind
    691 .type	se_handler,\@abi-omnipotent
    692 .align	16
    693 se_handler:
    694 	push	%rsi
    695 	push	%rdi
    696 	push	%rbx
    697 	push	%rbp
    698 	push	%r12
    699 	push	%r13
    700 	push	%r14
    701 	push	%r15
    702 	pushfq
    703 	sub	\$64,%rsp
    704 
    705 	mov	120($context),%rax	# pull context->Rax
    706 	mov	248($context),%rbx	# pull context->Rip
    707 
    708 	mov	8($disp),%rsi		# disp->ImageBase
    709 	mov	56($disp),%r11		# disp->HandlerData
    710 
    711 	mov	0(%r11),%r10d		# HandlerData[0]
    712 	lea	(%rsi,%r10),%r10	# prologue label
    713 	cmp	%r10,%rbx		# context->Rip<prologue label
    714 	jb	.Lin_prologue
    715 
    716 	mov	152($context),%rax	# pull context->Rsp
    717 
    718 	mov	4(%r11),%r10d		# HandlerData[1]
    719 	lea	(%rsi,%r10),%r10	# epilogue label
    720 	cmp	%r10,%rbx		# context->Rip>=epilogue label
    721 	jae	.Lin_prologue
    722 
    723 	lea	24(%rax),%rax		# adjust "rsp"
    724 
    725 	mov	-8(%rax),%rbx
    726 	mov	-16(%rax),%rbp
    727 	mov	-24(%rax),%r12
    728 	mov	%rbx,144($context)	# restore context->Rbx
    729 	mov	%rbp,160($context)	# restore context->Rbp
    730 	mov	%r12,216($context)	# restore context->R12
    731 
    732 .Lin_prologue:
    733 	mov	8(%rax),%rdi
    734 	mov	16(%rax),%rsi
    735 	mov	%rax,152($context)	# restore context->Rsp
    736 	mov	%rsi,168($context)	# restore context->Rsi
    737 	mov	%rdi,176($context)	# restore context->Rdi
    738 
    739 	mov	40($disp),%rdi		# disp->ContextRecord
    740 	mov	$context,%rsi		# context
    741 	mov	\$`1232/8`,%ecx		# sizeof(CONTEXT)
    742 	.long	0xa548f3fc		# cld; rep movsq
    743 
    744 	mov	$disp,%rsi
    745 	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
    746 	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
    747 	mov	0(%rsi),%r8		# arg3, disp->ControlPc
    748 	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
    749 	mov	40(%rsi),%r10		# disp->ContextRecord
    750 	lea	56(%rsi),%r11		# &disp->HandlerData
    751 	lea	24(%rsi),%r12		# &disp->EstablisherFrame
    752 	mov	%r10,32(%rsp)		# arg5
    753 	mov	%r11,40(%rsp)		# arg6
    754 	mov	%r12,48(%rsp)		# arg7
    755 	mov	%rcx,56(%rsp)		# arg8, (NULL)
    756 	call	*__imp_RtlVirtualUnwind(%rip)
    757 
    758 	mov	\$1,%eax		# ExceptionContinueSearch
    759 	add	\$64,%rsp
    760 	popfq
    761 	pop	%r15
    762 	pop	%r14
    763 	pop	%r13
    764 	pop	%r12
    765 	pop	%rbp
    766 	pop	%rbx
    767 	pop	%rdi
    768 	pop	%rsi
    769 	ret
    770 .size	se_handler,.-se_handler
    771 
    772 .section	.pdata
    773 .align	4
    774 	.rva	.LSEH_begin_gcm_gmult_4bit
    775 	.rva	.LSEH_end_gcm_gmult_4bit
    776 	.rva	.LSEH_info_gcm_gmult_4bit
    777 
    778 	.rva	.LSEH_begin_gcm_ghash_4bit
    779 	.rva	.LSEH_end_gcm_ghash_4bit
    780 	.rva	.LSEH_info_gcm_ghash_4bit
    781 
    782 	.rva	.LSEH_begin_gcm_ghash_clmul
    783 	.rva	.LSEH_end_gcm_ghash_clmul
    784 	.rva	.LSEH_info_gcm_ghash_clmul
    785 
    786 .section	.xdata
    787 .align	8
    788 .LSEH_info_gcm_gmult_4bit:
    789 	.byte	9,0,0,0
    790 	.rva	se_handler
    791 	.rva	.Lgmult_prologue,.Lgmult_epilogue	# HandlerData
    792 .LSEH_info_gcm_ghash_4bit:
    793 	.byte	9,0,0,0
    794 	.rva	se_handler
    795 	.rva	.Lghash_prologue,.Lghash_epilogue	# HandlerData
    796 .LSEH_info_gcm_ghash_clmul:
    797 	.byte	0x01,0x1f,0x0b,0x00
    798 	.byte	0x1f,0xa8,0x04,0x00	#movaps 0x40(rsp),xmm10
    799 	.byte	0x19,0x98,0x03,0x00	#movaps 0x30(rsp),xmm9
    800 	.byte	0x13,0x88,0x02,0x00	#movaps 0x20(rsp),xmm8
    801 	.byte	0x0d,0x78,0x01,0x00	#movaps 0x10(rsp),xmm7
    802 	.byte	0x08,0x68,0x00,0x00	#movaps (rsp),xmm6
    803 	.byte	0x04,0xa2,0x00,0x00	#sub	rsp,0x58
    804 ___
    805 }
    806 
    808 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
    809 
    810 print $code;
    811 
    812 close STDOUT;
    813