Home | History | Annotate | Download | only in asm
      1 #! /usr/bin/env perl
      2 # Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
      3 #
      4 # Licensed under the OpenSSL license (the "License").  You may not use
      5 # this file except in compliance with the License.  You can obtain a copy
      6 # in the file LICENSE in the source distribution or at
      7 # https://www.openssl.org/source/license.html
      8 
      9 #
     10 # ====================================================================
     11 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL
     12 # project. The module is, however, dual licensed under OpenSSL and
     13 # CRYPTOGAMS licenses depending on where you obtain it. For further
     14 # details see http://www.openssl.org/~appro/cryptogams/.
     15 # ====================================================================
     16 #
     17 # April 2010
     18 #
     19 # The module implements "4-bit" GCM GHASH function and underlying
     20 # single multiplication operation in GF(2^128). "4-bit" means that it
     21 # uses 256 bytes per-key table [+32 bytes shared table]. There is no
     22 # experimental performance data available yet. The only approximation
     23 # that can be made at this point is based on code size. Inner loop is
     24 # 32 instructions long and on single-issue core should execute in <40
     25 # cycles. Having verified that gcc 3.4 didn't unroll corresponding
     26 # loop, this assembler loop body was found to be ~3x smaller than
     27 # compiler-generated one...
     28 #
     29 # July 2010
     30 #
     31 # Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
     32 # Cortex A8 core and ~25 cycles per processed byte (which was observed
     33 # to be ~3 times faster than gcc-generated code:-)
     34 #
     35 # February 2011
     36 #
     37 # Profiler-assisted and platform-specific optimization resulted in 7%
     38 # improvement on Cortex A8 core and ~23.5 cycles per byte.
     39 #
     40 # March 2011
     41 #
     42 # Add NEON implementation featuring polynomial multiplication, i.e. no
     43 # lookup tables involved. On Cortex A8 it was measured to process one
     44 # byte in 15 cycles or 55% faster than integer-only code.
     45 #
     46 # April 2014
     47 #
     48 # Switch to multiplication algorithm suggested in paper referred
     49 # below and combine it with reduction algorithm from x86 module.
     50 # Performance improvement over previous version varies from 65% on
     51 # Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
     52 # processes one byte in 8.45 cycles, A9 - in 10.2, A15 - in 7.63,
     53 # Snapdragon S4 - in 9.33.
     54 #
     55 # Cmara, D.; Gouva, C. P. L.; Lpez, J. & Dahab, R.: Fast Software
     56 # Polynomial Multiplication on ARM Processors using the NEON Engine.
     57 #
     58 # http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf
     59 
     60 # ====================================================================
     61 # Note about "528B" variant. In ARM case it makes lesser sense to
     62 # implement it for following reasons:
     63 #
     64 # - performance improvement won't be anywhere near 50%, because 128-
     65 #   bit shift operation is neatly fused with 128-bit xor here, and
     66 #   "538B" variant would eliminate only 4-5 instructions out of 32
     67 #   in the inner loop (meaning that estimated improvement is ~15%);
     68 # - ARM-based systems are often embedded ones and extra memory
     69 #   consumption might be unappreciated (for so little improvement);
     70 #
     71 # Byte order [in]dependence. =========================================
     72 #
     73 # Caller is expected to maintain specific *dword* order in Htable,
     74 # namely with *least* significant dword of 128-bit value at *lower*
     75 # address. This differs completely from C code and has everything to
     76 # do with ldm instruction and order in which dwords are "consumed" by
     77 # algorithm. *Byte* order within these dwords in turn is whatever
     78 # *native* byte order on current platform. See gcm128.c for working
     79 # example...
     80 
     81 $flavour = shift;
     82 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
     83 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
     84 
     85 if ($flavour && $flavour ne "void") {
     86     $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
     87     ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
     88     ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
     89     die "can't locate arm-xlate.pl";
     90 
     91     open STDOUT,"| \"$^X\" $xlate $flavour $output";
     92 } else {
     93     open STDOUT,">$output";
     94 }
     95 
     96 $Xi="r0";	# argument block
     97 $Htbl="r1";
     98 $inp="r2";
     99 $len="r3";
    100 
    101 $Zll="r4";	# variables
    102 $Zlh="r5";
    103 $Zhl="r6";
    104 $Zhh="r7";
    105 $Tll="r8";
    106 $Tlh="r9";
    107 $Thl="r10";
    108 $Thh="r11";
    109 $nlo="r12";
    110 ################# r13 is stack pointer
    111 $nhi="r14";
    112 ################# r15 is program counter
    113 
    114 $rem_4bit=$inp;	# used in gcm_gmult_4bit
    115 $cnt=$len;
    116 
    117 sub Zsmash() {
    118   my $i=12;
    119   my @args=@_;
    120   for ($Zll,$Zlh,$Zhl,$Zhh) {
    121     $code.=<<___;
    122 #if __ARM_ARCH__>=7 && defined(__ARMEL__)
    123 	rev	$_,$_
    124 	str	$_,[$Xi,#$i]
    125 #elif defined(__ARMEB__)
    126 	str	$_,[$Xi,#$i]
    127 #else
    128 	mov	$Tlh,$_,lsr#8
    129 	strb	$_,[$Xi,#$i+3]
    130 	mov	$Thl,$_,lsr#16
    131 	strb	$Tlh,[$Xi,#$i+2]
    132 	mov	$Thh,$_,lsr#24
    133 	strb	$Thl,[$Xi,#$i+1]
    134 	strb	$Thh,[$Xi,#$i]
    135 #endif
    136 ___
    137     $code.="\t".shift(@args)."\n";
    138     $i-=4;
    139   }
    140 }
    141 
    142 $code=<<___;
    143 #include <openssl/arm_arch.h>
    144 
    145 @ Silence ARMv8 deprecated IT instruction warnings. This file is used by both
    146 @ ARMv7 and ARMv8 processors and does not use ARMv8 instructions. (ARMv8 PMULL
    147 @ instructions are in aesv8-armx.pl.)
    148 .arch  armv7-a
    149 
    150 .text
    151 #if defined(__thumb2__) || defined(__clang__)
    152 .syntax	unified
    153 #define ldrplb  ldrbpl
    154 #define ldrneb  ldrbne
    155 #endif
    156 #if defined(__thumb2__)
    157 .thumb
    158 #else
    159 .code	32
    160 #endif
    161 
    162 .type	rem_4bit,%object
    163 .align	5
    164 rem_4bit:
    165 .short	0x0000,0x1C20,0x3840,0x2460
    166 .short	0x7080,0x6CA0,0x48C0,0x54E0
    167 .short	0xE100,0xFD20,0xD940,0xC560
    168 .short	0x9180,0x8DA0,0xA9C0,0xB5E0
    169 .size	rem_4bit,.-rem_4bit
    170 
    171 .type	rem_4bit_get,%function
    172 rem_4bit_get:
    173 #if defined(__thumb2__)
    174 	adr	$rem_4bit,rem_4bit
    175 #else
    176 	sub	$rem_4bit,pc,#8+32	@ &rem_4bit
    177 #endif
    178 	b	.Lrem_4bit_got
    179 	nop
    180 	nop
    181 .size	rem_4bit_get,.-rem_4bit_get
    182 
    183 .global	gcm_ghash_4bit
    184 .type	gcm_ghash_4bit,%function
    185 .align	4
    186 gcm_ghash_4bit:
    187 #if defined(__thumb2__)
    188 	adr	r12,rem_4bit
    189 #else
    190 	sub	r12,pc,#8+48		@ &rem_4bit
    191 #endif
    192 	add	$len,$inp,$len		@ $len to point at the end
    193 	stmdb	sp!,{r3-r11,lr}		@ save $len/end too
    194 
    195 	ldmia	r12,{r4-r11}		@ copy rem_4bit ...
    196 	stmdb	sp!,{r4-r11}		@ ... to stack
    197 
    198 	ldrb	$nlo,[$inp,#15]
    199 	ldrb	$nhi,[$Xi,#15]
    200 .Louter:
    201 	eor	$nlo,$nlo,$nhi
    202 	and	$nhi,$nlo,#0xf0
    203 	and	$nlo,$nlo,#0x0f
    204 	mov	$cnt,#14
    205 
    206 	add	$Zhh,$Htbl,$nlo,lsl#4
    207 	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
    208 	add	$Thh,$Htbl,$nhi
    209 	ldrb	$nlo,[$inp,#14]
    210 
    211 	and	$nhi,$Zll,#0xf		@ rem
    212 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    213 	add	$nhi,$nhi,$nhi
    214 	eor	$Zll,$Tll,$Zll,lsr#4
    215 	ldrh	$Tll,[sp,$nhi]		@ rem_4bit[rem]
    216 	eor	$Zll,$Zll,$Zlh,lsl#28
    217 	ldrb	$nhi,[$Xi,#14]
    218 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    219 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    220 	eor	$Zhl,$Thl,$Zhl,lsr#4
    221 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    222 	eor	$Zhh,$Thh,$Zhh,lsr#4
    223 	eor	$nlo,$nlo,$nhi
    224 	and	$nhi,$nlo,#0xf0
    225 	and	$nlo,$nlo,#0x0f
    226 	eor	$Zhh,$Zhh,$Tll,lsl#16
    227 
    228 .Linner:
    229 	add	$Thh,$Htbl,$nlo,lsl#4
    230 	and	$nlo,$Zll,#0xf		@ rem
    231 	subs	$cnt,$cnt,#1
    232 	add	$nlo,$nlo,$nlo
    233 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
    234 	eor	$Zll,$Tll,$Zll,lsr#4
    235 	eor	$Zll,$Zll,$Zlh,lsl#28
    236 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    237 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    238 	ldrh	$Tll,[sp,$nlo]		@ rem_4bit[rem]
    239 	eor	$Zhl,$Thl,$Zhl,lsr#4
    240 #ifdef	__thumb2__
    241 	it	pl
    242 #endif
    243 	ldrplb	$nlo,[$inp,$cnt]
    244 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    245 	eor	$Zhh,$Thh,$Zhh,lsr#4
    246 
    247 	add	$Thh,$Htbl,$nhi
    248 	and	$nhi,$Zll,#0xf		@ rem
    249 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    250 	add	$nhi,$nhi,$nhi
    251 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    252 	eor	$Zll,$Tll,$Zll,lsr#4
    253 #ifdef	__thumb2__
    254 	it	pl
    255 #endif
    256 	ldrplb	$Tll,[$Xi,$cnt]
    257 	eor	$Zll,$Zll,$Zlh,lsl#28
    258 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    259 	ldrh	$Tlh,[sp,$nhi]
    260 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    261 	eor	$Zhl,$Thl,$Zhl,lsr#4
    262 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    263 #ifdef	__thumb2__
    264 	it	pl
    265 #endif
    266 	eorpl	$nlo,$nlo,$Tll
    267 	eor	$Zhh,$Thh,$Zhh,lsr#4
    268 #ifdef	__thumb2__
    269 	itt	pl
    270 #endif
    271 	andpl	$nhi,$nlo,#0xf0
    272 	andpl	$nlo,$nlo,#0x0f
    273 	eor	$Zhh,$Zhh,$Tlh,lsl#16	@ ^= rem_4bit[rem]
    274 	bpl	.Linner
    275 
    276 	ldr	$len,[sp,#32]		@ re-load $len/end
    277 	add	$inp,$inp,#16
    278 	mov	$nhi,$Zll
    279 ___
    280 	&Zsmash("cmp\t$inp,$len","\n".
    281 				 "#ifdef __thumb2__\n".
    282 				 "	it	ne\n".
    283 				 "#endif\n".
    284 				 "	ldrneb	$nlo,[$inp,#15]");
    285 $code.=<<___;
    286 	bne	.Louter
    287 
    288 	add	sp,sp,#36
    289 #if __ARM_ARCH__>=5
    290 	ldmia	sp!,{r4-r11,pc}
    291 #else
    292 	ldmia	sp!,{r4-r11,lr}
    293 	tst	lr,#1
    294 	moveq	pc,lr			@ be binary compatible with V4, yet
    295 	bx	lr			@ interoperable with Thumb ISA:-)
    296 #endif
    297 .size	gcm_ghash_4bit,.-gcm_ghash_4bit
    298 
    299 .global	gcm_gmult_4bit
    300 .type	gcm_gmult_4bit,%function
    301 gcm_gmult_4bit:
    302 	stmdb	sp!,{r4-r11,lr}
    303 	ldrb	$nlo,[$Xi,#15]
    304 	b	rem_4bit_get
    305 .Lrem_4bit_got:
    306 	and	$nhi,$nlo,#0xf0
    307 	and	$nlo,$nlo,#0x0f
    308 	mov	$cnt,#14
    309 
    310 	add	$Zhh,$Htbl,$nlo,lsl#4
    311 	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
    312 	ldrb	$nlo,[$Xi,#14]
    313 
    314 	add	$Thh,$Htbl,$nhi
    315 	and	$nhi,$Zll,#0xf		@ rem
    316 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    317 	add	$nhi,$nhi,$nhi
    318 	eor	$Zll,$Tll,$Zll,lsr#4
    319 	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
    320 	eor	$Zll,$Zll,$Zlh,lsl#28
    321 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    322 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    323 	eor	$Zhl,$Thl,$Zhl,lsr#4
    324 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    325 	eor	$Zhh,$Thh,$Zhh,lsr#4
    326 	and	$nhi,$nlo,#0xf0
    327 	eor	$Zhh,$Zhh,$Tll,lsl#16
    328 	and	$nlo,$nlo,#0x0f
    329 
    330 .Loop:
    331 	add	$Thh,$Htbl,$nlo,lsl#4
    332 	and	$nlo,$Zll,#0xf		@ rem
    333 	subs	$cnt,$cnt,#1
    334 	add	$nlo,$nlo,$nlo
    335 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
    336 	eor	$Zll,$Tll,$Zll,lsr#4
    337 	eor	$Zll,$Zll,$Zlh,lsl#28
    338 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    339 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    340 	ldrh	$Tll,[$rem_4bit,$nlo]	@ rem_4bit[rem]
    341 	eor	$Zhl,$Thl,$Zhl,lsr#4
    342 #ifdef	__thumb2__
    343 	it	pl
    344 #endif
    345 	ldrplb	$nlo,[$Xi,$cnt]
    346 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    347 	eor	$Zhh,$Thh,$Zhh,lsr#4
    348 
    349 	add	$Thh,$Htbl,$nhi
    350 	and	$nhi,$Zll,#0xf		@ rem
    351 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    352 	add	$nhi,$nhi,$nhi
    353 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    354 	eor	$Zll,$Tll,$Zll,lsr#4
    355 	eor	$Zll,$Zll,$Zlh,lsl#28
    356 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    357 	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
    358 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    359 	eor	$Zhl,$Thl,$Zhl,lsr#4
    360 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    361 	eor	$Zhh,$Thh,$Zhh,lsr#4
    362 #ifdef	__thumb2__
    363 	itt	pl
    364 #endif
    365 	andpl	$nhi,$nlo,#0xf0
    366 	andpl	$nlo,$nlo,#0x0f
    367 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    368 	bpl	.Loop
    369 ___
    370 	&Zsmash();
    371 $code.=<<___;
    372 #if __ARM_ARCH__>=5
    373 	ldmia	sp!,{r4-r11,pc}
    374 #else
    375 	ldmia	sp!,{r4-r11,lr}
    376 	tst	lr,#1
    377 	moveq	pc,lr			@ be binary compatible with V4, yet
    378 	bx	lr			@ interoperable with Thumb ISA:-)
    379 #endif
    380 .size	gcm_gmult_4bit,.-gcm_gmult_4bit
    381 ___
    382 {
    383 my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
    384 my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
    385 my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));
    386 
    387 sub clmul64x64 {
    388 my ($r,$a,$b)=@_;
    389 $code.=<<___;
    390 	vext.8		$t0#lo, $a, $a, #1	@ A1
    391 	vmull.p8	$t0, $t0#lo, $b		@ F = A1*B
    392 	vext.8		$r#lo, $b, $b, #1	@ B1
    393 	vmull.p8	$r, $a, $r#lo		@ E = A*B1
    394 	vext.8		$t1#lo, $a, $a, #2	@ A2
    395 	vmull.p8	$t1, $t1#lo, $b		@ H = A2*B
    396 	vext.8		$t3#lo, $b, $b, #2	@ B2
    397 	vmull.p8	$t3, $a, $t3#lo		@ G = A*B2
    398 	vext.8		$t2#lo, $a, $a, #3	@ A3
    399 	veor		$t0, $t0, $r		@ L = E + F
    400 	vmull.p8	$t2, $t2#lo, $b		@ J = A3*B
    401 	vext.8		$r#lo, $b, $b, #3	@ B3
    402 	veor		$t1, $t1, $t3		@ M = G + H
    403 	vmull.p8	$r, $a, $r#lo		@ I = A*B3
    404 	veor		$t0#lo, $t0#lo, $t0#hi	@ t0 = (L) (P0 + P1) << 8
    405 	vand		$t0#hi, $t0#hi, $k48
    406 	vext.8		$t3#lo, $b, $b, #4	@ B4
    407 	veor		$t1#lo, $t1#lo, $t1#hi	@ t1 = (M) (P2 + P3) << 16
    408 	vand		$t1#hi, $t1#hi, $k32
    409 	vmull.p8	$t3, $a, $t3#lo		@ K = A*B4
    410 	veor		$t2, $t2, $r		@ N = I + J
    411 	veor		$t0#lo, $t0#lo, $t0#hi
    412 	veor		$t1#lo, $t1#lo, $t1#hi
    413 	veor		$t2#lo, $t2#lo, $t2#hi	@ t2 = (N) (P4 + P5) << 24
    414 	vand		$t2#hi, $t2#hi, $k16
    415 	vext.8		$t0, $t0, $t0, #15
    416 	veor		$t3#lo, $t3#lo, $t3#hi	@ t3 = (K) (P6 + P7) << 32
    417 	vmov.i64	$t3#hi, #0
    418 	vext.8		$t1, $t1, $t1, #14
    419 	veor		$t2#lo, $t2#lo, $t2#hi
    420 	vmull.p8	$r, $a, $b		@ D = A*B
    421 	vext.8		$t3, $t3, $t3, #12
    422 	vext.8		$t2, $t2, $t2, #13
    423 	veor		$t0, $t0, $t1
    424 	veor		$t2, $t2, $t3
    425 	veor		$r, $r, $t0
    426 	veor		$r, $r, $t2
    427 ___
    428 }
    429 
    430 $code.=<<___;
    431 #if __ARM_MAX_ARCH__>=7
    432 .arch	armv7-a
    433 .fpu	neon
    434 
    435 .global	gcm_init_neon
    436 .type	gcm_init_neon,%function
    437 .align	4
    438 gcm_init_neon:
    439 	vld1.64		$IN#hi,[r1]!		@ load H
    440 	vmov.i8		$t0,#0xe1
    441 	vld1.64		$IN#lo,[r1]
    442 	vshl.i64	$t0#hi,#57
    443 	vshr.u64	$t0#lo,#63		@ t0=0xc2....01
    444 	vdup.8		$t1,$IN#hi[7]
    445 	vshr.u64	$Hlo,$IN#lo,#63
    446 	vshr.s8		$t1,#7			@ broadcast carry bit
    447 	vshl.i64	$IN,$IN,#1
    448 	vand		$t0,$t0,$t1
    449 	vorr		$IN#hi,$Hlo		@ H<<<=1
    450 	veor		$IN,$IN,$t0		@ twisted H
    451 	vstmia		r0,{$IN}
    452 
    453 	ret					@ bx lr
    454 .size	gcm_init_neon,.-gcm_init_neon
    455 
    456 .global	gcm_gmult_neon
    457 .type	gcm_gmult_neon,%function
    458 .align	4
    459 gcm_gmult_neon:
    460 	vld1.64		$IN#hi,[$Xi]!		@ load Xi
    461 	vld1.64		$IN#lo,[$Xi]!
    462 	vmov.i64	$k48,#0x0000ffffffffffff
    463 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
    464 	vmov.i64	$k32,#0x00000000ffffffff
    465 #ifdef __ARMEL__
    466 	vrev64.8	$IN,$IN
    467 #endif
    468 	vmov.i64	$k16,#0x000000000000ffff
    469 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
    470 	mov		$len,#16
    471 	b		.Lgmult_neon
    472 .size	gcm_gmult_neon,.-gcm_gmult_neon
    473 
    474 .global	gcm_ghash_neon
    475 .type	gcm_ghash_neon,%function
    476 .align	4
    477 gcm_ghash_neon:
    478 	vld1.64		$Xl#hi,[$Xi]!		@ load Xi
    479 	vld1.64		$Xl#lo,[$Xi]!
    480 	vmov.i64	$k48,#0x0000ffffffffffff
    481 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
    482 	vmov.i64	$k32,#0x00000000ffffffff
    483 #ifdef __ARMEL__
    484 	vrev64.8	$Xl,$Xl
    485 #endif
    486 	vmov.i64	$k16,#0x000000000000ffff
    487 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
    488 
    489 .Loop_neon:
    490 	vld1.64		$IN#hi,[$inp]!		@ load inp
    491 	vld1.64		$IN#lo,[$inp]!
    492 #ifdef __ARMEL__
    493 	vrev64.8	$IN,$IN
    494 #endif
    495 	veor		$IN,$Xl			@ inp^=Xi
    496 .Lgmult_neon:
    497 ___
    498 	&clmul64x64	($Xl,$Hlo,"$IN#lo");	# H.loXi.lo
    499 $code.=<<___;
    500 	veor		$IN#lo,$IN#lo,$IN#hi	@ Karatsuba pre-processing
    501 ___
    502 	&clmul64x64	($Xm,$Hhl,"$IN#lo");	# (H.lo+H.hi)(Xi.lo+Xi.hi)
    503 	&clmul64x64	($Xh,$Hhi,"$IN#hi");	# H.hiXi.hi
    504 $code.=<<___;
    505 	veor		$Xm,$Xm,$Xl		@ Karatsuba post-processing
    506 	veor		$Xm,$Xm,$Xh
    507 	veor		$Xl#hi,$Xl#hi,$Xm#lo
    508 	veor		$Xh#lo,$Xh#lo,$Xm#hi	@ Xh|Xl - 256-bit result
    509 
    510 	@ equivalent of reduction_avx from ghash-x86_64.pl
    511 	vshl.i64	$t1,$Xl,#57		@ 1st phase
    512 	vshl.i64	$t2,$Xl,#62
    513 	veor		$t2,$t2,$t1		@
    514 	vshl.i64	$t1,$Xl,#63
    515 	veor		$t2, $t2, $t1		@
    516  	veor		$Xl#hi,$Xl#hi,$t2#lo	@
    517 	veor		$Xh#lo,$Xh#lo,$t2#hi
    518 
    519 	vshr.u64	$t2,$Xl,#1		@ 2nd phase
    520 	veor		$Xh,$Xh,$Xl
    521 	veor		$Xl,$Xl,$t2		@
    522 	vshr.u64	$t2,$t2,#6
    523 	vshr.u64	$Xl,$Xl,#1		@
    524 	veor		$Xl,$Xl,$Xh		@
    525 	veor		$Xl,$Xl,$t2		@
    526 
    527 	subs		$len,#16
    528 	bne		.Loop_neon
    529 
    530 #ifdef __ARMEL__
    531 	vrev64.8	$Xl,$Xl
    532 #endif
    533 	sub		$Xi,#16
    534 	vst1.64		$Xl#hi,[$Xi]!		@ write out Xi
    535 	vst1.64		$Xl#lo,[$Xi]
    536 
    537 	ret					@ bx lr
    538 .size	gcm_ghash_neon,.-gcm_ghash_neon
    539 #endif
    540 ___
    541 }
    542 $code.=<<___;
    543 .asciz  "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
    544 .align  2
    545 ___
    546 
    547 foreach (split("\n",$code)) {
    548 	s/\`([^\`]*)\`/eval $1/geo;
    549 
    550 	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
    551 	s/\bret\b/bx	lr/go		or
    552 	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;    # make it possible to compile with -march=armv4
    553 
    554 	print $_,"\n";
    555 }
    556 close STDOUT; # enforce flush
    557