Home | History | Annotate | Download | only in asm 
      1 #!/usr/bin/env perl
      2 #
      3 # ====================================================================
      4 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL
      5 # project. The module is, however, dual licensed under OpenSSL and
      6 # CRYPTOGAMS licenses depending on where you obtain it. For further
      7 # details see http://www.openssl.org/~appro/cryptogams/.
      8 # ====================================================================
      9 #
     10 # April 2010
     11 #
     12 # The module implements "4-bit" GCM GHASH function and underlying
     13 # single multiplication operation in GF(2^128). "4-bit" means that it
     14 # uses 256 bytes per-key table [+32 bytes shared table]. There is no
     15 # experimental performance data available yet. The only approximation
     16 # that can be made at this point is based on code size. Inner loop is
     17 # 32 instructions long and on single-issue core should execute in <40
     18 # cycles. Having verified that gcc 3.4 didn't unroll corresponding
     19 # loop, this assembler loop body was found to be ~3x smaller than
     20 # compiler-generated one...
     21 #
     22 # July 2010
     23 #
     24 # Rescheduling for dual-issue pipeline resulted in 8.5% improvement on
     25 # Cortex A8 core and ~25 cycles per processed byte (which was observed
     26 # to be ~3 times faster than gcc-generated code:-)
     27 #
     28 # February 2011
     29 #
     30 # Profiler-assisted and platform-specific optimization resulted in 7%
     31 # improvement on Cortex A8 core and ~23.5 cycles per byte.
     32 #
     33 # March 2011
     34 #
     35 # Add NEON implementation featuring polynomial multiplication, i.e. no
     36 # lookup tables involved. On Cortex A8 it was measured to process one
     37 # byte in 15 cycles or 55% faster than integer-only code.
     38 #
     39 # April 2014
     40 #
     41 # Switch to multiplication algorithm suggested in paper referred
     42 # below and combine it with reduction algorithm from x86 module.
     43 # Performance improvement over previous version varies from 65% on
     44 # Snapdragon S4 to 110% on Cortex A9. In absolute terms Cortex A8
     45 # processes one byte in 8.45 cycles, A9 - in 10.2, Snapdragon S4 -
     46 # in 9.33.
     47 #
     48 # Cmara, D.; Gouva, C. P. L.; Lpez, J. & Dahab, R.: Fast Software
     49 # Polynomial Multiplication on ARM Processors using the NEON Engine.
     50 # 
     51 # http://conradoplg.cryptoland.net/files/2010/12/mocrysen13.pdf
     52 
     53 # ====================================================================
     54 # Note about "528B" variant. In ARM case it makes lesser sense to
     55 # implement it for following reasons:
     56 #
     57 # - performance improvement won't be anywhere near 50%, because 128-
     58 #   bit shift operation is neatly fused with 128-bit xor here, and
     59 #   "538B" variant would eliminate only 4-5 instructions out of 32
     60 #   in the inner loop (meaning that estimated improvement is ~15%);
     61 # - ARM-based systems are often embedded ones and extra memory
     62 #   consumption might be unappreciated (for so little improvement);
     63 #
     64 # Byte order [in]dependence. =========================================
     65 #
     66 # Caller is expected to maintain specific *dword* order in Htable,
     67 # namely with *least* significant dword of 128-bit value at *lower*
     68 # address. This differs completely from C code and has everything to
     69 # do with ldm instruction and order in which dwords are "consumed" by
     70 # algorithm. *Byte* order within these dwords in turn is whatever
     71 # *native* byte order on current platform. See gcm128.c for working
     72 # example...
     73 
     74 while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
     75 open STDOUT,">$output";
     76 
     77 $Xi="r0";	# argument block
     78 $Htbl="r1";
     79 $inp="r2";
     80 $len="r3";
     81 
     82 $Zll="r4";	# variables
     83 $Zlh="r5";
     84 $Zhl="r6";
     85 $Zhh="r7";
     86 $Tll="r8";
     87 $Tlh="r9";
     88 $Thl="r10";
     89 $Thh="r11";
     90 $nlo="r12";
     91 ################# r13 is stack pointer
     92 $nhi="r14";
     93 ################# r15 is program counter
     94 
     95 $rem_4bit=$inp;	# used in gcm_gmult_4bit
     96 $cnt=$len;
     97 
     98 sub Zsmash() {
     99   my $i=12;
    100   my @args=@_;
    101   for ($Zll,$Zlh,$Zhl,$Zhh) {
    102     $code.=<<___;
    103 #if __ARM_ARCH__>=7 && defined(__ARMEL__)
    104 	rev	$_,$_
    105 	str	$_,[$Xi,#$i]
    106 #elif defined(__ARMEB__)
    107 	str	$_,[$Xi,#$i]
    108 #else
    109 	mov	$Tlh,$_,lsr#8
    110 	strb	$_,[$Xi,#$i+3]
    111 	mov	$Thl,$_,lsr#16
    112 	strb	$Tlh,[$Xi,#$i+2]
    113 	mov	$Thh,$_,lsr#24
    114 	strb	$Thl,[$Xi,#$i+1]
    115 	strb	$Thh,[$Xi,#$i]
    116 #endif
    117 ___
    118     $code.="\t".shift(@args)."\n";
    119     $i-=4;
    120   }
    121 }
    122 
    123 $code=<<___;
    124 #include "arm_arch.h"
    125 
    126 .text
    127 .code	32
    128 
    129 .type	rem_4bit,%object
    130 .align	5
    131 rem_4bit:
    132 .short	0x0000,0x1C20,0x3840,0x2460
    133 .short	0x7080,0x6CA0,0x48C0,0x54E0
    134 .short	0xE100,0xFD20,0xD940,0xC560
    135 .short	0x9180,0x8DA0,0xA9C0,0xB5E0
    136 .size	rem_4bit,.-rem_4bit
    137 
    138 .type	rem_4bit_get,%function
    139 rem_4bit_get:
    140 	sub	$rem_4bit,pc,#8
    141 	sub	$rem_4bit,$rem_4bit,#32	@ &rem_4bit
    142 	b	.Lrem_4bit_got
    143 	nop
    144 .size	rem_4bit_get,.-rem_4bit_get
    145 
    146 .global	gcm_ghash_4bit
    147 .type	gcm_ghash_4bit,%function
    148 gcm_ghash_4bit:
    149 	sub	r12,pc,#8
    150 	add	$len,$inp,$len		@ $len to point at the end
    151 	stmdb	sp!,{r3-r11,lr}		@ save $len/end too
    152 	sub	r12,r12,#48		@ &rem_4bit
    153 
    154 	ldmia	r12,{r4-r11}		@ copy rem_4bit ...
    155 	stmdb	sp!,{r4-r11}		@ ... to stack
    156 
    157 	ldrb	$nlo,[$inp,#15]
    158 	ldrb	$nhi,[$Xi,#15]
    159 .Louter:
    160 	eor	$nlo,$nlo,$nhi
    161 	and	$nhi,$nlo,#0xf0
    162 	and	$nlo,$nlo,#0x0f
    163 	mov	$cnt,#14
    164 
    165 	add	$Zhh,$Htbl,$nlo,lsl#4
    166 	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
    167 	add	$Thh,$Htbl,$nhi
    168 	ldrb	$nlo,[$inp,#14]
    169 
    170 	and	$nhi,$Zll,#0xf		@ rem
    171 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    172 	add	$nhi,$nhi,$nhi
    173 	eor	$Zll,$Tll,$Zll,lsr#4
    174 	ldrh	$Tll,[sp,$nhi]		@ rem_4bit[rem]
    175 	eor	$Zll,$Zll,$Zlh,lsl#28
    176 	ldrb	$nhi,[$Xi,#14]
    177 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    178 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    179 	eor	$Zhl,$Thl,$Zhl,lsr#4
    180 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    181 	eor	$Zhh,$Thh,$Zhh,lsr#4
    182 	eor	$nlo,$nlo,$nhi
    183 	and	$nhi,$nlo,#0xf0
    184 	and	$nlo,$nlo,#0x0f
    185 	eor	$Zhh,$Zhh,$Tll,lsl#16
    186 
    187 .Linner:
    188 	add	$Thh,$Htbl,$nlo,lsl#4
    189 	and	$nlo,$Zll,#0xf		@ rem
    190 	subs	$cnt,$cnt,#1
    191 	add	$nlo,$nlo,$nlo
    192 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
    193 	eor	$Zll,$Tll,$Zll,lsr#4
    194 	eor	$Zll,$Zll,$Zlh,lsl#28
    195 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    196 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    197 	ldrh	$Tll,[sp,$nlo]		@ rem_4bit[rem]
    198 	eor	$Zhl,$Thl,$Zhl,lsr#4
    199 	ldrplb	$nlo,[$inp,$cnt]
    200 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    201 	eor	$Zhh,$Thh,$Zhh,lsr#4
    202 
    203 	add	$Thh,$Htbl,$nhi
    204 	and	$nhi,$Zll,#0xf		@ rem
    205 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    206 	add	$nhi,$nhi,$nhi
    207 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    208 	eor	$Zll,$Tll,$Zll,lsr#4
    209 	ldrplb	$Tll,[$Xi,$cnt]
    210 	eor	$Zll,$Zll,$Zlh,lsl#28
    211 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    212 	ldrh	$Tlh,[sp,$nhi]
    213 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    214 	eor	$Zhl,$Thl,$Zhl,lsr#4
    215 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    216 	eorpl	$nlo,$nlo,$Tll
    217 	eor	$Zhh,$Thh,$Zhh,lsr#4
    218 	andpl	$nhi,$nlo,#0xf0
    219 	andpl	$nlo,$nlo,#0x0f
    220 	eor	$Zhh,$Zhh,$Tlh,lsl#16	@ ^= rem_4bit[rem]
    221 	bpl	.Linner
    222 
    223 	ldr	$len,[sp,#32]		@ re-load $len/end
    224 	add	$inp,$inp,#16
    225 	mov	$nhi,$Zll
    226 ___
    227 	&Zsmash("cmp\t$inp,$len","ldrneb\t$nlo,[$inp,#15]");
    228 $code.=<<___;
    229 	bne	.Louter
    230 
    231 	add	sp,sp,#36
    232 #if __ARM_ARCH__>=5
    233 	ldmia	sp!,{r4-r11,pc}
    234 #else
    235 	ldmia	sp!,{r4-r11,lr}
    236 	tst	lr,#1
    237 	moveq	pc,lr			@ be binary compatible with V4, yet
    238 	bx	lr			@ interoperable with Thumb ISA:-)
    239 #endif
    240 .size	gcm_ghash_4bit,.-gcm_ghash_4bit
    241 
    242 .global	gcm_gmult_4bit
    243 .type	gcm_gmult_4bit,%function
    244 gcm_gmult_4bit:
    245 	stmdb	sp!,{r4-r11,lr}
    246 	ldrb	$nlo,[$Xi,#15]
    247 	b	rem_4bit_get
    248 .Lrem_4bit_got:
    249 	and	$nhi,$nlo,#0xf0
    250 	and	$nlo,$nlo,#0x0f
    251 	mov	$cnt,#14
    252 
    253 	add	$Zhh,$Htbl,$nlo,lsl#4
    254 	ldmia	$Zhh,{$Zll-$Zhh}	@ load Htbl[nlo]
    255 	ldrb	$nlo,[$Xi,#14]
    256 
    257 	add	$Thh,$Htbl,$nhi
    258 	and	$nhi,$Zll,#0xf		@ rem
    259 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    260 	add	$nhi,$nhi,$nhi
    261 	eor	$Zll,$Tll,$Zll,lsr#4
    262 	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
    263 	eor	$Zll,$Zll,$Zlh,lsl#28
    264 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    265 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    266 	eor	$Zhl,$Thl,$Zhl,lsr#4
    267 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    268 	eor	$Zhh,$Thh,$Zhh,lsr#4
    269 	and	$nhi,$nlo,#0xf0
    270 	eor	$Zhh,$Zhh,$Tll,lsl#16
    271 	and	$nlo,$nlo,#0x0f
    272 
    273 .Loop:
    274 	add	$Thh,$Htbl,$nlo,lsl#4
    275 	and	$nlo,$Zll,#0xf		@ rem
    276 	subs	$cnt,$cnt,#1
    277 	add	$nlo,$nlo,$nlo
    278 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nlo]
    279 	eor	$Zll,$Tll,$Zll,lsr#4
    280 	eor	$Zll,$Zll,$Zlh,lsl#28
    281 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    282 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    283 	ldrh	$Tll,[$rem_4bit,$nlo]	@ rem_4bit[rem]
    284 	eor	$Zhl,$Thl,$Zhl,lsr#4
    285 	ldrplb	$nlo,[$Xi,$cnt]
    286 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    287 	eor	$Zhh,$Thh,$Zhh,lsr#4
    288 
    289 	add	$Thh,$Htbl,$nhi
    290 	and	$nhi,$Zll,#0xf		@ rem
    291 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    292 	add	$nhi,$nhi,$nhi
    293 	ldmia	$Thh,{$Tll-$Thh}	@ load Htbl[nhi]
    294 	eor	$Zll,$Tll,$Zll,lsr#4
    295 	eor	$Zll,$Zll,$Zlh,lsl#28
    296 	eor	$Zlh,$Tlh,$Zlh,lsr#4
    297 	ldrh	$Tll,[$rem_4bit,$nhi]	@ rem_4bit[rem]
    298 	eor	$Zlh,$Zlh,$Zhl,lsl#28
    299 	eor	$Zhl,$Thl,$Zhl,lsr#4
    300 	eor	$Zhl,$Zhl,$Zhh,lsl#28
    301 	eor	$Zhh,$Thh,$Zhh,lsr#4
    302 	andpl	$nhi,$nlo,#0xf0
    303 	andpl	$nlo,$nlo,#0x0f
    304 	eor	$Zhh,$Zhh,$Tll,lsl#16	@ ^= rem_4bit[rem]
    305 	bpl	.Loop
    306 ___
    307 	&Zsmash();
    308 $code.=<<___;
    309 #if __ARM_ARCH__>=5
    310 	ldmia	sp!,{r4-r11,pc}
    311 #else
    312 	ldmia	sp!,{r4-r11,lr}
    313 	tst	lr,#1
    314 	moveq	pc,lr			@ be binary compatible with V4, yet
    315 	bx	lr			@ interoperable with Thumb ISA:-)
    316 #endif
    317 .size	gcm_gmult_4bit,.-gcm_gmult_4bit
    318 ___
    319 {
    320 my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
    321 my ($t0,$t1,$t2,$t3)=map("q$_",(8..12));
    322 my ($Hlo,$Hhi,$Hhl,$k48,$k32,$k16)=map("d$_",(26..31));
    323 
    324 sub clmul64x64 {
    325 my ($r,$a,$b)=@_;
    326 $code.=<<___;
    327 	vext.8		$t0#lo, $a, $a, #1	@ A1
    328 	vmull.p8	$t0, $t0#lo, $b		@ F = A1*B
    329 	vext.8		$r#lo, $b, $b, #1	@ B1
    330 	vmull.p8	$r, $a, $r#lo		@ E = A*B1
    331 	vext.8		$t1#lo, $a, $a, #2	@ A2
    332 	vmull.p8	$t1, $t1#lo, $b		@ H = A2*B
    333 	vext.8		$t3#lo, $b, $b, #2	@ B2
    334 	vmull.p8	$t3, $a, $t3#lo		@ G = A*B2
    335 	vext.8		$t2#lo, $a, $a, #3	@ A3
    336 	veor		$t0, $t0, $r		@ L = E + F
    337 	vmull.p8	$t2, $t2#lo, $b		@ J = A3*B
    338 	vext.8		$r#lo, $b, $b, #3	@ B3
    339 	veor		$t1, $t1, $t3		@ M = G + H
    340 	vmull.p8	$r, $a, $r#lo		@ I = A*B3
    341 	veor		$t0#lo, $t0#lo, $t0#hi	@ t0 = (L) (P0 + P1) << 8
    342 	vand		$t0#hi, $t0#hi, $k48
    343 	vext.8		$t3#lo, $b, $b, #4	@ B4
    344 	veor		$t1#lo, $t1#lo, $t1#hi	@ t1 = (M) (P2 + P3) << 16
    345 	vand		$t1#hi, $t1#hi, $k32
    346 	vmull.p8	$t3, $a, $t3#lo		@ K = A*B4
    347 	veor		$t2, $t2, $r		@ N = I + J
    348 	veor		$t0#lo, $t0#lo, $t0#hi
    349 	veor		$t1#lo, $t1#lo, $t1#hi
    350 	veor		$t2#lo, $t2#lo, $t2#hi	@ t2 = (N) (P4 + P5) << 24
    351 	vand		$t2#hi, $t2#hi, $k16
    352 	vext.8		$t0, $t0, $t0, #15
    353 	veor		$t3#lo, $t3#lo, $t3#hi	@ t3 = (K) (P6 + P7) << 32
    354 	vmov.i64	$t3#hi, #0
    355 	vext.8		$t1, $t1, $t1, #14
    356 	veor		$t2#lo, $t2#lo, $t2#hi
    357 	vmull.p8	$r, $a, $b		@ D = A*B
    358 	vext.8		$t3, $t3, $t3, #12
    359 	vext.8		$t2, $t2, $t2, #13
    360 	veor		$t0, $t0, $t1
    361 	veor		$t2, $t2, $t3
    362 	veor		$r, $r, $t0
    363 	veor		$r, $r, $t2
    364 ___
    365 }
    366 
    367 $code.=<<___;
    368 #if __ARM_ARCH__>=7
    369 .fpu	neon
    370 
    371 .global	gcm_init_neon
    372 .type	gcm_init_neon,%function
    373 .align	4
    374 gcm_init_neon:
    375 	vld1.64		$IN#hi,[r1,:64]!	@ load H
    376 	vmov.i8		$t0,#0xe1
    377 	vld1.64		$IN#lo,[r1,:64]
    378 	vshl.i64	$t0#hi,#57
    379 	vshr.u64	$t0#lo,#63		@ t0=0xc2....01
    380 	vdup.8		$t1,$IN#hi[7]
    381 	vshr.u64	$Hlo,$IN#lo,#63
    382 	vshr.s8		$t1,#7			@ broadcast carry bit
    383 	vshl.i64	$IN,$IN,#1
    384 	vand		$t0,$t0,$t1
    385 	vorr		$IN#hi,$Hlo		@ H<<<=1
    386 	veor		$IN,$IN,$t0		@ twisted H
    387 	vstmia		r0,{$IN}
    388 
    389 	ret					@ bx lr
    390 .size	gcm_init_neon,.-gcm_init_neon
    391 
    392 .global	gcm_gmult_neon
    393 .type	gcm_gmult_neon,%function
    394 .align	4
    395 gcm_gmult_neon:
    396 	vld1.64		$IN#hi,[$Xi,:64]!	@ load Xi
    397 	vld1.64		$IN#lo,[$Xi,:64]!
    398 	vmov.i64	$k48,#0x0000ffffffffffff
    399 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
    400 	vmov.i64	$k32,#0x00000000ffffffff
    401 #ifdef __ARMEL__
    402 	vrev64.8	$IN,$IN
    403 #endif
    404 	vmov.i64	$k16,#0x000000000000ffff
    405 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
    406 	mov		$len,#16
    407 	b		.Lgmult_neon
    408 .size	gcm_gmult_neon,.-gcm_gmult_neon
    409 
    410 .global	gcm_ghash_neon
    411 .type	gcm_ghash_neon,%function
    412 .align	4
    413 gcm_ghash_neon:
    414 	vld1.64		$Xl#hi,[$Xi,:64]!	@ load Xi
    415 	vld1.64		$Xl#lo,[$Xi,:64]!
    416 	vmov.i64	$k48,#0x0000ffffffffffff
    417 	vldmia		$Htbl,{$Hlo-$Hhi}	@ load twisted H
    418 	vmov.i64	$k32,#0x00000000ffffffff
    419 #ifdef __ARMEL__
    420 	vrev64.8	$Xl,$Xl
    421 #endif
    422 	vmov.i64	$k16,#0x000000000000ffff
    423 	veor		$Hhl,$Hlo,$Hhi		@ Karatsuba pre-processing
    424 
    425 .Loop_neon:
    426 	vld1.64		$IN#hi,[$inp]!		@ load inp
    427 	vld1.64		$IN#lo,[$inp]!
    428 #ifdef __ARMEL__
    429 	vrev64.8	$IN,$IN
    430 #endif
    431 	veor		$IN,$Xl			@ inp^=Xi
    432 .Lgmult_neon:
    433 ___
    434 	&clmul64x64	($Xl,$Hlo,"$IN#lo");	# H.loXi.lo
    435 $code.=<<___;
    436 	veor		$IN#lo,$IN#lo,$IN#hi	@ Karatsuba pre-processing
    437 ___
    438 	&clmul64x64	($Xm,$Hhl,"$IN#lo");	# (H.lo+H.hi)(Xi.lo+Xi.hi)
    439 	&clmul64x64	($Xh,$Hhi,"$IN#hi");	# H.hiXi.hi
    440 $code.=<<___;
    441 	veor		$Xm,$Xm,$Xl		@ Karatsuba post-processing
    442 	veor		$Xm,$Xm,$Xh
    443 	veor		$Xl#hi,$Xl#hi,$Xm#lo
    444 	veor		$Xh#lo,$Xh#lo,$Xm#hi	@ Xh|Xl - 256-bit result
    445 
    446 	@ equivalent of reduction_avx from ghash-x86_64.pl
    447 	vshl.i64	$t1,$Xl,#57		@ 1st phase
    448 	vshl.i64	$t2,$Xl,#62
    449 	veor		$t2,$t2,$t1		@
    450 	vshl.i64	$t1,$Xl,#63
    451 	veor		$t2, $t2, $t1		@
    452  	veor		$Xl#hi,$Xl#hi,$t2#lo	@
    453 	veor		$Xh#lo,$Xh#lo,$t2#hi
    454 
    455 	vshr.u64	$t2,$Xl,#1		@ 2nd phase
    456 	veor		$Xh,$Xh,$Xl
    457 	veor		$Xl,$Xl,$t2		@
    458 	vshr.u64	$t2,$t2,#6
    459 	vshr.u64	$Xl,$Xl,#1		@
    460 	veor		$Xl,$Xl,$Xh		@
    461 	veor		$Xl,$Xl,$t2		@
    462 
    463 	subs		$len,#16
    464 	bne		.Loop_neon
    465 
    466 #ifdef __ARMEL__
    467 	vrev64.8	$Xl,$Xl
    468 #endif
    469 	sub		$Xi,#16	
    470 	vst1.64		$Xl#hi,[$Xi,:64]!	@ write out Xi
    471 	vst1.64		$Xl#lo,[$Xi,:64]
    472 
    473 	ret					@ bx lr
    474 .size	gcm_ghash_neon,.-gcm_ghash_neon
    475 #endif
    476 ___
    477 }
    478 $code.=<<___;
    479 .asciz  "GHASH for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
    480 .align  2
    481 ___
    482 
    483 foreach (split("\n",$code)) {
    484 	s/\`([^\`]*)\`/eval $1/geo;
    485 
    486 	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
    487 	s/\bret\b/bx	lr/go		or
    488 	s/\bbx\s+lr\b/.word\t0xe12fff1e/go;    # make it possible to compile with -march=armv4
    489 
    490 	print $_,"\n";
    491 }
    492 close STDOUT; # enforce flush
    493