Home | History | Annotate | Download | only in asm 
      1 #! /usr/bin/env perl
      2 # Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved.
      3 #
      4 # Licensed under the OpenSSL license (the "License").  You may not use
      5 # this file except in compliance with the License.  You can obtain a copy
      6 # in the file LICENSE in the source distribution or at
      7 # https://www.openssl.org/source/license.html
      8 
      9 
     10 ######################################################################
     11 ## Constant-time SSSE3 AES core implementation.
     12 ## version 0.1
     13 ##
     14 ## By Mike Hamburg (Stanford University), 2009
     15 ## Public domain.
     16 ##
     17 ## For details see http://shiftleft.org/papers/vector_aes/ and
     18 ## http://crypto.stanford.edu/vpaes/.
     19 ##
     20 ######################################################################
     21 # ARMv8 NEON adaptation by <appro (at] openssl.org>
     22 #
     23 # Reason for undertaken effort is that there is at least one popular
     24 # SoC based on Cortex-A53 that doesn't have crypto extensions.
     25 #
     26 #                   CBC enc     ECB enc/dec(*)   [bit-sliced enc/dec]
     27 # Cortex-A53        21.5        18.1/20.6        [17.5/19.8         ]
     28 # Cortex-A57        36.0(**)    20.4/24.9(**)    [14.4/16.6         ]
     29 # X-Gene            45.9(**)    45.8/57.7(**)    [33.1/37.6(**)     ]
     30 # Denver(***)       16.6(**)    15.1/17.8(**)    [8.80/9.93         ]
     31 # Apple A7(***)     22.7(**)    10.9/14.3        [8.45/10.0         ]
     32 # Mongoose(***)     26.3(**)    21.0/25.0(**)    [13.3/16.8         ]
     33 #
     34 # (*)	ECB denotes approximate result for parallelizable modes
     35 #	such as CBC decrypt, CTR, etc.;
     36 # (**)	these results are worse than scalar compiler-generated
     37 #	code, but it's constant-time and therefore preferred;
     38 # (***)	presented for reference/comparison purposes;
     39 
     40 $flavour = shift;
     41 while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
     42 
     43 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
     44 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
     45 ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
     46 die "can't locate arm-xlate.pl";
     47 
     48 open OUT,"| \"$^X\" $xlate $flavour $output";
     49 *STDOUT=*OUT;
     50 
     51 $code.=<<___;
     52 .section	.rodata
     53 
     54 .type	_vpaes_consts,%object
     55 .align	7	// totally strategic alignment
     56 _vpaes_consts:
     57 .Lk_mc_forward:	// mc_forward
     58 	.quad	0x0407060500030201, 0x0C0F0E0D080B0A09
     59 	.quad	0x080B0A0904070605, 0x000302010C0F0E0D
     60 	.quad	0x0C0F0E0D080B0A09, 0x0407060500030201
     61 	.quad	0x000302010C0F0E0D, 0x080B0A0904070605
     62 .Lk_mc_backward:// mc_backward
     63 	.quad	0x0605040702010003, 0x0E0D0C0F0A09080B
     64 	.quad	0x020100030E0D0C0F, 0x0A09080B06050407
     65 	.quad	0x0E0D0C0F0A09080B, 0x0605040702010003
     66 	.quad	0x0A09080B06050407, 0x020100030E0D0C0F
     67 .Lk_sr:		// sr
     68 	.quad	0x0706050403020100, 0x0F0E0D0C0B0A0908
     69 	.quad	0x030E09040F0A0500, 0x0B06010C07020D08
     70 	.quad	0x0F060D040B020900, 0x070E050C030A0108
     71 	.quad	0x0B0E0104070A0D00, 0x0306090C0F020508
     72 
     73 //
     74 // "Hot" constants
     75 //
     76 .Lk_inv:	// inv, inva
     77 	.quad	0x0E05060F0D080180, 0x040703090A0B0C02
     78 	.quad	0x01040A060F0B0780, 0x030D0E0C02050809
     79 .Lk_ipt:	// input transform (lo, hi)
     80 	.quad	0xC2B2E8985A2A7000, 0xCABAE09052227808
     81 	.quad	0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81
     82 .Lk_sbo:	// sbou, sbot
     83 	.quad	0xD0D26D176FBDC700, 0x15AABF7AC502A878
     84 	.quad	0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA
     85 .Lk_sb1:	// sb1u, sb1t
     86 	.quad	0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF
     87 	.quad	0xB19BE18FCB503E00, 0xA5DF7A6E142AF544
     88 .Lk_sb2:	// sb2u, sb2t
     89 	.quad	0x69EB88400AE12900, 0xC2A163C8AB82234A
     90 	.quad	0xE27A93C60B712400, 0x5EB7E955BC982FCD
     91 
     92 //
     93 //  Decryption stuff
     94 //
     95 .Lk_dipt:	// decryption input transform
     96 	.quad	0x0F505B040B545F00, 0x154A411E114E451A
     97 	.quad	0x86E383E660056500, 0x12771772F491F194
     98 .Lk_dsbo:	// decryption sbox final output
     99 	.quad	0x1387EA537EF94000, 0xC7AA6DB9D4943E2D
    100 	.quad	0x12D7560F93441D00, 0xCA4B8159D8C58E9C
    101 .Lk_dsb9:	// decryption sbox output *9*u, *9*t
    102 	.quad	0x851C03539A86D600, 0xCAD51F504F994CC9
    103 	.quad	0xC03B1789ECD74900, 0x725E2C9EB2FBA565
    104 .Lk_dsbd:	// decryption sbox output *D*u, *D*t
    105 	.quad	0x7D57CCDFE6B1A200, 0xF56E9B13882A4439
    106 	.quad	0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3
    107 .Lk_dsbb:	// decryption sbox output *B*u, *B*t
    108 	.quad	0xD022649296B44200, 0x602646F6B0F2D404
    109 	.quad	0xC19498A6CD596700, 0xF3FF0C3E3255AA6B
    110 .Lk_dsbe:	// decryption sbox output *E*u, *E*t
    111 	.quad	0x46F2929626D4D000, 0x2242600464B4F6B0
    112 	.quad	0x0C55A6CDFFAAC100, 0x9467F36B98593E32
    113 
    114 //
    115 //  Key schedule constants
    116 //
    117 .Lk_dksd:	// decryption key schedule: invskew x*D
    118 	.quad	0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9
    119 	.quad	0x41C277F4B5368300, 0x5FDC69EAAB289D1E
    120 .Lk_dksb:	// decryption key schedule: invskew x*B
    121 	.quad	0x9A4FCA1F8550D500, 0x03D653861CC94C99
    122 	.quad	0x115BEDA7B6FC4A00, 0xD993256F7E3482C8
    123 .Lk_dkse:	// decryption key schedule: invskew x*E + 0x63
    124 	.quad	0xD5031CCA1FC9D600, 0x53859A4C994F5086
    125 	.quad	0xA23196054FDC7BE8, 0xCD5EF96A20B31487
    126 .Lk_dks9:	// decryption key schedule: invskew x*9
    127 	.quad	0xB6116FC87ED9A700, 0x4AED933482255BFC
    128 	.quad	0x4576516227143300, 0x8BB89FACE9DAFDCE
    129 
    130 .Lk_rcon:	// rcon
    131 	.quad	0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81
    132 
    133 .Lk_opt:	// output transform
    134 	.quad	0xFF9F4929D6B66000, 0xF7974121DEBE6808
    135 	.quad	0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0
    136 .Lk_deskew:	// deskew tables: inverts the sbox's "skew"
    137 	.quad	0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A
    138 	.quad	0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77
    139 
    140 .asciz  "Vector Permutation AES for ARMv8, Mike Hamburg (Stanford University)"
    141 .size	_vpaes_consts,.-_vpaes_consts
    142 .align	6
    143 
    144 .text
    145 ___
    146 
    148 {
    149 my ($inp,$out,$key) = map("x$_",(0..2));
    150 
    151 my ($invlo,$invhi,$iptlo,$ipthi,$sbou,$sbot) = map("v$_.16b",(18..23));
    152 my ($sb1u,$sb1t,$sb2u,$sb2t) = map("v$_.16b",(24..27));
    153 my ($sb9u,$sb9t,$sbdu,$sbdt,$sbbu,$sbbt,$sbeu,$sbet)=map("v$_.16b",(24..31));
    154 
    155 $code.=<<___;
    156 ##
    157 ##  _aes_preheat
    158 ##
    159 ##  Fills register %r10 -> .aes_consts (so you can -fPIC)
    160 ##  and %xmm9-%xmm15 as specified below.
    161 ##
    162 .type	_vpaes_encrypt_preheat,%function
    163 .align	4
    164 _vpaes_encrypt_preheat:
    165 	adrp	x10, :pg_hi21:.Lk_inv
    166 	add	x10, x10, :lo12:.Lk_inv
    167 	movi	v17.16b, #0x0f
    168 	ld1	{v18.2d-v19.2d}, [x10],#32	// .Lk_inv
    169 	ld1	{v20.2d-v23.2d}, [x10],#64	// .Lk_ipt, .Lk_sbo
    170 	ld1	{v24.2d-v27.2d}, [x10]		// .Lk_sb1, .Lk_sb2
    171 	ret
    172 .size	_vpaes_encrypt_preheat,.-_vpaes_encrypt_preheat
    173 
    174 ##
    175 ##  _aes_encrypt_core
    176 ##
    177 ##  AES-encrypt %xmm0.
    178 ##
    179 ##  Inputs:
    180 ##     %xmm0 = input
    181 ##     %xmm9-%xmm15 as in _vpaes_preheat
    182 ##    (%rdx) = scheduled keys
    183 ##
    184 ##  Output in %xmm0
    185 ##  Clobbers  %xmm1-%xmm5, %r9, %r10, %r11, %rax
    186 ##  Preserves %xmm6 - %xmm8 so you get some local vectors
    187 ##
    188 ##
    189 .type	_vpaes_encrypt_core,%function
    190 .align 4
    191 _vpaes_encrypt_core:
    192 	mov	x9, $key
    193 	ldr	w8, [$key,#240]			// pull rounds
    194 	adrp	x11, :pg_hi21:.Lk_mc_forward+16
    195 	add	x11, x11, :lo12:.Lk_mc_forward+16
    196 						// vmovdqa	.Lk_ipt(%rip),	%xmm2	# iptlo
    197 	ld1	{v16.2d}, [x9], #16		// vmovdqu	(%r9),	%xmm5		# round0 key
    198 	and	v1.16b, v7.16b, v17.16b		// vpand	%xmm9,	%xmm0,	%xmm1
    199 	ushr	v0.16b, v7.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0
    200 	tbl	v1.16b, {$iptlo}, v1.16b	// vpshufb	%xmm1,	%xmm2,	%xmm1
    201 						// vmovdqa	.Lk_ipt+16(%rip), %xmm3	# ipthi
    202 	tbl	v2.16b, {$ipthi}, v0.16b	// vpshufb	%xmm0,	%xmm3,	%xmm2
    203 	eor	v0.16b, v1.16b, v16.16b		// vpxor	%xmm5,	%xmm1,	%xmm0
    204 	eor	v0.16b, v0.16b, v2.16b		// vpxor	%xmm2,	%xmm0,	%xmm0
    205 	b	.Lenc_entry
    206 
    207 .align 4
    208 .Lenc_loop:
    209 	// middle of middle round
    210 	add	x10, x11, #0x40
    211 	tbl	v4.16b, {$sb1t}, v2.16b		// vpshufb	%xmm2,	%xmm13,	%xmm4	# 4 = sb1u
    212 	ld1	{v1.2d}, [x11], #16		// vmovdqa	-0x40(%r11,%r10), %xmm1	# .Lk_mc_forward[]
    213 	tbl	v0.16b, {$sb1u}, v3.16b		// vpshufb	%xmm3,	%xmm12,	%xmm0	# 0 = sb1t
    214 	eor	v4.16b, v4.16b, v16.16b		// vpxor	%xmm5,	%xmm4,	%xmm4	# 4 = sb1u + k
    215 	tbl	v5.16b,	{$sb2t}, v2.16b		// vpshufb	%xmm2,	%xmm15,	%xmm5	# 4 = sb2u
    216 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0,	%xmm0	# 0 = A
    217 	tbl	v2.16b, {$sb2u}, v3.16b		// vpshufb	%xmm3,	%xmm14,	%xmm2	# 2 = sb2t
    218 	ld1	{v4.2d}, [x10]			// vmovdqa	(%r11,%r10), %xmm4	# .Lk_mc_backward[]
    219 	tbl	v3.16b, {v0.16b}, v1.16b	// vpshufb	%xmm1,	%xmm0,	%xmm3	# 0 = B
    220 	eor	v2.16b, v2.16b, v5.16b		// vpxor	%xmm5,	%xmm2,	%xmm2	# 2 = 2A
    221 	tbl	v0.16b, {v0.16b}, v4.16b	// vpshufb	%xmm4,	%xmm0,	%xmm0	# 3 = D
    222 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3	# 0 = 2A+B
    223 	tbl	v4.16b, {v3.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm4	# 0 = 2B+C
    224 	eor	v0.16b, v0.16b, v3.16b		// vpxor	%xmm3,	%xmm0,	%xmm0	# 3 = 2A+B+D
    225 	and	x11, x11, #~(1<<6)		// and		\$0x30,	%r11		# ... mod 4
    226 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0, %xmm0	# 0 = 2A+3B+C+D
    227 	sub	w8, w8, #1			// nr--
    228 
    229 .Lenc_entry:
    230 	// top of round
    231 	and	v1.16b, v0.16b, v17.16b		// vpand	%xmm0,	%xmm9,	%xmm1   # 0 = k
    232 	ushr	v0.16b, v0.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0	# 1 = i
    233 	tbl	v5.16b, {$invhi}, v1.16b	// vpshufb	%xmm1,	%xmm11,	%xmm5	# 2 = a/k
    234 	eor	v1.16b, v1.16b, v0.16b		// vpxor	%xmm0,	%xmm1,	%xmm1	# 0 = j
    235 	tbl	v3.16b, {$invlo}, v0.16b	// vpshufb	%xmm0, 	%xmm10,	%xmm3  	# 3 = 1/i
    236 	tbl	v4.16b, {$invlo}, v1.16b	// vpshufb	%xmm1, 	%xmm10,	%xmm4  	# 4 = 1/j
    237 	eor	v3.16b, v3.16b, v5.16b		// vpxor	%xmm5,	%xmm3,	%xmm3	# 3 = iak = 1/i + a/k
    238 	eor	v4.16b, v4.16b, v5.16b		// vpxor	%xmm5,	%xmm4,	%xmm4  	# 4 = jak = 1/j + a/k
    239 	tbl	v2.16b, {$invlo}, v3.16b	// vpshufb	%xmm3,	%xmm10,	%xmm2  	# 2 = 1/iak
    240 	tbl	v3.16b, {$invlo}, v4.16b	// vpshufb	%xmm4,	%xmm10,	%xmm3	# 3 = 1/jak
    241 	eor	v2.16b, v2.16b, v1.16b		// vpxor	%xmm1,	%xmm2,	%xmm2  	# 2 = io
    242 	eor	v3.16b, v3.16b, v0.16b		// vpxor	%xmm0,	%xmm3,	%xmm3	# 3 = jo
    243 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm5
    244 	cbnz	w8, .Lenc_loop
    245 
    246 	// middle of last round
    247 	add	x10, x11, #0x80
    248 						// vmovdqa	-0x60(%r10), %xmm4	# 3 : sbou	.Lk_sbo
    249 						// vmovdqa	-0x50(%r10), %xmm0	# 0 : sbot	.Lk_sbo+16
    250 	tbl	v4.16b, {$sbou}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4	# 4 = sbou
    251 	ld1	{v1.2d}, [x10]			// vmovdqa	0x40(%r11,%r10), %xmm1	# .Lk_sr[]
    252 	tbl	v0.16b, {$sbot}, v3.16b		// vpshufb	%xmm3,	%xmm0,	%xmm0	# 0 = sb1t
    253 	eor	v4.16b, v4.16b, v16.16b		// vpxor	%xmm5,	%xmm4,	%xmm4	# 4 = sb1u + k
    254 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0,	%xmm0	# 0 = A
    255 	tbl	v0.16b, {v0.16b}, v1.16b	// vpshufb	%xmm1,	%xmm0,	%xmm0
    256 	ret
    257 .size	_vpaes_encrypt_core,.-_vpaes_encrypt_core
    258 
    259 .globl	vpaes_encrypt
    260 .type	vpaes_encrypt,%function
    261 .align	4
    262 vpaes_encrypt:
    263 	stp	x29,x30,[sp,#-16]!
    264 	add	x29,sp,#0
    265 
    266 	ld1	{v7.16b}, [$inp]
    267 	bl	_vpaes_encrypt_preheat
    268 	bl	_vpaes_encrypt_core
    269 	st1	{v0.16b}, [$out]
    270 
    271 	ldp	x29,x30,[sp],#16
    272 	ret
    273 .size	vpaes_encrypt,.-vpaes_encrypt
    274 
    275 .type	_vpaes_encrypt_2x,%function
    276 .align 4
    277 _vpaes_encrypt_2x:
    278 	mov	x9, $key
    279 	ldr	w8, [$key,#240]			// pull rounds
    280 	adrp	x11, :pg_hi21:.Lk_mc_forward+16
    281 	add	x11, x11, :lo12:.Lk_mc_forward+16
    282 						// vmovdqa	.Lk_ipt(%rip),	%xmm2	# iptlo
    283 	ld1	{v16.2d}, [x9], #16		// vmovdqu	(%r9),	%xmm5		# round0 key
    284 	and	v1.16b,  v14.16b,  v17.16b	// vpand	%xmm9,	%xmm0,	%xmm1
    285 	ushr	v0.16b,  v14.16b,  #4		// vpsrlb	\$4,	%xmm0,	%xmm0
    286 	 and	v9.16b,  v15.16b,  v17.16b
    287 	 ushr	v8.16b,  v15.16b,  #4
    288 	tbl	v1.16b,  {$iptlo}, v1.16b	// vpshufb	%xmm1,	%xmm2,	%xmm1
    289 	 tbl	v9.16b,  {$iptlo}, v9.16b
    290 						// vmovdqa	.Lk_ipt+16(%rip), %xmm3	# ipthi
    291 	tbl	v2.16b,  {$ipthi}, v0.16b	// vpshufb	%xmm0,	%xmm3,	%xmm2
    292 	 tbl	v10.16b, {$ipthi}, v8.16b
    293 	eor	v0.16b,  v1.16b,   v16.16b	// vpxor	%xmm5,	%xmm1,	%xmm0
    294 	 eor	v8.16b,  v9.16b,   v16.16b
    295 	eor	v0.16b,  v0.16b,   v2.16b	// vpxor	%xmm2,	%xmm0,	%xmm0
    296 	 eor	v8.16b,  v8.16b,   v10.16b
    297 	b	.Lenc_2x_entry
    298 
    299 .align 4
    300 .Lenc_2x_loop:
    301 	// middle of middle round
    302 	add	x10, x11, #0x40
    303 	tbl	v4.16b,  {$sb1t}, v2.16b	// vpshufb	%xmm2,	%xmm13,	%xmm4	# 4 = sb1u
    304 	 tbl	v12.16b, {$sb1t}, v10.16b
    305 	ld1	{v1.2d}, [x11], #16		// vmovdqa	-0x40(%r11,%r10), %xmm1	# .Lk_mc_forward[]
    306 	tbl	v0.16b,  {$sb1u}, v3.16b	// vpshufb	%xmm3,	%xmm12,	%xmm0	# 0 = sb1t
    307 	 tbl	v8.16b,  {$sb1u}, v11.16b
    308 	eor	v4.16b,  v4.16b,  v16.16b	// vpxor	%xmm5,	%xmm4,	%xmm4	# 4 = sb1u + k
    309 	 eor	v12.16b, v12.16b, v16.16b
    310 	tbl	v5.16b,	 {$sb2t}, v2.16b	// vpshufb	%xmm2,	%xmm15,	%xmm5	# 4 = sb2u
    311 	 tbl	v13.16b, {$sb2t}, v10.16b
    312 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0,	%xmm0	# 0 = A
    313 	 eor	v8.16b,  v8.16b,  v12.16b
    314 	tbl	v2.16b,  {$sb2u}, v3.16b	// vpshufb	%xmm3,	%xmm14,	%xmm2	# 2 = sb2t
    315 	 tbl	v10.16b, {$sb2u}, v11.16b
    316 	ld1	{v4.2d}, [x10]			// vmovdqa	(%r11,%r10), %xmm4	# .Lk_mc_backward[]
    317 	tbl	v3.16b,  {v0.16b}, v1.16b	// vpshufb	%xmm1,	%xmm0,	%xmm3	# 0 = B
    318 	 tbl	v11.16b, {v8.16b}, v1.16b
    319 	eor	v2.16b,  v2.16b,  v5.16b	// vpxor	%xmm5,	%xmm2,	%xmm2	# 2 = 2A
    320 	 eor	v10.16b, v10.16b, v13.16b
    321 	tbl	v0.16b,  {v0.16b}, v4.16b	// vpshufb	%xmm4,	%xmm0,	%xmm0	# 3 = D
    322 	 tbl	v8.16b,  {v8.16b}, v4.16b
    323 	eor	v3.16b,  v3.16b,  v2.16b	// vpxor	%xmm2,	%xmm3,	%xmm3	# 0 = 2A+B
    324 	 eor	v11.16b, v11.16b, v10.16b
    325 	tbl	v4.16b,  {v3.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm4	# 0 = 2B+C
    326 	 tbl	v12.16b, {v11.16b},v1.16b
    327 	eor	v0.16b,  v0.16b,  v3.16b	// vpxor	%xmm3,	%xmm0,	%xmm0	# 3 = 2A+B+D
    328 	 eor	v8.16b,  v8.16b,  v11.16b
    329 	and	x11, x11, #~(1<<6)		// and		\$0x30,	%r11		# ... mod 4
    330 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0, %xmm0	# 0 = 2A+3B+C+D
    331 	 eor	v8.16b,  v8.16b,  v12.16b
    332 	sub	w8, w8, #1			// nr--
    333 
    334 .Lenc_2x_entry:
    335 	// top of round
    336 	and	v1.16b,  v0.16b, v17.16b	// vpand	%xmm0,	%xmm9,	%xmm1   # 0 = k
    337 	ushr	v0.16b,  v0.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0	# 1 = i
    338 	 and	v9.16b,  v8.16b, v17.16b
    339 	 ushr	v8.16b,  v8.16b, #4
    340 	tbl	v5.16b,  {$invhi},v1.16b	// vpshufb	%xmm1,	%xmm11,	%xmm5	# 2 = a/k
    341 	 tbl	v13.16b, {$invhi},v9.16b
    342 	eor	v1.16b,  v1.16b,  v0.16b	// vpxor	%xmm0,	%xmm1,	%xmm1	# 0 = j
    343 	 eor	v9.16b,  v9.16b,  v8.16b
    344 	tbl	v3.16b,  {$invlo},v0.16b	// vpshufb	%xmm0, 	%xmm10,	%xmm3  	# 3 = 1/i
    345 	 tbl	v11.16b, {$invlo},v8.16b
    346 	tbl	v4.16b,  {$invlo},v1.16b	// vpshufb	%xmm1, 	%xmm10,	%xmm4  	# 4 = 1/j
    347 	 tbl	v12.16b, {$invlo},v9.16b
    348 	eor	v3.16b,  v3.16b,  v5.16b	// vpxor	%xmm5,	%xmm3,	%xmm3	# 3 = iak = 1/i + a/k
    349 	 eor	v11.16b, v11.16b, v13.16b
    350 	eor	v4.16b,  v4.16b,  v5.16b	// vpxor	%xmm5,	%xmm4,	%xmm4  	# 4 = jak = 1/j + a/k
    351 	 eor	v12.16b, v12.16b, v13.16b
    352 	tbl	v2.16b,  {$invlo},v3.16b	// vpshufb	%xmm3,	%xmm10,	%xmm2  	# 2 = 1/iak
    353 	 tbl	v10.16b, {$invlo},v11.16b
    354 	tbl	v3.16b,  {$invlo},v4.16b	// vpshufb	%xmm4,	%xmm10,	%xmm3	# 3 = 1/jak
    355 	 tbl	v11.16b, {$invlo},v12.16b
    356 	eor	v2.16b,  v2.16b,  v1.16b	// vpxor	%xmm1,	%xmm2,	%xmm2  	# 2 = io
    357 	 eor	v10.16b, v10.16b, v9.16b
    358 	eor	v3.16b,  v3.16b,  v0.16b	// vpxor	%xmm0,	%xmm3,	%xmm3	# 3 = jo
    359 	 eor	v11.16b, v11.16b, v8.16b
    360 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm5
    361 	cbnz	w8, .Lenc_2x_loop
    362 
    363 	// middle of last round
    364 	add	x10, x11, #0x80
    365 						// vmovdqa	-0x60(%r10), %xmm4	# 3 : sbou	.Lk_sbo
    366 						// vmovdqa	-0x50(%r10), %xmm0	# 0 : sbot	.Lk_sbo+16
    367 	tbl	v4.16b,  {$sbou}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4	# 4 = sbou
    368 	 tbl	v12.16b, {$sbou}, v10.16b
    369 	ld1	{v1.2d}, [x10]			// vmovdqa	0x40(%r11,%r10), %xmm1	# .Lk_sr[]
    370 	tbl	v0.16b,  {$sbot}, v3.16b	// vpshufb	%xmm3,	%xmm0,	%xmm0	# 0 = sb1t
    371 	 tbl	v8.16b,  {$sbot}, v11.16b
    372 	eor	v4.16b,  v4.16b,  v16.16b	// vpxor	%xmm5,	%xmm4,	%xmm4	# 4 = sb1u + k
    373 	 eor	v12.16b, v12.16b, v16.16b
    374 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0,	%xmm0	# 0 = A
    375 	 eor	v8.16b,  v8.16b,  v12.16b
    376 	tbl	v0.16b,  {v0.16b},v1.16b	// vpshufb	%xmm1,	%xmm0,	%xmm0
    377 	 tbl	v1.16b,  {v8.16b},v1.16b
    378 	ret
    379 .size	_vpaes_encrypt_2x,.-_vpaes_encrypt_2x
    380 
    381 .type	_vpaes_decrypt_preheat,%function
    382 .align	4
    383 _vpaes_decrypt_preheat:
    384 	adrp	x10, :pg_hi21:.Lk_inv
    385 	add	x10, x10, :lo12:.Lk_inv
    386 	movi	v17.16b, #0x0f
    387 	adrp	x11, :pg_hi21:.Lk_dipt
    388 	add	x11, x11, :lo12:.Lk_dipt
    389 	ld1	{v18.2d-v19.2d}, [x10],#32	// .Lk_inv
    390 	ld1	{v20.2d-v23.2d}, [x11],#64	// .Lk_dipt, .Lk_dsbo
    391 	ld1	{v24.2d-v27.2d}, [x11],#64	// .Lk_dsb9, .Lk_dsbd
    392 	ld1	{v28.2d-v31.2d}, [x11]		// .Lk_dsbb, .Lk_dsbe
    393 	ret
    394 .size	_vpaes_decrypt_preheat,.-_vpaes_decrypt_preheat
    395 
    396 ##
    397 ##  Decryption core
    398 ##
    399 ##  Same API as encryption core.
    400 ##
    401 .type	_vpaes_decrypt_core,%function
    402 .align	4
    403 _vpaes_decrypt_core:
    404 	mov	x9, $key
    405 	ldr	w8, [$key,#240]			// pull rounds
    406 
    407 						// vmovdqa	.Lk_dipt(%rip), %xmm2	# iptlo
    408 	lsl	x11, x8, #4			// mov	%rax,	%r11;	shl	\$4, %r11
    409 	eor	x11, x11, #0x30			// xor		\$0x30,	%r11
    410 	adrp	x10, :pg_hi21:.Lk_sr
    411 	add	x10, x10, :lo12:.Lk_sr
    412 	and	x11, x11, #0x30			// and		\$0x30,	%r11
    413 	add	x11, x11, x10
    414 	adrp	x10, :pg_hi21:.Lk_mc_forward+48
    415 	add	x10, x10, :lo12:.Lk_mc_forward+48
    416 
    417 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm4		# round0 key
    418 	and	v1.16b, v7.16b, v17.16b		// vpand	%xmm9,	%xmm0,	%xmm1
    419 	ushr	v0.16b, v7.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0
    420 	tbl	v2.16b, {$iptlo}, v1.16b	// vpshufb	%xmm1,	%xmm2,	%xmm2
    421 	ld1	{v5.2d}, [x10]			// vmovdqa	.Lk_mc_forward+48(%rip), %xmm5
    422 						// vmovdqa	.Lk_dipt+16(%rip), %xmm1 # ipthi
    423 	tbl	v0.16b, {$ipthi}, v0.16b	// vpshufb	%xmm0,	%xmm1,	%xmm0
    424 	eor	v2.16b, v2.16b, v16.16b		// vpxor	%xmm4,	%xmm2,	%xmm2
    425 	eor	v0.16b, v0.16b, v2.16b		// vpxor	%xmm2,	%xmm0,	%xmm0
    426 	b	.Ldec_entry
    427 
    428 .align 4
    429 .Ldec_loop:
    430 //
    431 //  Inverse mix columns
    432 //
    433 						// vmovdqa	-0x20(%r10),%xmm4		# 4 : sb9u
    434 						// vmovdqa	-0x10(%r10),%xmm1		# 0 : sb9t
    435 	tbl	v4.16b, {$sb9u}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sb9u
    436 	tbl	v1.16b, {$sb9t}, v3.16b		// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sb9t
    437 	eor	v0.16b, v4.16b, v16.16b		// vpxor	%xmm4,	%xmm0,	%xmm0
    438 						// vmovdqa	0x00(%r10),%xmm4		# 4 : sbdu
    439 	eor	v0.16b, v0.16b, v1.16b		// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    440 						// vmovdqa	0x10(%r10),%xmm1		# 0 : sbdt
    441 
    442 	tbl	v4.16b, {$sbdu}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbdu
    443 	tbl 	v0.16b, {v0.16b}, v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    444 	tbl	v1.16b, {$sbdt}, v3.16b		// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbdt
    445 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    446 						// vmovdqa	0x20(%r10),	%xmm4		# 4 : sbbu
    447 	eor	v0.16b, v0.16b, v1.16b		// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    448 						// vmovdqa	0x30(%r10),	%xmm1		# 0 : sbbt
    449 
    450 	tbl	v4.16b, {$sbbu}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbbu
    451 	tbl	v0.16b, {v0.16b}, v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    452 	tbl	v1.16b, {$sbbt}, v3.16b		// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbbt
    453 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    454 						// vmovdqa	0x40(%r10),	%xmm4		# 4 : sbeu
    455 	eor	v0.16b, v0.16b, v1.16b		// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    456 						// vmovdqa	0x50(%r10),	%xmm1		# 0 : sbet
    457 
    458 	tbl	v4.16b, {$sbeu}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbeu
    459 	tbl	v0.16b, {v0.16b}, v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    460 	tbl	v1.16b, {$sbet}, v3.16b		// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbet
    461 	eor	v0.16b, v0.16b, v4.16b		// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    462 	ext	v5.16b, v5.16b, v5.16b, #12	// vpalignr \$12,	%xmm5,	%xmm5,	%xmm5
    463 	eor	v0.16b, v0.16b, v1.16b		// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    464 	sub	w8, w8, #1			// sub		\$1,%rax			# nr--
    465 
    466 .Ldec_entry:
    467 	// top of round
    468 	and	v1.16b, v0.16b, v17.16b		// vpand	%xmm9,	%xmm0,	%xmm1	# 0 = k
    469 	ushr	v0.16b, v0.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0	# 1 = i
    470 	tbl	v2.16b, {$invhi}, v1.16b	// vpshufb	%xmm1,	%xmm11,	%xmm2	# 2 = a/k
    471 	eor	v1.16b,	v1.16b, v0.16b		// vpxor	%xmm0,	%xmm1,	%xmm1	# 0 = j
    472 	tbl	v3.16b, {$invlo}, v0.16b	// vpshufb	%xmm0, 	%xmm10,	%xmm3	# 3 = 1/i
    473 	tbl	v4.16b, {$invlo}, v1.16b	// vpshufb	%xmm1,	%xmm10,	%xmm4	# 4 = 1/j
    474 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3	# 3 = iak = 1/i + a/k
    475 	eor	v4.16b, v4.16b, v2.16b		// vpxor	%xmm2, 	%xmm4,	%xmm4	# 4 = jak = 1/j + a/k
    476 	tbl	v2.16b, {$invlo}, v3.16b	// vpshufb	%xmm3,	%xmm10,	%xmm2	# 2 = 1/iak
    477 	tbl	v3.16b, {$invlo}, v4.16b	// vpshufb	%xmm4,  %xmm10,	%xmm3	# 3 = 1/jak
    478 	eor	v2.16b, v2.16b, v1.16b		// vpxor	%xmm1,	%xmm2,	%xmm2	# 2 = io
    479 	eor	v3.16b, v3.16b, v0.16b		// vpxor	%xmm0,  %xmm3,	%xmm3	# 3 = jo
    480 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm0
    481 	cbnz	w8, .Ldec_loop
    482 
    483 	// middle of last round
    484 						// vmovdqa	0x60(%r10),	%xmm4	# 3 : sbou
    485 	tbl	v4.16b, {$sbou}, v2.16b		// vpshufb	%xmm2,	%xmm4,	%xmm4	# 4 = sbou
    486 						// vmovdqa	0x70(%r10),	%xmm1	# 0 : sbot
    487 	ld1	{v2.2d}, [x11]			// vmovdqa	-0x160(%r11),	%xmm2	# .Lk_sr-.Lk_dsbd=-0x160
    488 	tbl	v1.16b, {$sbot}, v3.16b		// vpshufb	%xmm3,	%xmm1,	%xmm1	# 0 = sb1t
    489 	eor	v4.16b, v4.16b, v16.16b		// vpxor	%xmm0,	%xmm4,	%xmm4	# 4 = sb1u + k
    490 	eor	v0.16b, v1.16b, v4.16b		// vpxor	%xmm4,	%xmm1,	%xmm0	# 0 = A
    491 	tbl	v0.16b, {v0.16b}, v2.16b	// vpshufb	%xmm2,	%xmm0,	%xmm0
    492 	ret
    493 .size	_vpaes_decrypt_core,.-_vpaes_decrypt_core
    494 
    495 .globl	vpaes_decrypt
    496 .type	vpaes_decrypt,%function
    497 .align	4
    498 vpaes_decrypt:
    499 	stp	x29,x30,[sp,#-16]!
    500 	add	x29,sp,#0
    501 
    502 	ld1	{v7.16b}, [$inp]
    503 	bl	_vpaes_decrypt_preheat
    504 	bl	_vpaes_decrypt_core
    505 	st1	{v0.16b}, [$out]
    506 
    507 	ldp	x29,x30,[sp],#16
    508 	ret
    509 .size	vpaes_decrypt,.-vpaes_decrypt
    510 
    511 // v14-v15 input, v0-v1 output
    512 .type	_vpaes_decrypt_2x,%function
    513 .align	4
    514 _vpaes_decrypt_2x:
    515 	mov	x9, $key
    516 	ldr	w8, [$key,#240]			// pull rounds
    517 
    518 						// vmovdqa	.Lk_dipt(%rip), %xmm2	# iptlo
    519 	lsl	x11, x8, #4			// mov	%rax,	%r11;	shl	\$4, %r11
    520 	eor	x11, x11, #0x30			// xor		\$0x30,	%r11
    521 	adrp	x10, :pg_hi21:.Lk_sr
    522 	add	x10, x10, :lo12:.Lk_sr
    523 	and	x11, x11, #0x30			// and		\$0x30,	%r11
    524 	add	x11, x11, x10
    525 	adrp	x10, :pg_hi21:.Lk_mc_forward+48
    526 	add	x10, x10, :lo12:.Lk_mc_forward+48
    527 
    528 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm4		# round0 key
    529 	and	v1.16b,  v14.16b, v17.16b	// vpand	%xmm9,	%xmm0,	%xmm1
    530 	ushr	v0.16b,  v14.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0
    531 	 and	v9.16b,  v15.16b, v17.16b
    532 	 ushr	v8.16b,  v15.16b, #4
    533 	tbl	v2.16b,  {$iptlo},v1.16b	// vpshufb	%xmm1,	%xmm2,	%xmm2
    534 	 tbl	v10.16b, {$iptlo},v9.16b
    535 	ld1	{v5.2d}, [x10]			// vmovdqa	.Lk_mc_forward+48(%rip), %xmm5
    536 						// vmovdqa	.Lk_dipt+16(%rip), %xmm1 # ipthi
    537 	tbl	v0.16b,  {$ipthi},v0.16b	// vpshufb	%xmm0,	%xmm1,	%xmm0
    538 	 tbl	v8.16b,  {$ipthi},v8.16b
    539 	eor	v2.16b,  v2.16b,  v16.16b	// vpxor	%xmm4,	%xmm2,	%xmm2
    540 	 eor	v10.16b, v10.16b, v16.16b
    541 	eor	v0.16b,  v0.16b,  v2.16b	// vpxor	%xmm2,	%xmm0,	%xmm0
    542 	 eor	v8.16b,  v8.16b,  v10.16b
    543 	b	.Ldec_2x_entry
    544 
    545 .align 4
    546 .Ldec_2x_loop:
    547 //
    548 //  Inverse mix columns
    549 //
    550 						// vmovdqa	-0x20(%r10),%xmm4		# 4 : sb9u
    551 						// vmovdqa	-0x10(%r10),%xmm1		# 0 : sb9t
    552 	tbl	v4.16b,  {$sb9u}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sb9u
    553 	 tbl	v12.16b, {$sb9u}, v10.16b
    554 	tbl	v1.16b,  {$sb9t}, v3.16b	// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sb9t
    555 	 tbl	v9.16b,  {$sb9t}, v11.16b
    556 	eor	v0.16b,  v4.16b,  v16.16b	// vpxor	%xmm4,	%xmm0,	%xmm0
    557 	 eor	v8.16b,  v12.16b, v16.16b
    558 						// vmovdqa	0x00(%r10),%xmm4		# 4 : sbdu
    559 	eor	v0.16b,  v0.16b,  v1.16b	// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    560 	 eor	v8.16b,  v8.16b,  v9.16b	// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    561 						// vmovdqa	0x10(%r10),%xmm1		# 0 : sbdt
    562 
    563 	tbl	v4.16b,  {$sbdu}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbdu
    564 	 tbl	v12.16b, {$sbdu}, v10.16b
    565 	tbl 	v0.16b,  {v0.16b},v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    566 	 tbl 	v8.16b,  {v8.16b},v5.16b
    567 	tbl	v1.16b,  {$sbdt}, v3.16b	// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbdt
    568 	 tbl	v9.16b,  {$sbdt}, v11.16b
    569 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    570 	 eor	v8.16b,  v8.16b,  v12.16b
    571 						// vmovdqa	0x20(%r10),	%xmm4		# 4 : sbbu
    572 	eor	v0.16b,  v0.16b,  v1.16b	// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    573 	 eor	v8.16b,  v8.16b,  v9.16b
    574 						// vmovdqa	0x30(%r10),	%xmm1		# 0 : sbbt
    575 
    576 	tbl	v4.16b,  {$sbbu}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbbu
    577 	 tbl	v12.16b, {$sbbu}, v10.16b
    578 	tbl	v0.16b,  {v0.16b},v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    579 	 tbl	v8.16b,  {v8.16b},v5.16b
    580 	tbl	v1.16b,  {$sbbt}, v3.16b	// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbbt
    581 	 tbl	v9.16b,  {$sbbt}, v11.16b
    582 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    583 	 eor	v8.16b,  v8.16b,  v12.16b
    584 						// vmovdqa	0x40(%r10),	%xmm4		# 4 : sbeu
    585 	eor	v0.16b,  v0.16b,  v1.16b	// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    586 	 eor	v8.16b,  v8.16b,  v9.16b
    587 						// vmovdqa	0x50(%r10),	%xmm1		# 0 : sbet
    588 
    589 	tbl	v4.16b,  {$sbeu}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4		# 4 = sbeu
    590 	 tbl	v12.16b, {$sbeu}, v10.16b
    591 	tbl	v0.16b,  {v0.16b},v5.16b	// vpshufb	%xmm5,	%xmm0,	%xmm0		# MC ch
    592 	 tbl	v8.16b,  {v8.16b},v5.16b
    593 	tbl	v1.16b,  {$sbet}, v3.16b	// vpshufb	%xmm3,	%xmm1,	%xmm1		# 0 = sbet
    594 	 tbl	v9.16b,  {$sbet}, v11.16b
    595 	eor	v0.16b,  v0.16b,  v4.16b	// vpxor	%xmm4,	%xmm0,	%xmm0		# 4 = ch
    596 	 eor	v8.16b,  v8.16b,  v12.16b
    597 	ext	v5.16b,  v5.16b,  v5.16b, #12	// vpalignr \$12,	%xmm5,	%xmm5,	%xmm5
    598 	eor	v0.16b,  v0.16b,  v1.16b	// vpxor	%xmm1,	%xmm0,	%xmm0		# 0 = ch
    599 	 eor	v8.16b,  v8.16b,  v9.16b
    600 	sub	w8, w8, #1			// sub		\$1,%rax			# nr--
    601 
    602 .Ldec_2x_entry:
    603 	// top of round
    604 	and	v1.16b,  v0.16b,  v17.16b	// vpand	%xmm9,	%xmm0,	%xmm1	# 0 = k
    605 	ushr	v0.16b,  v0.16b,  #4		// vpsrlb	\$4,	%xmm0,	%xmm0	# 1 = i
    606 	 and	v9.16b,  v8.16b,  v17.16b
    607 	 ushr	v8.16b,  v8.16b,  #4
    608 	tbl	v2.16b,  {$invhi},v1.16b	// vpshufb	%xmm1,	%xmm11,	%xmm2	# 2 = a/k
    609 	 tbl	v10.16b, {$invhi},v9.16b
    610 	eor	v1.16b,	 v1.16b,  v0.16b	// vpxor	%xmm0,	%xmm1,	%xmm1	# 0 = j
    611 	 eor	v9.16b,	 v9.16b,  v8.16b
    612 	tbl	v3.16b,  {$invlo},v0.16b	// vpshufb	%xmm0, 	%xmm10,	%xmm3	# 3 = 1/i
    613 	 tbl	v11.16b, {$invlo},v8.16b
    614 	tbl	v4.16b,  {$invlo},v1.16b	// vpshufb	%xmm1,	%xmm10,	%xmm4	# 4 = 1/j
    615 	 tbl	v12.16b, {$invlo},v9.16b
    616 	eor	v3.16b,  v3.16b,  v2.16b	// vpxor	%xmm2,	%xmm3,	%xmm3	# 3 = iak = 1/i + a/k
    617 	 eor	v11.16b, v11.16b, v10.16b
    618 	eor	v4.16b,  v4.16b,  v2.16b	// vpxor	%xmm2, 	%xmm4,	%xmm4	# 4 = jak = 1/j + a/k
    619 	 eor	v12.16b, v12.16b, v10.16b
    620 	tbl	v2.16b,  {$invlo},v3.16b	// vpshufb	%xmm3,	%xmm10,	%xmm2	# 2 = 1/iak
    621 	 tbl	v10.16b, {$invlo},v11.16b
    622 	tbl	v3.16b,  {$invlo},v4.16b	// vpshufb	%xmm4,  %xmm10,	%xmm3	# 3 = 1/jak
    623 	 tbl	v11.16b, {$invlo},v12.16b
    624 	eor	v2.16b,  v2.16b,  v1.16b	// vpxor	%xmm1,	%xmm2,	%xmm2	# 2 = io
    625 	 eor	v10.16b, v10.16b, v9.16b
    626 	eor	v3.16b,  v3.16b,  v0.16b	// vpxor	%xmm0,  %xmm3,	%xmm3	# 3 = jo
    627 	 eor	v11.16b, v11.16b, v8.16b
    628 	ld1	{v16.2d}, [x9],#16		// vmovdqu	(%r9),	%xmm0
    629 	cbnz	w8, .Ldec_2x_loop
    630 
    631 	// middle of last round
    632 						// vmovdqa	0x60(%r10),	%xmm4	# 3 : sbou
    633 	tbl	v4.16b,  {$sbou}, v2.16b	// vpshufb	%xmm2,	%xmm4,	%xmm4	# 4 = sbou
    634 	 tbl	v12.16b, {$sbou}, v10.16b
    635 						// vmovdqa	0x70(%r10),	%xmm1	# 0 : sbot
    636 	tbl	v1.16b,  {$sbot}, v3.16b	// vpshufb	%xmm3,	%xmm1,	%xmm1	# 0 = sb1t
    637 	 tbl	v9.16b,  {$sbot}, v11.16b
    638 	ld1	{v2.2d}, [x11]			// vmovdqa	-0x160(%r11),	%xmm2	# .Lk_sr-.Lk_dsbd=-0x160
    639 	eor	v4.16b,  v4.16b,  v16.16b	// vpxor	%xmm0,	%xmm4,	%xmm4	# 4 = sb1u + k
    640 	 eor	v12.16b, v12.16b, v16.16b
    641 	eor	v0.16b,  v1.16b,  v4.16b	// vpxor	%xmm4,	%xmm1,	%xmm0	# 0 = A
    642 	 eor	v8.16b,  v9.16b,  v12.16b
    643 	tbl	v0.16b,  {v0.16b},v2.16b	// vpshufb	%xmm2,	%xmm0,	%xmm0
    644 	 tbl	v1.16b,  {v8.16b},v2.16b
    645 	ret
    646 .size	_vpaes_decrypt_2x,.-_vpaes_decrypt_2x
    647 ___
    648 }
    650 {
    651 my ($inp,$bits,$out,$dir)=("x0","w1","x2","w3");
    652 my ($invlo,$invhi,$iptlo,$ipthi,$rcon) = map("v$_.16b",(18..21,8));
    653 
    654 $code.=<<___;
    655 ########################################################
    656 ##                                                    ##
    657 ##                  AES key schedule                  ##
    658 ##                                                    ##
    659 ########################################################
    660 .type	_vpaes_key_preheat,%function
    661 .align	4
    662 _vpaes_key_preheat:
    663 	adrp	x10, :pg_hi21:.Lk_inv
    664 	add	x10, x10, :lo12:.Lk_inv
    665 	movi	v16.16b, #0x5b			// .Lk_s63
    666 	adrp	x11, :pg_hi21:.Lk_sb1
    667 	add	x11, x11, :lo12:.Lk_sb1
    668 	movi	v17.16b, #0x0f			// .Lk_s0F
    669 	ld1	{v18.2d-v21.2d}, [x10]		// .Lk_inv, .Lk_ipt
    670 	adrp	x10, :pg_hi21:.Lk_dksd
    671 	add	x10, x10, :lo12:.Lk_dksd
    672 	ld1	{v22.2d-v23.2d}, [x11]		// .Lk_sb1
    673 	adrp	x11, :pg_hi21:.Lk_mc_forward
    674 	add	x11, x11, :lo12:.Lk_mc_forward
    675 	ld1	{v24.2d-v27.2d}, [x10],#64	// .Lk_dksd, .Lk_dksb
    676 	ld1	{v28.2d-v31.2d}, [x10],#64	// .Lk_dkse, .Lk_dks9
    677 	ld1	{v8.2d}, [x10]			// .Lk_rcon
    678 	ld1	{v9.2d}, [x11]			// .Lk_mc_forward[0]
    679 	ret
    680 .size	_vpaes_key_preheat,.-_vpaes_key_preheat
    681 
    682 .type	_vpaes_schedule_core,%function
    683 .align	4
    684 _vpaes_schedule_core:
    685 	stp	x29, x30, [sp,#-16]!
    686 	add	x29,sp,#0
    687 
    688 	bl	_vpaes_key_preheat		// load the tables
    689 
    690 	ld1	{v0.16b}, [$inp],#16		// vmovdqu	(%rdi),	%xmm0		# load key (unaligned)
    691 
    692 	// input transform
    693 	mov	v3.16b, v0.16b			// vmovdqa	%xmm0,	%xmm3
    694 	bl	_vpaes_schedule_transform
    695 	mov	v7.16b, v0.16b			// vmovdqa	%xmm0,	%xmm7
    696 
    697 	adrp	x10, :pg_hi21:.Lk_sr		// lea	.Lk_sr(%rip),%r10
    698 	add	x10, x10, :lo12:.Lk_sr
    699 
    700 	add	x8, x8, x10
    701 	cbnz	$dir, .Lschedule_am_decrypting
    702 
    703 	// encrypting, output zeroth round key after transform
    704 	st1	{v0.2d}, [$out]			// vmovdqu	%xmm0,	(%rdx)
    705 	b	.Lschedule_go
    706 
    707 .Lschedule_am_decrypting:
    708 	// decrypting, output zeroth round key after shiftrows
    709 	ld1	{v1.2d}, [x8]			// vmovdqa	(%r8,%r10),	%xmm1
    710 	tbl	v3.16b, {v3.16b}, v1.16b	// vpshufb  %xmm1,	%xmm3,	%xmm3
    711 	st1	{v3.2d}, [$out]			// vmovdqu	%xmm3,	(%rdx)
    712 	eor	x8, x8, #0x30			// xor	\$0x30, %r8
    713 
    714 .Lschedule_go:
    715 	cmp	$bits, #192			// cmp	\$192,	%esi
    716 	b.hi	.Lschedule_256
    717 	b.eq	.Lschedule_192
    718 	// 128: fall though
    719 
    720 ##
    721 ##  .schedule_128
    722 ##
    723 ##  128-bit specific part of key schedule.
    724 ##
    725 ##  This schedule is really simple, because all its parts
    726 ##  are accomplished by the subroutines.
    727 ##
    728 .Lschedule_128:
    729 	mov	$inp, #10			// mov	\$10, %esi
    730 
    731 .Loop_schedule_128:
    732 	sub	$inp, $inp, #1			// dec	%esi
    733 	bl 	_vpaes_schedule_round
    734 	cbz 	$inp, .Lschedule_mangle_last
    735 	bl	_vpaes_schedule_mangle		// write output
    736 	b 	.Loop_schedule_128
    737 
    738 ##
    739 ##  .aes_schedule_192
    740 ##
    741 ##  192-bit specific part of key schedule.
    742 ##
    743 ##  The main body of this schedule is the same as the 128-bit
    744 ##  schedule, but with more smearing.  The long, high side is
    745 ##  stored in %xmm7 as before, and the short, low side is in
    746 ##  the high bits of %xmm6.
    747 ##
    748 ##  This schedule is somewhat nastier, however, because each
    749 ##  round produces 192 bits of key material, or 1.5 round keys.
    750 ##  Therefore, on each cycle we do 2 rounds and produce 3 round
    751 ##  keys.
    752 ##
    753 .align	4
    754 .Lschedule_192:
    755 	sub	$inp, $inp, #8
    756 	ld1	{v0.16b}, [$inp]		// vmovdqu	8(%rdi),%xmm0		# load key part 2 (very unaligned)
    757 	bl	_vpaes_schedule_transform	// input transform
    758 	mov	v6.16b, v0.16b			// vmovdqa	%xmm0,	%xmm6		# save short part
    759 	eor	v4.16b, v4.16b, v4.16b		// vpxor	%xmm4,	%xmm4, %xmm4	# clear 4
    760 	ins	v6.d[0], v4.d[0]		// vmovhlps	%xmm4,	%xmm6,	%xmm6		# clobber low side with zeros
    761 	mov	$inp, #4			// mov	\$4,	%esi
    762 
    763 .Loop_schedule_192:
    764 	sub	$inp, $inp, #1			// dec	%esi
    765 	bl	_vpaes_schedule_round
    766 	ext	v0.16b, v6.16b, v0.16b, #8	// vpalignr	\$8,%xmm6,%xmm0,%xmm0
    767 	bl	_vpaes_schedule_mangle		// save key n
    768 	bl	_vpaes_schedule_192_smear
    769 	bl	_vpaes_schedule_mangle		// save key n+1
    770 	bl	_vpaes_schedule_round
    771 	cbz 	$inp, .Lschedule_mangle_last
    772 	bl	_vpaes_schedule_mangle		// save key n+2
    773 	bl	_vpaes_schedule_192_smear
    774 	b	.Loop_schedule_192
    775 
    776 ##
    777 ##  .aes_schedule_256
    778 ##
    779 ##  256-bit specific part of key schedule.
    780 ##
    781 ##  The structure here is very similar to the 128-bit
    782 ##  schedule, but with an additional "low side" in
    783 ##  %xmm6.  The low side's rounds are the same as the
    784 ##  high side's, except no rcon and no rotation.
    785 ##
    786 .align	4
    787 .Lschedule_256:
    788 	ld1	{v0.16b}, [$inp]		// vmovdqu	16(%rdi),%xmm0		# load key part 2 (unaligned)
    789 	bl	_vpaes_schedule_transform	// input transform
    790 	mov	$inp, #7			// mov	\$7, %esi
    791 
    792 .Loop_schedule_256:
    793 	sub	$inp, $inp, #1			// dec	%esi
    794 	bl	_vpaes_schedule_mangle		// output low result
    795 	mov	v6.16b, v0.16b			// vmovdqa	%xmm0,	%xmm6		# save cur_lo in xmm6
    796 
    797 	// high round
    798 	bl	_vpaes_schedule_round
    799 	cbz 	$inp, .Lschedule_mangle_last
    800 	bl	_vpaes_schedule_mangle
    801 
    802 	// low round. swap xmm7 and xmm6
    803 	dup	v0.4s, v0.s[3]			// vpshufd	\$0xFF,	%xmm0,	%xmm0
    804 	movi	v4.16b, #0
    805 	mov	v5.16b, v7.16b			// vmovdqa	%xmm7,	%xmm5
    806 	mov	v7.16b, v6.16b			// vmovdqa	%xmm6,	%xmm7
    807 	bl	_vpaes_schedule_low_round
    808 	mov	v7.16b, v5.16b			// vmovdqa	%xmm5,	%xmm7
    809 
    810 	b	.Loop_schedule_256
    811 
    812 ##
    813 ##  .aes_schedule_mangle_last
    814 ##
    815 ##  Mangler for last round of key schedule
    816 ##  Mangles %xmm0
    817 ##    when encrypting, outputs out(%xmm0) ^ 63
    818 ##    when decrypting, outputs unskew(%xmm0)
    819 ##
    820 ##  Always called right before return... jumps to cleanup and exits
    821 ##
    822 .align	4
    823 .Lschedule_mangle_last:
    824 	// schedule last round key from xmm0
    825 	adrp	x11, :pg_hi21:.Lk_deskew	// lea	.Lk_deskew(%rip),%r11	# prepare to deskew
    826 	add	x11, x11, :lo12:.Lk_deskew
    827 
    828 	cbnz	$dir, .Lschedule_mangle_last_dec
    829 
    830 	// encrypting
    831 	ld1	{v1.2d}, [x8]			// vmovdqa	(%r8,%r10),%xmm1
    832 	adrp	x11, :pg_hi21:.Lk_opt		// lea	.Lk_opt(%rip),	%r11		# prepare to output transform
    833 	add	x11, x11, :lo12:.Lk_opt
    834 	add	$out, $out, #32			// add	\$32,	%rdx
    835 	tbl	v0.16b, {v0.16b}, v1.16b	// vpshufb	%xmm1,	%xmm0,	%xmm0		# output permute
    836 
    837 .Lschedule_mangle_last_dec:
    838 	ld1	{v20.2d-v21.2d}, [x11]		// reload constants
    839 	sub	$out, $out, #16			// add	\$-16,	%rdx
    840 	eor	v0.16b, v0.16b, v16.16b		// vpxor	.Lk_s63(%rip),	%xmm0,	%xmm0
    841 	bl	_vpaes_schedule_transform	// output transform
    842 	st1	{v0.2d}, [$out]			// vmovdqu	%xmm0,	(%rdx)		# save last key
    843 
    844 	// cleanup
    845 	eor	v0.16b, v0.16b, v0.16b		// vpxor	%xmm0,	%xmm0,	%xmm0
    846 	eor	v1.16b, v1.16b, v1.16b		// vpxor	%xmm1,	%xmm1,	%xmm1
    847 	eor	v2.16b, v2.16b, v2.16b		// vpxor	%xmm2,	%xmm2,	%xmm2
    848 	eor	v3.16b, v3.16b, v3.16b		// vpxor	%xmm3,	%xmm3,	%xmm3
    849 	eor	v4.16b, v4.16b, v4.16b		// vpxor	%xmm4,	%xmm4,	%xmm4
    850 	eor	v5.16b, v5.16b, v5.16b		// vpxor	%xmm5,	%xmm5,	%xmm5
    851 	eor	v6.16b, v6.16b, v6.16b		// vpxor	%xmm6,	%xmm6,	%xmm6
    852 	eor	v7.16b, v7.16b, v7.16b		// vpxor	%xmm7,	%xmm7,	%xmm7
    853 	ldp	x29, x30, [sp],#16
    854 	ret
    855 .size	_vpaes_schedule_core,.-_vpaes_schedule_core
    856 
    857 ##
    858 ##  .aes_schedule_192_smear
    859 ##
    860 ##  Smear the short, low side in the 192-bit key schedule.
    861 ##
    862 ##  Inputs:
    863 ##    %xmm7: high side, b  a  x  y
    864 ##    %xmm6:  low side, d  c  0  0
    865 ##    %xmm13: 0
    866 ##
    867 ##  Outputs:
    868 ##    %xmm6: b+c+d  b+c  0  0
    869 ##    %xmm0: b+c+d  b+c  b  a
    870 ##
    871 .type	_vpaes_schedule_192_smear,%function
    872 .align	4
    873 _vpaes_schedule_192_smear:
    874 	movi	v1.16b, #0
    875 	dup	v0.4s, v7.s[3]
    876 	ins	v1.s[3], v6.s[2]	// vpshufd	\$0x80,	%xmm6,	%xmm1	# d c 0 0 -> c 0 0 0
    877 	ins	v0.s[0], v7.s[2]	// vpshufd	\$0xFE,	%xmm7,	%xmm0	# b a _ _ -> b b b a
    878 	eor	v6.16b, v6.16b, v1.16b	// vpxor	%xmm1,	%xmm6,	%xmm6	# -> c+d c 0 0
    879 	eor	v1.16b, v1.16b, v1.16b	// vpxor	%xmm1,	%xmm1,	%xmm1
    880 	eor	v6.16b, v6.16b, v0.16b	// vpxor	%xmm0,	%xmm6,	%xmm6	# -> b+c+d b+c b a
    881 	mov	v0.16b, v6.16b		// vmovdqa	%xmm6,	%xmm0
    882 	ins	v6.d[0], v1.d[0]	// vmovhlps	%xmm1,	%xmm6,	%xmm6	# clobber low side with zeros
    883 	ret
    884 .size	_vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear
    885 
    886 ##
    887 ##  .aes_schedule_round
    888 ##
    889 ##  Runs one main round of the key schedule on %xmm0, %xmm7
    890 ##
    891 ##  Specifically, runs subbytes on the high dword of %xmm0
    892 ##  then rotates it by one byte and xors into the low dword of
    893 ##  %xmm7.
    894 ##
    895 ##  Adds rcon from low byte of %xmm8, then rotates %xmm8 for
    896 ##  next rcon.
    897 ##
    898 ##  Smears the dwords of %xmm7 by xoring the low into the
    899 ##  second low, result into third, result into highest.
    900 ##
    901 ##  Returns results in %xmm7 = %xmm0.
    902 ##  Clobbers %xmm1-%xmm4, %r11.
    903 ##
    904 .type	_vpaes_schedule_round,%function
    905 .align	4
    906 _vpaes_schedule_round:
    907 	// extract rcon from xmm8
    908 	movi	v4.16b, #0			// vpxor	%xmm4,	%xmm4,	%xmm4
    909 	ext	v1.16b, $rcon, v4.16b, #15	// vpalignr	\$15,	%xmm8,	%xmm4,	%xmm1
    910 	ext	$rcon, $rcon, $rcon, #15	// vpalignr	\$15,	%xmm8,	%xmm8,	%xmm8
    911 	eor	v7.16b, v7.16b, v1.16b		// vpxor	%xmm1,	%xmm7,	%xmm7
    912 
    913 	// rotate
    914 	dup	v0.4s, v0.s[3]			// vpshufd	\$0xFF,	%xmm0,	%xmm0
    915 	ext	v0.16b, v0.16b, v0.16b, #1	// vpalignr	\$1,	%xmm0,	%xmm0,	%xmm0
    916 
    917 	// fall through...
    918 
    919 	// low round: same as high round, but no rotation and no rcon.
    920 _vpaes_schedule_low_round:
    921 	// smear xmm7
    922 	ext	v1.16b, v4.16b, v7.16b, #12	// vpslldq	\$4,	%xmm7,	%xmm1
    923 	eor	v7.16b, v7.16b, v1.16b		// vpxor	%xmm1,	%xmm7,	%xmm7
    924 	ext	v4.16b, v4.16b, v7.16b, #8	// vpslldq	\$8,	%xmm7,	%xmm4
    925 
    926 	// subbytes
    927 	and	v1.16b, v0.16b, v17.16b		// vpand	%xmm9,	%xmm0,	%xmm1		# 0 = k
    928 	ushr	v0.16b, v0.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0		# 1 = i
    929 	 eor	v7.16b, v7.16b, v4.16b		// vpxor	%xmm4,	%xmm7,	%xmm7
    930 	tbl	v2.16b, {$invhi}, v1.16b	// vpshufb	%xmm1,	%xmm11,	%xmm2		# 2 = a/k
    931 	eor	v1.16b, v1.16b, v0.16b		// vpxor	%xmm0,	%xmm1,	%xmm1		# 0 = j
    932 	tbl	v3.16b, {$invlo}, v0.16b	// vpshufb	%xmm0, 	%xmm10,	%xmm3		# 3 = 1/i
    933 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3		# 3 = iak = 1/i + a/k
    934 	tbl	v4.16b, {$invlo}, v1.16b	// vpshufb	%xmm1,	%xmm10,	%xmm4		# 4 = 1/j
    935 	 eor	v7.16b, v7.16b, v16.16b		// vpxor	.Lk_s63(%rip),	%xmm7,	%xmm7
    936 	tbl	v3.16b, {$invlo}, v3.16b	// vpshufb	%xmm3,	%xmm10,	%xmm3		# 2 = 1/iak
    937 	eor	v4.16b, v4.16b, v2.16b		// vpxor	%xmm2,	%xmm4,	%xmm4		# 4 = jak = 1/j + a/k
    938 	tbl	v2.16b, {$invlo}, v4.16b	// vpshufb	%xmm4,	%xmm10,	%xmm2		# 3 = 1/jak
    939 	eor	v3.16b, v3.16b, v1.16b		// vpxor	%xmm1,	%xmm3,	%xmm3		# 2 = io
    940 	eor	v2.16b, v2.16b, v0.16b		// vpxor	%xmm0,	%xmm2,	%xmm2		# 3 = jo
    941 	tbl	v4.16b, {v23.16b}, v3.16b	// vpshufb	%xmm3,	%xmm13,	%xmm4		# 4 = sbou
    942 	tbl	v1.16b, {v22.16b}, v2.16b	// vpshufb	%xmm2,	%xmm12,	%xmm1		# 0 = sb1t
    943 	eor	v1.16b, v1.16b, v4.16b		// vpxor	%xmm4,	%xmm1,	%xmm1		# 0 = sbox output
    944 
    945 	// add in smeared stuff
    946 	eor	v0.16b, v1.16b, v7.16b		// vpxor	%xmm7,	%xmm1,	%xmm0
    947 	eor	v7.16b, v1.16b, v7.16b		// vmovdqa	%xmm0,	%xmm7
    948 	ret
    949 .size	_vpaes_schedule_round,.-_vpaes_schedule_round
    950 
    951 ##
    952 ##  .aes_schedule_transform
    953 ##
    954 ##  Linear-transform %xmm0 according to tables at (%r11)
    955 ##
    956 ##  Requires that %xmm9 = 0x0F0F... as in preheat
    957 ##  Output in %xmm0
    958 ##  Clobbers %xmm1, %xmm2
    959 ##
    960 .type	_vpaes_schedule_transform,%function
    961 .align	4
    962 _vpaes_schedule_transform:
    963 	and	v1.16b, v0.16b, v17.16b		// vpand	%xmm9,	%xmm0,	%xmm1
    964 	ushr	v0.16b, v0.16b, #4		// vpsrlb	\$4,	%xmm0,	%xmm0
    965 						// vmovdqa	(%r11),	%xmm2 	# lo
    966 	tbl	v2.16b, {$iptlo}, v1.16b	// vpshufb	%xmm1,	%xmm2,	%xmm2
    967 						// vmovdqa	16(%r11),	%xmm1 # hi
    968 	tbl	v0.16b, {$ipthi}, v0.16b	// vpshufb	%xmm0,	%xmm1,	%xmm0
    969 	eor	v0.16b, v0.16b, v2.16b		// vpxor	%xmm2,	%xmm0,	%xmm0
    970 	ret
    971 .size	_vpaes_schedule_transform,.-_vpaes_schedule_transform
    972 
    973 ##
    974 ##  .aes_schedule_mangle
    975 ##
    976 ##  Mangle xmm0 from (basis-transformed) standard version
    977 ##  to our version.
    978 ##
    979 ##  On encrypt,
    980 ##    xor with 0x63
    981 ##    multiply by circulant 0,1,1,1
    982 ##    apply shiftrows transform
    983 ##
    984 ##  On decrypt,
    985 ##    xor with 0x63
    986 ##    multiply by "inverse mixcolumns" circulant E,B,D,9
    987 ##    deskew
    988 ##    apply shiftrows transform
    989 ##
    990 ##
    991 ##  Writes out to (%rdx), and increments or decrements it
    992 ##  Keeps track of round number mod 4 in %r8
    993 ##  Preserves xmm0
    994 ##  Clobbers xmm1-xmm5
    995 ##
    996 .type	_vpaes_schedule_mangle,%function
    997 .align	4
    998 _vpaes_schedule_mangle:
    999 	mov	v4.16b, v0.16b			// vmovdqa	%xmm0,	%xmm4	# save xmm0 for later
   1000 						// vmovdqa	.Lk_mc_forward(%rip),%xmm5
   1001 	cbnz	$dir, .Lschedule_mangle_dec
   1002 
   1003 	// encrypting
   1004 	eor	v4.16b, v0.16b, v16.16b		// vpxor	.Lk_s63(%rip),	%xmm0,	%xmm4
   1005 	add	$out, $out, #16			// add	\$16,	%rdx
   1006 	tbl	v4.16b, {v4.16b}, v9.16b	// vpshufb	%xmm5,	%xmm4,	%xmm4
   1007 	tbl	v1.16b, {v4.16b}, v9.16b	// vpshufb	%xmm5,	%xmm4,	%xmm1
   1008 	tbl	v3.16b, {v1.16b}, v9.16b	// vpshufb	%xmm5,	%xmm1,	%xmm3
   1009 	eor	v4.16b, v4.16b, v1.16b		// vpxor	%xmm1,	%xmm4,	%xmm4
   1010 	ld1	{v1.2d}, [x8]			// vmovdqa	(%r8,%r10),	%xmm1
   1011 	eor	v3.16b, v3.16b, v4.16b		// vpxor	%xmm4,	%xmm3,	%xmm3
   1012 
   1013 	b	.Lschedule_mangle_both
   1014 .align	4
   1015 .Lschedule_mangle_dec:
   1016 	// inverse mix columns
   1017 						// lea	.Lk_dksd(%rip),%r11
   1018 	ushr	v1.16b, v4.16b, #4		// vpsrlb	\$4,	%xmm4,	%xmm1	# 1 = hi
   1019 	and	v4.16b, v4.16b, v17.16b		// vpand	%xmm9,	%xmm4,	%xmm4	# 4 = lo
   1020 
   1021 						// vmovdqa	0x00(%r11),	%xmm2
   1022 	tbl	v2.16b, {v24.16b}, v4.16b	// vpshufb	%xmm4,	%xmm2,	%xmm2
   1023 						// vmovdqa	0x10(%r11),	%xmm3
   1024 	tbl	v3.16b,	{v25.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm3
   1025 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3
   1026 	tbl	v3.16b, {v3.16b}, v9.16b	// vpshufb	%xmm5,	%xmm3,	%xmm3
   1027 
   1028 						// vmovdqa	0x20(%r11),	%xmm2
   1029 	tbl	v2.16b, {v26.16b}, v4.16b	// vpshufb	%xmm4,	%xmm2,	%xmm2
   1030 	eor	v2.16b, v2.16b, v3.16b		// vpxor	%xmm3,	%xmm2,	%xmm2
   1031 						// vmovdqa	0x30(%r11),	%xmm3
   1032 	tbl	v3.16b, {v27.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm3
   1033 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3
   1034 	tbl	v3.16b, {v3.16b}, v9.16b	// vpshufb	%xmm5,	%xmm3,	%xmm3
   1035 
   1036 						// vmovdqa	0x40(%r11),	%xmm2
   1037 	tbl	v2.16b, {v28.16b}, v4.16b	// vpshufb	%xmm4,	%xmm2,	%xmm2
   1038 	eor	v2.16b, v2.16b, v3.16b		// vpxor	%xmm3,	%xmm2,	%xmm2
   1039 						// vmovdqa	0x50(%r11),	%xmm3
   1040 	tbl	v3.16b, {v29.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm3
   1041 	eor	v3.16b, v3.16b, v2.16b		// vpxor	%xmm2,	%xmm3,	%xmm3
   1042 
   1043 						// vmovdqa	0x60(%r11),	%xmm2
   1044 	tbl	v2.16b, {v30.16b}, v4.16b	// vpshufb	%xmm4,	%xmm2,	%xmm2
   1045 	tbl	v3.16b, {v3.16b}, v9.16b	// vpshufb	%xmm5,	%xmm3,	%xmm3
   1046 						// vmovdqa	0x70(%r11),	%xmm4
   1047 	tbl	v4.16b, {v31.16b}, v1.16b	// vpshufb	%xmm1,	%xmm4,	%xmm4
   1048 	ld1	{v1.2d}, [x8]			// vmovdqa	(%r8,%r10),	%xmm1
   1049 	eor	v2.16b, v2.16b, v3.16b		// vpxor	%xmm3,	%xmm2,	%xmm2
   1050 	eor	v3.16b, v4.16b, v2.16b		// vpxor	%xmm2,	%xmm4,	%xmm3
   1051 
   1052 	sub	$out, $out, #16			// add	\$-16,	%rdx
   1053 
   1054 .Lschedule_mangle_both:
   1055 	tbl	v3.16b, {v3.16b}, v1.16b	// vpshufb	%xmm1,	%xmm3,	%xmm3
   1056 	add	x8, x8, #64-16			// add	\$-16,	%r8
   1057 	and	x8, x8, #~(1<<6)		// and	\$0x30,	%r8
   1058 	st1	{v3.2d}, [$out]			// vmovdqu	%xmm3,	(%rdx)
   1059 	ret
   1060 .size	_vpaes_schedule_mangle,.-_vpaes_schedule_mangle
   1061 
   1062 .globl	vpaes_set_encrypt_key
   1063 .type	vpaes_set_encrypt_key,%function
   1064 .align	4
   1065 vpaes_set_encrypt_key:
   1066 	stp	x29,x30,[sp,#-16]!
   1067 	add	x29,sp,#0
   1068 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1069 
   1070 	lsr	w9, $bits, #5		// shr	\$5,%eax
   1071 	add	w9, w9, #5		// \$5,%eax
   1072 	str	w9, [$out,#240]		// mov	%eax,240(%rdx)	# AES_KEY->rounds = nbits/32+5;
   1073 
   1074 	mov	$dir, #0		// mov	\$0,%ecx
   1075 	mov	x8, #0x30		// mov	\$0x30,%r8d
   1076 	bl	_vpaes_schedule_core
   1077 	eor	x0, x0, x0
   1078 
   1079 	ldp	d8,d9,[sp],#16
   1080 	ldp	x29,x30,[sp],#16
   1081 	ret
   1082 .size	vpaes_set_encrypt_key,.-vpaes_set_encrypt_key
   1083 
   1084 .globl	vpaes_set_decrypt_key
   1085 .type	vpaes_set_decrypt_key,%function
   1086 .align	4
   1087 vpaes_set_decrypt_key:
   1088 	stp	x29,x30,[sp,#-16]!
   1089 	add	x29,sp,#0
   1090 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1091 
   1092 	lsr	w9, $bits, #5		// shr	\$5,%eax
   1093 	add	w9, w9, #5		// \$5,%eax
   1094 	str	w9, [$out,#240]		// mov	%eax,240(%rdx)	# AES_KEY->rounds = nbits/32+5;
   1095 	lsl	w9, w9, #4		// shl	\$4,%eax
   1096 	add	$out, $out, #16		// lea	16(%rdx,%rax),%rdx
   1097 	add	$out, $out, x9
   1098 
   1099 	mov	$dir, #1		// mov	\$1,%ecx
   1100 	lsr	w8, $bits, #1		// shr	\$1,%r8d
   1101 	and	x8, x8, #32		// and	\$32,%r8d
   1102 	eor	x8, x8, #32		// xor	\$32,%r8d	# nbits==192?0:32
   1103 	bl	_vpaes_schedule_core
   1104 
   1105 	ldp	d8,d9,[sp],#16
   1106 	ldp	x29,x30,[sp],#16
   1107 	ret
   1108 .size	vpaes_set_decrypt_key,.-vpaes_set_decrypt_key
   1109 ___
   1110 }
   1111 {
   1112 my ($inp,$out,$len,$key,$ivec,$dir) = map("x$_",(0..5));
   1113 
   1114 $code.=<<___;
   1115 .globl	vpaes_cbc_encrypt
   1116 .type	vpaes_cbc_encrypt,%function
   1117 .align	4
   1118 vpaes_cbc_encrypt:
   1119 	cbz	$len, .Lcbc_abort
   1120 	cmp	w5, #0			// check direction
   1121 	b.eq	vpaes_cbc_decrypt
   1122 
   1123 	stp	x29,x30,[sp,#-16]!
   1124 	add	x29,sp,#0
   1125 
   1126 	mov	x17, $len		// reassign
   1127 	mov	x2,  $key		// reassign
   1128 
   1129 	ld1	{v0.16b}, [$ivec]	// load ivec
   1130 	bl	_vpaes_encrypt_preheat
   1131 	b	.Lcbc_enc_loop
   1132 
   1133 .align	4
   1134 .Lcbc_enc_loop:
   1135 	ld1	{v7.16b}, [$inp],#16	// load input
   1136 	eor	v7.16b, v7.16b, v0.16b	// xor with ivec
   1137 	bl	_vpaes_encrypt_core
   1138 	st1	{v0.16b}, [$out],#16	// save output
   1139 	subs	x17, x17, #16
   1140 	b.hi	.Lcbc_enc_loop
   1141 
   1142 	st1	{v0.16b}, [$ivec]	// write ivec
   1143 
   1144 	ldp	x29,x30,[sp],#16
   1145 .Lcbc_abort:
   1146 	ret
   1147 .size	vpaes_cbc_encrypt,.-vpaes_cbc_encrypt
   1148 
   1149 .type	vpaes_cbc_decrypt,%function
   1150 .align	4
   1151 vpaes_cbc_decrypt:
   1152 	stp	x29,x30,[sp,#-16]!
   1153 	add	x29,sp,#0
   1154 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1155 	stp	d10,d11,[sp,#-16]!
   1156 	stp	d12,d13,[sp,#-16]!
   1157 	stp	d14,d15,[sp,#-16]!
   1158 
   1159 	mov	x17, $len		// reassign
   1160 	mov	x2,  $key		// reassign
   1161 	ld1	{v6.16b}, [$ivec]	// load ivec
   1162 	bl	_vpaes_decrypt_preheat
   1163 	tst	x17, #16
   1164 	b.eq	.Lcbc_dec_loop2x
   1165 
   1166 	ld1	{v7.16b}, [$inp], #16	// load input
   1167 	bl	_vpaes_decrypt_core
   1168 	eor	v0.16b, v0.16b, v6.16b	// xor with ivec
   1169 	orr	v6.16b, v7.16b, v7.16b	// next ivec value
   1170 	st1	{v0.16b}, [$out], #16
   1171 	subs	x17, x17, #16
   1172 	b.ls	.Lcbc_dec_done
   1173 
   1174 .align	4
   1175 .Lcbc_dec_loop2x:
   1176 	ld1	{v14.16b,v15.16b}, [$inp], #32
   1177 	bl	_vpaes_decrypt_2x
   1178 	eor	v0.16b, v0.16b, v6.16b	// xor with ivec
   1179 	eor	v1.16b, v1.16b, v14.16b
   1180 	orr	v6.16b, v15.16b, v15.16b
   1181 	st1	{v0.16b,v1.16b}, [$out], #32
   1182 	subs	x17, x17, #32
   1183 	b.hi	.Lcbc_dec_loop2x
   1184 
   1185 .Lcbc_dec_done:
   1186 	st1	{v6.16b}, [$ivec]
   1187 
   1188 	ldp	d14,d15,[sp],#16
   1189 	ldp	d12,d13,[sp],#16
   1190 	ldp	d10,d11,[sp],#16
   1191 	ldp	d8,d9,[sp],#16
   1192 	ldp	x29,x30,[sp],#16
   1193 	ret
   1194 .size	vpaes_cbc_decrypt,.-vpaes_cbc_decrypt
   1195 ___
   1196 # We omit vpaes_ecb_* in BoringSSL. They are unused.
   1197 if (0) {
   1198 $code.=<<___;
   1199 .globl	vpaes_ecb_encrypt
   1200 .type	vpaes_ecb_encrypt,%function
   1201 .align	4
   1202 vpaes_ecb_encrypt:
   1203 	stp	x29,x30,[sp,#-16]!
   1204 	add	x29,sp,#0
   1205 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1206 	stp	d10,d11,[sp,#-16]!
   1207 	stp	d12,d13,[sp,#-16]!
   1208 	stp	d14,d15,[sp,#-16]!
   1209 
   1210 	mov	x17, $len
   1211 	mov	x2,  $key
   1212 	bl	_vpaes_encrypt_preheat
   1213 	tst	x17, #16
   1214 	b.eq	.Lecb_enc_loop
   1215 
   1216 	ld1	{v7.16b}, [$inp],#16
   1217 	bl	_vpaes_encrypt_core
   1218 	st1	{v0.16b}, [$out],#16
   1219 	subs	x17, x17, #16
   1220 	b.ls	.Lecb_enc_done
   1221 
   1222 .align	4
   1223 .Lecb_enc_loop:
   1224 	ld1	{v14.16b,v15.16b}, [$inp], #32
   1225 	bl	_vpaes_encrypt_2x
   1226 	st1	{v0.16b,v1.16b}, [$out], #32
   1227 	subs	x17, x17, #32
   1228 	b.hi	.Lecb_enc_loop
   1229 
   1230 .Lecb_enc_done:
   1231 	ldp	d14,d15,[sp],#16
   1232 	ldp	d12,d13,[sp],#16
   1233 	ldp	d10,d11,[sp],#16
   1234 	ldp	d8,d9,[sp],#16
   1235 	ldp	x29,x30,[sp],#16
   1236 	ret
   1237 .size	vpaes_ecb_encrypt,.-vpaes_ecb_encrypt
   1238 
   1239 .globl	vpaes_ecb_decrypt
   1240 .type	vpaes_ecb_decrypt,%function
   1241 .align	4
   1242 vpaes_ecb_decrypt:
   1243 	stp	x29,x30,[sp,#-16]!
   1244 	add	x29,sp,#0
   1245 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1246 	stp	d10,d11,[sp,#-16]!
   1247 	stp	d12,d13,[sp,#-16]!
   1248 	stp	d14,d15,[sp,#-16]!
   1249 
   1250 	mov	x17, $len
   1251 	mov	x2,  $key
   1252 	bl	_vpaes_decrypt_preheat
   1253 	tst	x17, #16
   1254 	b.eq	.Lecb_dec_loop
   1255 
   1256 	ld1	{v7.16b}, [$inp],#16
   1257 	bl	_vpaes_encrypt_core
   1258 	st1	{v0.16b}, [$out],#16
   1259 	subs	x17, x17, #16
   1260 	b.ls	.Lecb_dec_done
   1261 
   1262 .align	4
   1263 .Lecb_dec_loop:
   1264 	ld1	{v14.16b,v15.16b}, [$inp], #32
   1265 	bl	_vpaes_decrypt_2x
   1266 	st1	{v0.16b,v1.16b}, [$out], #32
   1267 	subs	x17, x17, #32
   1268 	b.hi	.Lecb_dec_loop
   1269 
   1270 .Lecb_dec_done:
   1271 	ldp	d14,d15,[sp],#16
   1272 	ldp	d12,d13,[sp],#16
   1273 	ldp	d10,d11,[sp],#16
   1274 	ldp	d8,d9,[sp],#16
   1275 	ldp	x29,x30,[sp],#16
   1276 	ret
   1277 .size	vpaes_ecb_decrypt,.-vpaes_ecb_decrypt
   1278 ___
   1279 }
   1280 
   1281 my ($ctr, $ctr_tmp) = ("w6", "w7");
   1282 
   1283 # void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len,
   1284 #                                 const AES_KEY *key, const uint8_t ivec[16]);
   1285 $code.=<<___;
   1286 .globl	vpaes_ctr32_encrypt_blocks
   1287 .type	vpaes_ctr32_encrypt_blocks,%function
   1288 .align	4
   1289 vpaes_ctr32_encrypt_blocks:
   1290 	stp	x29,x30,[sp,#-16]!
   1291 	add	x29,sp,#0
   1292 	stp	d8,d9,[sp,#-16]!	// ABI spec says so
   1293 	stp	d10,d11,[sp,#-16]!
   1294 	stp	d12,d13,[sp,#-16]!
   1295 	stp	d14,d15,[sp,#-16]!
   1296 
   1297 	cbz	$len, .Lctr32_done
   1298 
   1299 	// Note, unlike the other functions, $len here is measured in blocks,
   1300 	// not bytes.
   1301 	mov	x17, $len
   1302 	mov	x2,  $key
   1303 
   1304 	// Load the IV and counter portion.
   1305 	ldr	$ctr, [$ivec, #12]
   1306 	ld1	{v7.16b}, [$ivec]
   1307 
   1308 	bl	_vpaes_encrypt_preheat
   1309 	tst	x17, #1
   1310 	rev	$ctr, $ctr		// The counter is big-endian.
   1311 	b.eq	.Lctr32_prep_loop
   1312 
   1313 	// Handle one block so the remaining block count is even for
   1314 	// _vpaes_encrypt_2x.
   1315 	ld1	{v6.16b}, [$inp], #16	// Load input ahead of time
   1316 	bl	_vpaes_encrypt_core
   1317 	eor	v0.16b, v0.16b, v6.16b	// XOR input and result
   1318 	st1	{v0.16b}, [$out], #16
   1319 	subs	x17, x17, #1
   1320 	// Update the counter.
   1321 	add	$ctr, $ctr, #1
   1322 	rev	$ctr_tmp, $ctr
   1323 	mov	v7.s[3], $ctr_tmp
   1324 	b.ls	.Lctr32_done
   1325 
   1326 .Lctr32_prep_loop:
   1327 	// _vpaes_encrypt_core takes its input from v7, while _vpaes_encrypt_2x
   1328 	// uses v14 and v15.
   1329 	mov	v15.16b, v7.16b
   1330 	mov	v14.16b, v7.16b
   1331 	add	$ctr, $ctr, #1
   1332 	rev	$ctr_tmp, $ctr
   1333 	mov	v15.s[3], $ctr_tmp
   1334 
   1335 .Lctr32_loop:
   1336 	ld1	{v6.16b,v7.16b}, [$inp], #32	// Load input ahead of time
   1337 	bl	_vpaes_encrypt_2x
   1338 	eor	v0.16b, v0.16b, v6.16b		// XOR input and result
   1339 	eor	v1.16b, v1.16b, v7.16b		// XOR input and result (#2)
   1340 	st1	{v0.16b,v1.16b}, [$out], #32
   1341 	subs	x17, x17, #2
   1342 	// Update the counter.
   1343 	add	$ctr_tmp, $ctr, #1
   1344 	add	$ctr, $ctr, #2
   1345 	rev	$ctr_tmp, $ctr_tmp
   1346 	mov	v14.s[3], $ctr_tmp
   1347 	rev	$ctr_tmp, $ctr
   1348 	mov	v15.s[3], $ctr_tmp
   1349 	b.hi	.Lctr32_loop
   1350 
   1351 .Lctr32_done:
   1352 	ldp	d14,d15,[sp],#16
   1353 	ldp	d12,d13,[sp],#16
   1354 	ldp	d10,d11,[sp],#16
   1355 	ldp	d8,d9,[sp],#16
   1356 	ldp	x29,x30,[sp],#16
   1357 	ret
   1358 .size	vpaes_ctr32_encrypt_blocks,.-vpaes_ctr32_encrypt_blocks
   1359 ___
   1360 }
   1361 
   1362 print $code;
   1363 
   1364 close STDOUT;
   1365