1 #!/usr/bin/env perl 2 3 ################################################################### 4 ### AES-128 [originally in CTR mode] ### 5 ### bitsliced implementation for Intel Core 2 processors ### 6 ### requires support of SSE extensions up to SSSE3 ### 7 ### Author: Emilia Ksper and Peter Schwabe ### 8 ### Date: 2009-03-19 ### 9 ### Public domain ### 10 ### ### 11 ### See http://homes.esat.kuleuven.be/~ekasper/#software for ### 12 ### further information. ### 13 ################################################################### 14 # 15 # September 2011. 16 # 17 # Started as transliteration to "perlasm" the original code has 18 # undergone following changes: 19 # 20 # - code was made position-independent; 21 # - rounds were folded into a loop resulting in >5x size reduction 22 # from 12.5KB to 2.2KB; 23 # - above was possibile thanks to mixcolumns() modification that 24 # allowed to feed its output back to aesenc[last], this was 25 # achieved at cost of two additional inter-registers moves; 26 # - some instruction reordering and interleaving; 27 # - this module doesn't implement key setup subroutine, instead it 28 # relies on conversion of "conventional" key schedule as returned 29 # by AES_set_encrypt_key (see discussion below); 30 # - first and last round keys are treated differently, which allowed 31 # to skip one shiftrows(), reduce bit-sliced key schedule and 32 # speed-up conversion by 22%; 33 # - support for 192- and 256-bit keys was added; 34 # 35 # Resulting performance in CPU cycles spent to encrypt one byte out 36 # of 4096-byte buffer with 128-bit key is: 37 # 38 # Emilia's this(*) difference 39 # 40 # Core 2 9.30 8.69 +7% 41 # Nehalem(**) 7.63 6.98 +9% 42 # Atom 17.1 17.4 -2%(***) 43 # 44 # (*) Comparison is not completely fair, because "this" is ECB, 45 # i.e. no extra processing such as counter values calculation 46 # and xor-ing input as in Emilia's CTR implementation is 47 # performed. However, the CTR calculations stand for not more 48 # than 1% of total time, so comparison is *rather* fair. 49 # 50 # (**) Results were collected on Westmere, which is considered to 51 # be equivalent to Nehalem for this code. 52 # 53 # (***) Slowdown on Atom is rather strange per se, because original 54 # implementation has a number of 9+-bytes instructions, which 55 # are bad for Atom front-end, and which I eliminated completely. 56 # In attempt to address deterioration sbox() was tested in FP 57 # SIMD "domain" (movaps instead of movdqa, xorps instead of 58 # pxor, etc.). While it resulted in nominal 4% improvement on 59 # Atom, it hurted Westmere by more than 2x factor. 60 # 61 # As for key schedule conversion subroutine. Interface to OpenSSL 62 # relies on per-invocation on-the-fly conversion. This naturally 63 # has impact on performance, especially for short inputs. Conversion 64 # time in CPU cycles and its ratio to CPU cycles spent in 8x block 65 # function is: 66 # 67 # conversion conversion/8x block 68 # Core 2 240 0.22 69 # Nehalem 180 0.20 70 # Atom 430 0.19 71 # 72 # The ratio values mean that 128-byte blocks will be processed 73 # 16-18% slower, 256-byte blocks - 9-10%, 384-byte blocks - 6-7%, 74 # etc. Then keep in mind that input sizes not divisible by 128 are 75 # *effectively* slower, especially shortest ones, e.g. consecutive 76 # 144-byte blocks are processed 44% slower than one would expect, 77 # 272 - 29%, 400 - 22%, etc. Yet, despite all these "shortcomings" 78 # it's still faster than ["hyper-threading-safe" code path in] 79 # aes-x86_64.pl on all lengths above 64 bytes... 80 # 81 # October 2011. 82 # 83 # Add decryption procedure. Performance in CPU cycles spent to decrypt 84 # one byte out of 4096-byte buffer with 128-bit key is: 85 # 86 # Core 2 9.83 87 # Nehalem 7.74 88 # Atom 19.0 89 # 90 # November 2011. 91 # 92 # Add bsaes_xts_[en|de]crypt. Less-than-80-bytes-block performance is 93 # suboptimal, but XTS is meant to be used with larger blocks... 94 # 95 # <appro (at] openssl.org> 96 97 $flavour = shift; 98 $output = shift; 99 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 100 101 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 102 103 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 104 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 105 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 106 die "can't locate x86_64-xlate.pl"; 107 108 open OUT,"| \"$^X\" $xlate $flavour $output"; 109 *STDOUT=*OUT; 110 111 my ($inp,$out,$len,$key,$ivp)=("%rdi","%rsi","%rdx","%rcx"); 112 my @XMM=map("%xmm$_",(15,0..14)); # best on Atom, +10% over (0..15) 113 my $ecb=0; # suppress unreferenced ECB subroutines, spare some space... 114 115 { 116 my ($key,$rounds,$const)=("%rax","%r10d","%r11"); 117 118 sub Sbox { 119 # input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb 120 # output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb 121 my @b=@_[0..7]; 122 my @t=@_[8..11]; 123 my @s=@_[12..15]; 124 &InBasisChange (@b); 125 &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); 126 &OutBasisChange (@b[7,1,4,2,6,5,0,3]); 127 } 128 129 sub InBasisChange { 130 # input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb 131 # output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb 132 my @b=@_[0..7]; 133 $code.=<<___; 134 pxor @b[6], @b[5] 135 pxor @b[1], @b[2] 136 pxor @b[0], @b[3] 137 pxor @b[2], @b[6] 138 pxor @b[0], @b[5] 139 140 pxor @b[3], @b[6] 141 pxor @b[7], @b[3] 142 pxor @b[5], @b[7] 143 pxor @b[4], @b[3] 144 pxor @b[5], @b[4] 145 pxor @b[1], @b[3] 146 147 pxor @b[7], @b[2] 148 pxor @b[5], @b[1] 149 ___ 150 } 151 152 sub OutBasisChange { 153 # input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb 154 # output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb 155 my @b=@_[0..7]; 156 $code.=<<___; 157 pxor @b[6], @b[0] 158 pxor @b[4], @b[1] 159 pxor @b[0], @b[2] 160 pxor @b[6], @b[4] 161 pxor @b[1], @b[6] 162 163 pxor @b[5], @b[1] 164 pxor @b[3], @b[5] 165 pxor @b[7], @b[3] 166 pxor @b[5], @b[7] 167 pxor @b[5], @b[2] 168 169 pxor @b[7], @b[4] 170 ___ 171 } 172 173 sub InvSbox { 174 # input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb 175 # output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb 176 my @b=@_[0..7]; 177 my @t=@_[8..11]; 178 my @s=@_[12..15]; 179 &InvInBasisChange (@b); 180 &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); 181 &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); 182 } 183 184 sub InvInBasisChange { # OutBasisChange in reverse 185 my @b=@_[5,1,2,6,3,7,0,4]; 186 $code.=<<___ 187 pxor @b[7], @b[4] 188 189 pxor @b[5], @b[7] 190 pxor @b[5], @b[2] 191 pxor @b[7], @b[3] 192 pxor @b[3], @b[5] 193 pxor @b[5], @b[1] 194 195 pxor @b[1], @b[6] 196 pxor @b[0], @b[2] 197 pxor @b[6], @b[4] 198 pxor @b[6], @b[0] 199 pxor @b[4], @b[1] 200 ___ 201 } 202 203 sub InvOutBasisChange { # InBasisChange in reverse 204 my @b=@_[2,5,7,3,6,1,0,4]; 205 $code.=<<___; 206 pxor @b[5], @b[1] 207 pxor @b[7], @b[2] 208 209 pxor @b[1], @b[3] 210 pxor @b[5], @b[4] 211 pxor @b[5], @b[7] 212 pxor @b[4], @b[3] 213 pxor @b[0], @b[5] 214 pxor @b[7], @b[3] 215 pxor @b[2], @b[6] 216 pxor @b[1], @b[2] 217 pxor @b[3], @b[6] 218 219 pxor @b[0], @b[3] 220 pxor @b[6], @b[5] 221 ___ 222 } 223 224 sub Mul_GF4 { 225 #;************************************************************* 226 #;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * 227 #;************************************************************* 228 my ($x0,$x1,$y0,$y1,$t0)=@_; 229 $code.=<<___; 230 movdqa $y0, $t0 231 pxor $y1, $t0 232 pand $x0, $t0 233 pxor $x1, $x0 234 pand $y0, $x1 235 pand $y1, $x0 236 pxor $x1, $x0 237 pxor $t0, $x1 238 ___ 239 } 240 241 sub Mul_GF4_N { # not used, see next subroutine 242 # multiply and scale by N 243 my ($x0,$x1,$y0,$y1,$t0)=@_; 244 $code.=<<___; 245 movdqa $y0, $t0 246 pxor $y1, $t0 247 pand $x0, $t0 248 pxor $x1, $x0 249 pand $y0, $x1 250 pand $y1, $x0 251 pxor $x0, $x1 252 pxor $t0, $x0 253 ___ 254 } 255 256 sub Mul_GF4_N_GF4 { 257 # interleaved Mul_GF4_N and Mul_GF4 258 my ($x0,$x1,$y0,$y1,$t0, 259 $x2,$x3,$y2,$y3,$t1)=@_; 260 $code.=<<___; 261 movdqa $y0, $t0 262 movdqa $y2, $t1 263 pxor $y1, $t0 264 pxor $y3, $t1 265 pand $x0, $t0 266 pand $x2, $t1 267 pxor $x1, $x0 268 pxor $x3, $x2 269 pand $y0, $x1 270 pand $y2, $x3 271 pand $y1, $x0 272 pand $y3, $x2 273 pxor $x0, $x1 274 pxor $x3, $x2 275 pxor $t0, $x0 276 pxor $t1, $x3 277 ___ 278 } 279 sub Mul_GF16_2 { 280 my @x=@_[0..7]; 281 my @y=@_[8..11]; 282 my @t=@_[12..15]; 283 $code.=<<___; 284 movdqa @x[0], @t[0] 285 movdqa @x[1], @t[1] 286 ___ 287 &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2]); 288 $code.=<<___; 289 pxor @x[2], @t[0] 290 pxor @x[3], @t[1] 291 pxor @y[2], @y[0] 292 pxor @y[3], @y[1] 293 ___ 294 Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], 295 @x[2], @x[3], @y[2], @y[3], @t[2]); 296 $code.=<<___; 297 pxor @t[0], @x[0] 298 pxor @t[0], @x[2] 299 pxor @t[1], @x[1] 300 pxor @t[1], @x[3] 301 302 movdqa @x[4], @t[0] 303 movdqa @x[5], @t[1] 304 pxor @x[6], @t[0] 305 pxor @x[7], @t[1] 306 ___ 307 &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], 308 @x[6], @x[7], @y[2], @y[3], @t[2]); 309 $code.=<<___; 310 pxor @y[2], @y[0] 311 pxor @y[3], @y[1] 312 ___ 313 &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[3]); 314 $code.=<<___; 315 pxor @t[0], @x[4] 316 pxor @t[0], @x[6] 317 pxor @t[1], @x[5] 318 pxor @t[1], @x[7] 319 ___ 320 } 321 sub Inv_GF256 { 322 #;******************************************************************** 323 #;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * 324 #;******************************************************************** 325 my @x=@_[0..7]; 326 my @t=@_[8..11]; 327 my @s=@_[12..15]; 328 # direct optimizations from hardware 329 $code.=<<___; 330 movdqa @x[4], @t[3] 331 movdqa @x[5], @t[2] 332 movdqa @x[1], @t[1] 333 movdqa @x[7], @s[1] 334 movdqa @x[0], @s[0] 335 336 pxor @x[6], @t[3] 337 pxor @x[7], @t[2] 338 pxor @x[3], @t[1] 339 movdqa @t[3], @s[2] 340 pxor @x[6], @s[1] 341 movdqa @t[2], @t[0] 342 pxor @x[2], @s[0] 343 movdqa @t[3], @s[3] 344 345 por @t[1], @t[2] 346 por @s[0], @t[3] 347 pxor @t[0], @s[3] 348 pand @s[0], @s[2] 349 pxor @t[1], @s[0] 350 pand @t[1], @t[0] 351 pand @s[0], @s[3] 352 movdqa @x[3], @s[0] 353 pxor @x[2], @s[0] 354 pand @s[0], @s[1] 355 pxor @s[1], @t[3] 356 pxor @s[1], @t[2] 357 movdqa @x[4], @s[1] 358 movdqa @x[1], @s[0] 359 pxor @x[5], @s[1] 360 pxor @x[0], @s[0] 361 movdqa @s[1], @t[1] 362 pand @s[0], @s[1] 363 por @s[0], @t[1] 364 pxor @s[1], @t[0] 365 pxor @s[3], @t[3] 366 pxor @s[2], @t[2] 367 pxor @s[3], @t[1] 368 movdqa @x[7], @s[0] 369 pxor @s[2], @t[0] 370 movdqa @x[6], @s[1] 371 pxor @s[2], @t[1] 372 movdqa @x[5], @s[2] 373 pand @x[3], @s[0] 374 movdqa @x[4], @s[3] 375 pand @x[2], @s[1] 376 pand @x[1], @s[2] 377 por @x[0], @s[3] 378 pxor @s[0], @t[3] 379 pxor @s[1], @t[2] 380 pxor @s[2], @t[1] 381 pxor @s[3], @t[0] 382 383 #Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 384 385 # new smaller inversion 386 387 movdqa @t[3], @s[0] 388 pand @t[1], @t[3] 389 pxor @t[2], @s[0] 390 391 movdqa @t[0], @s[2] 392 movdqa @s[0], @s[3] 393 pxor @t[3], @s[2] 394 pand @s[2], @s[3] 395 396 movdqa @t[1], @s[1] 397 pxor @t[2], @s[3] 398 pxor @t[0], @s[1] 399 400 pxor @t[2], @t[3] 401 402 pand @t[3], @s[1] 403 404 movdqa @s[2], @t[2] 405 pxor @t[0], @s[1] 406 407 pxor @s[1], @t[2] 408 pxor @s[1], @t[1] 409 410 pand @t[0], @t[2] 411 412 pxor @t[2], @s[2] 413 pxor @t[2], @t[1] 414 415 pand @s[3], @s[2] 416 417 pxor @s[0], @s[2] 418 ___ 419 # output in s3, s2, s1, t1 420 421 # Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 422 423 # Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 424 &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); 425 426 ### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb 427 } 428 429 # AES linear components 430 431 sub ShiftRows { 432 my @x=@_[0..7]; 433 my $mask=pop; 434 $code.=<<___; 435 pxor 0x00($key),@x[0] 436 pxor 0x10($key),@x[1] 437 pshufb $mask,@x[0] 438 pxor 0x20($key),@x[2] 439 pshufb $mask,@x[1] 440 pxor 0x30($key),@x[3] 441 pshufb $mask,@x[2] 442 pxor 0x40($key),@x[4] 443 pshufb $mask,@x[3] 444 pxor 0x50($key),@x[5] 445 pshufb $mask,@x[4] 446 pxor 0x60($key),@x[6] 447 pshufb $mask,@x[5] 448 pxor 0x70($key),@x[7] 449 pshufb $mask,@x[6] 450 lea 0x80($key),$key 451 pshufb $mask,@x[7] 452 ___ 453 } 454 455 sub MixColumns { 456 # modified to emit output in order suitable for feeding back to aesenc[last] 457 my @x=@_[0..7]; 458 my @t=@_[8..15]; 459 my $inv=@_[16]; # optional 460 $code.=<<___; 461 pshufd \$0x93, @x[0], @t[0] # x0 <<< 32 462 pshufd \$0x93, @x[1], @t[1] 463 pxor @t[0], @x[0] # x0 ^ (x0 <<< 32) 464 pshufd \$0x93, @x[2], @t[2] 465 pxor @t[1], @x[1] 466 pshufd \$0x93, @x[3], @t[3] 467 pxor @t[2], @x[2] 468 pshufd \$0x93, @x[4], @t[4] 469 pxor @t[3], @x[3] 470 pshufd \$0x93, @x[5], @t[5] 471 pxor @t[4], @x[4] 472 pshufd \$0x93, @x[6], @t[6] 473 pxor @t[5], @x[5] 474 pshufd \$0x93, @x[7], @t[7] 475 pxor @t[6], @x[6] 476 pxor @t[7], @x[7] 477 478 pxor @x[0], @t[1] 479 pxor @x[7], @t[0] 480 pxor @x[7], @t[1] 481 pshufd \$0x4E, @x[0], @x[0] # (x0 ^ (x0 <<< 32)) <<< 64) 482 pxor @x[1], @t[2] 483 pshufd \$0x4E, @x[1], @x[1] 484 pxor @x[4], @t[5] 485 pxor @t[0], @x[0] 486 pxor @x[5], @t[6] 487 pxor @t[1], @x[1] 488 pxor @x[3], @t[4] 489 pshufd \$0x4E, @x[4], @t[0] 490 pxor @x[6], @t[7] 491 pshufd \$0x4E, @x[5], @t[1] 492 pxor @x[2], @t[3] 493 pshufd \$0x4E, @x[3], @x[4] 494 pxor @x[7], @t[3] 495 pshufd \$0x4E, @x[7], @x[5] 496 pxor @x[7], @t[4] 497 pshufd \$0x4E, @x[6], @x[3] 498 pxor @t[4], @t[0] 499 pshufd \$0x4E, @x[2], @x[6] 500 pxor @t[5], @t[1] 501 ___ 502 $code.=<<___ if (!$inv); 503 pxor @t[3], @x[4] 504 pxor @t[7], @x[5] 505 pxor @t[6], @x[3] 506 movdqa @t[0], @x[2] 507 pxor @t[2], @x[6] 508 movdqa @t[1], @x[7] 509 ___ 510 $code.=<<___ if ($inv); 511 pxor @x[4], @t[3] 512 pxor @t[7], @x[5] 513 pxor @x[3], @t[6] 514 movdqa @t[0], @x[3] 515 pxor @t[2], @x[6] 516 movdqa @t[6], @x[2] 517 movdqa @t[1], @x[7] 518 movdqa @x[6], @x[4] 519 movdqa @t[3], @x[6] 520 ___ 521 } 522 523 sub InvMixColumns_orig { 524 my @x=@_[0..7]; 525 my @t=@_[8..15]; 526 527 $code.=<<___; 528 # multiplication by 0x0e 529 pshufd \$0x93, @x[7], @t[7] 530 movdqa @x[2], @t[2] 531 pxor @x[5], @x[7] # 7 5 532 pxor @x[5], @x[2] # 2 5 533 pshufd \$0x93, @x[0], @t[0] 534 movdqa @x[5], @t[5] 535 pxor @x[0], @x[5] # 5 0 [1] 536 pxor @x[1], @x[0] # 0 1 537 pshufd \$0x93, @x[1], @t[1] 538 pxor @x[2], @x[1] # 1 25 539 pxor @x[6], @x[0] # 01 6 [2] 540 pxor @x[3], @x[1] # 125 3 [4] 541 pshufd \$0x93, @x[3], @t[3] 542 pxor @x[0], @x[2] # 25 016 [3] 543 pxor @x[7], @x[3] # 3 75 544 pxor @x[6], @x[7] # 75 6 [0] 545 pshufd \$0x93, @x[6], @t[6] 546 movdqa @x[4], @t[4] 547 pxor @x[4], @x[6] # 6 4 548 pxor @x[3], @x[4] # 4 375 [6] 549 pxor @x[7], @x[3] # 375 756=36 550 pxor @t[5], @x[6] # 64 5 [7] 551 pxor @t[2], @x[3] # 36 2 552 pxor @t[4], @x[3] # 362 4 [5] 553 pshufd \$0x93, @t[5], @t[5] 554 ___ 555 my @y = @x[7,5,0,2,1,3,4,6]; 556 $code.=<<___; 557 # multiplication by 0x0b 558 pxor @y[0], @y[1] 559 pxor @t[0], @y[0] 560 pxor @t[1], @y[1] 561 pshufd \$0x93, @t[2], @t[2] 562 pxor @t[5], @y[0] 563 pxor @t[6], @y[1] 564 pxor @t[7], @y[0] 565 pshufd \$0x93, @t[4], @t[4] 566 pxor @t[6], @t[7] # clobber t[7] 567 pxor @y[0], @y[1] 568 569 pxor @t[0], @y[3] 570 pshufd \$0x93, @t[0], @t[0] 571 pxor @t[1], @y[2] 572 pxor @t[1], @y[4] 573 pxor @t[2], @y[2] 574 pshufd \$0x93, @t[1], @t[1] 575 pxor @t[2], @y[3] 576 pxor @t[2], @y[5] 577 pxor @t[7], @y[2] 578 pshufd \$0x93, @t[2], @t[2] 579 pxor @t[3], @y[3] 580 pxor @t[3], @y[6] 581 pxor @t[3], @y[4] 582 pshufd \$0x93, @t[3], @t[3] 583 pxor @t[4], @y[7] 584 pxor @t[4], @y[5] 585 pxor @t[7], @y[7] 586 pxor @t[5], @y[3] 587 pxor @t[4], @y[4] 588 pxor @t[5], @t[7] # clobber t[7] even more 589 590 pxor @t[7], @y[5] 591 pshufd \$0x93, @t[4], @t[4] 592 pxor @t[7], @y[6] 593 pxor @t[7], @y[4] 594 595 pxor @t[5], @t[7] 596 pshufd \$0x93, @t[5], @t[5] 597 pxor @t[6], @t[7] # restore t[7] 598 599 # multiplication by 0x0d 600 pxor @y[7], @y[4] 601 pxor @t[4], @y[7] 602 pshufd \$0x93, @t[6], @t[6] 603 pxor @t[0], @y[2] 604 pxor @t[5], @y[7] 605 pxor @t[2], @y[2] 606 pshufd \$0x93, @t[7], @t[7] 607 608 pxor @y[1], @y[3] 609 pxor @t[1], @y[1] 610 pxor @t[0], @y[0] 611 pxor @t[0], @y[3] 612 pxor @t[5], @y[1] 613 pxor @t[5], @y[0] 614 pxor @t[7], @y[1] 615 pshufd \$0x93, @t[0], @t[0] 616 pxor @t[6], @y[0] 617 pxor @y[1], @y[3] 618 pxor @t[1], @y[4] 619 pshufd \$0x93, @t[1], @t[1] 620 621 pxor @t[7], @y[7] 622 pxor @t[2], @y[4] 623 pxor @t[2], @y[5] 624 pshufd \$0x93, @t[2], @t[2] 625 pxor @t[6], @y[2] 626 pxor @t[3], @t[6] # clobber t[6] 627 pxor @y[7], @y[4] 628 pxor @t[6], @y[3] 629 630 pxor @t[6], @y[6] 631 pxor @t[5], @y[5] 632 pxor @t[4], @y[6] 633 pshufd \$0x93, @t[4], @t[4] 634 pxor @t[6], @y[5] 635 pxor @t[7], @y[6] 636 pxor @t[3], @t[6] # restore t[6] 637 638 pshufd \$0x93, @t[5], @t[5] 639 pshufd \$0x93, @t[6], @t[6] 640 pshufd \$0x93, @t[7], @t[7] 641 pshufd \$0x93, @t[3], @t[3] 642 643 # multiplication by 0x09 644 pxor @y[1], @y[4] 645 pxor @y[1], @t[1] # t[1]=y[1] 646 pxor @t[5], @t[0] # clobber t[0] 647 pxor @t[5], @t[1] 648 pxor @t[0], @y[3] 649 pxor @y[0], @t[0] # t[0]=y[0] 650 pxor @t[6], @t[1] 651 pxor @t[7], @t[6] # clobber t[6] 652 pxor @t[1], @y[4] 653 pxor @t[4], @y[7] 654 pxor @y[4], @t[4] # t[4]=y[4] 655 pxor @t[3], @y[6] 656 pxor @y[3], @t[3] # t[3]=y[3] 657 pxor @t[2], @y[5] 658 pxor @y[2], @t[2] # t[2]=y[2] 659 pxor @t[7], @t[3] 660 pxor @y[5], @t[5] # t[5]=y[5] 661 pxor @t[6], @t[2] 662 pxor @t[6], @t[5] 663 pxor @y[6], @t[6] # t[6]=y[6] 664 pxor @y[7], @t[7] # t[7]=y[7] 665 666 movdqa @t[0],@XMM[0] 667 movdqa @t[1],@XMM[1] 668 movdqa @t[2],@XMM[2] 669 movdqa @t[3],@XMM[3] 670 movdqa @t[4],@XMM[4] 671 movdqa @t[5],@XMM[5] 672 movdqa @t[6],@XMM[6] 673 movdqa @t[7],@XMM[7] 674 ___ 675 } 676 677 sub InvMixColumns { 678 my @x=@_[0..7]; 679 my @t=@_[8..15]; 680 681 # Thanks to Jussi Kivilinna for providing pointer to 682 # 683 # | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | 684 # | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | 685 # | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | 686 # | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | 687 688 $code.=<<___; 689 # multiplication by 0x05-0x00-0x04-0x00 690 pshufd \$0x4E, @x[0], @t[0] 691 pshufd \$0x4E, @x[6], @t[6] 692 pxor @x[0], @t[0] 693 pshufd \$0x4E, @x[7], @t[7] 694 pxor @x[6], @t[6] 695 pshufd \$0x4E, @x[1], @t[1] 696 pxor @x[7], @t[7] 697 pshufd \$0x4E, @x[2], @t[2] 698 pxor @x[1], @t[1] 699 pshufd \$0x4E, @x[3], @t[3] 700 pxor @x[2], @t[2] 701 pxor @t[6], @x[0] 702 pxor @t[6], @x[1] 703 pshufd \$0x4E, @x[4], @t[4] 704 pxor @x[3], @t[3] 705 pxor @t[0], @x[2] 706 pxor @t[1], @x[3] 707 pshufd \$0x4E, @x[5], @t[5] 708 pxor @x[4], @t[4] 709 pxor @t[7], @x[1] 710 pxor @t[2], @x[4] 711 pxor @x[5], @t[5] 712 713 pxor @t[7], @x[2] 714 pxor @t[6], @x[3] 715 pxor @t[6], @x[4] 716 pxor @t[3], @x[5] 717 pxor @t[4], @x[6] 718 pxor @t[7], @x[4] 719 pxor @t[7], @x[5] 720 pxor @t[5], @x[7] 721 ___ 722 &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 723 } 724 725 sub aesenc { # not used 726 my @b=@_[0..7]; 727 my @t=@_[8..15]; 728 $code.=<<___; 729 movdqa 0x30($const),@t[0] # .LSR 730 ___ 731 &ShiftRows (@b,@t[0]); 732 &Sbox (@b,@t); 733 &MixColumns (@b[0,1,4,6,3,7,2,5],@t); 734 } 735 736 sub aesenclast { # not used 737 my @b=@_[0..7]; 738 my @t=@_[8..15]; 739 $code.=<<___; 740 movdqa 0x40($const),@t[0] # .LSRM0 741 ___ 742 &ShiftRows (@b,@t[0]); 743 &Sbox (@b,@t); 744 $code.=<<___ 745 pxor 0x00($key),@b[0] 746 pxor 0x10($key),@b[1] 747 pxor 0x20($key),@b[4] 748 pxor 0x30($key),@b[6] 749 pxor 0x40($key),@b[3] 750 pxor 0x50($key),@b[7] 751 pxor 0x60($key),@b[2] 752 pxor 0x70($key),@b[5] 753 ___ 754 } 755 756 sub swapmove { 757 my ($a,$b,$n,$mask,$t)=@_; 758 $code.=<<___; 759 movdqa $b,$t 760 psrlq \$$n,$b 761 pxor $a,$b 762 pand $mask,$b 763 pxor $b,$a 764 psllq \$$n,$b 765 pxor $t,$b 766 ___ 767 } 768 sub swapmove2x { 769 my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; 770 $code.=<<___; 771 movdqa $b0,$t0 772 psrlq \$$n,$b0 773 movdqa $b1,$t1 774 psrlq \$$n,$b1 775 pxor $a0,$b0 776 pxor $a1,$b1 777 pand $mask,$b0 778 pand $mask,$b1 779 pxor $b0,$a0 780 psllq \$$n,$b0 781 pxor $b1,$a1 782 psllq \$$n,$b1 783 pxor $t0,$b0 784 pxor $t1,$b1 785 ___ 786 } 787 788 sub bitslice { 789 my @x=reverse(@_[0..7]); 790 my ($t0,$t1,$t2,$t3)=@_[8..11]; 791 $code.=<<___; 792 movdqa 0x00($const),$t0 # .LBS0 793 movdqa 0x10($const),$t1 # .LBS1 794 ___ 795 &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); 796 &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); 797 $code.=<<___; 798 movdqa 0x20($const),$t0 # .LBS2 799 ___ 800 &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); 801 &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); 802 803 &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); 804 &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); 805 } 806 807 $code.=<<___; 808 .text 809 810 .extern asm_AES_encrypt 811 .extern asm_AES_decrypt 812 813 .type _bsaes_encrypt8,\@abi-omnipotent 814 .align 64 815 _bsaes_encrypt8: 816 lea .LBS0(%rip), $const # constants table 817 818 movdqa ($key), @XMM[9] # round 0 key 819 lea 0x10($key), $key 820 movdqa 0x50($const), @XMM[8] # .LM0SR 821 pxor @XMM[9], @XMM[0] # xor with round0 key 822 pxor @XMM[9], @XMM[1] 823 pshufb @XMM[8], @XMM[0] 824 pxor @XMM[9], @XMM[2] 825 pshufb @XMM[8], @XMM[1] 826 pxor @XMM[9], @XMM[3] 827 pshufb @XMM[8], @XMM[2] 828 pxor @XMM[9], @XMM[4] 829 pshufb @XMM[8], @XMM[3] 830 pxor @XMM[9], @XMM[5] 831 pshufb @XMM[8], @XMM[4] 832 pxor @XMM[9], @XMM[6] 833 pshufb @XMM[8], @XMM[5] 834 pxor @XMM[9], @XMM[7] 835 pshufb @XMM[8], @XMM[6] 836 pshufb @XMM[8], @XMM[7] 837 _bsaes_encrypt8_bitslice: 838 ___ 839 &bitslice (@XMM[0..7, 8..11]); 840 $code.=<<___; 841 dec $rounds 842 jmp .Lenc_sbox 843 .align 16 844 .Lenc_loop: 845 ___ 846 &ShiftRows (@XMM[0..7, 8]); 847 $code.=".Lenc_sbox:\n"; 848 &Sbox (@XMM[0..7, 8..15]); 849 $code.=<<___; 850 dec $rounds 851 jl .Lenc_done 852 ___ 853 &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); 854 $code.=<<___; 855 movdqa 0x30($const), @XMM[8] # .LSR 856 jnz .Lenc_loop 857 movdqa 0x40($const), @XMM[8] # .LSRM0 858 jmp .Lenc_loop 859 .align 16 860 .Lenc_done: 861 ___ 862 # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb 863 &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); 864 $code.=<<___; 865 movdqa ($key), @XMM[8] # last round key 866 pxor @XMM[8], @XMM[4] 867 pxor @XMM[8], @XMM[6] 868 pxor @XMM[8], @XMM[3] 869 pxor @XMM[8], @XMM[7] 870 pxor @XMM[8], @XMM[2] 871 pxor @XMM[8], @XMM[5] 872 pxor @XMM[8], @XMM[0] 873 pxor @XMM[8], @XMM[1] 874 ret 875 .size _bsaes_encrypt8,.-_bsaes_encrypt8 876 877 .type _bsaes_decrypt8,\@abi-omnipotent 878 .align 64 879 _bsaes_decrypt8: 880 lea .LBS0(%rip), $const # constants table 881 882 movdqa ($key), @XMM[9] # round 0 key 883 lea 0x10($key), $key 884 movdqa -0x30($const), @XMM[8] # .LM0ISR 885 pxor @XMM[9], @XMM[0] # xor with round0 key 886 pxor @XMM[9], @XMM[1] 887 pshufb @XMM[8], @XMM[0] 888 pxor @XMM[9], @XMM[2] 889 pshufb @XMM[8], @XMM[1] 890 pxor @XMM[9], @XMM[3] 891 pshufb @XMM[8], @XMM[2] 892 pxor @XMM[9], @XMM[4] 893 pshufb @XMM[8], @XMM[3] 894 pxor @XMM[9], @XMM[5] 895 pshufb @XMM[8], @XMM[4] 896 pxor @XMM[9], @XMM[6] 897 pshufb @XMM[8], @XMM[5] 898 pxor @XMM[9], @XMM[7] 899 pshufb @XMM[8], @XMM[6] 900 pshufb @XMM[8], @XMM[7] 901 ___ 902 &bitslice (@XMM[0..7, 8..11]); 903 $code.=<<___; 904 dec $rounds 905 jmp .Ldec_sbox 906 .align 16 907 .Ldec_loop: 908 ___ 909 &ShiftRows (@XMM[0..7, 8]); 910 $code.=".Ldec_sbox:\n"; 911 &InvSbox (@XMM[0..7, 8..15]); 912 $code.=<<___; 913 dec $rounds 914 jl .Ldec_done 915 ___ 916 &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); 917 $code.=<<___; 918 movdqa -0x10($const), @XMM[8] # .LISR 919 jnz .Ldec_loop 920 movdqa -0x20($const), @XMM[8] # .LISRM0 921 jmp .Ldec_loop 922 .align 16 923 .Ldec_done: 924 ___ 925 &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); 926 $code.=<<___; 927 movdqa ($key), @XMM[8] # last round key 928 pxor @XMM[8], @XMM[6] 929 pxor @XMM[8], @XMM[4] 930 pxor @XMM[8], @XMM[2] 931 pxor @XMM[8], @XMM[7] 932 pxor @XMM[8], @XMM[3] 933 pxor @XMM[8], @XMM[5] 934 pxor @XMM[8], @XMM[0] 935 pxor @XMM[8], @XMM[1] 936 ret 937 .size _bsaes_decrypt8,.-_bsaes_decrypt8 938 ___ 939 } 940 { 941 my ($out,$inp,$rounds,$const)=("%rax","%rcx","%r10d","%r11"); 942 943 sub bitslice_key { 944 my @x=reverse(@_[0..7]); 945 my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; 946 947 &swapmove (@x[0,1],1,$bs0,$t2,$t3); 948 $code.=<<___; 949 #&swapmove(@x[2,3],1,$t0,$t2,$t3); 950 movdqa @x[0], @x[2] 951 movdqa @x[1], @x[3] 952 ___ 953 #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); 954 955 &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); 956 $code.=<<___; 957 #&swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); 958 movdqa @x[0], @x[4] 959 movdqa @x[2], @x[6] 960 movdqa @x[1], @x[5] 961 movdqa @x[3], @x[7] 962 ___ 963 &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); 964 &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); 965 } 966 967 $code.=<<___; 968 .type _bsaes_key_convert,\@abi-omnipotent 969 .align 16 970 _bsaes_key_convert: 971 lea .Lmasks(%rip), $const 972 movdqu ($inp), %xmm7 # load round 0 key 973 lea 0x10($inp), $inp 974 movdqa 0x00($const), %xmm0 # 0x01... 975 movdqa 0x10($const), %xmm1 # 0x02... 976 movdqa 0x20($const), %xmm2 # 0x04... 977 movdqa 0x30($const), %xmm3 # 0x08... 978 movdqa 0x40($const), %xmm4 # .LM0 979 pcmpeqd %xmm5, %xmm5 # .LNOT 980 981 movdqu ($inp), %xmm6 # load round 1 key 982 movdqa %xmm7, ($out) # save round 0 key 983 lea 0x10($out), $out 984 dec $rounds 985 jmp .Lkey_loop 986 .align 16 987 .Lkey_loop: 988 pshufb %xmm4, %xmm6 # .LM0 989 990 movdqa %xmm0, %xmm8 991 movdqa %xmm1, %xmm9 992 993 pand %xmm6, %xmm8 994 pand %xmm6, %xmm9 995 movdqa %xmm2, %xmm10 996 pcmpeqb %xmm0, %xmm8 997 psllq \$4, %xmm0 # 0x10... 998 movdqa %xmm3, %xmm11 999 pcmpeqb %xmm1, %xmm9 1000 psllq \$4, %xmm1 # 0x20... 1001 1002 pand %xmm6, %xmm10 1003 pand %xmm6, %xmm11 1004 movdqa %xmm0, %xmm12 1005 pcmpeqb %xmm2, %xmm10 1006 psllq \$4, %xmm2 # 0x40... 1007 movdqa %xmm1, %xmm13 1008 pcmpeqb %xmm3, %xmm11 1009 psllq \$4, %xmm3 # 0x80... 1010 1011 movdqa %xmm2, %xmm14 1012 movdqa %xmm3, %xmm15 1013 pxor %xmm5, %xmm8 # "pnot" 1014 pxor %xmm5, %xmm9 1015 1016 pand %xmm6, %xmm12 1017 pand %xmm6, %xmm13 1018 movdqa %xmm8, 0x00($out) # write bit-sliced round key 1019 pcmpeqb %xmm0, %xmm12 1020 psrlq \$4, %xmm0 # 0x01... 1021 movdqa %xmm9, 0x10($out) 1022 pcmpeqb %xmm1, %xmm13 1023 psrlq \$4, %xmm1 # 0x02... 1024 lea 0x10($inp), $inp 1025 1026 pand %xmm6, %xmm14 1027 pand %xmm6, %xmm15 1028 movdqa %xmm10, 0x20($out) 1029 pcmpeqb %xmm2, %xmm14 1030 psrlq \$4, %xmm2 # 0x04... 1031 movdqa %xmm11, 0x30($out) 1032 pcmpeqb %xmm3, %xmm15 1033 psrlq \$4, %xmm3 # 0x08... 1034 movdqu ($inp), %xmm6 # load next round key 1035 1036 pxor %xmm5, %xmm13 # "pnot" 1037 pxor %xmm5, %xmm14 1038 movdqa %xmm12, 0x40($out) 1039 movdqa %xmm13, 0x50($out) 1040 movdqa %xmm14, 0x60($out) 1041 movdqa %xmm15, 0x70($out) 1042 lea 0x80($out),$out 1043 dec $rounds 1044 jnz .Lkey_loop 1045 1046 movdqa 0x50($const), %xmm7 # .L63 1047 #movdqa %xmm6, ($out) # don't save last round key 1048 ret 1049 .size _bsaes_key_convert,.-_bsaes_key_convert 1050 ___ 1051 } 1052 1053 if (0 && !$win64) { # following four functions are unsupported interface 1054 # used for benchmarking... 1055 $code.=<<___; 1056 .globl bsaes_enc_key_convert 1057 .type bsaes_enc_key_convert,\@function,2 1058 .align 16 1059 bsaes_enc_key_convert: 1060 mov 240($inp),%r10d # pass rounds 1061 mov $inp,%rcx # pass key 1062 mov $out,%rax # pass key schedule 1063 call _bsaes_key_convert 1064 pxor %xmm6,%xmm7 # fix up last round key 1065 movdqa %xmm7,(%rax) # save last round key 1066 ret 1067 .size bsaes_enc_key_convert,.-bsaes_enc_key_convert 1068 1069 .globl bsaes_encrypt_128 1070 .type bsaes_encrypt_128,\@function,4 1071 .align 16 1072 bsaes_encrypt_128: 1073 .Lenc128_loop: 1074 movdqu 0x00($inp), @XMM[0] # load input 1075 movdqu 0x10($inp), @XMM[1] 1076 movdqu 0x20($inp), @XMM[2] 1077 movdqu 0x30($inp), @XMM[3] 1078 movdqu 0x40($inp), @XMM[4] 1079 movdqu 0x50($inp), @XMM[5] 1080 movdqu 0x60($inp), @XMM[6] 1081 movdqu 0x70($inp), @XMM[7] 1082 mov $key, %rax # pass the $key 1083 lea 0x80($inp), $inp 1084 mov \$10,%r10d 1085 1086 call _bsaes_encrypt8 1087 1088 movdqu @XMM[0], 0x00($out) # write output 1089 movdqu @XMM[1], 0x10($out) 1090 movdqu @XMM[4], 0x20($out) 1091 movdqu @XMM[6], 0x30($out) 1092 movdqu @XMM[3], 0x40($out) 1093 movdqu @XMM[7], 0x50($out) 1094 movdqu @XMM[2], 0x60($out) 1095 movdqu @XMM[5], 0x70($out) 1096 lea 0x80($out), $out 1097 sub \$0x80,$len 1098 ja .Lenc128_loop 1099 ret 1100 .size bsaes_encrypt_128,.-bsaes_encrypt_128 1101 1102 .globl bsaes_dec_key_convert 1103 .type bsaes_dec_key_convert,\@function,2 1104 .align 16 1105 bsaes_dec_key_convert: 1106 mov 240($inp),%r10d # pass rounds 1107 mov $inp,%rcx # pass key 1108 mov $out,%rax # pass key schedule 1109 call _bsaes_key_convert 1110 pxor ($out),%xmm7 # fix up round 0 key 1111 movdqa %xmm6,(%rax) # save last round key 1112 movdqa %xmm7,($out) 1113 ret 1114 .size bsaes_dec_key_convert,.-bsaes_dec_key_convert 1115 1116 .globl bsaes_decrypt_128 1117 .type bsaes_decrypt_128,\@function,4 1118 .align 16 1119 bsaes_decrypt_128: 1120 .Ldec128_loop: 1121 movdqu 0x00($inp), @XMM[0] # load input 1122 movdqu 0x10($inp), @XMM[1] 1123 movdqu 0x20($inp), @XMM[2] 1124 movdqu 0x30($inp), @XMM[3] 1125 movdqu 0x40($inp), @XMM[4] 1126 movdqu 0x50($inp), @XMM[5] 1127 movdqu 0x60($inp), @XMM[6] 1128 movdqu 0x70($inp), @XMM[7] 1129 mov $key, %rax # pass the $key 1130 lea 0x80($inp), $inp 1131 mov \$10,%r10d 1132 1133 call _bsaes_decrypt8 1134 1135 movdqu @XMM[0], 0x00($out) # write output 1136 movdqu @XMM[1], 0x10($out) 1137 movdqu @XMM[6], 0x20($out) 1138 movdqu @XMM[4], 0x30($out) 1139 movdqu @XMM[2], 0x40($out) 1140 movdqu @XMM[7], 0x50($out) 1141 movdqu @XMM[3], 0x60($out) 1142 movdqu @XMM[5], 0x70($out) 1143 lea 0x80($out), $out 1144 sub \$0x80,$len 1145 ja .Ldec128_loop 1146 ret 1147 .size bsaes_decrypt_128,.-bsaes_decrypt_128 1148 ___ 1149 } 1150 { 1151 ###################################################################### 1152 # 1153 # OpenSSL interface 1154 # 1155 my ($arg1,$arg2,$arg3,$arg4,$arg5,$arg6)=$win64 ? ("%rcx","%rdx","%r8","%r9","%r10","%r11d") 1156 : ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d"); 1157 my ($inp,$out,$len,$key)=("%r12","%r13","%r14","%r15"); 1158 1159 if ($ecb) { 1160 $code.=<<___; 1161 .globl bsaes_ecb_encrypt_blocks 1162 .type bsaes_ecb_encrypt_blocks,\@abi-omnipotent 1163 .align 16 1164 bsaes_ecb_encrypt_blocks: 1165 mov %rsp, %rax 1166 .Lecb_enc_prologue: 1167 push %rbp 1168 push %rbx 1169 push %r12 1170 push %r13 1171 push %r14 1172 push %r15 1173 lea -0x48(%rsp),%rsp 1174 ___ 1175 $code.=<<___ if ($win64); 1176 lea -0xa0(%rsp), %rsp 1177 movaps %xmm6, 0x40(%rsp) 1178 movaps %xmm7, 0x50(%rsp) 1179 movaps %xmm8, 0x60(%rsp) 1180 movaps %xmm9, 0x70(%rsp) 1181 movaps %xmm10, 0x80(%rsp) 1182 movaps %xmm11, 0x90(%rsp) 1183 movaps %xmm12, 0xa0(%rsp) 1184 movaps %xmm13, 0xb0(%rsp) 1185 movaps %xmm14, 0xc0(%rsp) 1186 movaps %xmm15, 0xd0(%rsp) 1187 .Lecb_enc_body: 1188 ___ 1189 $code.=<<___; 1190 mov %rsp,%rbp # backup %rsp 1191 mov 240($arg4),%eax # rounds 1192 mov $arg1,$inp # backup arguments 1193 mov $arg2,$out 1194 mov $arg3,$len 1195 mov $arg4,$key 1196 cmp \$8,$arg3 1197 jb .Lecb_enc_short 1198 1199 mov %eax,%ebx # backup rounds 1200 shl \$7,%rax # 128 bytes per inner round key 1201 sub \$`128-32`,%rax # size of bit-sliced key schedule 1202 sub %rax,%rsp 1203 mov %rsp,%rax # pass key schedule 1204 mov $key,%rcx # pass key 1205 mov %ebx,%r10d # pass rounds 1206 call _bsaes_key_convert 1207 pxor %xmm6,%xmm7 # fix up last round key 1208 movdqa %xmm7,(%rax) # save last round key 1209 1210 sub \$8,$len 1211 .Lecb_enc_loop: 1212 movdqu 0x00($inp), @XMM[0] # load input 1213 movdqu 0x10($inp), @XMM[1] 1214 movdqu 0x20($inp), @XMM[2] 1215 movdqu 0x30($inp), @XMM[3] 1216 movdqu 0x40($inp), @XMM[4] 1217 movdqu 0x50($inp), @XMM[5] 1218 mov %rsp, %rax # pass key schedule 1219 movdqu 0x60($inp), @XMM[6] 1220 mov %ebx,%r10d # pass rounds 1221 movdqu 0x70($inp), @XMM[7] 1222 lea 0x80($inp), $inp 1223 1224 call _bsaes_encrypt8 1225 1226 movdqu @XMM[0], 0x00($out) # write output 1227 movdqu @XMM[1], 0x10($out) 1228 movdqu @XMM[4], 0x20($out) 1229 movdqu @XMM[6], 0x30($out) 1230 movdqu @XMM[3], 0x40($out) 1231 movdqu @XMM[7], 0x50($out) 1232 movdqu @XMM[2], 0x60($out) 1233 movdqu @XMM[5], 0x70($out) 1234 lea 0x80($out), $out 1235 sub \$8,$len 1236 jnc .Lecb_enc_loop 1237 1238 add \$8,$len 1239 jz .Lecb_enc_done 1240 1241 movdqu 0x00($inp), @XMM[0] # load input 1242 mov %rsp, %rax # pass key schedule 1243 mov %ebx,%r10d # pass rounds 1244 cmp \$2,$len 1245 jb .Lecb_enc_one 1246 movdqu 0x10($inp), @XMM[1] 1247 je .Lecb_enc_two 1248 movdqu 0x20($inp), @XMM[2] 1249 cmp \$4,$len 1250 jb .Lecb_enc_three 1251 movdqu 0x30($inp), @XMM[3] 1252 je .Lecb_enc_four 1253 movdqu 0x40($inp), @XMM[4] 1254 cmp \$6,$len 1255 jb .Lecb_enc_five 1256 movdqu 0x50($inp), @XMM[5] 1257 je .Lecb_enc_six 1258 movdqu 0x60($inp), @XMM[6] 1259 call _bsaes_encrypt8 1260 movdqu @XMM[0], 0x00($out) # write output 1261 movdqu @XMM[1], 0x10($out) 1262 movdqu @XMM[4], 0x20($out) 1263 movdqu @XMM[6], 0x30($out) 1264 movdqu @XMM[3], 0x40($out) 1265 movdqu @XMM[7], 0x50($out) 1266 movdqu @XMM[2], 0x60($out) 1267 jmp .Lecb_enc_done 1268 .align 16 1269 .Lecb_enc_six: 1270 call _bsaes_encrypt8 1271 movdqu @XMM[0], 0x00($out) # write output 1272 movdqu @XMM[1], 0x10($out) 1273 movdqu @XMM[4], 0x20($out) 1274 movdqu @XMM[6], 0x30($out) 1275 movdqu @XMM[3], 0x40($out) 1276 movdqu @XMM[7], 0x50($out) 1277 jmp .Lecb_enc_done 1278 .align 16 1279 .Lecb_enc_five: 1280 call _bsaes_encrypt8 1281 movdqu @XMM[0], 0x00($out) # write output 1282 movdqu @XMM[1], 0x10($out) 1283 movdqu @XMM[4], 0x20($out) 1284 movdqu @XMM[6], 0x30($out) 1285 movdqu @XMM[3], 0x40($out) 1286 jmp .Lecb_enc_done 1287 .align 16 1288 .Lecb_enc_four: 1289 call _bsaes_encrypt8 1290 movdqu @XMM[0], 0x00($out) # write output 1291 movdqu @XMM[1], 0x10($out) 1292 movdqu @XMM[4], 0x20($out) 1293 movdqu @XMM[6], 0x30($out) 1294 jmp .Lecb_enc_done 1295 .align 16 1296 .Lecb_enc_three: 1297 call _bsaes_encrypt8 1298 movdqu @XMM[0], 0x00($out) # write output 1299 movdqu @XMM[1], 0x10($out) 1300 movdqu @XMM[4], 0x20($out) 1301 jmp .Lecb_enc_done 1302 .align 16 1303 .Lecb_enc_two: 1304 call _bsaes_encrypt8 1305 movdqu @XMM[0], 0x00($out) # write output 1306 movdqu @XMM[1], 0x10($out) 1307 jmp .Lecb_enc_done 1308 .align 16 1309 .Lecb_enc_one: 1310 call _bsaes_encrypt8 1311 movdqu @XMM[0], 0x00($out) # write output 1312 jmp .Lecb_enc_done 1313 .align 16 1314 .Lecb_enc_short: 1315 lea ($inp), $arg1 1316 lea ($out), $arg2 1317 lea ($key), $arg3 1318 call asm_AES_encrypt 1319 lea 16($inp), $inp 1320 lea 16($out), $out 1321 dec $len 1322 jnz .Lecb_enc_short 1323 1324 .Lecb_enc_done: 1325 lea (%rsp),%rax 1326 pxor %xmm0, %xmm0 1327 .Lecb_enc_bzero: # wipe key schedule [if any] 1328 movdqa %xmm0, 0x00(%rax) 1329 movdqa %xmm0, 0x10(%rax) 1330 lea 0x20(%rax), %rax 1331 cmp %rax, %rbp 1332 jb .Lecb_enc_bzero 1333 1334 lea (%rbp),%rsp # restore %rsp 1335 ___ 1336 $code.=<<___ if ($win64); 1337 movaps 0x40(%rbp), %xmm6 1338 movaps 0x50(%rbp), %xmm7 1339 movaps 0x60(%rbp), %xmm8 1340 movaps 0x70(%rbp), %xmm9 1341 movaps 0x80(%rbp), %xmm10 1342 movaps 0x90(%rbp), %xmm11 1343 movaps 0xa0(%rbp), %xmm12 1344 movaps 0xb0(%rbp), %xmm13 1345 movaps 0xc0(%rbp), %xmm14 1346 movaps 0xd0(%rbp), %xmm15 1347 lea 0xa0(%rbp), %rsp 1348 ___ 1349 $code.=<<___; 1350 mov 0x48(%rsp), %r15 1351 mov 0x50(%rsp), %r14 1352 mov 0x58(%rsp), %r13 1353 mov 0x60(%rsp), %r12 1354 mov 0x68(%rsp), %rbx 1355 mov 0x70(%rsp), %rax 1356 lea 0x78(%rsp), %rsp 1357 mov %rax, %rbp 1358 .Lecb_enc_epilogue: 1359 ret 1360 .size bsaes_ecb_encrypt_blocks,.-bsaes_ecb_encrypt_blocks 1361 1362 .globl bsaes_ecb_decrypt_blocks 1363 .type bsaes_ecb_decrypt_blocks,\@abi-omnipotent 1364 .align 16 1365 bsaes_ecb_decrypt_blocks: 1366 mov %rsp, %rax 1367 .Lecb_dec_prologue: 1368 push %rbp 1369 push %rbx 1370 push %r12 1371 push %r13 1372 push %r14 1373 push %r15 1374 lea -0x48(%rsp),%rsp 1375 ___ 1376 $code.=<<___ if ($win64); 1377 lea -0xa0(%rsp), %rsp 1378 movaps %xmm6, 0x40(%rsp) 1379 movaps %xmm7, 0x50(%rsp) 1380 movaps %xmm8, 0x60(%rsp) 1381 movaps %xmm9, 0x70(%rsp) 1382 movaps %xmm10, 0x80(%rsp) 1383 movaps %xmm11, 0x90(%rsp) 1384 movaps %xmm12, 0xa0(%rsp) 1385 movaps %xmm13, 0xb0(%rsp) 1386 movaps %xmm14, 0xc0(%rsp) 1387 movaps %xmm15, 0xd0(%rsp) 1388 .Lecb_dec_body: 1389 ___ 1390 $code.=<<___; 1391 mov %rsp,%rbp # backup %rsp 1392 mov 240($arg4),%eax # rounds 1393 mov $arg1,$inp # backup arguments 1394 mov $arg2,$out 1395 mov $arg3,$len 1396 mov $arg4,$key 1397 cmp \$8,$arg3 1398 jb .Lecb_dec_short 1399 1400 mov %eax,%ebx # backup rounds 1401 shl \$7,%rax # 128 bytes per inner round key 1402 sub \$`128-32`,%rax # size of bit-sliced key schedule 1403 sub %rax,%rsp 1404 mov %rsp,%rax # pass key schedule 1405 mov $key,%rcx # pass key 1406 mov %ebx,%r10d # pass rounds 1407 call _bsaes_key_convert 1408 pxor (%rsp),%xmm7 # fix up 0 round key 1409 movdqa %xmm6,(%rax) # save last round key 1410 movdqa %xmm7,(%rsp) 1411 1412 sub \$8,$len 1413 .Lecb_dec_loop: 1414 movdqu 0x00($inp), @XMM[0] # load input 1415 movdqu 0x10($inp), @XMM[1] 1416 movdqu 0x20($inp), @XMM[2] 1417 movdqu 0x30($inp), @XMM[3] 1418 movdqu 0x40($inp), @XMM[4] 1419 movdqu 0x50($inp), @XMM[5] 1420 mov %rsp, %rax # pass key schedule 1421 movdqu 0x60($inp), @XMM[6] 1422 mov %ebx,%r10d # pass rounds 1423 movdqu 0x70($inp), @XMM[7] 1424 lea 0x80($inp), $inp 1425 1426 call _bsaes_decrypt8 1427 1428 movdqu @XMM[0], 0x00($out) # write output 1429 movdqu @XMM[1], 0x10($out) 1430 movdqu @XMM[6], 0x20($out) 1431 movdqu @XMM[4], 0x30($out) 1432 movdqu @XMM[2], 0x40($out) 1433 movdqu @XMM[7], 0x50($out) 1434 movdqu @XMM[3], 0x60($out) 1435 movdqu @XMM[5], 0x70($out) 1436 lea 0x80($out), $out 1437 sub \$8,$len 1438 jnc .Lecb_dec_loop 1439 1440 add \$8,$len 1441 jz .Lecb_dec_done 1442 1443 movdqu 0x00($inp), @XMM[0] # load input 1444 mov %rsp, %rax # pass key schedule 1445 mov %ebx,%r10d # pass rounds 1446 cmp \$2,$len 1447 jb .Lecb_dec_one 1448 movdqu 0x10($inp), @XMM[1] 1449 je .Lecb_dec_two 1450 movdqu 0x20($inp), @XMM[2] 1451 cmp \$4,$len 1452 jb .Lecb_dec_three 1453 movdqu 0x30($inp), @XMM[3] 1454 je .Lecb_dec_four 1455 movdqu 0x40($inp), @XMM[4] 1456 cmp \$6,$len 1457 jb .Lecb_dec_five 1458 movdqu 0x50($inp), @XMM[5] 1459 je .Lecb_dec_six 1460 movdqu 0x60($inp), @XMM[6] 1461 call _bsaes_decrypt8 1462 movdqu @XMM[0], 0x00($out) # write output 1463 movdqu @XMM[1], 0x10($out) 1464 movdqu @XMM[6], 0x20($out) 1465 movdqu @XMM[4], 0x30($out) 1466 movdqu @XMM[2], 0x40($out) 1467 movdqu @XMM[7], 0x50($out) 1468 movdqu @XMM[3], 0x60($out) 1469 jmp .Lecb_dec_done 1470 .align 16 1471 .Lecb_dec_six: 1472 call _bsaes_decrypt8 1473 movdqu @XMM[0], 0x00($out) # write output 1474 movdqu @XMM[1], 0x10($out) 1475 movdqu @XMM[6], 0x20($out) 1476 movdqu @XMM[4], 0x30($out) 1477 movdqu @XMM[2], 0x40($out) 1478 movdqu @XMM[7], 0x50($out) 1479 jmp .Lecb_dec_done 1480 .align 16 1481 .Lecb_dec_five: 1482 call _bsaes_decrypt8 1483 movdqu @XMM[0], 0x00($out) # write output 1484 movdqu @XMM[1], 0x10($out) 1485 movdqu @XMM[6], 0x20($out) 1486 movdqu @XMM[4], 0x30($out) 1487 movdqu @XMM[2], 0x40($out) 1488 jmp .Lecb_dec_done 1489 .align 16 1490 .Lecb_dec_four: 1491 call _bsaes_decrypt8 1492 movdqu @XMM[0], 0x00($out) # write output 1493 movdqu @XMM[1], 0x10($out) 1494 movdqu @XMM[6], 0x20($out) 1495 movdqu @XMM[4], 0x30($out) 1496 jmp .Lecb_dec_done 1497 .align 16 1498 .Lecb_dec_three: 1499 call _bsaes_decrypt8 1500 movdqu @XMM[0], 0x00($out) # write output 1501 movdqu @XMM[1], 0x10($out) 1502 movdqu @XMM[6], 0x20($out) 1503 jmp .Lecb_dec_done 1504 .align 16 1505 .Lecb_dec_two: 1506 call _bsaes_decrypt8 1507 movdqu @XMM[0], 0x00($out) # write output 1508 movdqu @XMM[1], 0x10($out) 1509 jmp .Lecb_dec_done 1510 .align 16 1511 .Lecb_dec_one: 1512 call _bsaes_decrypt8 1513 movdqu @XMM[0], 0x00($out) # write output 1514 jmp .Lecb_dec_done 1515 .align 16 1516 .Lecb_dec_short: 1517 lea ($inp), $arg1 1518 lea ($out), $arg2 1519 lea ($key), $arg3 1520 call asm_AES_decrypt 1521 lea 16($inp), $inp 1522 lea 16($out), $out 1523 dec $len 1524 jnz .Lecb_dec_short 1525 1526 .Lecb_dec_done: 1527 lea (%rsp),%rax 1528 pxor %xmm0, %xmm0 1529 .Lecb_dec_bzero: # wipe key schedule [if any] 1530 movdqa %xmm0, 0x00(%rax) 1531 movdqa %xmm0, 0x10(%rax) 1532 lea 0x20(%rax), %rax 1533 cmp %rax, %rbp 1534 jb .Lecb_dec_bzero 1535 1536 lea (%rbp),%rsp # restore %rsp 1537 ___ 1538 $code.=<<___ if ($win64); 1539 movaps 0x40(%rbp), %xmm6 1540 movaps 0x50(%rbp), %xmm7 1541 movaps 0x60(%rbp), %xmm8 1542 movaps 0x70(%rbp), %xmm9 1543 movaps 0x80(%rbp), %xmm10 1544 movaps 0x90(%rbp), %xmm11 1545 movaps 0xa0(%rbp), %xmm12 1546 movaps 0xb0(%rbp), %xmm13 1547 movaps 0xc0(%rbp), %xmm14 1548 movaps 0xd0(%rbp), %xmm15 1549 lea 0xa0(%rbp), %rsp 1550 ___ 1551 $code.=<<___; 1552 mov 0x48(%rsp), %r15 1553 mov 0x50(%rsp), %r14 1554 mov 0x58(%rsp), %r13 1555 mov 0x60(%rsp), %r12 1556 mov 0x68(%rsp), %rbx 1557 mov 0x70(%rsp), %rax 1558 lea 0x78(%rsp), %rsp 1559 mov %rax, %rbp 1560 .Lecb_dec_epilogue: 1561 ret 1562 .size bsaes_ecb_decrypt_blocks,.-bsaes_ecb_decrypt_blocks 1563 ___ 1564 } 1565 $code.=<<___; 1566 .extern asm_AES_cbc_encrypt 1567 .globl bsaes_cbc_encrypt 1568 .type bsaes_cbc_encrypt,\@abi-omnipotent 1569 .align 16 1570 bsaes_cbc_encrypt: 1571 ___ 1572 $code.=<<___ if ($win64); 1573 mov 48(%rsp),$arg6 # pull direction flag 1574 ___ 1575 $code.=<<___; 1576 cmp \$0,$arg6 1577 jne asm_AES_cbc_encrypt 1578 cmp \$128,$arg3 1579 jb asm_AES_cbc_encrypt 1580 1581 mov %rsp, %rax 1582 .Lcbc_dec_prologue: 1583 push %rbp 1584 push %rbx 1585 push %r12 1586 push %r13 1587 push %r14 1588 push %r15 1589 lea -0x48(%rsp), %rsp 1590 ___ 1591 $code.=<<___ if ($win64); 1592 mov 0xa0(%rsp),$arg5 # pull ivp 1593 lea -0xa0(%rsp), %rsp 1594 movaps %xmm6, 0x40(%rsp) 1595 movaps %xmm7, 0x50(%rsp) 1596 movaps %xmm8, 0x60(%rsp) 1597 movaps %xmm9, 0x70(%rsp) 1598 movaps %xmm10, 0x80(%rsp) 1599 movaps %xmm11, 0x90(%rsp) 1600 movaps %xmm12, 0xa0(%rsp) 1601 movaps %xmm13, 0xb0(%rsp) 1602 movaps %xmm14, 0xc0(%rsp) 1603 movaps %xmm15, 0xd0(%rsp) 1604 .Lcbc_dec_body: 1605 ___ 1606 $code.=<<___; 1607 mov %rsp, %rbp # backup %rsp 1608 mov 240($arg4), %eax # rounds 1609 mov $arg1, $inp # backup arguments 1610 mov $arg2, $out 1611 mov $arg3, $len 1612 mov $arg4, $key 1613 mov $arg5, %rbx 1614 shr \$4, $len # bytes to blocks 1615 1616 mov %eax, %edx # rounds 1617 shl \$7, %rax # 128 bytes per inner round key 1618 sub \$`128-32`, %rax # size of bit-sliced key schedule 1619 sub %rax, %rsp 1620 1621 mov %rsp, %rax # pass key schedule 1622 mov $key, %rcx # pass key 1623 mov %edx, %r10d # pass rounds 1624 call _bsaes_key_convert 1625 pxor (%rsp),%xmm7 # fix up 0 round key 1626 movdqa %xmm6,(%rax) # save last round key 1627 movdqa %xmm7,(%rsp) 1628 1629 movdqu (%rbx), @XMM[15] # load IV 1630 sub \$8,$len 1631 .Lcbc_dec_loop: 1632 movdqu 0x00($inp), @XMM[0] # load input 1633 movdqu 0x10($inp), @XMM[1] 1634 movdqu 0x20($inp), @XMM[2] 1635 movdqu 0x30($inp), @XMM[3] 1636 movdqu 0x40($inp), @XMM[4] 1637 movdqu 0x50($inp), @XMM[5] 1638 mov %rsp, %rax # pass key schedule 1639 movdqu 0x60($inp), @XMM[6] 1640 mov %edx,%r10d # pass rounds 1641 movdqu 0x70($inp), @XMM[7] 1642 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1643 1644 call _bsaes_decrypt8 1645 1646 pxor 0x20(%rbp), @XMM[0] # ^= IV 1647 movdqu 0x00($inp), @XMM[8] # re-load input 1648 movdqu 0x10($inp), @XMM[9] 1649 pxor @XMM[8], @XMM[1] 1650 movdqu 0x20($inp), @XMM[10] 1651 pxor @XMM[9], @XMM[6] 1652 movdqu 0x30($inp), @XMM[11] 1653 pxor @XMM[10], @XMM[4] 1654 movdqu 0x40($inp), @XMM[12] 1655 pxor @XMM[11], @XMM[2] 1656 movdqu 0x50($inp), @XMM[13] 1657 pxor @XMM[12], @XMM[7] 1658 movdqu 0x60($inp), @XMM[14] 1659 pxor @XMM[13], @XMM[3] 1660 movdqu 0x70($inp), @XMM[15] # IV 1661 pxor @XMM[14], @XMM[5] 1662 movdqu @XMM[0], 0x00($out) # write output 1663 lea 0x80($inp), $inp 1664 movdqu @XMM[1], 0x10($out) 1665 movdqu @XMM[6], 0x20($out) 1666 movdqu @XMM[4], 0x30($out) 1667 movdqu @XMM[2], 0x40($out) 1668 movdqu @XMM[7], 0x50($out) 1669 movdqu @XMM[3], 0x60($out) 1670 movdqu @XMM[5], 0x70($out) 1671 lea 0x80($out), $out 1672 sub \$8,$len 1673 jnc .Lcbc_dec_loop 1674 1675 add \$8,$len 1676 jz .Lcbc_dec_done 1677 1678 movdqu 0x00($inp), @XMM[0] # load input 1679 mov %rsp, %rax # pass key schedule 1680 mov %edx, %r10d # pass rounds 1681 cmp \$2,$len 1682 jb .Lcbc_dec_one 1683 movdqu 0x10($inp), @XMM[1] 1684 je .Lcbc_dec_two 1685 movdqu 0x20($inp), @XMM[2] 1686 cmp \$4,$len 1687 jb .Lcbc_dec_three 1688 movdqu 0x30($inp), @XMM[3] 1689 je .Lcbc_dec_four 1690 movdqu 0x40($inp), @XMM[4] 1691 cmp \$6,$len 1692 jb .Lcbc_dec_five 1693 movdqu 0x50($inp), @XMM[5] 1694 je .Lcbc_dec_six 1695 movdqu 0x60($inp), @XMM[6] 1696 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1697 call _bsaes_decrypt8 1698 pxor 0x20(%rbp), @XMM[0] # ^= IV 1699 movdqu 0x00($inp), @XMM[8] # re-load input 1700 movdqu 0x10($inp), @XMM[9] 1701 pxor @XMM[8], @XMM[1] 1702 movdqu 0x20($inp), @XMM[10] 1703 pxor @XMM[9], @XMM[6] 1704 movdqu 0x30($inp), @XMM[11] 1705 pxor @XMM[10], @XMM[4] 1706 movdqu 0x40($inp), @XMM[12] 1707 pxor @XMM[11], @XMM[2] 1708 movdqu 0x50($inp), @XMM[13] 1709 pxor @XMM[12], @XMM[7] 1710 movdqu 0x60($inp), @XMM[15] # IV 1711 pxor @XMM[13], @XMM[3] 1712 movdqu @XMM[0], 0x00($out) # write output 1713 movdqu @XMM[1], 0x10($out) 1714 movdqu @XMM[6], 0x20($out) 1715 movdqu @XMM[4], 0x30($out) 1716 movdqu @XMM[2], 0x40($out) 1717 movdqu @XMM[7], 0x50($out) 1718 movdqu @XMM[3], 0x60($out) 1719 jmp .Lcbc_dec_done 1720 .align 16 1721 .Lcbc_dec_six: 1722 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1723 call _bsaes_decrypt8 1724 pxor 0x20(%rbp), @XMM[0] # ^= IV 1725 movdqu 0x00($inp), @XMM[8] # re-load input 1726 movdqu 0x10($inp), @XMM[9] 1727 pxor @XMM[8], @XMM[1] 1728 movdqu 0x20($inp), @XMM[10] 1729 pxor @XMM[9], @XMM[6] 1730 movdqu 0x30($inp), @XMM[11] 1731 pxor @XMM[10], @XMM[4] 1732 movdqu 0x40($inp), @XMM[12] 1733 pxor @XMM[11], @XMM[2] 1734 movdqu 0x50($inp), @XMM[15] # IV 1735 pxor @XMM[12], @XMM[7] 1736 movdqu @XMM[0], 0x00($out) # write output 1737 movdqu @XMM[1], 0x10($out) 1738 movdqu @XMM[6], 0x20($out) 1739 movdqu @XMM[4], 0x30($out) 1740 movdqu @XMM[2], 0x40($out) 1741 movdqu @XMM[7], 0x50($out) 1742 jmp .Lcbc_dec_done 1743 .align 16 1744 .Lcbc_dec_five: 1745 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1746 call _bsaes_decrypt8 1747 pxor 0x20(%rbp), @XMM[0] # ^= IV 1748 movdqu 0x00($inp), @XMM[8] # re-load input 1749 movdqu 0x10($inp), @XMM[9] 1750 pxor @XMM[8], @XMM[1] 1751 movdqu 0x20($inp), @XMM[10] 1752 pxor @XMM[9], @XMM[6] 1753 movdqu 0x30($inp), @XMM[11] 1754 pxor @XMM[10], @XMM[4] 1755 movdqu 0x40($inp), @XMM[15] # IV 1756 pxor @XMM[11], @XMM[2] 1757 movdqu @XMM[0], 0x00($out) # write output 1758 movdqu @XMM[1], 0x10($out) 1759 movdqu @XMM[6], 0x20($out) 1760 movdqu @XMM[4], 0x30($out) 1761 movdqu @XMM[2], 0x40($out) 1762 jmp .Lcbc_dec_done 1763 .align 16 1764 .Lcbc_dec_four: 1765 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1766 call _bsaes_decrypt8 1767 pxor 0x20(%rbp), @XMM[0] # ^= IV 1768 movdqu 0x00($inp), @XMM[8] # re-load input 1769 movdqu 0x10($inp), @XMM[9] 1770 pxor @XMM[8], @XMM[1] 1771 movdqu 0x20($inp), @XMM[10] 1772 pxor @XMM[9], @XMM[6] 1773 movdqu 0x30($inp), @XMM[15] # IV 1774 pxor @XMM[10], @XMM[4] 1775 movdqu @XMM[0], 0x00($out) # write output 1776 movdqu @XMM[1], 0x10($out) 1777 movdqu @XMM[6], 0x20($out) 1778 movdqu @XMM[4], 0x30($out) 1779 jmp .Lcbc_dec_done 1780 .align 16 1781 .Lcbc_dec_three: 1782 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1783 call _bsaes_decrypt8 1784 pxor 0x20(%rbp), @XMM[0] # ^= IV 1785 movdqu 0x00($inp), @XMM[8] # re-load input 1786 movdqu 0x10($inp), @XMM[9] 1787 pxor @XMM[8], @XMM[1] 1788 movdqu 0x20($inp), @XMM[15] # IV 1789 pxor @XMM[9], @XMM[6] 1790 movdqu @XMM[0], 0x00($out) # write output 1791 movdqu @XMM[1], 0x10($out) 1792 movdqu @XMM[6], 0x20($out) 1793 jmp .Lcbc_dec_done 1794 .align 16 1795 .Lcbc_dec_two: 1796 movdqa @XMM[15], 0x20(%rbp) # put aside IV 1797 call _bsaes_decrypt8 1798 pxor 0x20(%rbp), @XMM[0] # ^= IV 1799 movdqu 0x00($inp), @XMM[8] # re-load input 1800 movdqu 0x10($inp), @XMM[15] # IV 1801 pxor @XMM[8], @XMM[1] 1802 movdqu @XMM[0], 0x00($out) # write output 1803 movdqu @XMM[1], 0x10($out) 1804 jmp .Lcbc_dec_done 1805 .align 16 1806 .Lcbc_dec_one: 1807 lea ($inp), $arg1 1808 lea 0x20(%rbp), $arg2 # buffer output 1809 lea ($key), $arg3 1810 call asm_AES_decrypt # doesn't touch %xmm 1811 pxor 0x20(%rbp), @XMM[15] # ^= IV 1812 movdqu @XMM[15], ($out) # write output 1813 movdqa @XMM[0], @XMM[15] # IV 1814 1815 .Lcbc_dec_done: 1816 movdqu @XMM[15], (%rbx) # return IV 1817 lea (%rsp), %rax 1818 pxor %xmm0, %xmm0 1819 .Lcbc_dec_bzero: # wipe key schedule [if any] 1820 movdqa %xmm0, 0x00(%rax) 1821 movdqa %xmm0, 0x10(%rax) 1822 lea 0x20(%rax), %rax 1823 cmp %rax, %rbp 1824 ja .Lcbc_dec_bzero 1825 1826 lea (%rbp),%rsp # restore %rsp 1827 ___ 1828 $code.=<<___ if ($win64); 1829 movaps 0x40(%rbp), %xmm6 1830 movaps 0x50(%rbp), %xmm7 1831 movaps 0x60(%rbp), %xmm8 1832 movaps 0x70(%rbp), %xmm9 1833 movaps 0x80(%rbp), %xmm10 1834 movaps 0x90(%rbp), %xmm11 1835 movaps 0xa0(%rbp), %xmm12 1836 movaps 0xb0(%rbp), %xmm13 1837 movaps 0xc0(%rbp), %xmm14 1838 movaps 0xd0(%rbp), %xmm15 1839 lea 0xa0(%rbp), %rsp 1840 ___ 1841 $code.=<<___; 1842 mov 0x48(%rsp), %r15 1843 mov 0x50(%rsp), %r14 1844 mov 0x58(%rsp), %r13 1845 mov 0x60(%rsp), %r12 1846 mov 0x68(%rsp), %rbx 1847 mov 0x70(%rsp), %rax 1848 lea 0x78(%rsp), %rsp 1849 mov %rax, %rbp 1850 .Lcbc_dec_epilogue: 1851 ret 1852 .size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt 1853 1854 .globl bsaes_ctr32_encrypt_blocks 1855 .type bsaes_ctr32_encrypt_blocks,\@abi-omnipotent 1856 .align 16 1857 bsaes_ctr32_encrypt_blocks: 1858 mov %rsp, %rax 1859 .Lctr_enc_prologue: 1860 push %rbp 1861 push %rbx 1862 push %r12 1863 push %r13 1864 push %r14 1865 push %r15 1866 lea -0x48(%rsp), %rsp 1867 ___ 1868 $code.=<<___ if ($win64); 1869 mov 0xa0(%rsp),$arg5 # pull ivp 1870 lea -0xa0(%rsp), %rsp 1871 movaps %xmm6, 0x40(%rsp) 1872 movaps %xmm7, 0x50(%rsp) 1873 movaps %xmm8, 0x60(%rsp) 1874 movaps %xmm9, 0x70(%rsp) 1875 movaps %xmm10, 0x80(%rsp) 1876 movaps %xmm11, 0x90(%rsp) 1877 movaps %xmm12, 0xa0(%rsp) 1878 movaps %xmm13, 0xb0(%rsp) 1879 movaps %xmm14, 0xc0(%rsp) 1880 movaps %xmm15, 0xd0(%rsp) 1881 .Lctr_enc_body: 1882 ___ 1883 $code.=<<___; 1884 mov %rsp, %rbp # backup %rsp 1885 movdqu ($arg5), %xmm0 # load counter 1886 mov 240($arg4), %eax # rounds 1887 mov $arg1, $inp # backup arguments 1888 mov $arg2, $out 1889 mov $arg3, $len 1890 mov $arg4, $key 1891 movdqa %xmm0, 0x20(%rbp) # copy counter 1892 cmp \$8, $arg3 1893 jb .Lctr_enc_short 1894 1895 mov %eax, %ebx # rounds 1896 shl \$7, %rax # 128 bytes per inner round key 1897 sub \$`128-32`, %rax # size of bit-sliced key schedule 1898 sub %rax, %rsp 1899 1900 mov %rsp, %rax # pass key schedule 1901 mov $key, %rcx # pass key 1902 mov %ebx, %r10d # pass rounds 1903 call _bsaes_key_convert 1904 pxor %xmm6,%xmm7 # fix up last round key 1905 movdqa %xmm7,(%rax) # save last round key 1906 1907 movdqa (%rsp), @XMM[9] # load round0 key 1908 lea .LADD1(%rip), %r11 1909 movdqa 0x20(%rbp), @XMM[0] # counter copy 1910 movdqa -0x20(%r11), @XMM[8] # .LSWPUP 1911 pshufb @XMM[8], @XMM[9] # byte swap upper part 1912 pshufb @XMM[8], @XMM[0] 1913 movdqa @XMM[9], (%rsp) # save adjusted round0 key 1914 jmp .Lctr_enc_loop 1915 .align 16 1916 .Lctr_enc_loop: 1917 movdqa @XMM[0], 0x20(%rbp) # save counter 1918 movdqa @XMM[0], @XMM[1] # prepare 8 counter values 1919 movdqa @XMM[0], @XMM[2] 1920 paddd 0x00(%r11), @XMM[1] # .LADD1 1921 movdqa @XMM[0], @XMM[3] 1922 paddd 0x10(%r11), @XMM[2] # .LADD2 1923 movdqa @XMM[0], @XMM[4] 1924 paddd 0x20(%r11), @XMM[3] # .LADD3 1925 movdqa @XMM[0], @XMM[5] 1926 paddd 0x30(%r11), @XMM[4] # .LADD4 1927 movdqa @XMM[0], @XMM[6] 1928 paddd 0x40(%r11), @XMM[5] # .LADD5 1929 movdqa @XMM[0], @XMM[7] 1930 paddd 0x50(%r11), @XMM[6] # .LADD6 1931 paddd 0x60(%r11), @XMM[7] # .LADD7 1932 1933 # Borrow prologue from _bsaes_encrypt8 to use the opportunity 1934 # to flip byte order in 32-bit counter 1935 movdqa (%rsp), @XMM[9] # round 0 key 1936 lea 0x10(%rsp), %rax # pass key schedule 1937 movdqa -0x10(%r11), @XMM[8] # .LSWPUPM0SR 1938 pxor @XMM[9], @XMM[0] # xor with round0 key 1939 pxor @XMM[9], @XMM[1] 1940 pshufb @XMM[8], @XMM[0] 1941 pxor @XMM[9], @XMM[2] 1942 pshufb @XMM[8], @XMM[1] 1943 pxor @XMM[9], @XMM[3] 1944 pshufb @XMM[8], @XMM[2] 1945 pxor @XMM[9], @XMM[4] 1946 pshufb @XMM[8], @XMM[3] 1947 pxor @XMM[9], @XMM[5] 1948 pshufb @XMM[8], @XMM[4] 1949 pxor @XMM[9], @XMM[6] 1950 pshufb @XMM[8], @XMM[5] 1951 pxor @XMM[9], @XMM[7] 1952 pshufb @XMM[8], @XMM[6] 1953 lea .LBS0(%rip), %r11 # constants table 1954 pshufb @XMM[8], @XMM[7] 1955 mov %ebx,%r10d # pass rounds 1956 1957 call _bsaes_encrypt8_bitslice 1958 1959 sub \$8,$len 1960 jc .Lctr_enc_loop_done 1961 1962 movdqu 0x00($inp), @XMM[8] # load input 1963 movdqu 0x10($inp), @XMM[9] 1964 movdqu 0x20($inp), @XMM[10] 1965 movdqu 0x30($inp), @XMM[11] 1966 movdqu 0x40($inp), @XMM[12] 1967 movdqu 0x50($inp), @XMM[13] 1968 movdqu 0x60($inp), @XMM[14] 1969 movdqu 0x70($inp), @XMM[15] 1970 lea 0x80($inp),$inp 1971 pxor @XMM[0], @XMM[8] 1972 movdqa 0x20(%rbp), @XMM[0] # load counter 1973 pxor @XMM[9], @XMM[1] 1974 movdqu @XMM[8], 0x00($out) # write output 1975 pxor @XMM[10], @XMM[4] 1976 movdqu @XMM[1], 0x10($out) 1977 pxor @XMM[11], @XMM[6] 1978 movdqu @XMM[4], 0x20($out) 1979 pxor @XMM[12], @XMM[3] 1980 movdqu @XMM[6], 0x30($out) 1981 pxor @XMM[13], @XMM[7] 1982 movdqu @XMM[3], 0x40($out) 1983 pxor @XMM[14], @XMM[2] 1984 movdqu @XMM[7], 0x50($out) 1985 pxor @XMM[15], @XMM[5] 1986 movdqu @XMM[2], 0x60($out) 1987 lea .LADD1(%rip), %r11 1988 movdqu @XMM[5], 0x70($out) 1989 lea 0x80($out), $out 1990 paddd 0x70(%r11), @XMM[0] # .LADD8 1991 jnz .Lctr_enc_loop 1992 1993 jmp .Lctr_enc_done 1994 .align 16 1995 .Lctr_enc_loop_done: 1996 add \$8, $len 1997 movdqu 0x00($inp), @XMM[8] # load input 1998 pxor @XMM[8], @XMM[0] 1999 movdqu @XMM[0], 0x00($out) # write output 2000 cmp \$2,$len 2001 jb .Lctr_enc_done 2002 movdqu 0x10($inp), @XMM[9] 2003 pxor @XMM[9], @XMM[1] 2004 movdqu @XMM[1], 0x10($out) 2005 je .Lctr_enc_done 2006 movdqu 0x20($inp), @XMM[10] 2007 pxor @XMM[10], @XMM[4] 2008 movdqu @XMM[4], 0x20($out) 2009 cmp \$4,$len 2010 jb .Lctr_enc_done 2011 movdqu 0x30($inp), @XMM[11] 2012 pxor @XMM[11], @XMM[6] 2013 movdqu @XMM[6], 0x30($out) 2014 je .Lctr_enc_done 2015 movdqu 0x40($inp), @XMM[12] 2016 pxor @XMM[12], @XMM[3] 2017 movdqu @XMM[3], 0x40($out) 2018 cmp \$6,$len 2019 jb .Lctr_enc_done 2020 movdqu 0x50($inp), @XMM[13] 2021 pxor @XMM[13], @XMM[7] 2022 movdqu @XMM[7], 0x50($out) 2023 je .Lctr_enc_done 2024 movdqu 0x60($inp), @XMM[14] 2025 pxor @XMM[14], @XMM[2] 2026 movdqu @XMM[2], 0x60($out) 2027 jmp .Lctr_enc_done 2028 2029 .align 16 2030 .Lctr_enc_short: 2031 lea 0x20(%rbp), $arg1 2032 lea 0x30(%rbp), $arg2 2033 lea ($key), $arg3 2034 call asm_AES_encrypt 2035 movdqu ($inp), @XMM[1] 2036 lea 16($inp), $inp 2037 mov 0x2c(%rbp), %eax # load 32-bit counter 2038 bswap %eax 2039 pxor 0x30(%rbp), @XMM[1] 2040 inc %eax # increment 2041 movdqu @XMM[1], ($out) 2042 bswap %eax 2043 lea 16($out), $out 2044 mov %eax, 0x2c(%rsp) # save 32-bit counter 2045 dec $len 2046 jnz .Lctr_enc_short 2047 2048 .Lctr_enc_done: 2049 lea (%rsp), %rax 2050 pxor %xmm0, %xmm0 2051 .Lctr_enc_bzero: # wipe key schedule [if any] 2052 movdqa %xmm0, 0x00(%rax) 2053 movdqa %xmm0, 0x10(%rax) 2054 lea 0x20(%rax), %rax 2055 cmp %rax, %rbp 2056 ja .Lctr_enc_bzero 2057 2058 lea (%rbp),%rsp # restore %rsp 2059 ___ 2060 $code.=<<___ if ($win64); 2061 movaps 0x40(%rbp), %xmm6 2062 movaps 0x50(%rbp), %xmm7 2063 movaps 0x60(%rbp), %xmm8 2064 movaps 0x70(%rbp), %xmm9 2065 movaps 0x80(%rbp), %xmm10 2066 movaps 0x90(%rbp), %xmm11 2067 movaps 0xa0(%rbp), %xmm12 2068 movaps 0xb0(%rbp), %xmm13 2069 movaps 0xc0(%rbp), %xmm14 2070 movaps 0xd0(%rbp), %xmm15 2071 lea 0xa0(%rbp), %rsp 2072 ___ 2073 $code.=<<___; 2074 mov 0x48(%rsp), %r15 2075 mov 0x50(%rsp), %r14 2076 mov 0x58(%rsp), %r13 2077 mov 0x60(%rsp), %r12 2078 mov 0x68(%rsp), %rbx 2079 mov 0x70(%rsp), %rax 2080 lea 0x78(%rsp), %rsp 2081 mov %rax, %rbp 2082 .Lctr_enc_epilogue: 2083 ret 2084 .size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks 2085 ___ 2086 ###################################################################### 2087 # void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, 2088 # const AES_KEY *key1, const AES_KEY *key2, 2089 # const unsigned char iv[16]); 2090 # 2091 my ($twmask,$twres,$twtmp)=@XMM[13..15]; 2092 $arg6=~s/d$//; 2093 2094 $code.=<<___; 2095 .globl bsaes_xts_encrypt 2096 .type bsaes_xts_encrypt,\@abi-omnipotent 2097 .align 16 2098 bsaes_xts_encrypt: 2099 mov %rsp, %rax 2100 .Lxts_enc_prologue: 2101 push %rbp 2102 push %rbx 2103 push %r12 2104 push %r13 2105 push %r14 2106 push %r15 2107 lea -0x48(%rsp), %rsp 2108 ___ 2109 $code.=<<___ if ($win64); 2110 mov 0xa0(%rsp),$arg5 # pull key2 2111 mov 0xa8(%rsp),$arg6 # pull ivp 2112 lea -0xa0(%rsp), %rsp 2113 movaps %xmm6, 0x40(%rsp) 2114 movaps %xmm7, 0x50(%rsp) 2115 movaps %xmm8, 0x60(%rsp) 2116 movaps %xmm9, 0x70(%rsp) 2117 movaps %xmm10, 0x80(%rsp) 2118 movaps %xmm11, 0x90(%rsp) 2119 movaps %xmm12, 0xa0(%rsp) 2120 movaps %xmm13, 0xb0(%rsp) 2121 movaps %xmm14, 0xc0(%rsp) 2122 movaps %xmm15, 0xd0(%rsp) 2123 .Lxts_enc_body: 2124 ___ 2125 $code.=<<___; 2126 mov %rsp, %rbp # backup %rsp 2127 mov $arg1, $inp # backup arguments 2128 mov $arg2, $out 2129 mov $arg3, $len 2130 mov $arg4, $key 2131 2132 lea ($arg6), $arg1 2133 lea 0x20(%rbp), $arg2 2134 lea ($arg5), $arg3 2135 call asm_AES_encrypt # generate initial tweak 2136 2137 mov 240($key), %eax # rounds 2138 mov $len, %rbx # backup $len 2139 2140 mov %eax, %edx # rounds 2141 shl \$7, %rax # 128 bytes per inner round key 2142 sub \$`128-32`, %rax # size of bit-sliced key schedule 2143 sub %rax, %rsp 2144 2145 mov %rsp, %rax # pass key schedule 2146 mov $key, %rcx # pass key 2147 mov %edx, %r10d # pass rounds 2148 call _bsaes_key_convert 2149 pxor %xmm6, %xmm7 # fix up last round key 2150 movdqa %xmm7, (%rax) # save last round key 2151 2152 and \$-16, $len 2153 sub \$0x80, %rsp # place for tweak[8] 2154 movdqa 0x20(%rbp), @XMM[7] # initial tweak 2155 2156 pxor $twtmp, $twtmp 2157 movdqa .Lxts_magic(%rip), $twmask 2158 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2159 2160 sub \$0x80, $len 2161 jc .Lxts_enc_short 2162 jmp .Lxts_enc_loop 2163 2164 .align 16 2165 .Lxts_enc_loop: 2166 ___ 2167 for ($i=0;$i<7;$i++) { 2168 $code.=<<___; 2169 pshufd \$0x13, $twtmp, $twres 2170 pxor $twtmp, $twtmp 2171 movdqa @XMM[7], @XMM[$i] 2172 movdqa @XMM[7], `0x10*$i`(%rsp)# save tweak[$i] 2173 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2174 pand $twmask, $twres # isolate carry and residue 2175 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2176 pxor $twres, @XMM[7] 2177 ___ 2178 $code.=<<___ if ($i>=1); 2179 movdqu `0x10*($i-1)`($inp), @XMM[8+$i-1] 2180 ___ 2181 $code.=<<___ if ($i>=2); 2182 pxor @XMM[8+$i-2], @XMM[$i-2]# input[] ^ tweak[] 2183 ___ 2184 } 2185 $code.=<<___; 2186 movdqu 0x60($inp), @XMM[8+6] 2187 pxor @XMM[8+5], @XMM[5] 2188 movdqu 0x70($inp), @XMM[8+7] 2189 lea 0x80($inp), $inp 2190 movdqa @XMM[7], 0x70(%rsp) 2191 pxor @XMM[8+6], @XMM[6] 2192 lea 0x80(%rsp), %rax # pass key schedule 2193 pxor @XMM[8+7], @XMM[7] 2194 mov %edx, %r10d # pass rounds 2195 2196 call _bsaes_encrypt8 2197 2198 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2199 pxor 0x10(%rsp), @XMM[1] 2200 movdqu @XMM[0], 0x00($out) # write output 2201 pxor 0x20(%rsp), @XMM[4] 2202 movdqu @XMM[1], 0x10($out) 2203 pxor 0x30(%rsp), @XMM[6] 2204 movdqu @XMM[4], 0x20($out) 2205 pxor 0x40(%rsp), @XMM[3] 2206 movdqu @XMM[6], 0x30($out) 2207 pxor 0x50(%rsp), @XMM[7] 2208 movdqu @XMM[3], 0x40($out) 2209 pxor 0x60(%rsp), @XMM[2] 2210 movdqu @XMM[7], 0x50($out) 2211 pxor 0x70(%rsp), @XMM[5] 2212 movdqu @XMM[2], 0x60($out) 2213 movdqu @XMM[5], 0x70($out) 2214 lea 0x80($out), $out 2215 2216 movdqa 0x70(%rsp), @XMM[7] # prepare next iteration tweak 2217 pxor $twtmp, $twtmp 2218 movdqa .Lxts_magic(%rip), $twmask 2219 pcmpgtd @XMM[7], $twtmp 2220 pshufd \$0x13, $twtmp, $twres 2221 pxor $twtmp, $twtmp 2222 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2223 pand $twmask, $twres # isolate carry and residue 2224 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2225 pxor $twres, @XMM[7] 2226 2227 sub \$0x80,$len 2228 jnc .Lxts_enc_loop 2229 2230 .Lxts_enc_short: 2231 add \$0x80, $len 2232 jz .Lxts_enc_done 2233 ___ 2234 for ($i=0;$i<7;$i++) { 2235 $code.=<<___; 2236 pshufd \$0x13, $twtmp, $twres 2237 pxor $twtmp, $twtmp 2238 movdqa @XMM[7], @XMM[$i] 2239 movdqa @XMM[7], `0x10*$i`(%rsp)# save tweak[$i] 2240 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2241 pand $twmask, $twres # isolate carry and residue 2242 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2243 pxor $twres, @XMM[7] 2244 ___ 2245 $code.=<<___ if ($i>=1); 2246 movdqu `0x10*($i-1)`($inp), @XMM[8+$i-1] 2247 cmp \$`0x10*$i`,$len 2248 je .Lxts_enc_$i 2249 ___ 2250 $code.=<<___ if ($i>=2); 2251 pxor @XMM[8+$i-2], @XMM[$i-2]# input[] ^ tweak[] 2252 ___ 2253 } 2254 $code.=<<___; 2255 movdqu 0x60($inp), @XMM[8+6] 2256 pxor @XMM[8+5], @XMM[5] 2257 movdqa @XMM[7], 0x70(%rsp) 2258 lea 0x70($inp), $inp 2259 pxor @XMM[8+6], @XMM[6] 2260 lea 0x80(%rsp), %rax # pass key schedule 2261 mov %edx, %r10d # pass rounds 2262 2263 call _bsaes_encrypt8 2264 2265 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2266 pxor 0x10(%rsp), @XMM[1] 2267 movdqu @XMM[0], 0x00($out) # write output 2268 pxor 0x20(%rsp), @XMM[4] 2269 movdqu @XMM[1], 0x10($out) 2270 pxor 0x30(%rsp), @XMM[6] 2271 movdqu @XMM[4], 0x20($out) 2272 pxor 0x40(%rsp), @XMM[3] 2273 movdqu @XMM[6], 0x30($out) 2274 pxor 0x50(%rsp), @XMM[7] 2275 movdqu @XMM[3], 0x40($out) 2276 pxor 0x60(%rsp), @XMM[2] 2277 movdqu @XMM[7], 0x50($out) 2278 movdqu @XMM[2], 0x60($out) 2279 lea 0x70($out), $out 2280 2281 movdqa 0x70(%rsp), @XMM[7] # next iteration tweak 2282 jmp .Lxts_enc_done 2283 .align 16 2284 .Lxts_enc_6: 2285 pxor @XMM[8+4], @XMM[4] 2286 lea 0x60($inp), $inp 2287 pxor @XMM[8+5], @XMM[5] 2288 lea 0x80(%rsp), %rax # pass key schedule 2289 mov %edx, %r10d # pass rounds 2290 2291 call _bsaes_encrypt8 2292 2293 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2294 pxor 0x10(%rsp), @XMM[1] 2295 movdqu @XMM[0], 0x00($out) # write output 2296 pxor 0x20(%rsp), @XMM[4] 2297 movdqu @XMM[1], 0x10($out) 2298 pxor 0x30(%rsp), @XMM[6] 2299 movdqu @XMM[4], 0x20($out) 2300 pxor 0x40(%rsp), @XMM[3] 2301 movdqu @XMM[6], 0x30($out) 2302 pxor 0x50(%rsp), @XMM[7] 2303 movdqu @XMM[3], 0x40($out) 2304 movdqu @XMM[7], 0x50($out) 2305 lea 0x60($out), $out 2306 2307 movdqa 0x60(%rsp), @XMM[7] # next iteration tweak 2308 jmp .Lxts_enc_done 2309 .align 16 2310 .Lxts_enc_5: 2311 pxor @XMM[8+3], @XMM[3] 2312 lea 0x50($inp), $inp 2313 pxor @XMM[8+4], @XMM[4] 2314 lea 0x80(%rsp), %rax # pass key schedule 2315 mov %edx, %r10d # pass rounds 2316 2317 call _bsaes_encrypt8 2318 2319 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2320 pxor 0x10(%rsp), @XMM[1] 2321 movdqu @XMM[0], 0x00($out) # write output 2322 pxor 0x20(%rsp), @XMM[4] 2323 movdqu @XMM[1], 0x10($out) 2324 pxor 0x30(%rsp), @XMM[6] 2325 movdqu @XMM[4], 0x20($out) 2326 pxor 0x40(%rsp), @XMM[3] 2327 movdqu @XMM[6], 0x30($out) 2328 movdqu @XMM[3], 0x40($out) 2329 lea 0x50($out), $out 2330 2331 movdqa 0x50(%rsp), @XMM[7] # next iteration tweak 2332 jmp .Lxts_enc_done 2333 .align 16 2334 .Lxts_enc_4: 2335 pxor @XMM[8+2], @XMM[2] 2336 lea 0x40($inp), $inp 2337 pxor @XMM[8+3], @XMM[3] 2338 lea 0x80(%rsp), %rax # pass key schedule 2339 mov %edx, %r10d # pass rounds 2340 2341 call _bsaes_encrypt8 2342 2343 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2344 pxor 0x10(%rsp), @XMM[1] 2345 movdqu @XMM[0], 0x00($out) # write output 2346 pxor 0x20(%rsp), @XMM[4] 2347 movdqu @XMM[1], 0x10($out) 2348 pxor 0x30(%rsp), @XMM[6] 2349 movdqu @XMM[4], 0x20($out) 2350 movdqu @XMM[6], 0x30($out) 2351 lea 0x40($out), $out 2352 2353 movdqa 0x40(%rsp), @XMM[7] # next iteration tweak 2354 jmp .Lxts_enc_done 2355 .align 16 2356 .Lxts_enc_3: 2357 pxor @XMM[8+1], @XMM[1] 2358 lea 0x30($inp), $inp 2359 pxor @XMM[8+2], @XMM[2] 2360 lea 0x80(%rsp), %rax # pass key schedule 2361 mov %edx, %r10d # pass rounds 2362 2363 call _bsaes_encrypt8 2364 2365 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2366 pxor 0x10(%rsp), @XMM[1] 2367 movdqu @XMM[0], 0x00($out) # write output 2368 pxor 0x20(%rsp), @XMM[4] 2369 movdqu @XMM[1], 0x10($out) 2370 movdqu @XMM[4], 0x20($out) 2371 lea 0x30($out), $out 2372 2373 movdqa 0x30(%rsp), @XMM[7] # next iteration tweak 2374 jmp .Lxts_enc_done 2375 .align 16 2376 .Lxts_enc_2: 2377 pxor @XMM[8+0], @XMM[0] 2378 lea 0x20($inp), $inp 2379 pxor @XMM[8+1], @XMM[1] 2380 lea 0x80(%rsp), %rax # pass key schedule 2381 mov %edx, %r10d # pass rounds 2382 2383 call _bsaes_encrypt8 2384 2385 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2386 pxor 0x10(%rsp), @XMM[1] 2387 movdqu @XMM[0], 0x00($out) # write output 2388 movdqu @XMM[1], 0x10($out) 2389 lea 0x20($out), $out 2390 2391 movdqa 0x20(%rsp), @XMM[7] # next iteration tweak 2392 jmp .Lxts_enc_done 2393 .align 16 2394 .Lxts_enc_1: 2395 pxor @XMM[0], @XMM[8] 2396 lea 0x10($inp), $inp 2397 movdqa @XMM[8], 0x20(%rbp) 2398 lea 0x20(%rbp), $arg1 2399 lea 0x20(%rbp), $arg2 2400 lea ($key), $arg3 2401 call asm_AES_encrypt # doesn't touch %xmm 2402 pxor 0x20(%rbp), @XMM[0] # ^= tweak[] 2403 #pxor @XMM[8], @XMM[0] 2404 #lea 0x80(%rsp), %rax # pass key schedule 2405 #mov %edx, %r10d # pass rounds 2406 #call _bsaes_encrypt8 2407 #pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2408 movdqu @XMM[0], 0x00($out) # write output 2409 lea 0x10($out), $out 2410 2411 movdqa 0x10(%rsp), @XMM[7] # next iteration tweak 2412 2413 .Lxts_enc_done: 2414 and \$15, %ebx 2415 jz .Lxts_enc_ret 2416 mov $out, %rdx 2417 2418 .Lxts_enc_steal: 2419 movzb ($inp), %eax 2420 movzb -16(%rdx), %ecx 2421 lea 1($inp), $inp 2422 mov %al, -16(%rdx) 2423 mov %cl, 0(%rdx) 2424 lea 1(%rdx), %rdx 2425 sub \$1,%ebx 2426 jnz .Lxts_enc_steal 2427 2428 movdqu -16($out), @XMM[0] 2429 lea 0x20(%rbp), $arg1 2430 pxor @XMM[7], @XMM[0] 2431 lea 0x20(%rbp), $arg2 2432 movdqa @XMM[0], 0x20(%rbp) 2433 lea ($key), $arg3 2434 call asm_AES_encrypt # doesn't touch %xmm 2435 pxor 0x20(%rbp), @XMM[7] 2436 movdqu @XMM[7], -16($out) 2437 2438 .Lxts_enc_ret: 2439 lea (%rsp), %rax 2440 pxor %xmm0, %xmm0 2441 .Lxts_enc_bzero: # wipe key schedule [if any] 2442 movdqa %xmm0, 0x00(%rax) 2443 movdqa %xmm0, 0x10(%rax) 2444 lea 0x20(%rax), %rax 2445 cmp %rax, %rbp 2446 ja .Lxts_enc_bzero 2447 2448 lea (%rbp),%rsp # restore %rsp 2449 ___ 2450 $code.=<<___ if ($win64); 2451 movaps 0x40(%rbp), %xmm6 2452 movaps 0x50(%rbp), %xmm7 2453 movaps 0x60(%rbp), %xmm8 2454 movaps 0x70(%rbp), %xmm9 2455 movaps 0x80(%rbp), %xmm10 2456 movaps 0x90(%rbp), %xmm11 2457 movaps 0xa0(%rbp), %xmm12 2458 movaps 0xb0(%rbp), %xmm13 2459 movaps 0xc0(%rbp), %xmm14 2460 movaps 0xd0(%rbp), %xmm15 2461 lea 0xa0(%rbp), %rsp 2462 ___ 2463 $code.=<<___; 2464 mov 0x48(%rsp), %r15 2465 mov 0x50(%rsp), %r14 2466 mov 0x58(%rsp), %r13 2467 mov 0x60(%rsp), %r12 2468 mov 0x68(%rsp), %rbx 2469 mov 0x70(%rsp), %rax 2470 lea 0x78(%rsp), %rsp 2471 mov %rax, %rbp 2472 .Lxts_enc_epilogue: 2473 ret 2474 .size bsaes_xts_encrypt,.-bsaes_xts_encrypt 2475 2476 .globl bsaes_xts_decrypt 2477 .type bsaes_xts_decrypt,\@abi-omnipotent 2478 .align 16 2479 bsaes_xts_decrypt: 2480 mov %rsp, %rax 2481 .Lxts_dec_prologue: 2482 push %rbp 2483 push %rbx 2484 push %r12 2485 push %r13 2486 push %r14 2487 push %r15 2488 lea -0x48(%rsp), %rsp 2489 ___ 2490 $code.=<<___ if ($win64); 2491 mov 0xa0(%rsp),$arg5 # pull key2 2492 mov 0xa8(%rsp),$arg6 # pull ivp 2493 lea -0xa0(%rsp), %rsp 2494 movaps %xmm6, 0x40(%rsp) 2495 movaps %xmm7, 0x50(%rsp) 2496 movaps %xmm8, 0x60(%rsp) 2497 movaps %xmm9, 0x70(%rsp) 2498 movaps %xmm10, 0x80(%rsp) 2499 movaps %xmm11, 0x90(%rsp) 2500 movaps %xmm12, 0xa0(%rsp) 2501 movaps %xmm13, 0xb0(%rsp) 2502 movaps %xmm14, 0xc0(%rsp) 2503 movaps %xmm15, 0xd0(%rsp) 2504 .Lxts_dec_body: 2505 ___ 2506 $code.=<<___; 2507 mov %rsp, %rbp # backup %rsp 2508 mov $arg1, $inp # backup arguments 2509 mov $arg2, $out 2510 mov $arg3, $len 2511 mov $arg4, $key 2512 2513 lea ($arg6), $arg1 2514 lea 0x20(%rbp), $arg2 2515 lea ($arg5), $arg3 2516 call asm_AES_encrypt # generate initial tweak 2517 2518 mov 240($key), %eax # rounds 2519 mov $len, %rbx # backup $len 2520 2521 mov %eax, %edx # rounds 2522 shl \$7, %rax # 128 bytes per inner round key 2523 sub \$`128-32`, %rax # size of bit-sliced key schedule 2524 sub %rax, %rsp 2525 2526 mov %rsp, %rax # pass key schedule 2527 mov $key, %rcx # pass key 2528 mov %edx, %r10d # pass rounds 2529 call _bsaes_key_convert 2530 pxor (%rsp), %xmm7 # fix up round 0 key 2531 movdqa %xmm6, (%rax) # save last round key 2532 movdqa %xmm7, (%rsp) 2533 2534 xor %eax, %eax # if ($len%16) len-=16; 2535 and \$-16, $len 2536 test \$15, %ebx 2537 setnz %al 2538 shl \$4, %rax 2539 sub %rax, $len 2540 2541 sub \$0x80, %rsp # place for tweak[8] 2542 movdqa 0x20(%rbp), @XMM[7] # initial tweak 2543 2544 pxor $twtmp, $twtmp 2545 movdqa .Lxts_magic(%rip), $twmask 2546 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2547 2548 sub \$0x80, $len 2549 jc .Lxts_dec_short 2550 jmp .Lxts_dec_loop 2551 2552 .align 16 2553 .Lxts_dec_loop: 2554 ___ 2555 for ($i=0;$i<7;$i++) { 2556 $code.=<<___; 2557 pshufd \$0x13, $twtmp, $twres 2558 pxor $twtmp, $twtmp 2559 movdqa @XMM[7], @XMM[$i] 2560 movdqa @XMM[7], `0x10*$i`(%rsp)# save tweak[$i] 2561 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2562 pand $twmask, $twres # isolate carry and residue 2563 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2564 pxor $twres, @XMM[7] 2565 ___ 2566 $code.=<<___ if ($i>=1); 2567 movdqu `0x10*($i-1)`($inp), @XMM[8+$i-1] 2568 ___ 2569 $code.=<<___ if ($i>=2); 2570 pxor @XMM[8+$i-2], @XMM[$i-2]# input[] ^ tweak[] 2571 ___ 2572 } 2573 $code.=<<___; 2574 movdqu 0x60($inp), @XMM[8+6] 2575 pxor @XMM[8+5], @XMM[5] 2576 movdqu 0x70($inp), @XMM[8+7] 2577 lea 0x80($inp), $inp 2578 movdqa @XMM[7], 0x70(%rsp) 2579 pxor @XMM[8+6], @XMM[6] 2580 lea 0x80(%rsp), %rax # pass key schedule 2581 pxor @XMM[8+7], @XMM[7] 2582 mov %edx, %r10d # pass rounds 2583 2584 call _bsaes_decrypt8 2585 2586 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2587 pxor 0x10(%rsp), @XMM[1] 2588 movdqu @XMM[0], 0x00($out) # write output 2589 pxor 0x20(%rsp), @XMM[6] 2590 movdqu @XMM[1], 0x10($out) 2591 pxor 0x30(%rsp), @XMM[4] 2592 movdqu @XMM[6], 0x20($out) 2593 pxor 0x40(%rsp), @XMM[2] 2594 movdqu @XMM[4], 0x30($out) 2595 pxor 0x50(%rsp), @XMM[7] 2596 movdqu @XMM[2], 0x40($out) 2597 pxor 0x60(%rsp), @XMM[3] 2598 movdqu @XMM[7], 0x50($out) 2599 pxor 0x70(%rsp), @XMM[5] 2600 movdqu @XMM[3], 0x60($out) 2601 movdqu @XMM[5], 0x70($out) 2602 lea 0x80($out), $out 2603 2604 movdqa 0x70(%rsp), @XMM[7] # prepare next iteration tweak 2605 pxor $twtmp, $twtmp 2606 movdqa .Lxts_magic(%rip), $twmask 2607 pcmpgtd @XMM[7], $twtmp 2608 pshufd \$0x13, $twtmp, $twres 2609 pxor $twtmp, $twtmp 2610 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2611 pand $twmask, $twres # isolate carry and residue 2612 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2613 pxor $twres, @XMM[7] 2614 2615 sub \$0x80,$len 2616 jnc .Lxts_dec_loop 2617 2618 .Lxts_dec_short: 2619 add \$0x80, $len 2620 jz .Lxts_dec_done 2621 ___ 2622 for ($i=0;$i<7;$i++) { 2623 $code.=<<___; 2624 pshufd \$0x13, $twtmp, $twres 2625 pxor $twtmp, $twtmp 2626 movdqa @XMM[7], @XMM[$i] 2627 movdqa @XMM[7], `0x10*$i`(%rsp)# save tweak[$i] 2628 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2629 pand $twmask, $twres # isolate carry and residue 2630 pcmpgtd @XMM[7], $twtmp # broadcast upper bits 2631 pxor $twres, @XMM[7] 2632 ___ 2633 $code.=<<___ if ($i>=1); 2634 movdqu `0x10*($i-1)`($inp), @XMM[8+$i-1] 2635 cmp \$`0x10*$i`,$len 2636 je .Lxts_dec_$i 2637 ___ 2638 $code.=<<___ if ($i>=2); 2639 pxor @XMM[8+$i-2], @XMM[$i-2]# input[] ^ tweak[] 2640 ___ 2641 } 2642 $code.=<<___; 2643 movdqu 0x60($inp), @XMM[8+6] 2644 pxor @XMM[8+5], @XMM[5] 2645 movdqa @XMM[7], 0x70(%rsp) 2646 lea 0x70($inp), $inp 2647 pxor @XMM[8+6], @XMM[6] 2648 lea 0x80(%rsp), %rax # pass key schedule 2649 mov %edx, %r10d # pass rounds 2650 2651 call _bsaes_decrypt8 2652 2653 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2654 pxor 0x10(%rsp), @XMM[1] 2655 movdqu @XMM[0], 0x00($out) # write output 2656 pxor 0x20(%rsp), @XMM[6] 2657 movdqu @XMM[1], 0x10($out) 2658 pxor 0x30(%rsp), @XMM[4] 2659 movdqu @XMM[6], 0x20($out) 2660 pxor 0x40(%rsp), @XMM[2] 2661 movdqu @XMM[4], 0x30($out) 2662 pxor 0x50(%rsp), @XMM[7] 2663 movdqu @XMM[2], 0x40($out) 2664 pxor 0x60(%rsp), @XMM[3] 2665 movdqu @XMM[7], 0x50($out) 2666 movdqu @XMM[3], 0x60($out) 2667 lea 0x70($out), $out 2668 2669 movdqa 0x70(%rsp), @XMM[7] # next iteration tweak 2670 jmp .Lxts_dec_done 2671 .align 16 2672 .Lxts_dec_6: 2673 pxor @XMM[8+4], @XMM[4] 2674 lea 0x60($inp), $inp 2675 pxor @XMM[8+5], @XMM[5] 2676 lea 0x80(%rsp), %rax # pass key schedule 2677 mov %edx, %r10d # pass rounds 2678 2679 call _bsaes_decrypt8 2680 2681 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2682 pxor 0x10(%rsp), @XMM[1] 2683 movdqu @XMM[0], 0x00($out) # write output 2684 pxor 0x20(%rsp), @XMM[6] 2685 movdqu @XMM[1], 0x10($out) 2686 pxor 0x30(%rsp), @XMM[4] 2687 movdqu @XMM[6], 0x20($out) 2688 pxor 0x40(%rsp), @XMM[2] 2689 movdqu @XMM[4], 0x30($out) 2690 pxor 0x50(%rsp), @XMM[7] 2691 movdqu @XMM[2], 0x40($out) 2692 movdqu @XMM[7], 0x50($out) 2693 lea 0x60($out), $out 2694 2695 movdqa 0x60(%rsp), @XMM[7] # next iteration tweak 2696 jmp .Lxts_dec_done 2697 .align 16 2698 .Lxts_dec_5: 2699 pxor @XMM[8+3], @XMM[3] 2700 lea 0x50($inp), $inp 2701 pxor @XMM[8+4], @XMM[4] 2702 lea 0x80(%rsp), %rax # pass key schedule 2703 mov %edx, %r10d # pass rounds 2704 2705 call _bsaes_decrypt8 2706 2707 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2708 pxor 0x10(%rsp), @XMM[1] 2709 movdqu @XMM[0], 0x00($out) # write output 2710 pxor 0x20(%rsp), @XMM[6] 2711 movdqu @XMM[1], 0x10($out) 2712 pxor 0x30(%rsp), @XMM[4] 2713 movdqu @XMM[6], 0x20($out) 2714 pxor 0x40(%rsp), @XMM[2] 2715 movdqu @XMM[4], 0x30($out) 2716 movdqu @XMM[2], 0x40($out) 2717 lea 0x50($out), $out 2718 2719 movdqa 0x50(%rsp), @XMM[7] # next iteration tweak 2720 jmp .Lxts_dec_done 2721 .align 16 2722 .Lxts_dec_4: 2723 pxor @XMM[8+2], @XMM[2] 2724 lea 0x40($inp), $inp 2725 pxor @XMM[8+3], @XMM[3] 2726 lea 0x80(%rsp), %rax # pass key schedule 2727 mov %edx, %r10d # pass rounds 2728 2729 call _bsaes_decrypt8 2730 2731 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2732 pxor 0x10(%rsp), @XMM[1] 2733 movdqu @XMM[0], 0x00($out) # write output 2734 pxor 0x20(%rsp), @XMM[6] 2735 movdqu @XMM[1], 0x10($out) 2736 pxor 0x30(%rsp), @XMM[4] 2737 movdqu @XMM[6], 0x20($out) 2738 movdqu @XMM[4], 0x30($out) 2739 lea 0x40($out), $out 2740 2741 movdqa 0x40(%rsp), @XMM[7] # next iteration tweak 2742 jmp .Lxts_dec_done 2743 .align 16 2744 .Lxts_dec_3: 2745 pxor @XMM[8+1], @XMM[1] 2746 lea 0x30($inp), $inp 2747 pxor @XMM[8+2], @XMM[2] 2748 lea 0x80(%rsp), %rax # pass key schedule 2749 mov %edx, %r10d # pass rounds 2750 2751 call _bsaes_decrypt8 2752 2753 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2754 pxor 0x10(%rsp), @XMM[1] 2755 movdqu @XMM[0], 0x00($out) # write output 2756 pxor 0x20(%rsp), @XMM[6] 2757 movdqu @XMM[1], 0x10($out) 2758 movdqu @XMM[6], 0x20($out) 2759 lea 0x30($out), $out 2760 2761 movdqa 0x30(%rsp), @XMM[7] # next iteration tweak 2762 jmp .Lxts_dec_done 2763 .align 16 2764 .Lxts_dec_2: 2765 pxor @XMM[8+0], @XMM[0] 2766 lea 0x20($inp), $inp 2767 pxor @XMM[8+1], @XMM[1] 2768 lea 0x80(%rsp), %rax # pass key schedule 2769 mov %edx, %r10d # pass rounds 2770 2771 call _bsaes_decrypt8 2772 2773 pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2774 pxor 0x10(%rsp), @XMM[1] 2775 movdqu @XMM[0], 0x00($out) # write output 2776 movdqu @XMM[1], 0x10($out) 2777 lea 0x20($out), $out 2778 2779 movdqa 0x20(%rsp), @XMM[7] # next iteration tweak 2780 jmp .Lxts_dec_done 2781 .align 16 2782 .Lxts_dec_1: 2783 pxor @XMM[0], @XMM[8] 2784 lea 0x10($inp), $inp 2785 movdqa @XMM[8], 0x20(%rbp) 2786 lea 0x20(%rbp), $arg1 2787 lea 0x20(%rbp), $arg2 2788 lea ($key), $arg3 2789 call asm_AES_decrypt # doesn't touch %xmm 2790 pxor 0x20(%rbp), @XMM[0] # ^= tweak[] 2791 #pxor @XMM[8], @XMM[0] 2792 #lea 0x80(%rsp), %rax # pass key schedule 2793 #mov %edx, %r10d # pass rounds 2794 #call _bsaes_decrypt8 2795 #pxor 0x00(%rsp), @XMM[0] # ^= tweak[] 2796 movdqu @XMM[0], 0x00($out) # write output 2797 lea 0x10($out), $out 2798 2799 movdqa 0x10(%rsp), @XMM[7] # next iteration tweak 2800 2801 .Lxts_dec_done: 2802 and \$15, %ebx 2803 jz .Lxts_dec_ret 2804 2805 pxor $twtmp, $twtmp 2806 movdqa .Lxts_magic(%rip), $twmask 2807 pcmpgtd @XMM[7], $twtmp 2808 pshufd \$0x13, $twtmp, $twres 2809 movdqa @XMM[7], @XMM[6] 2810 paddq @XMM[7], @XMM[7] # psllq 1,$tweak 2811 pand $twmask, $twres # isolate carry and residue 2812 movdqu ($inp), @XMM[0] 2813 pxor $twres, @XMM[7] 2814 2815 lea 0x20(%rbp), $arg1 2816 pxor @XMM[7], @XMM[0] 2817 lea 0x20(%rbp), $arg2 2818 movdqa @XMM[0], 0x20(%rbp) 2819 lea ($key), $arg3 2820 call asm_AES_decrypt # doesn't touch %xmm 2821 pxor 0x20(%rbp), @XMM[7] 2822 mov $out, %rdx 2823 movdqu @XMM[7], ($out) 2824 2825 .Lxts_dec_steal: 2826 movzb 16($inp), %eax 2827 movzb (%rdx), %ecx 2828 lea 1($inp), $inp 2829 mov %al, (%rdx) 2830 mov %cl, 16(%rdx) 2831 lea 1(%rdx), %rdx 2832 sub \$1,%ebx 2833 jnz .Lxts_dec_steal 2834 2835 movdqu ($out), @XMM[0] 2836 lea 0x20(%rbp), $arg1 2837 pxor @XMM[6], @XMM[0] 2838 lea 0x20(%rbp), $arg2 2839 movdqa @XMM[0], 0x20(%rbp) 2840 lea ($key), $arg3 2841 call asm_AES_decrypt # doesn't touch %xmm 2842 pxor 0x20(%rbp), @XMM[6] 2843 movdqu @XMM[6], ($out) 2844 2845 .Lxts_dec_ret: 2846 lea (%rsp), %rax 2847 pxor %xmm0, %xmm0 2848 .Lxts_dec_bzero: # wipe key schedule [if any] 2849 movdqa %xmm0, 0x00(%rax) 2850 movdqa %xmm0, 0x10(%rax) 2851 lea 0x20(%rax), %rax 2852 cmp %rax, %rbp 2853 ja .Lxts_dec_bzero 2854 2855 lea (%rbp),%rsp # restore %rsp 2856 ___ 2857 $code.=<<___ if ($win64); 2858 movaps 0x40(%rbp), %xmm6 2859 movaps 0x50(%rbp), %xmm7 2860 movaps 0x60(%rbp), %xmm8 2861 movaps 0x70(%rbp), %xmm9 2862 movaps 0x80(%rbp), %xmm10 2863 movaps 0x90(%rbp), %xmm11 2864 movaps 0xa0(%rbp), %xmm12 2865 movaps 0xb0(%rbp), %xmm13 2866 movaps 0xc0(%rbp), %xmm14 2867 movaps 0xd0(%rbp), %xmm15 2868 lea 0xa0(%rbp), %rsp 2869 ___ 2870 $code.=<<___; 2871 mov 0x48(%rsp), %r15 2872 mov 0x50(%rsp), %r14 2873 mov 0x58(%rsp), %r13 2874 mov 0x60(%rsp), %r12 2875 mov 0x68(%rsp), %rbx 2876 mov 0x70(%rsp), %rax 2877 lea 0x78(%rsp), %rsp 2878 mov %rax, %rbp 2879 .Lxts_dec_epilogue: 2880 ret 2881 .size bsaes_xts_decrypt,.-bsaes_xts_decrypt 2882 ___ 2883 } 2884 $code.=<<___; 2885 .type _bsaes_const,\@object 2886 .align 64 2887 _bsaes_const: 2888 .LM0ISR: # InvShiftRows constants 2889 .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 2890 .LISRM0: 2891 .quad 0x01040b0e0205080f, 0x0306090c00070a0d 2892 .LISR: 2893 .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 2894 .LBS0: # bit-slice constants 2895 .quad 0x5555555555555555, 0x5555555555555555 2896 .LBS1: 2897 .quad 0x3333333333333333, 0x3333333333333333 2898 .LBS2: 2899 .quad 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f 2900 .LSR: # shiftrows constants 2901 .quad 0x0504070600030201, 0x0f0e0d0c0a09080b 2902 .LSRM0: 2903 .quad 0x0304090e00050a0f, 0x01060b0c0207080d 2904 .LM0SR: 2905 .quad 0x0a0e02060f03070b, 0x0004080c05090d01 2906 .LSWPUP: # byte-swap upper dword 2907 .quad 0x0706050403020100, 0x0c0d0e0f0b0a0908 2908 .LSWPUPM0SR: 2909 .quad 0x0a0d02060c03070b, 0x0004080f05090e01 2910 .LADD1: # counter increment constants 2911 .quad 0x0000000000000000, 0x0000000100000000 2912 .LADD2: 2913 .quad 0x0000000000000000, 0x0000000200000000 2914 .LADD3: 2915 .quad 0x0000000000000000, 0x0000000300000000 2916 .LADD4: 2917 .quad 0x0000000000000000, 0x0000000400000000 2918 .LADD5: 2919 .quad 0x0000000000000000, 0x0000000500000000 2920 .LADD6: 2921 .quad 0x0000000000000000, 0x0000000600000000 2922 .LADD7: 2923 .quad 0x0000000000000000, 0x0000000700000000 2924 .LADD8: 2925 .quad 0x0000000000000000, 0x0000000800000000 2926 .Lxts_magic: 2927 .long 0x87,0,1,0 2928 .Lmasks: 2929 .quad 0x0101010101010101, 0x0101010101010101 2930 .quad 0x0202020202020202, 0x0202020202020202 2931 .quad 0x0404040404040404, 0x0404040404040404 2932 .quad 0x0808080808080808, 0x0808080808080808 2933 .LM0: 2934 .quad 0x02060a0e03070b0f, 0x0004080c0105090d 2935 .L63: 2936 .quad 0x6363636363636363, 0x6363636363636363 2937 .asciz "Bit-sliced AES for x86_64/SSSE3, Emilia Ksper, Peter Schwabe, Andy Polyakov" 2938 .align 64 2939 .size _bsaes_const,.-_bsaes_const 2940 ___ 2941 2942 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 2943 # CONTEXT *context,DISPATCHER_CONTEXT *disp) 2944 if ($win64) { 2945 $rec="%rcx"; 2946 $frame="%rdx"; 2947 $context="%r8"; 2948 $disp="%r9"; 2949 2950 $code.=<<___; 2951 .extern __imp_RtlVirtualUnwind 2952 .type se_handler,\@abi-omnipotent 2953 .align 16 2954 se_handler: 2955 push %rsi 2956 push %rdi 2957 push %rbx 2958 push %rbp 2959 push %r12 2960 push %r13 2961 push %r14 2962 push %r15 2963 pushfq 2964 sub \$64,%rsp 2965 2966 mov 120($context),%rax # pull context->Rax 2967 mov 248($context),%rbx # pull context->Rip 2968 2969 mov 8($disp),%rsi # disp->ImageBase 2970 mov 56($disp),%r11 # disp->HandlerData 2971 2972 mov 0(%r11),%r10d # HandlerData[0] 2973 lea (%rsi,%r10),%r10 # prologue label 2974 cmp %r10,%rbx # context->Rip<prologue label 2975 jb .Lin_prologue 2976 2977 mov 152($context),%rax # pull context->Rsp 2978 2979 mov 4(%r11),%r10d # HandlerData[1] 2980 lea (%rsi,%r10),%r10 # epilogue label 2981 cmp %r10,%rbx # context->Rip>=epilogue label 2982 jae .Lin_prologue 2983 2984 mov 160($context),%rax # pull context->Rbp 2985 2986 lea 0x40(%rax),%rsi # %xmm save area 2987 lea 512($context),%rdi # &context.Xmm6 2988 mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) 2989 .long 0xa548f3fc # cld; rep movsq 2990 lea 0xa0(%rax),%rax # adjust stack pointer 2991 2992 mov 0x70(%rax),%rbp 2993 mov 0x68(%rax),%rbx 2994 mov 0x60(%rax),%r12 2995 mov 0x58(%rax),%r13 2996 mov 0x50(%rax),%r14 2997 mov 0x48(%rax),%r15 2998 lea 0x78(%rax),%rax # adjust stack pointer 2999 mov %rbx,144($context) # restore context->Rbx 3000 mov %rbp,160($context) # restore context->Rbp 3001 mov %r12,216($context) # restore context->R12 3002 mov %r13,224($context) # restore context->R13 3003 mov %r14,232($context) # restore context->R14 3004 mov %r15,240($context) # restore context->R15 3005 3006 .Lin_prologue: 3007 mov %rax,152($context) # restore context->Rsp 3008 3009 mov 40($disp),%rdi # disp->ContextRecord 3010 mov $context,%rsi # context 3011 mov \$`1232/8`,%ecx # sizeof(CONTEXT) 3012 .long 0xa548f3fc # cld; rep movsq 3013 3014 mov $disp,%rsi 3015 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 3016 mov 8(%rsi),%rdx # arg2, disp->ImageBase 3017 mov 0(%rsi),%r8 # arg3, disp->ControlPc 3018 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 3019 mov 40(%rsi),%r10 # disp->ContextRecord 3020 lea 56(%rsi),%r11 # &disp->HandlerData 3021 lea 24(%rsi),%r12 # &disp->EstablisherFrame 3022 mov %r10,32(%rsp) # arg5 3023 mov %r11,40(%rsp) # arg6 3024 mov %r12,48(%rsp) # arg7 3025 mov %rcx,56(%rsp) # arg8, (NULL) 3026 call *__imp_RtlVirtualUnwind(%rip) 3027 3028 mov \$1,%eax # ExceptionContinueSearch 3029 add \$64,%rsp 3030 popfq 3031 pop %r15 3032 pop %r14 3033 pop %r13 3034 pop %r12 3035 pop %rbp 3036 pop %rbx 3037 pop %rdi 3038 pop %rsi 3039 ret 3040 .size se_handler,.-se_handler 3041 3042 .section .pdata 3043 .align 4 3044 ___ 3045 $code.=<<___ if ($ecb); 3046 .rva .Lecb_enc_prologue 3047 .rva .Lecb_enc_epilogue 3048 .rva .Lecb_enc_info 3049 3050 .rva .Lecb_dec_prologue 3051 .rva .Lecb_dec_epilogue 3052 .rva .Lecb_dec_info 3053 ___ 3054 $code.=<<___; 3055 .rva .Lcbc_dec_prologue 3056 .rva .Lcbc_dec_epilogue 3057 .rva .Lcbc_dec_info 3058 3059 .rva .Lctr_enc_prologue 3060 .rva .Lctr_enc_epilogue 3061 .rva .Lctr_enc_info 3062 3063 .rva .Lxts_enc_prologue 3064 .rva .Lxts_enc_epilogue 3065 .rva .Lxts_enc_info 3066 3067 .rva .Lxts_dec_prologue 3068 .rva .Lxts_dec_epilogue 3069 .rva .Lxts_dec_info 3070 3071 .section .xdata 3072 .align 8 3073 ___ 3074 $code.=<<___ if ($ecb); 3075 .Lecb_enc_info: 3076 .byte 9,0,0,0 3077 .rva se_handler 3078 .rva .Lecb_enc_body,.Lecb_enc_epilogue # HandlerData[] 3079 .Lecb_dec_info: 3080 .byte 9,0,0,0 3081 .rva se_handler 3082 .rva .Lecb_dec_body,.Lecb_dec_epilogue # HandlerData[] 3083 ___ 3084 $code.=<<___; 3085 .Lcbc_dec_info: 3086 .byte 9,0,0,0 3087 .rva se_handler 3088 .rva .Lcbc_dec_body,.Lcbc_dec_epilogue # HandlerData[] 3089 .Lctr_enc_info: 3090 .byte 9,0,0,0 3091 .rva se_handler 3092 .rva .Lctr_enc_body,.Lctr_enc_epilogue # HandlerData[] 3093 .Lxts_enc_info: 3094 .byte 9,0,0,0 3095 .rva se_handler 3096 .rva .Lxts_enc_body,.Lxts_enc_epilogue # HandlerData[] 3097 .Lxts_dec_info: 3098 .byte 9,0,0,0 3099 .rva se_handler 3100 .rva .Lxts_dec_body,.Lxts_dec_epilogue # HandlerData[] 3101 ___ 3102 } 3103 3104 $code =~ s/\`([^\`]*)\`/eval($1)/gem; 3105 3106 print $code; 3107 3108 close STDOUT; 3109
