1 #! /usr/bin/env perl 2 # Copyright 1998-2016 The OpenSSL Project Authors. All Rights Reserved. 3 # 4 # Licensed under the OpenSSL license (the "License"). You may not use 5 # this file except in compliance with the License. You can obtain a copy 6 # in the file LICENSE in the source distribution or at 7 # https://www.openssl.org/source/license.html 8 9 10 # ==================================================================== 11 # [Re]written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL 12 # project. The module is, however, dual licensed under OpenSSL and 13 # CRYPTOGAMS licenses depending on where you obtain it. For further 14 # details see http://www.openssl.org/~appro/cryptogams/. 15 # ==================================================================== 16 17 # "[Re]written" was achieved in two major overhauls. In 2004 BODY_* 18 # functions were re-implemented to address P4 performance issue [see 19 # commentary below], and in 2006 the rest was rewritten in order to 20 # gain freedom to liberate licensing terms. 21 22 # January, September 2004. 23 # 24 # It was noted that Intel IA-32 C compiler generates code which 25 # performs ~30% *faster* on P4 CPU than original *hand-coded* 26 # SHA1 assembler implementation. To address this problem (and 27 # prove that humans are still better than machines:-), the 28 # original code was overhauled, which resulted in following 29 # performance changes: 30 # 31 # compared with original compared with Intel cc 32 # assembler impl. generated code 33 # Pentium -16% +48% 34 # PIII/AMD +8% +16% 35 # P4 +85%(!) +45% 36 # 37 # As you can see Pentium came out as looser:-( Yet I reckoned that 38 # improvement on P4 outweighs the loss and incorporate this 39 # re-tuned code to 0.9.7 and later. 40 # ---------------------------------------------------------------- 41 42 # August 2009. 43 # 44 # George Spelvin has tipped that F_40_59(b,c,d) can be rewritten as 45 # '(c&d) + (b&(c^d))', which allows to accumulate partial results 46 # and lighten "pressure" on scratch registers. This resulted in 47 # >12% performance improvement on contemporary AMD cores (with no 48 # degradation on other CPUs:-). Also, the code was revised to maximize 49 # "distance" between instructions producing input to 'lea' instruction 50 # and the 'lea' instruction itself, which is essential for Intel Atom 51 # core and resulted in ~15% improvement. 52 53 # October 2010. 54 # 55 # Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it 56 # is to offload message schedule denoted by Wt in NIST specification, 57 # or Xupdate in OpenSSL source, to SIMD unit. The idea is not novel, 58 # and in SSE2 context was first explored by Dean Gaudet in 2004, see 59 # http://arctic.org/~dean/crypto/sha1.html. Since then several things 60 # have changed that made it interesting again: 61 # 62 # a) XMM units became faster and wider; 63 # b) instruction set became more versatile; 64 # c) an important observation was made by Max Locktykhin, which made 65 # it possible to reduce amount of instructions required to perform 66 # the operation in question, for further details see 67 # http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/. 68 69 # April 2011. 70 # 71 # Add AVX code path, probably most controversial... The thing is that 72 # switch to AVX alone improves performance by as little as 4% in 73 # comparison to SSSE3 code path. But below result doesn't look like 74 # 4% improvement... Trouble is that Sandy Bridge decodes 'ro[rl]' as 75 # pair of -ops, and it's the additional -ops, two per round, that 76 # make it run slower than Core2 and Westmere. But 'sh[rl]d' is decoded 77 # as single -op by Sandy Bridge and it's replacing 'ro[rl]' with 78 # equivalent 'sh[rl]d' that is responsible for the impressive 5.1 79 # cycles per processed byte. But 'sh[rl]d' is not something that used 80 # to be fast, nor does it appear to be fast in upcoming Bulldozer 81 # [according to its optimization manual]. Which is why AVX code path 82 # is guarded by *both* AVX and synthetic bit denoting Intel CPUs. 83 # One can argue that it's unfair to AMD, but without 'sh[rl]d' it 84 # makes no sense to keep the AVX code path. If somebody feels that 85 # strongly, it's probably more appropriate to discuss possibility of 86 # using vector rotate XOP on AMD... 87 88 # March 2014. 89 # 90 # Add support for Intel SHA Extensions. 91 92 ###################################################################### 93 # Current performance is summarized in following table. Numbers are 94 # CPU clock cycles spent to process single byte (less is better). 95 # 96 # x86 SSSE3 AVX 97 # Pentium 15.7 - 98 # PIII 11.5 - 99 # P4 10.6 - 100 # AMD K8 7.1 - 101 # Core2 7.3 6.0/+22% - 102 # Westmere 7.3 5.5/+33% - 103 # Sandy Bridge 8.8 6.2/+40% 5.1(**)/+73% 104 # Ivy Bridge 7.2 4.8/+51% 4.7(**)/+53% 105 # Haswell 6.5 4.3/+51% 4.1(**)/+58% 106 # Skylake 6.4 4.1/+55% 4.1(**)/+55% 107 # Bulldozer 11.6 6.0/+92% 108 # VIA Nano 10.6 7.5/+41% 109 # Atom 12.5 9.3(*)/+35% 110 # Silvermont 14.5 9.9(*)/+46% 111 # Goldmont 8.8 6.7/+30% 1.7(***)/+415% 112 # 113 # (*) Loop is 1056 instructions long and expected result is ~8.25. 114 # The discrepancy is because of front-end limitations, so 115 # called MS-ROM penalties, and on Silvermont even rotate's 116 # limited parallelism. 117 # 118 # (**) As per above comment, the result is for AVX *plus* sh[rl]d. 119 # 120 # (***) SHAEXT result 121 122 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 123 push(@INC,"${dir}","${dir}../../../perlasm"); 124 require "x86asm.pl"; 125 126 $output=pop; 127 open STDOUT,">$output"; 128 129 &asm_init($ARGV[0],$ARGV[$#ARGV] eq "386"); 130 131 $xmm=$ymm=0; 132 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); } 133 134 # In upstream, this is controlled by shelling out to the compiler to check 135 # versions, but BoringSSL is intended to be used with pre-generated perlasm 136 # output, so this isn't useful anyway. 137 $ymm = 1; 138 139 $ymm = 0 unless ($xmm); 140 141 $shaext=$xmm; ### set to zero if compiling for 1.0.1 142 143 # TODO(davidben): Consider enabling the Intel SHA Extensions code once it's 144 # been tested. 145 $shaext = 0; 146 147 &external_label("OPENSSL_ia32cap_P") if ($xmm); 148 149 150 $A="eax"; 151 $B="ebx"; 152 $C="ecx"; 153 $D="edx"; 154 $E="edi"; 155 $T="esi"; 156 $tmp1="ebp"; 157 158 @V=($A,$B,$C,$D,$E,$T); 159 160 $alt=0; # 1 denotes alternative IALU implementation, which performs 161 # 8% *worse* on P4, same on Westmere and Atom, 2% better on 162 # Sandy Bridge... 163 164 sub BODY_00_15 165 { 166 local($n,$a,$b,$c,$d,$e,$f)=@_; 167 168 &comment("00_15 $n"); 169 170 &mov($f,$c); # f to hold F_00_19(b,c,d) 171 if ($n==0) { &mov($tmp1,$a); } 172 else { &mov($a,$tmp1); } 173 &rotl($tmp1,5); # tmp1=ROTATE(a,5) 174 &xor($f,$d); 175 &add($tmp1,$e); # tmp1+=e; 176 &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded 177 # with xi, also note that e becomes 178 # f in next round... 179 &and($f,$b); 180 &rotr($b,2); # b=ROTATE(b,30) 181 &xor($f,$d); # f holds F_00_19(b,c,d) 182 &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi 183 184 if ($n==15) { &mov($e,&swtmp(($n+1)%16));# pre-fetch f for next round 185 &add($f,$tmp1); } # f+=tmp1 186 else { &add($tmp1,$f); } # f becomes a in next round 187 &mov($tmp1,$a) if ($alt && $n==15); 188 } 189 190 sub BODY_16_19 191 { 192 local($n,$a,$b,$c,$d,$e,$f)=@_; 193 194 &comment("16_19 $n"); 195 196 if ($alt) { 197 &xor($c,$d); 198 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 199 &and($tmp1,$c); # tmp1 to hold F_00_19(b,c,d), b&=c^d 200 &xor($f,&swtmp(($n+8)%16)); 201 &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) 202 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 203 &rotl($f,1); # f=ROTATE(f,1) 204 &add($e,$tmp1); # e+=F_00_19(b,c,d) 205 &xor($c,$d); # restore $c 206 &mov($tmp1,$a); # b in next round 207 &rotr($b,$n==16?2:7); # b=ROTATE(b,30) 208 &mov(&swtmp($n%16),$f); # xi=f 209 &rotl($a,5); # ROTATE(a,5) 210 &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e 211 &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round 212 &add($f,$a); # f+=ROTATE(a,5) 213 } else { 214 &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d) 215 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 216 &xor($tmp1,$d); 217 &xor($f,&swtmp(($n+8)%16)); 218 &and($tmp1,$b); 219 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 220 &rotl($f,1); # f=ROTATE(f,1) 221 &xor($tmp1,$d); # tmp1=F_00_19(b,c,d) 222 &add($e,$tmp1); # e+=F_00_19(b,c,d) 223 &mov($tmp1,$a); 224 &rotr($b,2); # b=ROTATE(b,30) 225 &mov(&swtmp($n%16),$f); # xi=f 226 &rotl($tmp1,5); # ROTATE(a,5) 227 &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e 228 &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round 229 &add($f,$tmp1); # f+=ROTATE(a,5) 230 } 231 } 232 233 sub BODY_20_39 234 { 235 local($n,$a,$b,$c,$d,$e,$f)=@_; 236 local $K=($n<40)?0x6ed9eba1:0xca62c1d6; 237 238 &comment("20_39 $n"); 239 240 if ($alt) { 241 &xor($tmp1,$c); # tmp1 to hold F_20_39(b,c,d), b^=c 242 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 243 &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) 244 &xor($f,&swtmp(($n+8)%16)); 245 &add($e,$tmp1); # e+=F_20_39(b,c,d) 246 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 247 &rotl($f,1); # f=ROTATE(f,1) 248 &mov($tmp1,$a); # b in next round 249 &rotr($b,7); # b=ROTATE(b,30) 250 &mov(&swtmp($n%16),$f) if($n<77);# xi=f 251 &rotl($a,5); # ROTATE(a,5) 252 &xor($b,$c) if($n==39);# warm up for BODY_40_59 253 &and($tmp1,$b) if($n==39); 254 &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY 255 &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round 256 &add($f,$a); # f+=ROTATE(a,5) 257 &rotr($a,5) if ($n==79); 258 } else { 259 &mov($tmp1,$b); # tmp1 to hold F_20_39(b,c,d) 260 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 261 &xor($tmp1,$c); 262 &xor($f,&swtmp(($n+8)%16)); 263 &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d) 264 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 265 &rotl($f,1); # f=ROTATE(f,1) 266 &add($e,$tmp1); # e+=F_20_39(b,c,d) 267 &rotr($b,2); # b=ROTATE(b,30) 268 &mov($tmp1,$a); 269 &rotl($tmp1,5); # ROTATE(a,5) 270 &mov(&swtmp($n%16),$f) if($n<77);# xi=f 271 &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY 272 &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round 273 &add($f,$tmp1); # f+=ROTATE(a,5) 274 } 275 } 276 277 sub BODY_40_59 278 { 279 local($n,$a,$b,$c,$d,$e,$f)=@_; 280 281 &comment("40_59 $n"); 282 283 if ($alt) { 284 &add($e,$tmp1); # e+=b&(c^d) 285 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 286 &mov($tmp1,$d); 287 &xor($f,&swtmp(($n+8)%16)); 288 &xor($c,$d); # restore $c 289 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 290 &rotl($f,1); # f=ROTATE(f,1) 291 &and($tmp1,$c); 292 &rotr($b,7); # b=ROTATE(b,30) 293 &add($e,$tmp1); # e+=c&d 294 &mov($tmp1,$a); # b in next round 295 &mov(&swtmp($n%16),$f); # xi=f 296 &rotl($a,5); # ROTATE(a,5) 297 &xor($b,$c) if ($n<59); 298 &and($tmp1,$b) if ($n<59);# tmp1 to hold F_40_59(b,c,d) 299 &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e+(b&(c^d)) 300 &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round 301 &add($f,$a); # f+=ROTATE(a,5) 302 } else { 303 &mov($tmp1,$c); # tmp1 to hold F_40_59(b,c,d) 304 &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd) 305 &xor($tmp1,$d); 306 &xor($f,&swtmp(($n+8)%16)); 307 &and($tmp1,$b); 308 &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd 309 &rotl($f,1); # f=ROTATE(f,1) 310 &add($tmp1,$e); # b&(c^d)+=e 311 &rotr($b,2); # b=ROTATE(b,30) 312 &mov($e,$a); # e becomes volatile 313 &rotl($e,5); # ROTATE(a,5) 314 &mov(&swtmp($n%16),$f); # xi=f 315 &lea($f,&DWP(0x8f1bbcdc,$f,$tmp1));# f+=K_40_59+e+(b&(c^d)) 316 &mov($tmp1,$c); 317 &add($f,$e); # f+=ROTATE(a,5) 318 &and($tmp1,$d); 319 &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round 320 &add($f,$tmp1); # f+=c&d 321 } 322 } 323 324 &function_begin("sha1_block_data_order"); 325 if ($xmm) { 326 &static_label("shaext_shortcut") if ($shaext); 327 &static_label("ssse3_shortcut"); 328 &static_label("avx_shortcut") if ($ymm); 329 &static_label("K_XX_XX"); 330 331 &call (&label("pic_point")); # make it PIC! 332 &set_label("pic_point"); 333 &blindpop($tmp1); 334 &picmeup($T,"OPENSSL_ia32cap_P",$tmp1,&label("pic_point")); 335 &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); 336 337 &mov ($A,&DWP(0,$T)); 338 &mov ($D,&DWP(4,$T)); 339 &test ($D,1<<9); # check SSSE3 bit 340 &jz (&label("x86")); 341 &mov ($C,&DWP(8,$T)); 342 &test ($A,1<<24); # check FXSR bit 343 &jz (&label("x86")); 344 if ($shaext) { 345 &test ($C,1<<29); # check SHA bit 346 &jnz (&label("shaext_shortcut")); 347 } 348 if ($ymm) { 349 &and ($D,1<<28); # mask AVX bit 350 &and ($A,1<<30); # mask "Intel CPU" bit 351 &or ($A,$D); 352 &cmp ($A,1<<28|1<<30); 353 &je (&label("avx_shortcut")); 354 } 355 &jmp (&label("ssse3_shortcut")); 356 &set_label("x86",16); 357 } 358 &mov($tmp1,&wparam(0)); # SHA_CTX *c 359 &mov($T,&wparam(1)); # const void *input 360 &mov($A,&wparam(2)); # size_t num 361 &stack_push(16+3); # allocate X[16] 362 &shl($A,6); 363 &add($A,$T); 364 &mov(&wparam(2),$A); # pointer beyond the end of input 365 &mov($E,&DWP(16,$tmp1));# pre-load E 366 &jmp(&label("loop")); 367 368 &set_label("loop",16); 369 370 # copy input chunk to X, but reversing byte order! 371 for ($i=0; $i<16; $i+=4) 372 { 373 &mov($A,&DWP(4*($i+0),$T)); 374 &mov($B,&DWP(4*($i+1),$T)); 375 &mov($C,&DWP(4*($i+2),$T)); 376 &mov($D,&DWP(4*($i+3),$T)); 377 &bswap($A); 378 &bswap($B); 379 &bswap($C); 380 &bswap($D); 381 &mov(&swtmp($i+0),$A); 382 &mov(&swtmp($i+1),$B); 383 &mov(&swtmp($i+2),$C); 384 &mov(&swtmp($i+3),$D); 385 } 386 &mov(&wparam(1),$T); # redundant in 1st spin 387 388 &mov($A,&DWP(0,$tmp1)); # load SHA_CTX 389 &mov($B,&DWP(4,$tmp1)); 390 &mov($C,&DWP(8,$tmp1)); 391 &mov($D,&DWP(12,$tmp1)); 392 # E is pre-loaded 393 394 for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } 395 for(;$i<20;$i++) { &BODY_16_19($i,@V); unshift(@V,pop(@V)); } 396 for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } 397 for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); } 398 for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); } 399 400 (($V[5] eq $D) and ($V[0] eq $E)) or die; # double-check 401 402 &mov($tmp1,&wparam(0)); # re-load SHA_CTX* 403 &mov($D,&wparam(1)); # D is last "T" and is discarded 404 405 &add($E,&DWP(0,$tmp1)); # E is last "A"... 406 &add($T,&DWP(4,$tmp1)); 407 &add($A,&DWP(8,$tmp1)); 408 &add($B,&DWP(12,$tmp1)); 409 &add($C,&DWP(16,$tmp1)); 410 411 &mov(&DWP(0,$tmp1),$E); # update SHA_CTX 412 &add($D,64); # advance input pointer 413 &mov(&DWP(4,$tmp1),$T); 414 &cmp($D,&wparam(2)); # have we reached the end yet? 415 &mov(&DWP(8,$tmp1),$A); 416 &mov($E,$C); # C is last "E" which needs to be "pre-loaded" 417 &mov(&DWP(12,$tmp1),$B); 418 &mov($T,$D); # input pointer 419 &mov(&DWP(16,$tmp1),$C); 420 &jb(&label("loop")); 421 422 &stack_pop(16+3); 423 &function_end("sha1_block_data_order"); 424 425 if ($xmm) { 426 if ($shaext) { 427 ###################################################################### 428 # Intel SHA Extensions implementation of SHA1 update function. 429 # 430 my ($ctx,$inp,$num)=("edi","esi","ecx"); 431 my ($ABCD,$E,$E_,$BSWAP)=map("xmm$_",(0..3)); 432 my @MSG=map("xmm$_",(4..7)); 433 434 sub sha1rnds4 { 435 my ($dst,$src,$imm)=@_; 436 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) 437 { &data_byte(0x0f,0x3a,0xcc,0xc0|($1<<3)|$2,$imm); } 438 } 439 sub sha1op38 { 440 my ($opcodelet,$dst,$src)=@_; 441 if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/) 442 { &data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2); } 443 } 444 sub sha1nexte { sha1op38(0xc8,@_); } 445 sub sha1msg1 { sha1op38(0xc9,@_); } 446 sub sha1msg2 { sha1op38(0xca,@_); } 447 448 &function_begin("_sha1_block_data_order_shaext"); 449 &call (&label("pic_point")); # make it PIC! 450 &set_label("pic_point"); 451 &blindpop($tmp1); 452 &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); 453 &set_label("shaext_shortcut"); 454 &mov ($ctx,&wparam(0)); 455 &mov ("ebx","esp"); 456 &mov ($inp,&wparam(1)); 457 &mov ($num,&wparam(2)); 458 &sub ("esp",32); 459 460 &movdqu ($ABCD,&QWP(0,$ctx)); 461 &movd ($E,&DWP(16,$ctx)); 462 &and ("esp",-32); 463 &movdqa ($BSWAP,&QWP(0x50,$tmp1)); # byte-n-word swap 464 465 &movdqu (@MSG[0],&QWP(0,$inp)); 466 &pshufd ($ABCD,$ABCD,0b00011011); # flip word order 467 &movdqu (@MSG[1],&QWP(0x10,$inp)); 468 &pshufd ($E,$E,0b00011011); # flip word order 469 &movdqu (@MSG[2],&QWP(0x20,$inp)); 470 &pshufb (@MSG[0],$BSWAP); 471 &movdqu (@MSG[3],&QWP(0x30,$inp)); 472 &pshufb (@MSG[1],$BSWAP); 473 &pshufb (@MSG[2],$BSWAP); 474 &pshufb (@MSG[3],$BSWAP); 475 &jmp (&label("loop_shaext")); 476 477 &set_label("loop_shaext",16); 478 &dec ($num); 479 &lea ("eax",&DWP(0x40,$inp)); 480 &movdqa (&QWP(0,"esp"),$E); # offload $E 481 &paddd ($E,@MSG[0]); 482 &cmovne ($inp,"eax"); 483 &movdqa (&QWP(16,"esp"),$ABCD); # offload $ABCD 484 485 for($i=0;$i<20-4;$i+=2) { 486 &sha1msg1 (@MSG[0],@MSG[1]); 487 &movdqa ($E_,$ABCD); 488 &sha1rnds4 ($ABCD,$E,int($i/5)); # 0-3... 489 &sha1nexte ($E_,@MSG[1]); 490 &pxor (@MSG[0],@MSG[2]); 491 &sha1msg1 (@MSG[1],@MSG[2]); 492 &sha1msg2 (@MSG[0],@MSG[3]); 493 494 &movdqa ($E,$ABCD); 495 &sha1rnds4 ($ABCD,$E_,int(($i+1)/5)); 496 &sha1nexte ($E,@MSG[2]); 497 &pxor (@MSG[1],@MSG[3]); 498 &sha1msg2 (@MSG[1],@MSG[0]); 499 500 push(@MSG,shift(@MSG)); push(@MSG,shift(@MSG)); 501 } 502 &movdqu (@MSG[0],&QWP(0,$inp)); 503 &movdqa ($E_,$ABCD); 504 &sha1rnds4 ($ABCD,$E,3); # 64-67 505 &sha1nexte ($E_,@MSG[1]); 506 &movdqu (@MSG[1],&QWP(0x10,$inp)); 507 &pshufb (@MSG[0],$BSWAP); 508 509 &movdqa ($E,$ABCD); 510 &sha1rnds4 ($ABCD,$E_,3); # 68-71 511 &sha1nexte ($E,@MSG[2]); 512 &movdqu (@MSG[2],&QWP(0x20,$inp)); 513 &pshufb (@MSG[1],$BSWAP); 514 515 &movdqa ($E_,$ABCD); 516 &sha1rnds4 ($ABCD,$E,3); # 72-75 517 &sha1nexte ($E_,@MSG[3]); 518 &movdqu (@MSG[3],&QWP(0x30,$inp)); 519 &pshufb (@MSG[2],$BSWAP); 520 521 &movdqa ($E,$ABCD); 522 &sha1rnds4 ($ABCD,$E_,3); # 76-79 523 &movdqa ($E_,&QWP(0,"esp")); 524 &pshufb (@MSG[3],$BSWAP); 525 &sha1nexte ($E,$E_); 526 &paddd ($ABCD,&QWP(16,"esp")); 527 528 &jnz (&label("loop_shaext")); 529 530 &pshufd ($ABCD,$ABCD,0b00011011); 531 &pshufd ($E,$E,0b00011011); 532 &movdqu (&QWP(0,$ctx),$ABCD) 533 &movd (&DWP(16,$ctx),$E); 534 &mov ("esp","ebx"); 535 &function_end("_sha1_block_data_order_shaext"); 536 } 537 ###################################################################### 538 # The SSSE3 implementation. 539 # 540 # %xmm[0-7] are used as ring @X[] buffer containing quadruples of last 541 # 32 elements of the message schedule or Xupdate outputs. First 4 542 # quadruples are simply byte-swapped input, next 4 are calculated 543 # according to method originally suggested by Dean Gaudet (modulo 544 # being implemented in SSSE3). Once 8 quadruples or 32 elements are 545 # collected, it switches to routine proposed by Max Locktyukhin. 546 # 547 # Calculations inevitably require temporary registers, and there are 548 # no %xmm registers left to spare. For this reason part of the ring 549 # buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring 550 # buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] - 551 # X[-5], and X[4] - X[-4]... 552 # 553 # Another notable optimization is aggressive stack frame compression 554 # aiming to minimize amount of 9-byte instructions... 555 # 556 # Yet another notable optimization is "jumping" $B variable. It means 557 # that there is no register permanently allocated for $B value. This 558 # allowed to eliminate one instruction from body_20_39... 559 # 560 my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded 561 my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 562 my @V=($A,$B,$C,$D,$E); 563 my $j=0; # hash round 564 my $rx=0; 565 my @T=($T,$tmp1); 566 my $inp; 567 568 my $_rol=sub { &rol(@_) }; 569 my $_ror=sub { &ror(@_) }; 570 571 &function_begin("_sha1_block_data_order_ssse3"); 572 &call (&label("pic_point")); # make it PIC! 573 &set_label("pic_point"); 574 &blindpop($tmp1); 575 &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); 576 &set_label("ssse3_shortcut"); 577 578 &movdqa (@X[3],&QWP(0,$tmp1)); # K_00_19 579 &movdqa (@X[4],&QWP(16,$tmp1)); # K_20_39 580 &movdqa (@X[5],&QWP(32,$tmp1)); # K_40_59 581 &movdqa (@X[6],&QWP(48,$tmp1)); # K_60_79 582 &movdqa (@X[2],&QWP(64,$tmp1)); # pbswap mask 583 584 &mov ($E,&wparam(0)); # load argument block 585 &mov ($inp=@T[1],&wparam(1)); 586 &mov ($D,&wparam(2)); 587 &mov (@T[0],"esp"); 588 589 # stack frame layout 590 # 591 # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area 592 # X[4]+K X[5]+K X[6]+K X[7]+K 593 # X[8]+K X[9]+K X[10]+K X[11]+K 594 # X[12]+K X[13]+K X[14]+K X[15]+K 595 # 596 # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area 597 # X[4] X[5] X[6] X[7] 598 # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 599 # 600 # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants 601 # K_40_59 K_40_59 K_40_59 K_40_59 602 # K_60_79 K_60_79 K_60_79 K_60_79 603 # K_00_19 K_00_19 K_00_19 K_00_19 604 # pbswap mask 605 # 606 # +192 ctx # argument block 607 # +196 inp 608 # +200 end 609 # +204 esp 610 &sub ("esp",208); 611 &and ("esp",-64); 612 613 &movdqa (&QWP(112+0,"esp"),@X[4]); # copy constants 614 &movdqa (&QWP(112+16,"esp"),@X[5]); 615 &movdqa (&QWP(112+32,"esp"),@X[6]); 616 &shl ($D,6); # len*64 617 &movdqa (&QWP(112+48,"esp"),@X[3]); 618 &add ($D,$inp); # end of input 619 &movdqa (&QWP(112+64,"esp"),@X[2]); 620 &add ($inp,64); 621 &mov (&DWP(192+0,"esp"),$E); # save argument block 622 &mov (&DWP(192+4,"esp"),$inp); 623 &mov (&DWP(192+8,"esp"),$D); 624 &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp 625 626 &mov ($A,&DWP(0,$E)); # load context 627 &mov ($B,&DWP(4,$E)); 628 &mov ($C,&DWP(8,$E)); 629 &mov ($D,&DWP(12,$E)); 630 &mov ($E,&DWP(16,$E)); 631 &mov (@T[0],$B); # magic seed 632 633 &movdqu (@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] 634 &movdqu (@X[-3&7],&QWP(-48,$inp)); 635 &movdqu (@X[-2&7],&QWP(-32,$inp)); 636 &movdqu (@X[-1&7],&QWP(-16,$inp)); 637 &pshufb (@X[-4&7],@X[2]); # byte swap 638 &pshufb (@X[-3&7],@X[2]); 639 &pshufb (@X[-2&7],@X[2]); 640 &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot 641 &pshufb (@X[-1&7],@X[2]); 642 &paddd (@X[-4&7],@X[3]); # add K_00_19 643 &paddd (@X[-3&7],@X[3]); 644 &paddd (@X[-2&7],@X[3]); 645 &movdqa (&QWP(0,"esp"),@X[-4&7]); # X[]+K xfer to IALU 646 &psubd (@X[-4&7],@X[3]); # restore X[] 647 &movdqa (&QWP(0+16,"esp"),@X[-3&7]); 648 &psubd (@X[-3&7],@X[3]); 649 &movdqa (&QWP(0+32,"esp"),@X[-2&7]); 650 &mov (@T[1],$C); 651 &psubd (@X[-2&7],@X[3]); 652 &xor (@T[1],$D); 653 &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); 654 &and (@T[0],@T[1]); 655 &jmp (&label("loop")); 656 657 ###################################################################### 658 # SSE instruction sequence is first broken to groups of independent 659 # instructions, independent in respect to their inputs and shifter 660 # (not all architectures have more than one). Then IALU instructions 661 # are "knitted in" between the SSE groups. Distance is maintained for 662 # SSE latency of 2 in hope that it fits better upcoming AMD Bulldozer 663 # [which allegedly also implements SSSE3]... 664 # 665 # Temporary registers usage. X[2] is volatile at the entry and at the 666 # end is restored from backtrace ring buffer. X[3] is expected to 667 # contain current K_XX_XX constant and is used to calculate X[-1]+K 668 # from previous round, it becomes volatile the moment the value is 669 # saved to stack for transfer to IALU. X[4] becomes volatile whenever 670 # X[-4] is accumulated and offloaded to backtrace ring buffer, at the 671 # end it is loaded with next K_XX_XX [which becomes X[3] in next 672 # round]... 673 # 674 sub Xupdate_ssse3_16_31() # recall that $Xi starts with 4 675 { use integer; 676 my $body = shift; 677 my @insns = (&$body,&$body,&$body,&$body); # 40 instructions 678 my ($a,$b,$c,$d,$e); 679 680 eval(shift(@insns)); # ror 681 eval(shift(@insns)); 682 eval(shift(@insns)); 683 &punpcklqdq(@X[0],@X[-3&7]); # compose "X[-14]" in "X[0]", was &palignr(@X[0],@X[-4&7],8); 684 &movdqa (@X[2],@X[-1&7]); 685 eval(shift(@insns)); 686 eval(shift(@insns)); 687 688 &paddd (@X[3],@X[-1&7]); 689 &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer 690 eval(shift(@insns)); # rol 691 eval(shift(@insns)); 692 &psrldq (@X[2],4); # "X[-3]", 3 dwords 693 eval(shift(@insns)); 694 eval(shift(@insns)); 695 &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]" 696 eval(shift(@insns)); 697 eval(shift(@insns)); # ror 698 699 &pxor (@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" 700 eval(shift(@insns)); 701 eval(shift(@insns)); 702 eval(shift(@insns)); 703 704 &pxor (@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" 705 eval(shift(@insns)); 706 eval(shift(@insns)); # rol 707 &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU 708 eval(shift(@insns)); 709 eval(shift(@insns)); 710 711 &movdqa (@X[4],@X[0]); 712 eval(shift(@insns)); 713 eval(shift(@insns)); 714 eval(shift(@insns)); # ror 715 &movdqa (@X[2],@X[0]); 716 eval(shift(@insns)); 717 718 &pslldq (@X[4],12); # "X[0]"<<96, extract one dword 719 &paddd (@X[0],@X[0]); 720 eval(shift(@insns)); 721 eval(shift(@insns)); 722 723 &psrld (@X[2],31); 724 eval(shift(@insns)); 725 eval(shift(@insns)); # rol 726 &movdqa (@X[3],@X[4]); 727 eval(shift(@insns)); 728 eval(shift(@insns)); 729 eval(shift(@insns)); 730 731 &psrld (@X[4],30); 732 eval(shift(@insns)); 733 eval(shift(@insns)); # ror 734 &por (@X[0],@X[2]); # "X[0]"<<<=1 735 eval(shift(@insns)); 736 &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer 737 eval(shift(@insns)); 738 eval(shift(@insns)); 739 740 &pslld (@X[3],2); 741 eval(shift(@insns)); 742 eval(shift(@insns)); # rol 743 &pxor (@X[0],@X[4]); 744 &movdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX 745 eval(shift(@insns)); 746 eval(shift(@insns)); 747 748 &pxor (@X[0],@X[3]); # "X[0]"^=("X[0]"<<96)<<<2 749 &pshufd (@X[1],@X[-3&7],0xee) if ($Xi<7); # was &movdqa (@X[1],@X[-2&7]) 750 &pshufd (@X[3],@X[-1&7],0xee) if ($Xi==7); 751 eval(shift(@insns)); 752 eval(shift(@insns)); 753 754 foreach (@insns) { eval; } # remaining instructions [if any] 755 756 $Xi++; push(@X,shift(@X)); # "rotate" X[] 757 } 758 759 sub Xupdate_ssse3_32_79() 760 { use integer; 761 my $body = shift; 762 my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions 763 my ($a,$b,$c,$d,$e); 764 765 eval(shift(@insns)); # body_20_39 766 &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" 767 &punpcklqdq(@X[2],@X[-1&7]); # compose "X[-6]", was &palignr(@X[2],@X[-2&7],8) 768 eval(shift(@insns)); 769 eval(shift(@insns)); 770 eval(shift(@insns)); # rol 771 772 &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]" 773 &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer 774 eval(shift(@insns)); 775 eval(shift(@insns)); 776 eval(shift(@insns)) if (@insns[0] =~ /_rol/); 777 if ($Xi%5) { 778 &movdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... 779 } else { # ... or load next one 780 &movdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); 781 } 782 eval(shift(@insns)); # ror 783 &paddd (@X[3],@X[-1&7]); 784 eval(shift(@insns)); 785 786 &pxor (@X[0],@X[2]); # "X[0]"^="X[-6]" 787 eval(shift(@insns)); # body_20_39 788 eval(shift(@insns)); 789 eval(shift(@insns)); 790 eval(shift(@insns)); # rol 791 792 &movdqa (@X[2],@X[0]); 793 &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU 794 eval(shift(@insns)); 795 eval(shift(@insns)); 796 eval(shift(@insns)); # ror 797 eval(shift(@insns)); 798 eval(shift(@insns)) if (@insns[0] =~ /_rol/); 799 800 &pslld (@X[0],2); 801 eval(shift(@insns)); # body_20_39 802 eval(shift(@insns)); 803 &psrld (@X[2],30); 804 eval(shift(@insns)); 805 eval(shift(@insns)); # rol 806 eval(shift(@insns)); 807 eval(shift(@insns)); 808 eval(shift(@insns)); # ror 809 eval(shift(@insns)); 810 eval(shift(@insns)) if (@insns[1] =~ /_rol/); 811 eval(shift(@insns)) if (@insns[0] =~ /_rol/); 812 813 &por (@X[0],@X[2]); # "X[0]"<<<=2 814 eval(shift(@insns)); # body_20_39 815 eval(shift(@insns)); 816 &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer 817 eval(shift(@insns)); 818 eval(shift(@insns)); # rol 819 eval(shift(@insns)); 820 eval(shift(@insns)); 821 eval(shift(@insns)); # ror 822 &pshufd (@X[3],@X[-1],0xee) if ($Xi<19); # was &movdqa (@X[3],@X[0]) 823 eval(shift(@insns)); 824 825 foreach (@insns) { eval; } # remaining instructions 826 827 $Xi++; push(@X,shift(@X)); # "rotate" X[] 828 } 829 830 sub Xuplast_ssse3_80() 831 { use integer; 832 my $body = shift; 833 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 834 my ($a,$b,$c,$d,$e); 835 836 eval(shift(@insns)); 837 eval(shift(@insns)); 838 eval(shift(@insns)); 839 eval(shift(@insns)); 840 eval(shift(@insns)); 841 eval(shift(@insns)); 842 eval(shift(@insns)); 843 &paddd (@X[3],@X[-1&7]); 844 eval(shift(@insns)); 845 eval(shift(@insns)); 846 eval(shift(@insns)); 847 eval(shift(@insns)); 848 849 &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU 850 851 foreach (@insns) { eval; } # remaining instructions 852 853 &mov ($inp=@T[1],&DWP(192+4,"esp")); 854 &cmp ($inp,&DWP(192+8,"esp")); 855 &je (&label("done")); 856 857 &movdqa (@X[3],&QWP(112+48,"esp")); # K_00_19 858 &movdqa (@X[2],&QWP(112+64,"esp")); # pbswap mask 859 &movdqu (@X[-4&7],&QWP(0,$inp)); # load input 860 &movdqu (@X[-3&7],&QWP(16,$inp)); 861 &movdqu (@X[-2&7],&QWP(32,$inp)); 862 &movdqu (@X[-1&7],&QWP(48,$inp)); 863 &add ($inp,64); 864 &pshufb (@X[-4&7],@X[2]); # byte swap 865 &mov (&DWP(192+4,"esp"),$inp); 866 &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot 867 868 $Xi=0; 869 } 870 871 sub Xloop_ssse3() 872 { use integer; 873 my $body = shift; 874 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 875 my ($a,$b,$c,$d,$e); 876 877 eval(shift(@insns)); 878 eval(shift(@insns)); 879 eval(shift(@insns)); 880 eval(shift(@insns)); 881 eval(shift(@insns)); 882 eval(shift(@insns)); 883 eval(shift(@insns)); 884 &pshufb (@X[($Xi-3)&7],@X[2]); 885 eval(shift(@insns)); 886 eval(shift(@insns)); 887 eval(shift(@insns)); 888 eval(shift(@insns)); 889 &paddd (@X[($Xi-4)&7],@X[3]); 890 eval(shift(@insns)); 891 eval(shift(@insns)); 892 eval(shift(@insns)); 893 eval(shift(@insns)); 894 &movdqa (&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]); # X[]+K xfer to IALU 895 eval(shift(@insns)); 896 eval(shift(@insns)); 897 eval(shift(@insns)); 898 eval(shift(@insns)); 899 &psubd (@X[($Xi-4)&7],@X[3]); 900 901 foreach (@insns) { eval; } 902 $Xi++; 903 } 904 905 sub Xtail_ssse3() 906 { use integer; 907 my $body = shift; 908 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 909 my ($a,$b,$c,$d,$e); 910 911 foreach (@insns) { eval; } 912 } 913 914 sub body_00_19 () { # ((c^d)&b)^d 915 # on start @T[0]=(c^d)&b 916 return &body_20_39() if ($rx==19); $rx++; 917 ( 918 '($a,$b,$c,$d,$e)=@V;'. 919 '&$_ror ($b,$j?7:2);', # $b>>>2 920 '&xor (@T[0],$d);', 921 '&mov (@T[1],$a);', # $b in next round 922 923 '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer 924 '&xor ($b,$c);', # $c^$d for next round 925 926 '&$_rol ($a,5);', 927 '&add ($e,@T[0]);', 928 '&and (@T[1],$b);', # ($b&($c^$d)) for next round 929 930 '&xor ($b,$c);', # restore $b 931 '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 932 ); 933 } 934 935 sub body_20_39 () { # b^d^c 936 # on entry @T[0]=b^d 937 return &body_40_59() if ($rx==39); $rx++; 938 ( 939 '($a,$b,$c,$d,$e)=@V;'. 940 '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer 941 '&xor (@T[0],$d) if($j==19);'. 942 '&xor (@T[0],$c) if($j> 19);', # ($b^$d^$c) 943 '&mov (@T[1],$a);', # $b in next round 944 945 '&$_rol ($a,5);', 946 '&add ($e,@T[0]);', 947 '&xor (@T[1],$c) if ($j< 79);', # $b^$d for next round 948 949 '&$_ror ($b,7);', # $b>>>2 950 '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 951 ); 952 } 953 954 sub body_40_59 () { # ((b^c)&(c^d))^c 955 # on entry @T[0]=(b^c), (c^=d) 956 $rx++; 957 ( 958 '($a,$b,$c,$d,$e)=@V;'. 959 '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer 960 '&and (@T[0],$c) if ($j>=40);', # (b^c)&(c^d) 961 '&xor ($c,$d) if ($j>=40);', # restore $c 962 963 '&$_ror ($b,7);', # $b>>>2 964 '&mov (@T[1],$a);', # $b for next round 965 '&xor (@T[0],$c);', 966 967 '&$_rol ($a,5);', 968 '&add ($e,@T[0]);', 969 '&xor (@T[1],$c) if ($j==59);'. 970 '&xor (@T[1],$b) if ($j< 59);', # b^c for next round 971 972 '&xor ($b,$c) if ($j< 59);', # c^d for next round 973 '&add ($e,$a);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 974 ); 975 } 976 ###### 977 sub bodyx_00_19 () { # ((c^d)&b)^d 978 # on start @T[0]=(b&c)^(~b&d), $e+=X[]+K 979 return &bodyx_20_39() if ($rx==19); $rx++; 980 ( 981 '($a,$b,$c,$d,$e)=@V;'. 982 983 '&rorx ($b,$b,2) if ($j==0);'. # $b>>>2 984 '&rorx ($b,@T[1],7) if ($j!=0);', # $b>>>2 985 '&lea ($e,&DWP(0,$e,@T[0]));', 986 '&rorx (@T[0],$a,5);', 987 988 '&andn (@T[1],$a,$c);', 989 '&and ($a,$b)', 990 '&add ($d,&DWP(4*(($j+1)&15),"esp"));', # X[]+K xfer 991 992 '&xor (@T[1],$a)', 993 '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 994 ); 995 } 996 997 sub bodyx_20_39 () { # b^d^c 998 # on start $b=b^c^d 999 return &bodyx_40_59() if ($rx==39); $rx++; 1000 ( 1001 '($a,$b,$c,$d,$e)=@V;'. 1002 1003 '&add ($e,($j==19?@T[0]:$b))', 1004 '&rorx ($b,@T[1],7);', # $b>>>2 1005 '&rorx (@T[0],$a,5);', 1006 1007 '&xor ($a,$b) if ($j<79);', 1008 '&add ($d,&DWP(4*(($j+1)&15),"esp")) if ($j<79);', # X[]+K xfer 1009 '&xor ($a,$c) if ($j<79);', 1010 '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 1011 ); 1012 } 1013 1014 sub bodyx_40_59 () { # ((b^c)&(c^d))^c 1015 # on start $b=((b^c)&(c^d))^c 1016 return &bodyx_20_39() if ($rx==59); $rx++; 1017 ( 1018 '($a,$b,$c,$d,$e)=@V;'. 1019 1020 '&rorx (@T[0],$a,5)', 1021 '&lea ($e,&DWP(0,$e,$b))', 1022 '&rorx ($b,@T[1],7)', # $b>>>2 1023 '&add ($d,&DWP(4*(($j+1)&15),"esp"))', # X[]+K xfer 1024 1025 '&mov (@T[1],$c)', 1026 '&xor ($a,$b)', # b^c for next round 1027 '&xor (@T[1],$b)', # c^d for next round 1028 1029 '&and ($a,@T[1])', 1030 '&add ($e,@T[0])', 1031 '&xor ($a,$b)' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));' 1032 ); 1033 } 1034 1035 &set_label("loop",16); 1036 &Xupdate_ssse3_16_31(\&body_00_19); 1037 &Xupdate_ssse3_16_31(\&body_00_19); 1038 &Xupdate_ssse3_16_31(\&body_00_19); 1039 &Xupdate_ssse3_16_31(\&body_00_19); 1040 &Xupdate_ssse3_32_79(\&body_00_19); 1041 &Xupdate_ssse3_32_79(\&body_20_39); 1042 &Xupdate_ssse3_32_79(\&body_20_39); 1043 &Xupdate_ssse3_32_79(\&body_20_39); 1044 &Xupdate_ssse3_32_79(\&body_20_39); 1045 &Xupdate_ssse3_32_79(\&body_20_39); 1046 &Xupdate_ssse3_32_79(\&body_40_59); 1047 &Xupdate_ssse3_32_79(\&body_40_59); 1048 &Xupdate_ssse3_32_79(\&body_40_59); 1049 &Xupdate_ssse3_32_79(\&body_40_59); 1050 &Xupdate_ssse3_32_79(\&body_40_59); 1051 &Xupdate_ssse3_32_79(\&body_20_39); 1052 &Xuplast_ssse3_80(\&body_20_39); # can jump to "done" 1053 1054 $saved_j=$j; @saved_V=@V; 1055 1056 &Xloop_ssse3(\&body_20_39); 1057 &Xloop_ssse3(\&body_20_39); 1058 &Xloop_ssse3(\&body_20_39); 1059 1060 &mov (@T[1],&DWP(192,"esp")); # update context 1061 &add ($A,&DWP(0,@T[1])); 1062 &add (@T[0],&DWP(4,@T[1])); # $b 1063 &add ($C,&DWP(8,@T[1])); 1064 &mov (&DWP(0,@T[1]),$A); 1065 &add ($D,&DWP(12,@T[1])); 1066 &mov (&DWP(4,@T[1]),@T[0]); 1067 &add ($E,&DWP(16,@T[1])); 1068 &mov (&DWP(8,@T[1]),$C); 1069 &mov ($B,$C); 1070 &mov (&DWP(12,@T[1]),$D); 1071 &xor ($B,$D); 1072 &mov (&DWP(16,@T[1]),$E); 1073 &mov (@T[1],@T[0]); 1074 &pshufd (@X[0],@X[-4&7],0xee); # was &movdqa (@X[0],@X[-3&7]); 1075 &and (@T[0],$B); 1076 &mov ($B,$T[1]); 1077 1078 &jmp (&label("loop")); 1079 1080 &set_label("done",16); $j=$saved_j; @V=@saved_V; 1081 1082 &Xtail_ssse3(\&body_20_39); 1083 &Xtail_ssse3(\&body_20_39); 1084 &Xtail_ssse3(\&body_20_39); 1085 1086 &mov (@T[1],&DWP(192,"esp")); # update context 1087 &add ($A,&DWP(0,@T[1])); 1088 &mov ("esp",&DWP(192+12,"esp")); # restore %esp 1089 &add (@T[0],&DWP(4,@T[1])); # $b 1090 &add ($C,&DWP(8,@T[1])); 1091 &mov (&DWP(0,@T[1]),$A); 1092 &add ($D,&DWP(12,@T[1])); 1093 &mov (&DWP(4,@T[1]),@T[0]); 1094 &add ($E,&DWP(16,@T[1])); 1095 &mov (&DWP(8,@T[1]),$C); 1096 &mov (&DWP(12,@T[1]),$D); 1097 &mov (&DWP(16,@T[1]),$E); 1098 1099 &function_end("_sha1_block_data_order_ssse3"); 1100 1101 $rx=0; # reset 1102 1103 if ($ymm) { 1104 my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded 1105 my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4 1106 my @V=($A,$B,$C,$D,$E); 1107 my $j=0; # hash round 1108 my @T=($T,$tmp1); 1109 my $inp; 1110 1111 my $_rol=sub { &shld(@_[0],@_) }; 1112 my $_ror=sub { &shrd(@_[0],@_) }; 1113 1114 &function_begin("_sha1_block_data_order_avx"); 1115 &call (&label("pic_point")); # make it PIC! 1116 &set_label("pic_point"); 1117 &blindpop($tmp1); 1118 &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1)); 1119 &set_label("avx_shortcut"); 1120 &vzeroall(); 1121 1122 &vmovdqa(@X[3],&QWP(0,$tmp1)); # K_00_19 1123 &vmovdqa(@X[4],&QWP(16,$tmp1)); # K_20_39 1124 &vmovdqa(@X[5],&QWP(32,$tmp1)); # K_40_59 1125 &vmovdqa(@X[6],&QWP(48,$tmp1)); # K_60_79 1126 &vmovdqa(@X[2],&QWP(64,$tmp1)); # pbswap mask 1127 1128 &mov ($E,&wparam(0)); # load argument block 1129 &mov ($inp=@T[1],&wparam(1)); 1130 &mov ($D,&wparam(2)); 1131 &mov (@T[0],"esp"); 1132 1133 # stack frame layout 1134 # 1135 # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area 1136 # X[4]+K X[5]+K X[6]+K X[7]+K 1137 # X[8]+K X[9]+K X[10]+K X[11]+K 1138 # X[12]+K X[13]+K X[14]+K X[15]+K 1139 # 1140 # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area 1141 # X[4] X[5] X[6] X[7] 1142 # X[8] X[9] X[10] X[11] # even borrowed for K_00_19 1143 # 1144 # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants 1145 # K_40_59 K_40_59 K_40_59 K_40_59 1146 # K_60_79 K_60_79 K_60_79 K_60_79 1147 # K_00_19 K_00_19 K_00_19 K_00_19 1148 # pbswap mask 1149 # 1150 # +192 ctx # argument block 1151 # +196 inp 1152 # +200 end 1153 # +204 esp 1154 &sub ("esp",208); 1155 &and ("esp",-64); 1156 1157 &vmovdqa(&QWP(112+0,"esp"),@X[4]); # copy constants 1158 &vmovdqa(&QWP(112+16,"esp"),@X[5]); 1159 &vmovdqa(&QWP(112+32,"esp"),@X[6]); 1160 &shl ($D,6); # len*64 1161 &vmovdqa(&QWP(112+48,"esp"),@X[3]); 1162 &add ($D,$inp); # end of input 1163 &vmovdqa(&QWP(112+64,"esp"),@X[2]); 1164 &add ($inp,64); 1165 &mov (&DWP(192+0,"esp"),$E); # save argument block 1166 &mov (&DWP(192+4,"esp"),$inp); 1167 &mov (&DWP(192+8,"esp"),$D); 1168 &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp 1169 1170 &mov ($A,&DWP(0,$E)); # load context 1171 &mov ($B,&DWP(4,$E)); 1172 &mov ($C,&DWP(8,$E)); 1173 &mov ($D,&DWP(12,$E)); 1174 &mov ($E,&DWP(16,$E)); 1175 &mov (@T[0],$B); # magic seed 1176 1177 &vmovdqu(@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3] 1178 &vmovdqu(@X[-3&7],&QWP(-48,$inp)); 1179 &vmovdqu(@X[-2&7],&QWP(-32,$inp)); 1180 &vmovdqu(@X[-1&7],&QWP(-16,$inp)); 1181 &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap 1182 &vpshufb(@X[-3&7],@X[-3&7],@X[2]); 1183 &vpshufb(@X[-2&7],@X[-2&7],@X[2]); 1184 &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot 1185 &vpshufb(@X[-1&7],@X[-1&7],@X[2]); 1186 &vpaddd (@X[0],@X[-4&7],@X[3]); # add K_00_19 1187 &vpaddd (@X[1],@X[-3&7],@X[3]); 1188 &vpaddd (@X[2],@X[-2&7],@X[3]); 1189 &vmovdqa(&QWP(0,"esp"),@X[0]); # X[]+K xfer to IALU 1190 &mov (@T[1],$C); 1191 &vmovdqa(&QWP(0+16,"esp"),@X[1]); 1192 &xor (@T[1],$D); 1193 &vmovdqa(&QWP(0+32,"esp"),@X[2]); 1194 &and (@T[0],@T[1]); 1195 &jmp (&label("loop")); 1196 1197 sub Xupdate_avx_16_31() # recall that $Xi starts with 4 1198 { use integer; 1199 my $body = shift; 1200 my @insns = (&$body,&$body,&$body,&$body); # 40 instructions 1201 my ($a,$b,$c,$d,$e); 1202 1203 eval(shift(@insns)); 1204 eval(shift(@insns)); 1205 &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]" 1206 eval(shift(@insns)); 1207 eval(shift(@insns)); 1208 1209 &vpaddd (@X[3],@X[3],@X[-1&7]); 1210 &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer 1211 eval(shift(@insns)); 1212 eval(shift(@insns)); 1213 &vpsrldq(@X[2],@X[-1&7],4); # "X[-3]", 3 dwords 1214 eval(shift(@insns)); 1215 eval(shift(@insns)); 1216 &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]" 1217 eval(shift(@insns)); 1218 eval(shift(@insns)); 1219 1220 &vpxor (@X[2],@X[2],@X[-2&7]); # "X[-3]"^"X[-8]" 1221 eval(shift(@insns)); 1222 eval(shift(@insns)); 1223 &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU 1224 eval(shift(@insns)); 1225 eval(shift(@insns)); 1226 1227 &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]" 1228 eval(shift(@insns)); 1229 eval(shift(@insns)); 1230 eval(shift(@insns)); 1231 eval(shift(@insns)); 1232 1233 &vpsrld (@X[2],@X[0],31); 1234 eval(shift(@insns)); 1235 eval(shift(@insns)); 1236 eval(shift(@insns)); 1237 eval(shift(@insns)); 1238 1239 &vpslldq(@X[4],@X[0],12); # "X[0]"<<96, extract one dword 1240 &vpaddd (@X[0],@X[0],@X[0]); 1241 eval(shift(@insns)); 1242 eval(shift(@insns)); 1243 eval(shift(@insns)); 1244 eval(shift(@insns)); 1245 1246 &vpsrld (@X[3],@X[4],30); 1247 &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=1 1248 eval(shift(@insns)); 1249 eval(shift(@insns)); 1250 eval(shift(@insns)); 1251 eval(shift(@insns)); 1252 1253 &vpslld (@X[4],@X[4],2); 1254 &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer 1255 eval(shift(@insns)); 1256 eval(shift(@insns)); 1257 &vpxor (@X[0],@X[0],@X[3]); 1258 eval(shift(@insns)); 1259 eval(shift(@insns)); 1260 eval(shift(@insns)); 1261 eval(shift(@insns)); 1262 1263 &vpxor (@X[0],@X[0],@X[4]); # "X[0]"^=("X[0]"<<96)<<<2 1264 eval(shift(@insns)); 1265 eval(shift(@insns)); 1266 &vmovdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX 1267 eval(shift(@insns)); 1268 eval(shift(@insns)); 1269 1270 foreach (@insns) { eval; } # remaining instructions [if any] 1271 1272 $Xi++; push(@X,shift(@X)); # "rotate" X[] 1273 } 1274 1275 sub Xupdate_avx_32_79() 1276 { use integer; 1277 my $body = shift; 1278 my @insns = (&$body,&$body,&$body,&$body); # 32 to 44 instructions 1279 my ($a,$b,$c,$d,$e); 1280 1281 &vpalignr(@X[2],@X[-1&7],@X[-2&7],8); # compose "X[-6]" 1282 &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]" 1283 eval(shift(@insns)); # body_20_39 1284 eval(shift(@insns)); 1285 eval(shift(@insns)); 1286 eval(shift(@insns)); # rol 1287 1288 &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]" 1289 &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer 1290 eval(shift(@insns)); 1291 eval(shift(@insns)); 1292 if ($Xi%5) { 1293 &vmovdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX... 1294 } else { # ... or load next one 1295 &vmovdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp")); 1296 } 1297 &vpaddd (@X[3],@X[3],@X[-1&7]); 1298 eval(shift(@insns)); # ror 1299 eval(shift(@insns)); 1300 1301 &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-6]" 1302 eval(shift(@insns)); # body_20_39 1303 eval(shift(@insns)); 1304 eval(shift(@insns)); 1305 eval(shift(@insns)); # rol 1306 1307 &vpsrld (@X[2],@X[0],30); 1308 &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU 1309 eval(shift(@insns)); 1310 eval(shift(@insns)); 1311 eval(shift(@insns)); # ror 1312 eval(shift(@insns)); 1313 1314 &vpslld (@X[0],@X[0],2); 1315 eval(shift(@insns)); # body_20_39 1316 eval(shift(@insns)); 1317 eval(shift(@insns)); 1318 eval(shift(@insns)); # rol 1319 eval(shift(@insns)); 1320 eval(shift(@insns)); 1321 eval(shift(@insns)); # ror 1322 eval(shift(@insns)); 1323 1324 &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=2 1325 eval(shift(@insns)); # body_20_39 1326 eval(shift(@insns)); 1327 &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer 1328 eval(shift(@insns)); 1329 eval(shift(@insns)); # rol 1330 eval(shift(@insns)); 1331 eval(shift(@insns)); 1332 eval(shift(@insns)); # ror 1333 eval(shift(@insns)); 1334 1335 foreach (@insns) { eval; } # remaining instructions 1336 1337 $Xi++; push(@X,shift(@X)); # "rotate" X[] 1338 } 1339 1340 sub Xuplast_avx_80() 1341 { use integer; 1342 my $body = shift; 1343 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 1344 my ($a,$b,$c,$d,$e); 1345 1346 eval(shift(@insns)); 1347 &vpaddd (@X[3],@X[3],@X[-1&7]); 1348 eval(shift(@insns)); 1349 eval(shift(@insns)); 1350 eval(shift(@insns)); 1351 eval(shift(@insns)); 1352 1353 &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU 1354 1355 foreach (@insns) { eval; } # remaining instructions 1356 1357 &mov ($inp=@T[1],&DWP(192+4,"esp")); 1358 &cmp ($inp,&DWP(192+8,"esp")); 1359 &je (&label("done")); 1360 1361 &vmovdqa(@X[3],&QWP(112+48,"esp")); # K_00_19 1362 &vmovdqa(@X[2],&QWP(112+64,"esp")); # pbswap mask 1363 &vmovdqu(@X[-4&7],&QWP(0,$inp)); # load input 1364 &vmovdqu(@X[-3&7],&QWP(16,$inp)); 1365 &vmovdqu(@X[-2&7],&QWP(32,$inp)); 1366 &vmovdqu(@X[-1&7],&QWP(48,$inp)); 1367 &add ($inp,64); 1368 &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap 1369 &mov (&DWP(192+4,"esp"),$inp); 1370 &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot 1371 1372 $Xi=0; 1373 } 1374 1375 sub Xloop_avx() 1376 { use integer; 1377 my $body = shift; 1378 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 1379 my ($a,$b,$c,$d,$e); 1380 1381 eval(shift(@insns)); 1382 eval(shift(@insns)); 1383 &vpshufb (@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]); 1384 eval(shift(@insns)); 1385 eval(shift(@insns)); 1386 &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@X[3]); 1387 eval(shift(@insns)); 1388 eval(shift(@insns)); 1389 eval(shift(@insns)); 1390 eval(shift(@insns)); 1391 &vmovdqa (&QWP(0+16*$Xi,"esp"),@X[$Xi&7]); # X[]+K xfer to IALU 1392 eval(shift(@insns)); 1393 eval(shift(@insns)); 1394 1395 foreach (@insns) { eval; } 1396 $Xi++; 1397 } 1398 1399 sub Xtail_avx() 1400 { use integer; 1401 my $body = shift; 1402 my @insns = (&$body,&$body,&$body,&$body); # 32 instructions 1403 my ($a,$b,$c,$d,$e); 1404 1405 foreach (@insns) { eval; } 1406 } 1407 1408 &set_label("loop",16); 1409 &Xupdate_avx_16_31(\&body_00_19); 1410 &Xupdate_avx_16_31(\&body_00_19); 1411 &Xupdate_avx_16_31(\&body_00_19); 1412 &Xupdate_avx_16_31(\&body_00_19); 1413 &Xupdate_avx_32_79(\&body_00_19); 1414 &Xupdate_avx_32_79(\&body_20_39); 1415 &Xupdate_avx_32_79(\&body_20_39); 1416 &Xupdate_avx_32_79(\&body_20_39); 1417 &Xupdate_avx_32_79(\&body_20_39); 1418 &Xupdate_avx_32_79(\&body_20_39); 1419 &Xupdate_avx_32_79(\&body_40_59); 1420 &Xupdate_avx_32_79(\&body_40_59); 1421 &Xupdate_avx_32_79(\&body_40_59); 1422 &Xupdate_avx_32_79(\&body_40_59); 1423 &Xupdate_avx_32_79(\&body_40_59); 1424 &Xupdate_avx_32_79(\&body_20_39); 1425 &Xuplast_avx_80(\&body_20_39); # can jump to "done" 1426 1427 $saved_j=$j; @saved_V=@V; 1428 1429 &Xloop_avx(\&body_20_39); 1430 &Xloop_avx(\&body_20_39); 1431 &Xloop_avx(\&body_20_39); 1432 1433 &mov (@T[1],&DWP(192,"esp")); # update context 1434 &add ($A,&DWP(0,@T[1])); 1435 &add (@T[0],&DWP(4,@T[1])); # $b 1436 &add ($C,&DWP(8,@T[1])); 1437 &mov (&DWP(0,@T[1]),$A); 1438 &add ($D,&DWP(12,@T[1])); 1439 &mov (&DWP(4,@T[1]),@T[0]); 1440 &add ($E,&DWP(16,@T[1])); 1441 &mov ($B,$C); 1442 &mov (&DWP(8,@T[1]),$C); 1443 &xor ($B,$D); 1444 &mov (&DWP(12,@T[1]),$D); 1445 &mov (&DWP(16,@T[1]),$E); 1446 &mov (@T[1],@T[0]); 1447 &and (@T[0],$B); 1448 &mov ($B,@T[1]); 1449 1450 &jmp (&label("loop")); 1451 1452 &set_label("done",16); $j=$saved_j; @V=@saved_V; 1453 1454 &Xtail_avx(\&body_20_39); 1455 &Xtail_avx(\&body_20_39); 1456 &Xtail_avx(\&body_20_39); 1457 1458 &vzeroall(); 1459 1460 &mov (@T[1],&DWP(192,"esp")); # update context 1461 &add ($A,&DWP(0,@T[1])); 1462 &mov ("esp",&DWP(192+12,"esp")); # restore %esp 1463 &add (@T[0],&DWP(4,@T[1])); # $b 1464 &add ($C,&DWP(8,@T[1])); 1465 &mov (&DWP(0,@T[1]),$A); 1466 &add ($D,&DWP(12,@T[1])); 1467 &mov (&DWP(4,@T[1]),@T[0]); 1468 &add ($E,&DWP(16,@T[1])); 1469 &mov (&DWP(8,@T[1]),$C); 1470 &mov (&DWP(12,@T[1]),$D); 1471 &mov (&DWP(16,@T[1]),$E); 1472 &function_end("_sha1_block_data_order_avx"); 1473 } 1474 &set_label("K_XX_XX",64); 1475 &data_word(0x5a827999,0x5a827999,0x5a827999,0x5a827999); # K_00_19 1476 &data_word(0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1); # K_20_39 1477 &data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59 1478 &data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79 1479 &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask 1480 &data_byte(0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0); 1481 } 1482 &asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>"); 1483 1484 &asm_finish(); 1485 1486 close STDOUT; 1487
