1 #!/usr/bin/env perl 2 # 3 # ==================================================================== 4 # Written by Andy Polyakov <appro (at] fy.chalmers.se> for the OpenSSL 5 # project. Rights for redistribution and usage in source and binary 6 # forms are granted according to the OpenSSL license. 7 # ==================================================================== 8 # 9 # SHA512_Transform_SSE2. 10 # 11 # As the name suggests, this is an IA-32 SSE2 implementation of 12 # SHA512_Transform. Motivating factor for the undertaken effort was that 13 # SHA512 was observed to *consistently* perform *significantly* poorer 14 # than SHA256 [2x and slower is common] on 32-bit platforms. On 64-bit 15 # platforms on the other hand SHA512 tend to outperform SHA256 [~50% 16 # seem to be common improvement factor]. All this is perfectly natural, 17 # as SHA512 is a 64-bit algorithm. But isn't IA-32 SSE2 essentially 18 # a 64-bit instruction set? Is it rich enough to implement SHA512? 19 # If answer was "no," then you wouldn't have been reading this... 20 # 21 # Throughput performance in MBps (larger is better): 22 # 23 # 2.4GHz P4 1.4GHz AMD32 1.4GHz AMD64(*) 24 # SHA256/gcc(*) 54 43 59 25 # SHA512/gcc 17 23 92 26 # SHA512/sse2 61(**) 57(**) 27 # SHA512/icc 26 28 28 # SHA256/icc(*) 65 54 29 # 30 # (*) AMD64 and SHA256 numbers are presented mostly for amusement or 31 # reference purposes. 32 # (**) I.e. it gives ~2-3x speed-up if compared with compiler generated 33 # code. One can argue that hand-coded *non*-SSE2 implementation 34 # would perform better than compiler generated one as well, and 35 # that comparison is therefore not exactly fair. Well, as SHA512 36 # puts enormous pressure on IA-32 GP register bank, I reckon that 37 # hand-coded version wouldn't perform significantly better than 38 # one compiled with icc, ~20% perhaps... So that this code would 39 # still outperform it with distinguishing marginal. But feel free 40 # to prove me wrong:-) 41 # <appro (at] fy.chalmers.se> 42 push(@INC,"perlasm","../../perlasm"); 43 require "x86asm.pl"; 44 45 &asm_init($ARGV[0],"sha512-sse2.pl",$ARGV[$#ARGV] eq "386"); 46 47 $K512="esi"; # K512[80] table, found at the end... 48 #$W512="esp"; # $W512 is not just W512[16]: it comprises *two* copies 49 # of W512[16] and a copy of A-H variables... 50 $W512_SZ=8*(16+16+8); # see above... 51 #$Kidx="ebx"; # index in K512 table, advances from 0 to 80... 52 $Widx="edx"; # index in W512, wraps around at 16... 53 $data="edi"; # 16 qwords of input data... 54 $A="mm0"; # B-D and 55 $E="mm1"; # F-H are allocated dynamically... 56 $Aoff=256+0; # A-H offsets relative to $W512... 57 $Boff=256+8; 58 $Coff=256+16; 59 $Doff=256+24; 60 $Eoff=256+32; 61 $Foff=256+40; 62 $Goff=256+48; 63 $Hoff=256+56; 64 65 sub SHA2_ROUND() 66 { local ($kidx,$widx)=@_; 67 68 # One can argue that one could reorder instructions for better 69 # performance. Well, I tried and it doesn't seem to make any 70 # noticeable difference. Modern out-of-order execution cores 71 # reorder instructions to their liking in either case and they 72 # apparently do decent job. So we can keep the code more 73 # readable/regular/comprehensible:-) 74 75 # I adhere to 64-bit %mmX registers in order to avoid/not care 76 # about #GP exceptions on misaligned 128-bit access, most 77 # notably in paddq with memory operand. Not to mention that 78 # SSE2 intructions operating on %mmX can be scheduled every 79 # cycle [and not every second one if operating on %xmmN]. 80 81 &movq ("mm4",&QWP($Foff,$W512)); # load f 82 &movq ("mm5",&QWP($Goff,$W512)); # load g 83 &movq ("mm6",&QWP($Hoff,$W512)); # load h 84 85 &movq ("mm2",$E); # %mm2 is sliding right 86 &movq ("mm3",$E); # %mm3 is sliding left 87 &psrlq ("mm2",14); 88 &psllq ("mm3",23); 89 &movq ("mm7","mm2"); # %mm7 is T1 90 &pxor ("mm7","mm3"); 91 &psrlq ("mm2",4); 92 &psllq ("mm3",23); 93 &pxor ("mm7","mm2"); 94 &pxor ("mm7","mm3"); 95 &psrlq ("mm2",23); 96 &psllq ("mm3",4); 97 &pxor ("mm7","mm2"); 98 &pxor ("mm7","mm3"); # T1=Sigma1_512(e) 99 100 &movq (&QWP($Foff,$W512),$E); # f = e 101 &movq (&QWP($Goff,$W512),"mm4"); # g = f 102 &movq (&QWP($Hoff,$W512),"mm5"); # h = g 103 104 &pxor ("mm4","mm5"); # f^=g 105 &pand ("mm4",$E); # f&=e 106 &pxor ("mm4","mm5"); # f^=g 107 &paddq ("mm7","mm4"); # T1+=Ch(e,f,g) 108 109 &movq ("mm2",&QWP($Boff,$W512)); # load b 110 &movq ("mm3",&QWP($Coff,$W512)); # load c 111 &movq ($E,&QWP($Doff,$W512)); # e = d 112 113 &paddq ("mm7","mm6"); # T1+=h 114 &paddq ("mm7",&QWP(0,$K512,$kidx,8)); # T1+=K512[i] 115 &paddq ("mm7",&QWP(0,$W512,$widx,8)); # T1+=W512[i] 116 &paddq ($E,"mm7"); # e += T1 117 118 &movq ("mm4",$A); # %mm4 is sliding right 119 &movq ("mm5",$A); # %mm5 is sliding left 120 &psrlq ("mm4",28); 121 &psllq ("mm5",25); 122 &movq ("mm6","mm4"); # %mm6 is T2 123 &pxor ("mm6","mm5"); 124 &psrlq ("mm4",6); 125 &psllq ("mm5",5); 126 &pxor ("mm6","mm4"); 127 &pxor ("mm6","mm5"); 128 &psrlq ("mm4",5); 129 &psllq ("mm5",6); 130 &pxor ("mm6","mm4"); 131 &pxor ("mm6","mm5"); # T2=Sigma0_512(a) 132 133 &movq (&QWP($Boff,$W512),$A); # b = a 134 &movq (&QWP($Coff,$W512),"mm2"); # c = b 135 &movq (&QWP($Doff,$W512),"mm3"); # d = c 136 137 &movq ("mm4",$A); # %mm4=a 138 &por ($A,"mm3"); # a=a|c 139 &pand ("mm4","mm3"); # %mm4=a&c 140 &pand ($A,"mm2"); # a=(a|c)&b 141 &por ("mm4",$A); # %mm4=(a&c)|((a|c)&b) 142 &paddq ("mm6","mm4"); # T2+=Maj(a,b,c) 143 144 &movq ($A,"mm7"); # a=T1 145 &paddq ($A,"mm6"); # a+=T2 146 } 147 148 $func="sha512_block_sse2"; 149 150 &function_begin_B($func); 151 if (0) {# Caller is expected to check if it's appropriate to 152 # call this routine. Below 3 lines are retained for 153 # debugging purposes... 154 &picmeup("eax","OPENSSL_ia32cap"); 155 &bt (&DWP(0,"eax"),26); 156 &jnc ("SHA512_Transform"); 157 } 158 159 &push ("ebp"); 160 &mov ("ebp","esp"); 161 &push ("ebx"); 162 &push ("esi"); 163 &push ("edi"); 164 165 &mov ($Widx,&DWP(8,"ebp")); # A-H state, 1st arg 166 &mov ($data,&DWP(12,"ebp")); # input data, 2nd arg 167 &call (&label("pic_point")); # make it PIC! 168 &set_label("pic_point"); 169 &blindpop($K512); 170 &lea ($K512,&DWP(&label("K512")."-".&label("pic_point"),$K512)); 171 172 $W512 = "esp"; # start using %esp as W512 173 &sub ($W512,$W512_SZ); 174 &and ($W512,-16); # ensure 128-bit alignment 175 176 # make private copy of A-H 177 # v assume the worst and stick to unaligned load 178 &movdqu ("xmm0",&QWP(0,$Widx)); 179 &movdqu ("xmm1",&QWP(16,$Widx)); 180 &movdqu ("xmm2",&QWP(32,$Widx)); 181 &movdqu ("xmm3",&QWP(48,$Widx)); 182 183 &align(8); 184 &set_label("_chunk_loop"); 185 186 &movdqa (&QWP($Aoff,$W512),"xmm0"); # a,b 187 &movdqa (&QWP($Coff,$W512),"xmm1"); # c,d 188 &movdqa (&QWP($Eoff,$W512),"xmm2"); # e,f 189 &movdqa (&QWP($Goff,$W512),"xmm3"); # g,h 190 191 &xor ($Widx,$Widx); 192 193 &movdq2q($A,"xmm0"); # load a 194 &movdq2q($E,"xmm2"); # load e 195 196 # Why aren't loops unrolled? It makes sense to unroll if 197 # execution time for loop body is comparable with branch 198 # penalties and/or if whole data-set resides in register bank. 199 # Neither is case here... Well, it would be possible to 200 # eliminate few store operations, but it would hardly affect 201 # so to say stop-watch performance, as there is a lot of 202 # available memory slots to fill. It will only relieve some 203 # pressure off memory bus... 204 205 # flip input stream byte order... 206 &mov ("eax",&DWP(0,$data,$Widx,8)); 207 &mov ("ebx",&DWP(4,$data,$Widx,8)); 208 &bswap ("eax"); 209 &bswap ("ebx"); 210 &mov (&DWP(0,$W512,$Widx,8),"ebx"); # W512[i] 211 &mov (&DWP(4,$W512,$Widx,8),"eax"); 212 &mov (&DWP(128+0,$W512,$Widx,8),"ebx"); # copy of W512[i] 213 &mov (&DWP(128+4,$W512,$Widx,8),"eax"); 214 215 &align(8); 216 &set_label("_1st_loop"); # 0-15 217 # flip input stream byte order... 218 &mov ("eax",&DWP(0+8,$data,$Widx,8)); 219 &mov ("ebx",&DWP(4+8,$data,$Widx,8)); 220 &bswap ("eax"); 221 &bswap ("ebx"); 222 &mov (&DWP(0+8,$W512,$Widx,8),"ebx"); # W512[i] 223 &mov (&DWP(4+8,$W512,$Widx,8),"eax"); 224 &mov (&DWP(128+0+8,$W512,$Widx,8),"ebx"); # copy of W512[i] 225 &mov (&DWP(128+4+8,$W512,$Widx,8),"eax"); 226 &set_label("_1st_looplet"); 227 &SHA2_ROUND($Widx,$Widx); &inc($Widx); 228 229 &cmp ($Widx,15) 230 &jl (&label("_1st_loop")); 231 &je (&label("_1st_looplet")); # playing similar trick on 2nd loop 232 # does not improve performance... 233 234 $Kidx = "ebx"; # start using %ebx as Kidx 235 &mov ($Kidx,$Widx); 236 237 &align(8); 238 &set_label("_2nd_loop"); # 16-79 239 &and($Widx,0xf); 240 241 # 128-bit fragment! I update W512[i] and W512[i+1] in 242 # parallel:-) Note that I refer to W512[(i&0xf)+N] and not to 243 # W512[(i+N)&0xf]! This is exactly what I maintain the second 244 # copy of W512[16] for... 245 &movdqu ("xmm0",&QWP(8*1,$W512,$Widx,8)); # s0=W512[i+1] 246 &movdqa ("xmm2","xmm0"); # %xmm2 is sliding right 247 &movdqa ("xmm3","xmm0"); # %xmm3 is sliding left 248 &psrlq ("xmm2",1); 249 &psllq ("xmm3",56); 250 &movdqa ("xmm0","xmm2"); 251 &pxor ("xmm0","xmm3"); 252 &psrlq ("xmm2",6); 253 &psllq ("xmm3",7); 254 &pxor ("xmm0","xmm2"); 255 &pxor ("xmm0","xmm3"); 256 &psrlq ("xmm2",1); 257 &pxor ("xmm0","xmm2"); # s0 = sigma0_512(s0); 258 259 &movdqa ("xmm1",&QWP(8*14,$W512,$Widx,8)); # s1=W512[i+14] 260 &movdqa ("xmm4","xmm1"); # %xmm4 is sliding right 261 &movdqa ("xmm5","xmm1"); # %xmm5 is sliding left 262 &psrlq ("xmm4",6); 263 &psllq ("xmm5",3); 264 &movdqa ("xmm1","xmm4"); 265 &pxor ("xmm1","xmm5"); 266 &psrlq ("xmm4",13); 267 &psllq ("xmm5",42); 268 &pxor ("xmm1","xmm4"); 269 &pxor ("xmm1","xmm5"); 270 &psrlq ("xmm4",42); 271 &pxor ("xmm1","xmm4"); # s1 = sigma1_512(s1); 272 273 # + have to explictly load W512[i+9] as it's not 128-bit 274 # v aligned and paddq would throw an exception... 275 &movdqu ("xmm6",&QWP(8*9,$W512,$Widx,8)); 276 &paddq ("xmm0","xmm1"); # s0 += s1 277 &paddq ("xmm0","xmm6"); # s0 += W512[i+9] 278 &paddq ("xmm0",&QWP(0,$W512,$Widx,8)); # s0 += W512[i] 279 280 &movdqa (&QWP(0,$W512,$Widx,8),"xmm0"); # W512[i] = s0 281 &movdqa (&QWP(16*8,$W512,$Widx,8),"xmm0"); # copy of W512[i] 282 283 # as the above fragment was 128-bit, we "owe" 2 rounds... 284 &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx); 285 &SHA2_ROUND($Kidx,$Widx); &inc($Kidx); &inc($Widx); 286 287 &cmp ($Kidx,80); 288 &jl (&label("_2nd_loop")); 289 290 # update A-H state 291 &mov ($Widx,&DWP(8,"ebp")); # A-H state, 1st arg 292 &movq (&QWP($Aoff,$W512),$A); # write out a 293 &movq (&QWP($Eoff,$W512),$E); # write out e 294 &movdqu ("xmm0",&QWP(0,$Widx)); 295 &movdqu ("xmm1",&QWP(16,$Widx)); 296 &movdqu ("xmm2",&QWP(32,$Widx)); 297 &movdqu ("xmm3",&QWP(48,$Widx)); 298 &paddq ("xmm0",&QWP($Aoff,$W512)); # 128-bit additions... 299 &paddq ("xmm1",&QWP($Coff,$W512)); 300 &paddq ("xmm2",&QWP($Eoff,$W512)); 301 &paddq ("xmm3",&QWP($Goff,$W512)); 302 &movdqu (&QWP(0,$Widx),"xmm0"); 303 &movdqu (&QWP(16,$Widx),"xmm1"); 304 &movdqu (&QWP(32,$Widx),"xmm2"); 305 &movdqu (&QWP(48,$Widx),"xmm3"); 306 307 &add ($data,16*8); # advance input data pointer 308 &dec (&DWP(16,"ebp")); # decrement 3rd arg 309 &jnz (&label("_chunk_loop")); 310 311 # epilogue 312 &emms (); # required for at least ELF and Win32 ABIs 313 &mov ("edi",&DWP(-12,"ebp")); 314 &mov ("esi",&DWP(-8,"ebp")); 315 &mov ("ebx",&DWP(-4,"ebp")); 316 &leave (); 317 &ret (); 318 319 &align(64); 320 &set_label("K512"); # Yes! I keep it in the code segment! 321 &data_word(0xd728ae22,0x428a2f98); # u64 322 &data_word(0x23ef65cd,0x71374491); # u64 323 &data_word(0xec4d3b2f,0xb5c0fbcf); # u64 324 &data_word(0x8189dbbc,0xe9b5dba5); # u64 325 &data_word(0xf348b538,0x3956c25b); # u64 326 &data_word(0xb605d019,0x59f111f1); # u64 327 &data_word(0xaf194f9b,0x923f82a4); # u64 328 &data_word(0xda6d8118,0xab1c5ed5); # u64 329 &data_word(0xa3030242,0xd807aa98); # u64 330 &data_word(0x45706fbe,0x12835b01); # u64 331 &data_word(0x4ee4b28c,0x243185be); # u64 332 &data_word(0xd5ffb4e2,0x550c7dc3); # u64 333 &data_word(0xf27b896f,0x72be5d74); # u64 334 &data_word(0x3b1696b1,0x80deb1fe); # u64 335 &data_word(0x25c71235,0x9bdc06a7); # u64 336 &data_word(0xcf692694,0xc19bf174); # u64 337 &data_word(0x9ef14ad2,0xe49b69c1); # u64 338 &data_word(0x384f25e3,0xefbe4786); # u64 339 &data_word(0x8b8cd5b5,0x0fc19dc6); # u64 340 &data_word(0x77ac9c65,0x240ca1cc); # u64 341 &data_word(0x592b0275,0x2de92c6f); # u64 342 &data_word(0x6ea6e483,0x4a7484aa); # u64 343 &data_word(0xbd41fbd4,0x5cb0a9dc); # u64 344 &data_word(0x831153b5,0x76f988da); # u64 345 &data_word(0xee66dfab,0x983e5152); # u64 346 &data_word(0x2db43210,0xa831c66d); # u64 347 &data_word(0x98fb213f,0xb00327c8); # u64 348 &data_word(0xbeef0ee4,0xbf597fc7); # u64 349 &data_word(0x3da88fc2,0xc6e00bf3); # u64 350 &data_word(0x930aa725,0xd5a79147); # u64 351 &data_word(0xe003826f,0x06ca6351); # u64 352 &data_word(0x0a0e6e70,0x14292967); # u64 353 &data_word(0x46d22ffc,0x27b70a85); # u64 354 &data_word(0x5c26c926,0x2e1b2138); # u64 355 &data_word(0x5ac42aed,0x4d2c6dfc); # u64 356 &data_word(0x9d95b3df,0x53380d13); # u64 357 &data_word(0x8baf63de,0x650a7354); # u64 358 &data_word(0x3c77b2a8,0x766a0abb); # u64 359 &data_word(0x47edaee6,0x81c2c92e); # u64 360 &data_word(0x1482353b,0x92722c85); # u64 361 &data_word(0x4cf10364,0xa2bfe8a1); # u64 362 &data_word(0xbc423001,0xa81a664b); # u64 363 &data_word(0xd0f89791,0xc24b8b70); # u64 364 &data_word(0x0654be30,0xc76c51a3); # u64 365 &data_word(0xd6ef5218,0xd192e819); # u64 366 &data_word(0x5565a910,0xd6990624); # u64 367 &data_word(0x5771202a,0xf40e3585); # u64 368 &data_word(0x32bbd1b8,0x106aa070); # u64 369 &data_word(0xb8d2d0c8,0x19a4c116); # u64 370 &data_word(0x5141ab53,0x1e376c08); # u64 371 &data_word(0xdf8eeb99,0x2748774c); # u64 372 &data_word(0xe19b48a8,0x34b0bcb5); # u64 373 &data_word(0xc5c95a63,0x391c0cb3); # u64 374 &data_word(0xe3418acb,0x4ed8aa4a); # u64 375 &data_word(0x7763e373,0x5b9cca4f); # u64 376 &data_word(0xd6b2b8a3,0x682e6ff3); # u64 377 &data_word(0x5defb2fc,0x748f82ee); # u64 378 &data_word(0x43172f60,0x78a5636f); # u64 379 &data_word(0xa1f0ab72,0x84c87814); # u64 380 &data_word(0x1a6439ec,0x8cc70208); # u64 381 &data_word(0x23631e28,0x90befffa); # u64 382 &data_word(0xde82bde9,0xa4506ceb); # u64 383 &data_word(0xb2c67915,0xbef9a3f7); # u64 384 &data_word(0xe372532b,0xc67178f2); # u64 385 &data_word(0xea26619c,0xca273ece); # u64 386 &data_word(0x21c0c207,0xd186b8c7); # u64 387 &data_word(0xcde0eb1e,0xeada7dd6); # u64 388 &data_word(0xee6ed178,0xf57d4f7f); # u64 389 &data_word(0x72176fba,0x06f067aa); # u64 390 &data_word(0xa2c898a6,0x0a637dc5); # u64 391 &data_word(0xbef90dae,0x113f9804); # u64 392 &data_word(0x131c471b,0x1b710b35); # u64 393 &data_word(0x23047d84,0x28db77f5); # u64 394 &data_word(0x40c72493,0x32caab7b); # u64 395 &data_word(0x15c9bebc,0x3c9ebe0a); # u64 396 &data_word(0x9c100d4c,0x431d67c4); # u64 397 &data_word(0xcb3e42b6,0x4cc5d4be); # u64 398 &data_word(0xfc657e2a,0x597f299c); # u64 399 &data_word(0x3ad6faec,0x5fcb6fab); # u64 400 &data_word(0x4a475817,0x6c44198c); # u64 401 402 &function_end_B($func); 403 404 &asm_finish(); 405
