1 // Copyright 2013 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 #include "textflag.h" 6 7 // SHA256 block routine. See sha256block.go for Go equivalent. 8 // 9 // The algorithm is detailed in FIPS 180-4: 10 // 11 // http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf 12 13 // The avx2-version is described in an Intel White-Paper: 14 // "Fast SHA-256 Implementations on Intel Architecture Processors" 15 // To find it, surf to http://www.intel.com/p/en_US/embedded 16 // and search for that title. 17 // AVX2 version by Intel, same algorithm as code in Linux kernel: 18 // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha256-avx2-asm.S 19 // by 20 // James Guilford <james.guilford (at) intel.com> 21 // Kirk Yap <kirk.s.yap (at) intel.com> 22 // Tim Chen <tim.c.chen (at) linux.intel.com> 23 24 // Wt = Mt; for 0 <= t <= 15 25 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63 26 // 27 // a = H0 28 // b = H1 29 // c = H2 30 // d = H3 31 // e = H4 32 // f = H5 33 // g = H6 34 // h = H7 35 // 36 // for t = 0 to 63 { 37 // T1 = h + BIGSIGMA1(e) + Ch(e,f,g) + Kt + Wt 38 // T2 = BIGSIGMA0(a) + Maj(a,b,c) 39 // h = g 40 // g = f 41 // f = e 42 // e = d + T1 43 // d = c 44 // c = b 45 // b = a 46 // a = T1 + T2 47 // } 48 // 49 // H0 = a + H0 50 // H1 = b + H1 51 // H2 = c + H2 52 // H3 = d + H3 53 // H4 = e + H4 54 // H5 = f + H5 55 // H6 = g + H6 56 // H7 = h + H7 57 58 // Wt = Mt; for 0 <= t <= 15 59 #define MSGSCHEDULE0(index) \ 60 MOVL (index*4)(SI), AX; \ 61 BSWAPL AX; \ 62 MOVL AX, (index*4)(BP) 63 64 // Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63 65 // SIGMA0(x) = ROTR(7,x) XOR ROTR(18,x) XOR SHR(3,x) 66 // SIGMA1(x) = ROTR(17,x) XOR ROTR(19,x) XOR SHR(10,x) 67 #define MSGSCHEDULE1(index) \ 68 MOVL ((index-2)*4)(BP), AX; \ 69 MOVL AX, CX; \ 70 RORL $17, AX; \ 71 MOVL CX, DX; \ 72 RORL $19, CX; \ 73 SHRL $10, DX; \ 74 MOVL ((index-15)*4)(BP), BX; \ 75 XORL CX, AX; \ 76 MOVL BX, CX; \ 77 XORL DX, AX; \ 78 RORL $7, BX; \ 79 MOVL CX, DX; \ 80 SHRL $3, DX; \ 81 RORL $18, CX; \ 82 ADDL ((index-7)*4)(BP), AX; \ 83 XORL CX, BX; \ 84 XORL DX, BX; \ 85 ADDL ((index-16)*4)(BP), BX; \ 86 ADDL BX, AX; \ 87 MOVL AX, ((index)*4)(BP) 88 89 // Calculate T1 in AX - uses AX, CX and DX registers. 90 // h is also used as an accumulator. Wt is passed in AX. 91 // T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt 92 // BIGSIGMA1(x) = ROTR(6,x) XOR ROTR(11,x) XOR ROTR(25,x) 93 // Ch(x, y, z) = (x AND y) XOR (NOT x AND z) 94 #define SHA256T1(const, e, f, g, h) \ 95 ADDL AX, h; \ 96 MOVL e, AX; \ 97 ADDL $const, h; \ 98 MOVL e, CX; \ 99 RORL $6, AX; \ 100 MOVL e, DX; \ 101 RORL $11, CX; \ 102 XORL CX, AX; \ 103 MOVL e, CX; \ 104 RORL $25, DX; \ 105 ANDL f, CX; \ 106 XORL AX, DX; \ 107 MOVL e, AX; \ 108 NOTL AX; \ 109 ADDL DX, h; \ 110 ANDL g, AX; \ 111 XORL CX, AX; \ 112 ADDL h, AX 113 114 // Calculate T2 in BX - uses BX, CX, DX and DI registers. 115 // T2 = BIGSIGMA0(a) + Maj(a, b, c) 116 // BIGSIGMA0(x) = ROTR(2,x) XOR ROTR(13,x) XOR ROTR(22,x) 117 // Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z) 118 #define SHA256T2(a, b, c) \ 119 MOVL a, DI; \ 120 MOVL c, BX; \ 121 RORL $2, DI; \ 122 MOVL a, DX; \ 123 ANDL b, BX; \ 124 RORL $13, DX; \ 125 MOVL a, CX; \ 126 ANDL c, CX; \ 127 XORL DX, DI; \ 128 XORL CX, BX; \ 129 MOVL a, DX; \ 130 MOVL b, CX; \ 131 RORL $22, DX; \ 132 ANDL a, CX; \ 133 XORL CX, BX; \ 134 XORL DX, DI; \ 135 ADDL DI, BX 136 137 // Calculate T1 and T2, then e = d + T1 and a = T1 + T2. 138 // The values for e and a are stored in d and h, ready for rotation. 139 #define SHA256ROUND(index, const, a, b, c, d, e, f, g, h) \ 140 SHA256T1(const, e, f, g, h); \ 141 SHA256T2(a, b, c); \ 142 MOVL BX, h; \ 143 ADDL AX, d; \ 144 ADDL AX, h 145 146 #define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \ 147 MSGSCHEDULE0(index); \ 148 SHA256ROUND(index, const, a, b, c, d, e, f, g, h) 149 150 #define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \ 151 MSGSCHEDULE1(index); \ 152 SHA256ROUND(index, const, a, b, c, d, e, f, g, h) 153 154 155 // Definitions for AVX2 version 156 157 // addm (mem), reg 158 // Add reg to mem using reg-mem add and store 159 #define addm(P1, P2) \ 160 ADDL P2, P1; \ 161 MOVL P1, P2 162 163 #define XDWORD0 Y4 164 #define XDWORD1 Y5 165 #define XDWORD2 Y6 166 #define XDWORD3 Y7 167 168 #define XWORD0 X4 169 #define XWORD1 X5 170 #define XWORD2 X6 171 #define XWORD3 X7 172 173 #define XTMP0 Y0 174 #define XTMP1 Y1 175 #define XTMP2 Y2 176 #define XTMP3 Y3 177 #define XTMP4 Y8 178 #define XTMP5 Y11 179 180 #define XFER Y9 181 182 #define BYTE_FLIP_MASK Y13 // mask to convert LE -> BE 183 #define X_BYTE_FLIP_MASK X13 184 185 #define NUM_BYTES DX 186 #define INP DI 187 188 #define CTX SI // Beginning of digest in memory (a, b, c, ... , h) 189 190 #define a AX 191 #define b BX 192 #define c CX 193 #define d R8 194 #define e DX 195 #define f R9 196 #define g R10 197 #define h R11 198 199 #define old_h R11 200 201 #define TBL BP 202 203 #define SRND SI // SRND is same register as CTX 204 205 #define T1 R12 206 207 #define y0 R13 208 #define y1 R14 209 #define y2 R15 210 #define y3 DI 211 212 // Offsets 213 #define XFER_SIZE 2*64*4 214 #define INP_END_SIZE 8 215 #define INP_SIZE 8 216 217 #define _XFER 0 218 #define _INP_END _XFER + XFER_SIZE 219 #define _INP _INP_END + INP_END_SIZE 220 #define STACK_SIZE _INP + INP_SIZE 221 222 #define ROUND_AND_SCHED_N_0(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \ 223 ; \ // ############################# RND N + 0 ############################// 224 MOVL a, y3; \ // y3 = a // MAJA 225 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 226 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 227 ; \ 228 ADDL (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h // disp = k + w 229 ORL c, y3; \ // y3 = a|c // MAJA 230 VPALIGNR $4, XDWORD2, XDWORD3, XTMP0; \ // XTMP0 = W[-7] 231 MOVL f, y2; \ // y2 = f // CH 232 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 233 ; \ 234 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 235 XORL g, y2; \ // y2 = f^g // CH 236 VPADDD XDWORD0, XTMP0, XTMP0; \ // XTMP0 = W[-7] + W[-16] // y1 = (e >> 6) // S1 237 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 238 ; \ 239 ANDL e, y2; \ // y2 = (f^g)&e // CH 240 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 241 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 242 ADDL h, d; \ // d = k + w + h + d // -- 243 ; \ 244 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 245 VPALIGNR $4, XDWORD0, XDWORD1, XTMP1; \ // XTMP1 = W[-15] 246 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 247 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 248 ; \ 249 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 250 VPSRLD $7, XTMP1, XTMP2; \ 251 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 252 MOVL a, T1; \ // T1 = a // MAJB 253 ANDL c, T1; \ // T1 = a&c // MAJB 254 ; \ 255 ADDL y0, y2; \ // y2 = S1 + CH // -- 256 VPSLLD $(32-7), XTMP1, XTMP3; \ 257 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 258 ADDL y1, h; \ // h = k + w + h + S0 // -- 259 ; \ 260 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // -- 261 VPOR XTMP2, XTMP3, XTMP3; \ // XTMP3 = W[-15] ror 7 262 ; \ 263 VPSRLD $18, XTMP1, XTMP2; \ 264 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 265 ADDL y3, h // h = t1 + S0 + MAJ // -- 266 267 #define ROUND_AND_SCHED_N_1(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \ 268 ; \ // ################################### RND N + 1 ############################ 269 ; \ 270 MOVL a, y3; \ // y3 = a // MAJA 271 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 272 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 273 ADDL (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 274 ORL c, y3; \ // y3 = a|c // MAJA 275 ; \ 276 VPSRLD $3, XTMP1, XTMP4; \ // XTMP4 = W[-15] >> 3 277 MOVL f, y2; \ // y2 = f // CH 278 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 279 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 280 XORL g, y2; \ // y2 = f^g // CH 281 ; \ 282 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 283 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 284 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 285 ANDL e, y2; \ // y2 = (f^g)&e // CH 286 ADDL h, d; \ // d = k + w + h + d // -- 287 ; \ 288 VPSLLD $(32-18), XTMP1, XTMP1; \ 289 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 290 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 291 ; \ 292 VPXOR XTMP1, XTMP3, XTMP3; \ 293 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 294 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 295 ; \ 296 VPXOR XTMP2, XTMP3, XTMP3; \ // XTMP3 = W[-15] ror 7 ^ W[-15] ror 18 297 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 298 MOVL a, T1; \ // T1 = a // MAJB 299 ANDL c, T1; \ // T1 = a&c // MAJB 300 ADDL y0, y2; \ // y2 = S1 + CH // -- 301 ; \ 302 VPXOR XTMP4, XTMP3, XTMP1; \ // XTMP1 = s0 303 VPSHUFD $0xFA, XDWORD3, XTMP2; \ // XTMP2 = W[-2] {BBAA} 304 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 305 ADDL y1, h; \ // h = k + w + h + S0 // -- 306 ; \ 307 VPADDD XTMP1, XTMP0, XTMP0; \ // XTMP0 = W[-16] + W[-7] + s0 308 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // -- 309 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 310 ADDL y3, h; \ // h = t1 + S0 + MAJ // -- 311 ; \ 312 VPSRLD $10, XTMP2, XTMP4 // XTMP4 = W[-2] >> 10 {BBAA} 313 314 #define ROUND_AND_SCHED_N_2(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \ 315 ; \ // ################################### RND N + 2 ############################ 316 ; \ 317 MOVL a, y3; \ // y3 = a // MAJA 318 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 319 ADDL (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 320 ; \ 321 VPSRLQ $19, XTMP2, XTMP3; \ // XTMP3 = W[-2] ror 19 {xBxA} 322 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 323 ORL c, y3; \ // y3 = a|c // MAJA 324 MOVL f, y2; \ // y2 = f // CH 325 XORL g, y2; \ // y2 = f^g // CH 326 ; \ 327 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 328 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 329 VPSRLQ $17, XTMP2, XTMP2; \ // XTMP2 = W[-2] ror 17 {xBxA} 330 ANDL e, y2; \ // y2 = (f^g)&e // CH 331 ; \ 332 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 333 VPXOR XTMP3, XTMP2, XTMP2; \ 334 ADDL h, d; \ // d = k + w + h + d // -- 335 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 336 ; \ 337 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 338 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 339 VPXOR XTMP2, XTMP4, XTMP4; \ // XTMP4 = s1 {xBxA} 340 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 341 ; \ 342 VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4;\ // XTMP4 = s1 {00BA} 343 ; \ 344 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 345 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 346 VPADDD XTMP4, XTMP0, XTMP0; \ // XTMP0 = {..., ..., W[1], W[0]} 347 ; \ 348 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 349 MOVL a, T1; \ // T1 = a // MAJB 350 ANDL c, T1; \ // T1 = a&c // MAJB 351 ADDL y0, y2; \ // y2 = S1 + CH // -- 352 VPSHUFD $80, XTMP0, XTMP2; \ // XTMP2 = W[-2] {DDCC} 353 ; \ 354 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 355 ADDL y1, h; \ // h = k + w + h + S0 // -- 356 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // -- 357 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 358 ; \ 359 ADDL y3, h // h = t1 + S0 + MAJ // -- 360 361 #define ROUND_AND_SCHED_N_3(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \ 362 ; \ // ################################### RND N + 3 ############################ 363 ; \ 364 MOVL a, y3; \ // y3 = a // MAJA 365 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 366 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 367 ADDL (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 368 ORL c, y3; \ // y3 = a|c // MAJA 369 ; \ 370 VPSRLD $10, XTMP2, XTMP5; \ // XTMP5 = W[-2] >> 10 {DDCC} 371 MOVL f, y2; \ // y2 = f // CH 372 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 373 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 374 XORL g, y2; \ // y2 = f^g // CH 375 ; \ 376 VPSRLQ $19, XTMP2, XTMP3; \ // XTMP3 = W[-2] ror 19 {xDxC} 377 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 378 ANDL e, y2; \ // y2 = (f^g)&e // CH 379 ADDL h, d; \ // d = k + w + h + d // -- 380 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 381 ; \ 382 VPSRLQ $17, XTMP2, XTMP2; \ // XTMP2 = W[-2] ror 17 {xDxC} 383 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 384 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 385 ; \ 386 VPXOR XTMP3, XTMP2, XTMP2; \ 387 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 388 ADDL y0, y2; \ // y2 = S1 + CH // -- 389 ; \ 390 VPXOR XTMP2, XTMP5, XTMP5; \ // XTMP5 = s1 {xDxC} 391 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 392 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // -- 393 ; \ 394 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 395 ; \ 396 VPSHUFB shuff_DC00<>(SB), XTMP5, XTMP5;\ // XTMP5 = s1 {DC00} 397 ; \ 398 VPADDD XTMP0, XTMP5, XDWORD0; \ // XDWORD0 = {W[3], W[2], W[1], W[0]} 399 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 400 MOVL a, T1; \ // T1 = a // MAJB 401 ANDL c, T1; \ // T1 = a&c // MAJB 402 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 403 ; \ 404 ADDL y1, h; \ // h = k + w + h + S0 // -- 405 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 406 ADDL y3, h // h = t1 + S0 + MAJ // -- 407 408 #define DO_ROUND_N_0(disp, a, b, c, d, e, f, g, h, old_h) \ 409 ; \ // ################################### RND N + 0 ########################### 410 MOVL f, y2; \ // y2 = f // CH 411 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 412 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 413 XORL g, y2; \ // y2 = f^g // CH 414 ; \ 415 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 416 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 417 ANDL e, y2; \ // y2 = (f^g)&e // CH 418 ; \ 419 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 420 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 421 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 422 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 423 MOVL a, y3; \ // y3 = a // MAJA 424 ; \ 425 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 426 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 427 ADDL (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 428 ORL c, y3; \ // y3 = a|c // MAJA 429 ; \ 430 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 431 MOVL a, T1; \ // T1 = a // MAJB 432 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 433 ANDL c, T1; \ // T1 = a&c // MAJB 434 ADDL y0, y2; \ // y2 = S1 + CH // -- 435 ; \ 436 ADDL h, d; \ // d = k + w + h + d // -- 437 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 438 ADDL y1, h; \ // h = k + w + h + S0 // -- 439 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // -- 440 441 #define DO_ROUND_N_1(disp, a, b, c, d, e, f, g, h, old_h) \ 442 ; \ // ################################### RND N + 1 ########################### 443 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0 // -- 444 MOVL f, y2; \ // y2 = f // CH 445 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 446 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 447 XORL g, y2; \ // y2 = f^g // CH 448 ; \ 449 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 450 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 451 ANDL e, y2; \ // y2 = (f^g)&e // CH 452 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // -- 453 ; \ 454 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 455 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 456 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 457 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 458 MOVL a, y3; \ // y3 = a // MAJA 459 ; \ 460 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 461 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 462 ADDL (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 463 ORL c, y3; \ // y3 = a|c // MAJA 464 ; \ 465 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 466 MOVL a, T1; \ // T1 = a // MAJB 467 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 468 ANDL c, T1; \ // T1 = a&c // MAJB 469 ADDL y0, y2; \ // y2 = S1 + CH // -- 470 ; \ 471 ADDL h, d; \ // d = k + w + h + d // -- 472 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 473 ADDL y1, h; \ // h = k + w + h + S0 // -- 474 ; \ 475 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // -- 476 477 #define DO_ROUND_N_2(disp, a, b, c, d, e, f, g, h, old_h) \ 478 ; \ // ################################### RND N + 2 ############################## 479 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 480 MOVL f, y2; \ // y2 = f // CH 481 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 482 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 483 XORL g, y2; \ // y2 = f^g // CH 484 ; \ 485 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 486 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 487 ANDL e, y2; \ // y2 = (f^g)&e // CH 488 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // -- 489 ; \ 490 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 491 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 492 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 493 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 494 MOVL a, y3; \ // y3 = a // MAJA 495 ; \ 496 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 497 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 498 ADDL (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 499 ORL c, y3; \ // y3 = a|c // MAJA 500 ; \ 501 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 502 MOVL a, T1; \ // T1 = a // MAJB 503 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 504 ANDL c, T1; \ // T1 = a&c // MAJB 505 ADDL y0, y2; \ // y2 = S1 + CH // -- 506 ; \ 507 ADDL h, d; \ // d = k + w + h + d // -- 508 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 509 ADDL y1, h; \ // h = k + w + h + S0 // -- 510 ; \ 511 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // -- 512 513 #define DO_ROUND_N_3(disp, a, b, c, d, e, f, g, h, old_h) \ 514 ; \ // ################################### RND N + 3 ########################### 515 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 516 MOVL f, y2; \ // y2 = f // CH 517 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A 518 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B 519 XORL g, y2; \ // y2 = f^g // CH 520 ; \ 521 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1 522 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1 523 ANDL e, y2; \ // y2 = (f^g)&e // CH 524 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // -- 525 ; \ 526 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1 527 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B 528 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH 529 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A 530 MOVL a, y3; \ // y3 = a // MAJA 531 ; \ 532 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0 533 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0 534 ADDL (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h // -- 535 ORL c, y3; \ // y3 = a|c // MAJA 536 ; \ 537 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0 538 MOVL a, T1; \ // T1 = a // MAJB 539 ANDL b, y3; \ // y3 = (a|c)&b // MAJA 540 ANDL c, T1; \ // T1 = a&c // MAJB 541 ADDL y0, y2; \ // y2 = S1 + CH // -- 542 ; \ 543 ADDL h, d; \ // d = k + w + h + d // -- 544 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ 545 ADDL y1, h; \ // h = k + w + h + S0 // -- 546 ; \ 547 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // -- 548 ; \ 549 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// -- 550 ; \ 551 ADDL y3, h // h = t1 + S0 + MAJ // -- 552 553 TEXT block(SB), 0, $536-32 554 CMPB useAVX2(SB), $1 555 JE avx2 556 557 MOVQ p_base+8(FP), SI 558 MOVQ p_len+16(FP), DX 559 SHRQ $6, DX 560 SHLQ $6, DX 561 562 LEAQ (SI)(DX*1), DI 563 MOVQ DI, 256(SP) 564 CMPQ SI, DI 565 JEQ end 566 567 MOVQ dig+0(FP), BP 568 MOVL (0*4)(BP), R8 // a = H0 569 MOVL (1*4)(BP), R9 // b = H1 570 MOVL (2*4)(BP), R10 // c = H2 571 MOVL (3*4)(BP), R11 // d = H3 572 MOVL (4*4)(BP), R12 // e = H4 573 MOVL (5*4)(BP), R13 // f = H5 574 MOVL (6*4)(BP), R14 // g = H6 575 MOVL (7*4)(BP), R15 // h = H7 576 577 loop: 578 MOVQ SP, BP 579 580 SHA256ROUND0(0, 0x428a2f98, R8, R9, R10, R11, R12, R13, R14, R15) 581 SHA256ROUND0(1, 0x71374491, R15, R8, R9, R10, R11, R12, R13, R14) 582 SHA256ROUND0(2, 0xb5c0fbcf, R14, R15, R8, R9, R10, R11, R12, R13) 583 SHA256ROUND0(3, 0xe9b5dba5, R13, R14, R15, R8, R9, R10, R11, R12) 584 SHA256ROUND0(4, 0x3956c25b, R12, R13, R14, R15, R8, R9, R10, R11) 585 SHA256ROUND0(5, 0x59f111f1, R11, R12, R13, R14, R15, R8, R9, R10) 586 SHA256ROUND0(6, 0x923f82a4, R10, R11, R12, R13, R14, R15, R8, R9) 587 SHA256ROUND0(7, 0xab1c5ed5, R9, R10, R11, R12, R13, R14, R15, R8) 588 SHA256ROUND0(8, 0xd807aa98, R8, R9, R10, R11, R12, R13, R14, R15) 589 SHA256ROUND0(9, 0x12835b01, R15, R8, R9, R10, R11, R12, R13, R14) 590 SHA256ROUND0(10, 0x243185be, R14, R15, R8, R9, R10, R11, R12, R13) 591 SHA256ROUND0(11, 0x550c7dc3, R13, R14, R15, R8, R9, R10, R11, R12) 592 SHA256ROUND0(12, 0x72be5d74, R12, R13, R14, R15, R8, R9, R10, R11) 593 SHA256ROUND0(13, 0x80deb1fe, R11, R12, R13, R14, R15, R8, R9, R10) 594 SHA256ROUND0(14, 0x9bdc06a7, R10, R11, R12, R13, R14, R15, R8, R9) 595 SHA256ROUND0(15, 0xc19bf174, R9, R10, R11, R12, R13, R14, R15, R8) 596 597 SHA256ROUND1(16, 0xe49b69c1, R8, R9, R10, R11, R12, R13, R14, R15) 598 SHA256ROUND1(17, 0xefbe4786, R15, R8, R9, R10, R11, R12, R13, R14) 599 SHA256ROUND1(18, 0x0fc19dc6, R14, R15, R8, R9, R10, R11, R12, R13) 600 SHA256ROUND1(19, 0x240ca1cc, R13, R14, R15, R8, R9, R10, R11, R12) 601 SHA256ROUND1(20, 0x2de92c6f, R12, R13, R14, R15, R8, R9, R10, R11) 602 SHA256ROUND1(21, 0x4a7484aa, R11, R12, R13, R14, R15, R8, R9, R10) 603 SHA256ROUND1(22, 0x5cb0a9dc, R10, R11, R12, R13, R14, R15, R8, R9) 604 SHA256ROUND1(23, 0x76f988da, R9, R10, R11, R12, R13, R14, R15, R8) 605 SHA256ROUND1(24, 0x983e5152, R8, R9, R10, R11, R12, R13, R14, R15) 606 SHA256ROUND1(25, 0xa831c66d, R15, R8, R9, R10, R11, R12, R13, R14) 607 SHA256ROUND1(26, 0xb00327c8, R14, R15, R8, R9, R10, R11, R12, R13) 608 SHA256ROUND1(27, 0xbf597fc7, R13, R14, R15, R8, R9, R10, R11, R12) 609 SHA256ROUND1(28, 0xc6e00bf3, R12, R13, R14, R15, R8, R9, R10, R11) 610 SHA256ROUND1(29, 0xd5a79147, R11, R12, R13, R14, R15, R8, R9, R10) 611 SHA256ROUND1(30, 0x06ca6351, R10, R11, R12, R13, R14, R15, R8, R9) 612 SHA256ROUND1(31, 0x14292967, R9, R10, R11, R12, R13, R14, R15, R8) 613 SHA256ROUND1(32, 0x27b70a85, R8, R9, R10, R11, R12, R13, R14, R15) 614 SHA256ROUND1(33, 0x2e1b2138, R15, R8, R9, R10, R11, R12, R13, R14) 615 SHA256ROUND1(34, 0x4d2c6dfc, R14, R15, R8, R9, R10, R11, R12, R13) 616 SHA256ROUND1(35, 0x53380d13, R13, R14, R15, R8, R9, R10, R11, R12) 617 SHA256ROUND1(36, 0x650a7354, R12, R13, R14, R15, R8, R9, R10, R11) 618 SHA256ROUND1(37, 0x766a0abb, R11, R12, R13, R14, R15, R8, R9, R10) 619 SHA256ROUND1(38, 0x81c2c92e, R10, R11, R12, R13, R14, R15, R8, R9) 620 SHA256ROUND1(39, 0x92722c85, R9, R10, R11, R12, R13, R14, R15, R8) 621 SHA256ROUND1(40, 0xa2bfe8a1, R8, R9, R10, R11, R12, R13, R14, R15) 622 SHA256ROUND1(41, 0xa81a664b, R15, R8, R9, R10, R11, R12, R13, R14) 623 SHA256ROUND1(42, 0xc24b8b70, R14, R15, R8, R9, R10, R11, R12, R13) 624 SHA256ROUND1(43, 0xc76c51a3, R13, R14, R15, R8, R9, R10, R11, R12) 625 SHA256ROUND1(44, 0xd192e819, R12, R13, R14, R15, R8, R9, R10, R11) 626 SHA256ROUND1(45, 0xd6990624, R11, R12, R13, R14, R15, R8, R9, R10) 627 SHA256ROUND1(46, 0xf40e3585, R10, R11, R12, R13, R14, R15, R8, R9) 628 SHA256ROUND1(47, 0x106aa070, R9, R10, R11, R12, R13, R14, R15, R8) 629 SHA256ROUND1(48, 0x19a4c116, R8, R9, R10, R11, R12, R13, R14, R15) 630 SHA256ROUND1(49, 0x1e376c08, R15, R8, R9, R10, R11, R12, R13, R14) 631 SHA256ROUND1(50, 0x2748774c, R14, R15, R8, R9, R10, R11, R12, R13) 632 SHA256ROUND1(51, 0x34b0bcb5, R13, R14, R15, R8, R9, R10, R11, R12) 633 SHA256ROUND1(52, 0x391c0cb3, R12, R13, R14, R15, R8, R9, R10, R11) 634 SHA256ROUND1(53, 0x4ed8aa4a, R11, R12, R13, R14, R15, R8, R9, R10) 635 SHA256ROUND1(54, 0x5b9cca4f, R10, R11, R12, R13, R14, R15, R8, R9) 636 SHA256ROUND1(55, 0x682e6ff3, R9, R10, R11, R12, R13, R14, R15, R8) 637 SHA256ROUND1(56, 0x748f82ee, R8, R9, R10, R11, R12, R13, R14, R15) 638 SHA256ROUND1(57, 0x78a5636f, R15, R8, R9, R10, R11, R12, R13, R14) 639 SHA256ROUND1(58, 0x84c87814, R14, R15, R8, R9, R10, R11, R12, R13) 640 SHA256ROUND1(59, 0x8cc70208, R13, R14, R15, R8, R9, R10, R11, R12) 641 SHA256ROUND1(60, 0x90befffa, R12, R13, R14, R15, R8, R9, R10, R11) 642 SHA256ROUND1(61, 0xa4506ceb, R11, R12, R13, R14, R15, R8, R9, R10) 643 SHA256ROUND1(62, 0xbef9a3f7, R10, R11, R12, R13, R14, R15, R8, R9) 644 SHA256ROUND1(63, 0xc67178f2, R9, R10, R11, R12, R13, R14, R15, R8) 645 646 MOVQ dig+0(FP), BP 647 ADDL (0*4)(BP), R8 // H0 = a + H0 648 MOVL R8, (0*4)(BP) 649 ADDL (1*4)(BP), R9 // H1 = b + H1 650 MOVL R9, (1*4)(BP) 651 ADDL (2*4)(BP), R10 // H2 = c + H2 652 MOVL R10, (2*4)(BP) 653 ADDL (3*4)(BP), R11 // H3 = d + H3 654 MOVL R11, (3*4)(BP) 655 ADDL (4*4)(BP), R12 // H4 = e + H4 656 MOVL R12, (4*4)(BP) 657 ADDL (5*4)(BP), R13 // H5 = f + H5 658 MOVL R13, (5*4)(BP) 659 ADDL (6*4)(BP), R14 // H6 = g + H6 660 MOVL R14, (6*4)(BP) 661 ADDL (7*4)(BP), R15 // H7 = h + H7 662 MOVL R15, (7*4)(BP) 663 664 ADDQ $64, SI 665 CMPQ SI, 256(SP) 666 JB loop 667 668 end: 669 RET 670 671 avx2: 672 MOVQ dig+0(FP), CTX // d.h[8] 673 MOVQ p_base+8(FP), INP 674 MOVQ p_len+16(FP), NUM_BYTES 675 676 LEAQ -64(INP)(NUM_BYTES*1), NUM_BYTES // Pointer to the last block 677 MOVQ NUM_BYTES, _INP_END(SP) 678 679 CMPQ NUM_BYTES, INP 680 JE avx2_only_one_block 681 682 // Load initial digest 683 MOVL 0(CTX), a // a = H0 684 MOVL 4(CTX), b // b = H1 685 MOVL 8(CTX), c // c = H2 686 MOVL 12(CTX), d // d = H3 687 MOVL 16(CTX), e // e = H4 688 MOVL 20(CTX), f // f = H5 689 MOVL 24(CTX), g // g = H6 690 MOVL 28(CTX), h // h = H7 691 692 avx2_loop0: // at each iteration works with one block (512 bit) 693 694 VMOVDQU (0*32)(INP), XTMP0 695 VMOVDQU (1*32)(INP), XTMP1 696 VMOVDQU (2*32)(INP), XTMP2 697 VMOVDQU (3*32)(INP), XTMP3 698 699 VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK 700 701 // Apply Byte Flip Mask: LE -> BE 702 VPSHUFB BYTE_FLIP_MASK, XTMP0, XTMP0 703 VPSHUFB BYTE_FLIP_MASK, XTMP1, XTMP1 704 VPSHUFB BYTE_FLIP_MASK, XTMP2, XTMP2 705 VPSHUFB BYTE_FLIP_MASK, XTMP3, XTMP3 706 707 // Transpose data into high/low parts 708 VPERM2I128 $0x20, XTMP2, XTMP0, XDWORD0 // w3, w2, w1, w0 709 VPERM2I128 $0x31, XTMP2, XTMP0, XDWORD1 // w7, w6, w5, w4 710 VPERM2I128 $0x20, XTMP3, XTMP1, XDWORD2 // w11, w10, w9, w8 711 VPERM2I128 $0x31, XTMP3, XTMP1, XDWORD3 // w15, w14, w13, w12 712 713 MOVQ $K256<>(SB), TBL // Loading address of table with round-specific constants 714 715 avx2_last_block_enter: 716 ADDQ $64, INP 717 MOVQ INP, _INP(SP) 718 XORQ SRND, SRND 719 720 avx2_loop1: // for w0 - w47 721 // Do 4 rounds and scheduling 722 VPADDD 0*32(TBL)(SRND*1), XDWORD0, XFER 723 VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1) 724 ROUND_AND_SCHED_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) 725 ROUND_AND_SCHED_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, XDWORD0, XDWORD1, XDWORD2, XDWORD3) 726 ROUND_AND_SCHED_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, XDWORD0, XDWORD1, XDWORD2, XDWORD3) 727 ROUND_AND_SCHED_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, XDWORD0, XDWORD1, XDWORD2, XDWORD3) 728 729 // Do 4 rounds and scheduling 730 VPADDD 1*32(TBL)(SRND*1), XDWORD1, XFER 731 VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1) 732 ROUND_AND_SCHED_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, XDWORD1, XDWORD2, XDWORD3, XDWORD0) 733 ROUND_AND_SCHED_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, XDWORD1, XDWORD2, XDWORD3, XDWORD0) 734 ROUND_AND_SCHED_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, XDWORD1, XDWORD2, XDWORD3, XDWORD0) 735 ROUND_AND_SCHED_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, XDWORD1, XDWORD2, XDWORD3, XDWORD0) 736 737 // Do 4 rounds and scheduling 738 VPADDD 2*32(TBL)(SRND*1), XDWORD2, XFER 739 VMOVDQU XFER, (_XFER + 2*32)(SP)(SRND*1) 740 ROUND_AND_SCHED_N_0(_XFER + 2*32, a, b, c, d, e, f, g, h, XDWORD2, XDWORD3, XDWORD0, XDWORD1) 741 ROUND_AND_SCHED_N_1(_XFER + 2*32, h, a, b, c, d, e, f, g, XDWORD2, XDWORD3, XDWORD0, XDWORD1) 742 ROUND_AND_SCHED_N_2(_XFER + 2*32, g, h, a, b, c, d, e, f, XDWORD2, XDWORD3, XDWORD0, XDWORD1) 743 ROUND_AND_SCHED_N_3(_XFER + 2*32, f, g, h, a, b, c, d, e, XDWORD2, XDWORD3, XDWORD0, XDWORD1) 744 745 // Do 4 rounds and scheduling 746 VPADDD 3*32(TBL)(SRND*1), XDWORD3, XFER 747 VMOVDQU XFER, (_XFER + 3*32)(SP)(SRND*1) 748 ROUND_AND_SCHED_N_0(_XFER + 3*32, e, f, g, h, a, b, c, d, XDWORD3, XDWORD0, XDWORD1, XDWORD2) 749 ROUND_AND_SCHED_N_1(_XFER + 3*32, d, e, f, g, h, a, b, c, XDWORD3, XDWORD0, XDWORD1, XDWORD2) 750 ROUND_AND_SCHED_N_2(_XFER + 3*32, c, d, e, f, g, h, a, b, XDWORD3, XDWORD0, XDWORD1, XDWORD2) 751 ROUND_AND_SCHED_N_3(_XFER + 3*32, b, c, d, e, f, g, h, a, XDWORD3, XDWORD0, XDWORD1, XDWORD2) 752 753 ADDQ $4*32, SRND 754 CMPQ SRND, $3*4*32 755 JB avx2_loop1 756 757 avx2_loop2: 758 // w48 - w63 processed with no scheduliung (last 16 rounds) 759 VPADDD 0*32(TBL)(SRND*1), XDWORD0, XFER 760 VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1) 761 DO_ROUND_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, h) 762 DO_ROUND_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, h) 763 DO_ROUND_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, g) 764 DO_ROUND_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, f) 765 766 VPADDD 1*32(TBL)(SRND*1), XDWORD1, XFER 767 VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1) 768 DO_ROUND_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, e) 769 DO_ROUND_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, d) 770 DO_ROUND_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, c) 771 DO_ROUND_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, b) 772 773 ADDQ $2*32, SRND 774 775 VMOVDQU XDWORD2, XDWORD0 776 VMOVDQU XDWORD3, XDWORD1 777 778 CMPQ SRND, $4*4*32 779 JB avx2_loop2 780 781 MOVQ dig+0(FP), CTX // d.h[8] 782 MOVQ _INP(SP), INP 783 784 addm( 0(CTX), a) 785 addm( 4(CTX), b) 786 addm( 8(CTX), c) 787 addm( 12(CTX), d) 788 addm( 16(CTX), e) 789 addm( 20(CTX), f) 790 addm( 24(CTX), g) 791 addm( 28(CTX), h) 792 793 CMPQ _INP_END(SP), INP 794 JB done_hash 795 796 XORQ SRND, SRND 797 798 avx2_loop3: // Do second block using previously scheduled results 799 DO_ROUND_N_0(_XFER + 0*32 + 16, a, b, c, d, e, f, g, h, a) 800 DO_ROUND_N_1(_XFER + 0*32 + 16, h, a, b, c, d, e, f, g, h) 801 DO_ROUND_N_2(_XFER + 0*32 + 16, g, h, a, b, c, d, e, f, g) 802 DO_ROUND_N_3(_XFER + 0*32 + 16, f, g, h, a, b, c, d, e, f) 803 804 DO_ROUND_N_0(_XFER + 1*32 + 16, e, f, g, h, a, b, c, d, e) 805 DO_ROUND_N_1(_XFER + 1*32 + 16, d, e, f, g, h, a, b, c, d) 806 DO_ROUND_N_2(_XFER + 1*32 + 16, c, d, e, f, g, h, a, b, c) 807 DO_ROUND_N_3(_XFER + 1*32 + 16, b, c, d, e, f, g, h, a, b) 808 809 ADDQ $2*32, SRND 810 CMPQ SRND, $4*4*32 811 JB avx2_loop3 812 813 MOVQ dig+0(FP), CTX // d.h[8] 814 MOVQ _INP(SP), INP 815 ADDQ $64, INP 816 817 addm( 0(CTX), a) 818 addm( 4(CTX), b) 819 addm( 8(CTX), c) 820 addm( 12(CTX), d) 821 addm( 16(CTX), e) 822 addm( 20(CTX), f) 823 addm( 24(CTX), g) 824 addm( 28(CTX), h) 825 826 CMPQ _INP_END(SP), INP 827 JA avx2_loop0 828 JB done_hash 829 830 avx2_do_last_block: 831 832 VMOVDQU 0(INP), XWORD0 833 VMOVDQU 16(INP), XWORD1 834 VMOVDQU 32(INP), XWORD2 835 VMOVDQU 48(INP), XWORD3 836 837 VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK 838 839 VPSHUFB X_BYTE_FLIP_MASK, XWORD0, XWORD0 840 VPSHUFB X_BYTE_FLIP_MASK, XWORD1, XWORD1 841 VPSHUFB X_BYTE_FLIP_MASK, XWORD2, XWORD2 842 VPSHUFB X_BYTE_FLIP_MASK, XWORD3, XWORD3 843 844 MOVQ $K256<>(SB), TBL 845 846 JMP avx2_last_block_enter 847 848 avx2_only_one_block: 849 // Load initial digest 850 MOVL 0(CTX), a // a = H0 851 MOVL 4(CTX), b // b = H1 852 MOVL 8(CTX), c // c = H2 853 MOVL 12(CTX), d // d = H3 854 MOVL 16(CTX), e // e = H4 855 MOVL 20(CTX), f // f = H5 856 MOVL 24(CTX), g // g = H6 857 MOVL 28(CTX), h // h = H7 858 859 JMP avx2_do_last_block 860 861 done_hash: 862 VZEROUPPER 863 RET 864 865 // shuffle byte order from LE to BE 866 DATA flip_mask<>+0x00(SB)/8, $0x0405060700010203 867 DATA flip_mask<>+0x08(SB)/8, $0x0c0d0e0f08090a0b 868 DATA flip_mask<>+0x10(SB)/8, $0x0405060700010203 869 DATA flip_mask<>+0x18(SB)/8, $0x0c0d0e0f08090a0b 870 GLOBL flip_mask<>(SB), 8, $32 871 872 // shuffle xBxA -> 00BA 873 DATA shuff_00BA<>+0x00(SB)/8, $0x0b0a090803020100 874 DATA shuff_00BA<>+0x08(SB)/8, $0xFFFFFFFFFFFFFFFF 875 DATA shuff_00BA<>+0x10(SB)/8, $0x0b0a090803020100 876 DATA shuff_00BA<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF 877 GLOBL shuff_00BA<>(SB), 8, $32 878 879 // shuffle xDxC -> DC00 880 DATA shuff_DC00<>+0x00(SB)/8, $0xFFFFFFFFFFFFFFFF 881 DATA shuff_DC00<>+0x08(SB)/8, $0x0b0a090803020100 882 DATA shuff_DC00<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF 883 DATA shuff_DC00<>+0x18(SB)/8, $0x0b0a090803020100 884 GLOBL shuff_DC00<>(SB), 8, $32 885 886 // Round specific constants 887 DATA K256<>+0x00(SB)/4, $0x428a2f98 // k1 888 DATA K256<>+0x04(SB)/4, $0x71374491 // k2 889 DATA K256<>+0x08(SB)/4, $0xb5c0fbcf // k3 890 DATA K256<>+0x0c(SB)/4, $0xe9b5dba5 // k4 891 DATA K256<>+0x10(SB)/4, $0x428a2f98 // k1 892 DATA K256<>+0x14(SB)/4, $0x71374491 // k2 893 DATA K256<>+0x18(SB)/4, $0xb5c0fbcf // k3 894 DATA K256<>+0x1c(SB)/4, $0xe9b5dba5 // k4 895 896 DATA K256<>+0x20(SB)/4, $0x3956c25b // k5 - k8 897 DATA K256<>+0x24(SB)/4, $0x59f111f1 898 DATA K256<>+0x28(SB)/4, $0x923f82a4 899 DATA K256<>+0x2c(SB)/4, $0xab1c5ed5 900 DATA K256<>+0x30(SB)/4, $0x3956c25b 901 DATA K256<>+0x34(SB)/4, $0x59f111f1 902 DATA K256<>+0x38(SB)/4, $0x923f82a4 903 DATA K256<>+0x3c(SB)/4, $0xab1c5ed5 904 905 DATA K256<>+0x40(SB)/4, $0xd807aa98 // k9 - k12 906 DATA K256<>+0x44(SB)/4, $0x12835b01 907 DATA K256<>+0x48(SB)/4, $0x243185be 908 DATA K256<>+0x4c(SB)/4, $0x550c7dc3 909 DATA K256<>+0x50(SB)/4, $0xd807aa98 910 DATA K256<>+0x54(SB)/4, $0x12835b01 911 DATA K256<>+0x58(SB)/4, $0x243185be 912 DATA K256<>+0x5c(SB)/4, $0x550c7dc3 913 914 DATA K256<>+0x60(SB)/4, $0x72be5d74 // k13 - k16 915 DATA K256<>+0x64(SB)/4, $0x80deb1fe 916 DATA K256<>+0x68(SB)/4, $0x9bdc06a7 917 DATA K256<>+0x6c(SB)/4, $0xc19bf174 918 DATA K256<>+0x70(SB)/4, $0x72be5d74 919 DATA K256<>+0x74(SB)/4, $0x80deb1fe 920 DATA K256<>+0x78(SB)/4, $0x9bdc06a7 921 DATA K256<>+0x7c(SB)/4, $0xc19bf174 922 923 DATA K256<>+0x80(SB)/4, $0xe49b69c1 // k17 - k20 924 DATA K256<>+0x84(SB)/4, $0xefbe4786 925 DATA K256<>+0x88(SB)/4, $0x0fc19dc6 926 DATA K256<>+0x8c(SB)/4, $0x240ca1cc 927 DATA K256<>+0x90(SB)/4, $0xe49b69c1 928 DATA K256<>+0x94(SB)/4, $0xefbe4786 929 DATA K256<>+0x98(SB)/4, $0x0fc19dc6 930 DATA K256<>+0x9c(SB)/4, $0x240ca1cc 931 932 DATA K256<>+0xa0(SB)/4, $0x2de92c6f // k21 - k24 933 DATA K256<>+0xa4(SB)/4, $0x4a7484aa 934 DATA K256<>+0xa8(SB)/4, $0x5cb0a9dc 935 DATA K256<>+0xac(SB)/4, $0x76f988da 936 DATA K256<>+0xb0(SB)/4, $0x2de92c6f 937 DATA K256<>+0xb4(SB)/4, $0x4a7484aa 938 DATA K256<>+0xb8(SB)/4, $0x5cb0a9dc 939 DATA K256<>+0xbc(SB)/4, $0x76f988da 940 941 DATA K256<>+0xc0(SB)/4, $0x983e5152 // k25 - k28 942 DATA K256<>+0xc4(SB)/4, $0xa831c66d 943 DATA K256<>+0xc8(SB)/4, $0xb00327c8 944 DATA K256<>+0xcc(SB)/4, $0xbf597fc7 945 DATA K256<>+0xd0(SB)/4, $0x983e5152 946 DATA K256<>+0xd4(SB)/4, $0xa831c66d 947 DATA K256<>+0xd8(SB)/4, $0xb00327c8 948 DATA K256<>+0xdc(SB)/4, $0xbf597fc7 949 950 DATA K256<>+0xe0(SB)/4, $0xc6e00bf3 // k29 - k32 951 DATA K256<>+0xe4(SB)/4, $0xd5a79147 952 DATA K256<>+0xe8(SB)/4, $0x06ca6351 953 DATA K256<>+0xec(SB)/4, $0x14292967 954 DATA K256<>+0xf0(SB)/4, $0xc6e00bf3 955 DATA K256<>+0xf4(SB)/4, $0xd5a79147 956 DATA K256<>+0xf8(SB)/4, $0x06ca6351 957 DATA K256<>+0xfc(SB)/4, $0x14292967 958 959 DATA K256<>+0x100(SB)/4, $0x27b70a85 960 DATA K256<>+0x104(SB)/4, $0x2e1b2138 961 DATA K256<>+0x108(SB)/4, $0x4d2c6dfc 962 DATA K256<>+0x10c(SB)/4, $0x53380d13 963 DATA K256<>+0x110(SB)/4, $0x27b70a85 964 DATA K256<>+0x114(SB)/4, $0x2e1b2138 965 DATA K256<>+0x118(SB)/4, $0x4d2c6dfc 966 DATA K256<>+0x11c(SB)/4, $0x53380d13 967 968 DATA K256<>+0x120(SB)/4, $0x650a7354 969 DATA K256<>+0x124(SB)/4, $0x766a0abb 970 DATA K256<>+0x128(SB)/4, $0x81c2c92e 971 DATA K256<>+0x12c(SB)/4, $0x92722c85 972 DATA K256<>+0x130(SB)/4, $0x650a7354 973 DATA K256<>+0x134(SB)/4, $0x766a0abb 974 DATA K256<>+0x138(SB)/4, $0x81c2c92e 975 DATA K256<>+0x13c(SB)/4, $0x92722c85 976 977 DATA K256<>+0x140(SB)/4, $0xa2bfe8a1 978 DATA K256<>+0x144(SB)/4, $0xa81a664b 979 DATA K256<>+0x148(SB)/4, $0xc24b8b70 980 DATA K256<>+0x14c(SB)/4, $0xc76c51a3 981 DATA K256<>+0x150(SB)/4, $0xa2bfe8a1 982 DATA K256<>+0x154(SB)/4, $0xa81a664b 983 DATA K256<>+0x158(SB)/4, $0xc24b8b70 984 DATA K256<>+0x15c(SB)/4, $0xc76c51a3 985 986 DATA K256<>+0x160(SB)/4, $0xd192e819 987 DATA K256<>+0x164(SB)/4, $0xd6990624 988 DATA K256<>+0x168(SB)/4, $0xf40e3585 989 DATA K256<>+0x16c(SB)/4, $0x106aa070 990 DATA K256<>+0x170(SB)/4, $0xd192e819 991 DATA K256<>+0x174(SB)/4, $0xd6990624 992 DATA K256<>+0x178(SB)/4, $0xf40e3585 993 DATA K256<>+0x17c(SB)/4, $0x106aa070 994 995 DATA K256<>+0x180(SB)/4, $0x19a4c116 996 DATA K256<>+0x184(SB)/4, $0x1e376c08 997 DATA K256<>+0x188(SB)/4, $0x2748774c 998 DATA K256<>+0x18c(SB)/4, $0x34b0bcb5 999 DATA K256<>+0x190(SB)/4, $0x19a4c116 1000 DATA K256<>+0x194(SB)/4, $0x1e376c08 1001 DATA K256<>+0x198(SB)/4, $0x2748774c 1002 DATA K256<>+0x19c(SB)/4, $0x34b0bcb5 1003 1004 DATA K256<>+0x1a0(SB)/4, $0x391c0cb3 1005 DATA K256<>+0x1a4(SB)/4, $0x4ed8aa4a 1006 DATA K256<>+0x1a8(SB)/4, $0x5b9cca4f 1007 DATA K256<>+0x1ac(SB)/4, $0x682e6ff3 1008 DATA K256<>+0x1b0(SB)/4, $0x391c0cb3 1009 DATA K256<>+0x1b4(SB)/4, $0x4ed8aa4a 1010 DATA K256<>+0x1b8(SB)/4, $0x5b9cca4f 1011 DATA K256<>+0x1bc(SB)/4, $0x682e6ff3 1012 1013 DATA K256<>+0x1c0(SB)/4, $0x748f82ee 1014 DATA K256<>+0x1c4(SB)/4, $0x78a5636f 1015 DATA K256<>+0x1c8(SB)/4, $0x84c87814 1016 DATA K256<>+0x1cc(SB)/4, $0x8cc70208 1017 DATA K256<>+0x1d0(SB)/4, $0x748f82ee 1018 DATA K256<>+0x1d4(SB)/4, $0x78a5636f 1019 DATA K256<>+0x1d8(SB)/4, $0x84c87814 1020 DATA K256<>+0x1dc(SB)/4, $0x8cc70208 1021 1022 DATA K256<>+0x1e0(SB)/4, $0x90befffa 1023 DATA K256<>+0x1e4(SB)/4, $0xa4506ceb 1024 DATA K256<>+0x1e8(SB)/4, $0xbef9a3f7 1025 DATA K256<>+0x1ec(SB)/4, $0xc67178f2 1026 DATA K256<>+0x1f0(SB)/4, $0x90befffa 1027 DATA K256<>+0x1f4(SB)/4, $0xa4506ceb 1028 DATA K256<>+0x1f8(SB)/4, $0xbef9a3f7 1029 DATA K256<>+0x1fc(SB)/4, $0xc67178f2 1030 1031 GLOBL K256<>(SB), (NOPTR + RODATA), $512 1032
