1 #if defined(__i386__) 2 .file "src/crypto/aes/asm/aesni-x86.S" 3 .text 4 .globl aesni_encrypt 5 .hidden aesni_encrypt 6 .type aesni_encrypt,@function 7 .align 16 8 aesni_encrypt: 9 .L_aesni_encrypt_begin: 10 movl 4(%esp),%eax 11 movl 12(%esp),%edx 12 movups (%eax),%xmm2 13 movl 240(%edx),%ecx 14 movl 8(%esp),%eax 15 movups (%edx),%xmm0 16 movups 16(%edx),%xmm1 17 leal 32(%edx),%edx 18 xorps %xmm0,%xmm2 19 .L000enc1_loop_1: 20 .byte 102,15,56,220,209 21 decl %ecx 22 movups (%edx),%xmm1 23 leal 16(%edx),%edx 24 jnz .L000enc1_loop_1 25 .byte 102,15,56,221,209 26 pxor %xmm0,%xmm0 27 pxor %xmm1,%xmm1 28 movups %xmm2,(%eax) 29 pxor %xmm2,%xmm2 30 ret 31 .size aesni_encrypt,.-.L_aesni_encrypt_begin 32 .globl aesni_decrypt 33 .hidden aesni_decrypt 34 .type aesni_decrypt,@function 35 .align 16 36 aesni_decrypt: 37 .L_aesni_decrypt_begin: 38 movl 4(%esp),%eax 39 movl 12(%esp),%edx 40 movups (%eax),%xmm2 41 movl 240(%edx),%ecx 42 movl 8(%esp),%eax 43 movups (%edx),%xmm0 44 movups 16(%edx),%xmm1 45 leal 32(%edx),%edx 46 xorps %xmm0,%xmm2 47 .L001dec1_loop_2: 48 .byte 102,15,56,222,209 49 decl %ecx 50 movups (%edx),%xmm1 51 leal 16(%edx),%edx 52 jnz .L001dec1_loop_2 53 .byte 102,15,56,223,209 54 pxor %xmm0,%xmm0 55 pxor %xmm1,%xmm1 56 movups %xmm2,(%eax) 57 pxor %xmm2,%xmm2 58 ret 59 .size aesni_decrypt,.-.L_aesni_decrypt_begin 60 .hidden _aesni_encrypt2 61 .type _aesni_encrypt2,@function 62 .align 16 63 _aesni_encrypt2: 64 movups (%edx),%xmm0 65 shll $4,%ecx 66 movups 16(%edx),%xmm1 67 xorps %xmm0,%xmm2 68 pxor %xmm0,%xmm3 69 movups 32(%edx),%xmm0 70 leal 32(%edx,%ecx,1),%edx 71 negl %ecx 72 addl $16,%ecx 73 .L002enc2_loop: 74 .byte 102,15,56,220,209 75 .byte 102,15,56,220,217 76 movups (%edx,%ecx,1),%xmm1 77 addl $32,%ecx 78 .byte 102,15,56,220,208 79 .byte 102,15,56,220,216 80 movups -16(%edx,%ecx,1),%xmm0 81 jnz .L002enc2_loop 82 .byte 102,15,56,220,209 83 .byte 102,15,56,220,217 84 .byte 102,15,56,221,208 85 .byte 102,15,56,221,216 86 ret 87 .size _aesni_encrypt2,.-_aesni_encrypt2 88 .hidden _aesni_decrypt2 89 .type _aesni_decrypt2,@function 90 .align 16 91 _aesni_decrypt2: 92 movups (%edx),%xmm0 93 shll $4,%ecx 94 movups 16(%edx),%xmm1 95 xorps %xmm0,%xmm2 96 pxor %xmm0,%xmm3 97 movups 32(%edx),%xmm0 98 leal 32(%edx,%ecx,1),%edx 99 negl %ecx 100 addl $16,%ecx 101 .L003dec2_loop: 102 .byte 102,15,56,222,209 103 .byte 102,15,56,222,217 104 movups (%edx,%ecx,1),%xmm1 105 addl $32,%ecx 106 .byte 102,15,56,222,208 107 .byte 102,15,56,222,216 108 movups -16(%edx,%ecx,1),%xmm0 109 jnz .L003dec2_loop 110 .byte 102,15,56,222,209 111 .byte 102,15,56,222,217 112 .byte 102,15,56,223,208 113 .byte 102,15,56,223,216 114 ret 115 .size _aesni_decrypt2,.-_aesni_decrypt2 116 .hidden _aesni_encrypt3 117 .type _aesni_encrypt3,@function 118 .align 16 119 _aesni_encrypt3: 120 movups (%edx),%xmm0 121 shll $4,%ecx 122 movups 16(%edx),%xmm1 123 xorps %xmm0,%xmm2 124 pxor %xmm0,%xmm3 125 pxor %xmm0,%xmm4 126 movups 32(%edx),%xmm0 127 leal 32(%edx,%ecx,1),%edx 128 negl %ecx 129 addl $16,%ecx 130 .L004enc3_loop: 131 .byte 102,15,56,220,209 132 .byte 102,15,56,220,217 133 .byte 102,15,56,220,225 134 movups (%edx,%ecx,1),%xmm1 135 addl $32,%ecx 136 .byte 102,15,56,220,208 137 .byte 102,15,56,220,216 138 .byte 102,15,56,220,224 139 movups -16(%edx,%ecx,1),%xmm0 140 jnz .L004enc3_loop 141 .byte 102,15,56,220,209 142 .byte 102,15,56,220,217 143 .byte 102,15,56,220,225 144 .byte 102,15,56,221,208 145 .byte 102,15,56,221,216 146 .byte 102,15,56,221,224 147 ret 148 .size _aesni_encrypt3,.-_aesni_encrypt3 149 .hidden _aesni_decrypt3 150 .type _aesni_decrypt3,@function 151 .align 16 152 _aesni_decrypt3: 153 movups (%edx),%xmm0 154 shll $4,%ecx 155 movups 16(%edx),%xmm1 156 xorps %xmm0,%xmm2 157 pxor %xmm0,%xmm3 158 pxor %xmm0,%xmm4 159 movups 32(%edx),%xmm0 160 leal 32(%edx,%ecx,1),%edx 161 negl %ecx 162 addl $16,%ecx 163 .L005dec3_loop: 164 .byte 102,15,56,222,209 165 .byte 102,15,56,222,217 166 .byte 102,15,56,222,225 167 movups (%edx,%ecx,1),%xmm1 168 addl $32,%ecx 169 .byte 102,15,56,222,208 170 .byte 102,15,56,222,216 171 .byte 102,15,56,222,224 172 movups -16(%edx,%ecx,1),%xmm0 173 jnz .L005dec3_loop 174 .byte 102,15,56,222,209 175 .byte 102,15,56,222,217 176 .byte 102,15,56,222,225 177 .byte 102,15,56,223,208 178 .byte 102,15,56,223,216 179 .byte 102,15,56,223,224 180 ret 181 .size _aesni_decrypt3,.-_aesni_decrypt3 182 .hidden _aesni_encrypt4 183 .type _aesni_encrypt4,@function 184 .align 16 185 _aesni_encrypt4: 186 movups (%edx),%xmm0 187 movups 16(%edx),%xmm1 188 shll $4,%ecx 189 xorps %xmm0,%xmm2 190 pxor %xmm0,%xmm3 191 pxor %xmm0,%xmm4 192 pxor %xmm0,%xmm5 193 movups 32(%edx),%xmm0 194 leal 32(%edx,%ecx,1),%edx 195 negl %ecx 196 .byte 15,31,64,0 197 addl $16,%ecx 198 .L006enc4_loop: 199 .byte 102,15,56,220,209 200 .byte 102,15,56,220,217 201 .byte 102,15,56,220,225 202 .byte 102,15,56,220,233 203 movups (%edx,%ecx,1),%xmm1 204 addl $32,%ecx 205 .byte 102,15,56,220,208 206 .byte 102,15,56,220,216 207 .byte 102,15,56,220,224 208 .byte 102,15,56,220,232 209 movups -16(%edx,%ecx,1),%xmm0 210 jnz .L006enc4_loop 211 .byte 102,15,56,220,209 212 .byte 102,15,56,220,217 213 .byte 102,15,56,220,225 214 .byte 102,15,56,220,233 215 .byte 102,15,56,221,208 216 .byte 102,15,56,221,216 217 .byte 102,15,56,221,224 218 .byte 102,15,56,221,232 219 ret 220 .size _aesni_encrypt4,.-_aesni_encrypt4 221 .hidden _aesni_decrypt4 222 .type _aesni_decrypt4,@function 223 .align 16 224 _aesni_decrypt4: 225 movups (%edx),%xmm0 226 movups 16(%edx),%xmm1 227 shll $4,%ecx 228 xorps %xmm0,%xmm2 229 pxor %xmm0,%xmm3 230 pxor %xmm0,%xmm4 231 pxor %xmm0,%xmm5 232 movups 32(%edx),%xmm0 233 leal 32(%edx,%ecx,1),%edx 234 negl %ecx 235 .byte 15,31,64,0 236 addl $16,%ecx 237 .L007dec4_loop: 238 .byte 102,15,56,222,209 239 .byte 102,15,56,222,217 240 .byte 102,15,56,222,225 241 .byte 102,15,56,222,233 242 movups (%edx,%ecx,1),%xmm1 243 addl $32,%ecx 244 .byte 102,15,56,222,208 245 .byte 102,15,56,222,216 246 .byte 102,15,56,222,224 247 .byte 102,15,56,222,232 248 movups -16(%edx,%ecx,1),%xmm0 249 jnz .L007dec4_loop 250 .byte 102,15,56,222,209 251 .byte 102,15,56,222,217 252 .byte 102,15,56,222,225 253 .byte 102,15,56,222,233 254 .byte 102,15,56,223,208 255 .byte 102,15,56,223,216 256 .byte 102,15,56,223,224 257 .byte 102,15,56,223,232 258 ret 259 .size _aesni_decrypt4,.-_aesni_decrypt4 260 .hidden _aesni_encrypt6 261 .type _aesni_encrypt6,@function 262 .align 16 263 _aesni_encrypt6: 264 movups (%edx),%xmm0 265 shll $4,%ecx 266 movups 16(%edx),%xmm1 267 xorps %xmm0,%xmm2 268 pxor %xmm0,%xmm3 269 pxor %xmm0,%xmm4 270 .byte 102,15,56,220,209 271 pxor %xmm0,%xmm5 272 pxor %xmm0,%xmm6 273 .byte 102,15,56,220,217 274 leal 32(%edx,%ecx,1),%edx 275 negl %ecx 276 .byte 102,15,56,220,225 277 pxor %xmm0,%xmm7 278 movups (%edx,%ecx,1),%xmm0 279 addl $16,%ecx 280 jmp .L008_aesni_encrypt6_inner 281 .align 16 282 .L009enc6_loop: 283 .byte 102,15,56,220,209 284 .byte 102,15,56,220,217 285 .byte 102,15,56,220,225 286 .L008_aesni_encrypt6_inner: 287 .byte 102,15,56,220,233 288 .byte 102,15,56,220,241 289 .byte 102,15,56,220,249 290 .L_aesni_encrypt6_enter: 291 movups (%edx,%ecx,1),%xmm1 292 addl $32,%ecx 293 .byte 102,15,56,220,208 294 .byte 102,15,56,220,216 295 .byte 102,15,56,220,224 296 .byte 102,15,56,220,232 297 .byte 102,15,56,220,240 298 .byte 102,15,56,220,248 299 movups -16(%edx,%ecx,1),%xmm0 300 jnz .L009enc6_loop 301 .byte 102,15,56,220,209 302 .byte 102,15,56,220,217 303 .byte 102,15,56,220,225 304 .byte 102,15,56,220,233 305 .byte 102,15,56,220,241 306 .byte 102,15,56,220,249 307 .byte 102,15,56,221,208 308 .byte 102,15,56,221,216 309 .byte 102,15,56,221,224 310 .byte 102,15,56,221,232 311 .byte 102,15,56,221,240 312 .byte 102,15,56,221,248 313 ret 314 .size _aesni_encrypt6,.-_aesni_encrypt6 315 .hidden _aesni_decrypt6 316 .type _aesni_decrypt6,@function 317 .align 16 318 _aesni_decrypt6: 319 movups (%edx),%xmm0 320 shll $4,%ecx 321 movups 16(%edx),%xmm1 322 xorps %xmm0,%xmm2 323 pxor %xmm0,%xmm3 324 pxor %xmm0,%xmm4 325 .byte 102,15,56,222,209 326 pxor %xmm0,%xmm5 327 pxor %xmm0,%xmm6 328 .byte 102,15,56,222,217 329 leal 32(%edx,%ecx,1),%edx 330 negl %ecx 331 .byte 102,15,56,222,225 332 pxor %xmm0,%xmm7 333 movups (%edx,%ecx,1),%xmm0 334 addl $16,%ecx 335 jmp .L010_aesni_decrypt6_inner 336 .align 16 337 .L011dec6_loop: 338 .byte 102,15,56,222,209 339 .byte 102,15,56,222,217 340 .byte 102,15,56,222,225 341 .L010_aesni_decrypt6_inner: 342 .byte 102,15,56,222,233 343 .byte 102,15,56,222,241 344 .byte 102,15,56,222,249 345 .L_aesni_decrypt6_enter: 346 movups (%edx,%ecx,1),%xmm1 347 addl $32,%ecx 348 .byte 102,15,56,222,208 349 .byte 102,15,56,222,216 350 .byte 102,15,56,222,224 351 .byte 102,15,56,222,232 352 .byte 102,15,56,222,240 353 .byte 102,15,56,222,248 354 movups -16(%edx,%ecx,1),%xmm0 355 jnz .L011dec6_loop 356 .byte 102,15,56,222,209 357 .byte 102,15,56,222,217 358 .byte 102,15,56,222,225 359 .byte 102,15,56,222,233 360 .byte 102,15,56,222,241 361 .byte 102,15,56,222,249 362 .byte 102,15,56,223,208 363 .byte 102,15,56,223,216 364 .byte 102,15,56,223,224 365 .byte 102,15,56,223,232 366 .byte 102,15,56,223,240 367 .byte 102,15,56,223,248 368 ret 369 .size _aesni_decrypt6,.-_aesni_decrypt6 370 .globl aesni_ecb_encrypt 371 .hidden aesni_ecb_encrypt 372 .type aesni_ecb_encrypt,@function 373 .align 16 374 aesni_ecb_encrypt: 375 .L_aesni_ecb_encrypt_begin: 376 pushl %ebp 377 pushl %ebx 378 pushl %esi 379 pushl %edi 380 movl 20(%esp),%esi 381 movl 24(%esp),%edi 382 movl 28(%esp),%eax 383 movl 32(%esp),%edx 384 movl 36(%esp),%ebx 385 andl $-16,%eax 386 jz .L012ecb_ret 387 movl 240(%edx),%ecx 388 testl %ebx,%ebx 389 jz .L013ecb_decrypt 390 movl %edx,%ebp 391 movl %ecx,%ebx 392 cmpl $96,%eax 393 jb .L014ecb_enc_tail 394 movdqu (%esi),%xmm2 395 movdqu 16(%esi),%xmm3 396 movdqu 32(%esi),%xmm4 397 movdqu 48(%esi),%xmm5 398 movdqu 64(%esi),%xmm6 399 movdqu 80(%esi),%xmm7 400 leal 96(%esi),%esi 401 subl $96,%eax 402 jmp .L015ecb_enc_loop6_enter 403 .align 16 404 .L016ecb_enc_loop6: 405 movups %xmm2,(%edi) 406 movdqu (%esi),%xmm2 407 movups %xmm3,16(%edi) 408 movdqu 16(%esi),%xmm3 409 movups %xmm4,32(%edi) 410 movdqu 32(%esi),%xmm4 411 movups %xmm5,48(%edi) 412 movdqu 48(%esi),%xmm5 413 movups %xmm6,64(%edi) 414 movdqu 64(%esi),%xmm6 415 movups %xmm7,80(%edi) 416 leal 96(%edi),%edi 417 movdqu 80(%esi),%xmm7 418 leal 96(%esi),%esi 419 .L015ecb_enc_loop6_enter: 420 call _aesni_encrypt6 421 movl %ebp,%edx 422 movl %ebx,%ecx 423 subl $96,%eax 424 jnc .L016ecb_enc_loop6 425 movups %xmm2,(%edi) 426 movups %xmm3,16(%edi) 427 movups %xmm4,32(%edi) 428 movups %xmm5,48(%edi) 429 movups %xmm6,64(%edi) 430 movups %xmm7,80(%edi) 431 leal 96(%edi),%edi 432 addl $96,%eax 433 jz .L012ecb_ret 434 .L014ecb_enc_tail: 435 movups (%esi),%xmm2 436 cmpl $32,%eax 437 jb .L017ecb_enc_one 438 movups 16(%esi),%xmm3 439 je .L018ecb_enc_two 440 movups 32(%esi),%xmm4 441 cmpl $64,%eax 442 jb .L019ecb_enc_three 443 movups 48(%esi),%xmm5 444 je .L020ecb_enc_four 445 movups 64(%esi),%xmm6 446 xorps %xmm7,%xmm7 447 call _aesni_encrypt6 448 movups %xmm2,(%edi) 449 movups %xmm3,16(%edi) 450 movups %xmm4,32(%edi) 451 movups %xmm5,48(%edi) 452 movups %xmm6,64(%edi) 453 jmp .L012ecb_ret 454 .align 16 455 .L017ecb_enc_one: 456 movups (%edx),%xmm0 457 movups 16(%edx),%xmm1 458 leal 32(%edx),%edx 459 xorps %xmm0,%xmm2 460 .L021enc1_loop_3: 461 .byte 102,15,56,220,209 462 decl %ecx 463 movups (%edx),%xmm1 464 leal 16(%edx),%edx 465 jnz .L021enc1_loop_3 466 .byte 102,15,56,221,209 467 movups %xmm2,(%edi) 468 jmp .L012ecb_ret 469 .align 16 470 .L018ecb_enc_two: 471 call _aesni_encrypt2 472 movups %xmm2,(%edi) 473 movups %xmm3,16(%edi) 474 jmp .L012ecb_ret 475 .align 16 476 .L019ecb_enc_three: 477 call _aesni_encrypt3 478 movups %xmm2,(%edi) 479 movups %xmm3,16(%edi) 480 movups %xmm4,32(%edi) 481 jmp .L012ecb_ret 482 .align 16 483 .L020ecb_enc_four: 484 call _aesni_encrypt4 485 movups %xmm2,(%edi) 486 movups %xmm3,16(%edi) 487 movups %xmm4,32(%edi) 488 movups %xmm5,48(%edi) 489 jmp .L012ecb_ret 490 .align 16 491 .L013ecb_decrypt: 492 movl %edx,%ebp 493 movl %ecx,%ebx 494 cmpl $96,%eax 495 jb .L022ecb_dec_tail 496 movdqu (%esi),%xmm2 497 movdqu 16(%esi),%xmm3 498 movdqu 32(%esi),%xmm4 499 movdqu 48(%esi),%xmm5 500 movdqu 64(%esi),%xmm6 501 movdqu 80(%esi),%xmm7 502 leal 96(%esi),%esi 503 subl $96,%eax 504 jmp .L023ecb_dec_loop6_enter 505 .align 16 506 .L024ecb_dec_loop6: 507 movups %xmm2,(%edi) 508 movdqu (%esi),%xmm2 509 movups %xmm3,16(%edi) 510 movdqu 16(%esi),%xmm3 511 movups %xmm4,32(%edi) 512 movdqu 32(%esi),%xmm4 513 movups %xmm5,48(%edi) 514 movdqu 48(%esi),%xmm5 515 movups %xmm6,64(%edi) 516 movdqu 64(%esi),%xmm6 517 movups %xmm7,80(%edi) 518 leal 96(%edi),%edi 519 movdqu 80(%esi),%xmm7 520 leal 96(%esi),%esi 521 .L023ecb_dec_loop6_enter: 522 call _aesni_decrypt6 523 movl %ebp,%edx 524 movl %ebx,%ecx 525 subl $96,%eax 526 jnc .L024ecb_dec_loop6 527 movups %xmm2,(%edi) 528 movups %xmm3,16(%edi) 529 movups %xmm4,32(%edi) 530 movups %xmm5,48(%edi) 531 movups %xmm6,64(%edi) 532 movups %xmm7,80(%edi) 533 leal 96(%edi),%edi 534 addl $96,%eax 535 jz .L012ecb_ret 536 .L022ecb_dec_tail: 537 movups (%esi),%xmm2 538 cmpl $32,%eax 539 jb .L025ecb_dec_one 540 movups 16(%esi),%xmm3 541 je .L026ecb_dec_two 542 movups 32(%esi),%xmm4 543 cmpl $64,%eax 544 jb .L027ecb_dec_three 545 movups 48(%esi),%xmm5 546 je .L028ecb_dec_four 547 movups 64(%esi),%xmm6 548 xorps %xmm7,%xmm7 549 call _aesni_decrypt6 550 movups %xmm2,(%edi) 551 movups %xmm3,16(%edi) 552 movups %xmm4,32(%edi) 553 movups %xmm5,48(%edi) 554 movups %xmm6,64(%edi) 555 jmp .L012ecb_ret 556 .align 16 557 .L025ecb_dec_one: 558 movups (%edx),%xmm0 559 movups 16(%edx),%xmm1 560 leal 32(%edx),%edx 561 xorps %xmm0,%xmm2 562 .L029dec1_loop_4: 563 .byte 102,15,56,222,209 564 decl %ecx 565 movups (%edx),%xmm1 566 leal 16(%edx),%edx 567 jnz .L029dec1_loop_4 568 .byte 102,15,56,223,209 569 movups %xmm2,(%edi) 570 jmp .L012ecb_ret 571 .align 16 572 .L026ecb_dec_two: 573 call _aesni_decrypt2 574 movups %xmm2,(%edi) 575 movups %xmm3,16(%edi) 576 jmp .L012ecb_ret 577 .align 16 578 .L027ecb_dec_three: 579 call _aesni_decrypt3 580 movups %xmm2,(%edi) 581 movups %xmm3,16(%edi) 582 movups %xmm4,32(%edi) 583 jmp .L012ecb_ret 584 .align 16 585 .L028ecb_dec_four: 586 call _aesni_decrypt4 587 movups %xmm2,(%edi) 588 movups %xmm3,16(%edi) 589 movups %xmm4,32(%edi) 590 movups %xmm5,48(%edi) 591 .L012ecb_ret: 592 pxor %xmm0,%xmm0 593 pxor %xmm1,%xmm1 594 pxor %xmm2,%xmm2 595 pxor %xmm3,%xmm3 596 pxor %xmm4,%xmm4 597 pxor %xmm5,%xmm5 598 pxor %xmm6,%xmm6 599 pxor %xmm7,%xmm7 600 popl %edi 601 popl %esi 602 popl %ebx 603 popl %ebp 604 ret 605 .size aesni_ecb_encrypt,.-.L_aesni_ecb_encrypt_begin 606 .globl aesni_ccm64_encrypt_blocks 607 .hidden aesni_ccm64_encrypt_blocks 608 .type aesni_ccm64_encrypt_blocks,@function 609 .align 16 610 aesni_ccm64_encrypt_blocks: 611 .L_aesni_ccm64_encrypt_blocks_begin: 612 pushl %ebp 613 pushl %ebx 614 pushl %esi 615 pushl %edi 616 movl 20(%esp),%esi 617 movl 24(%esp),%edi 618 movl 28(%esp),%eax 619 movl 32(%esp),%edx 620 movl 36(%esp),%ebx 621 movl 40(%esp),%ecx 622 movl %esp,%ebp 623 subl $60,%esp 624 andl $-16,%esp 625 movl %ebp,48(%esp) 626 movdqu (%ebx),%xmm7 627 movdqu (%ecx),%xmm3 628 movl 240(%edx),%ecx 629 movl $202182159,(%esp) 630 movl $134810123,4(%esp) 631 movl $67438087,8(%esp) 632 movl $66051,12(%esp) 633 movl $1,%ebx 634 xorl %ebp,%ebp 635 movl %ebx,16(%esp) 636 movl %ebp,20(%esp) 637 movl %ebp,24(%esp) 638 movl %ebp,28(%esp) 639 shll $4,%ecx 640 movl $16,%ebx 641 leal (%edx),%ebp 642 movdqa (%esp),%xmm5 643 movdqa %xmm7,%xmm2 644 leal 32(%edx,%ecx,1),%edx 645 subl %ecx,%ebx 646 .byte 102,15,56,0,253 647 .L030ccm64_enc_outer: 648 movups (%ebp),%xmm0 649 movl %ebx,%ecx 650 movups (%esi),%xmm6 651 xorps %xmm0,%xmm2 652 movups 16(%ebp),%xmm1 653 xorps %xmm6,%xmm0 654 xorps %xmm0,%xmm3 655 movups 32(%ebp),%xmm0 656 .L031ccm64_enc2_loop: 657 .byte 102,15,56,220,209 658 .byte 102,15,56,220,217 659 movups (%edx,%ecx,1),%xmm1 660 addl $32,%ecx 661 .byte 102,15,56,220,208 662 .byte 102,15,56,220,216 663 movups -16(%edx,%ecx,1),%xmm0 664 jnz .L031ccm64_enc2_loop 665 .byte 102,15,56,220,209 666 .byte 102,15,56,220,217 667 paddq 16(%esp),%xmm7 668 decl %eax 669 .byte 102,15,56,221,208 670 .byte 102,15,56,221,216 671 leal 16(%esi),%esi 672 xorps %xmm2,%xmm6 673 movdqa %xmm7,%xmm2 674 movups %xmm6,(%edi) 675 .byte 102,15,56,0,213 676 leal 16(%edi),%edi 677 jnz .L030ccm64_enc_outer 678 movl 48(%esp),%esp 679 movl 40(%esp),%edi 680 movups %xmm3,(%edi) 681 pxor %xmm0,%xmm0 682 pxor %xmm1,%xmm1 683 pxor %xmm2,%xmm2 684 pxor %xmm3,%xmm3 685 pxor %xmm4,%xmm4 686 pxor %xmm5,%xmm5 687 pxor %xmm6,%xmm6 688 pxor %xmm7,%xmm7 689 popl %edi 690 popl %esi 691 popl %ebx 692 popl %ebp 693 ret 694 .size aesni_ccm64_encrypt_blocks,.-.L_aesni_ccm64_encrypt_blocks_begin 695 .globl aesni_ccm64_decrypt_blocks 696 .hidden aesni_ccm64_decrypt_blocks 697 .type aesni_ccm64_decrypt_blocks,@function 698 .align 16 699 aesni_ccm64_decrypt_blocks: 700 .L_aesni_ccm64_decrypt_blocks_begin: 701 pushl %ebp 702 pushl %ebx 703 pushl %esi 704 pushl %edi 705 movl 20(%esp),%esi 706 movl 24(%esp),%edi 707 movl 28(%esp),%eax 708 movl 32(%esp),%edx 709 movl 36(%esp),%ebx 710 movl 40(%esp),%ecx 711 movl %esp,%ebp 712 subl $60,%esp 713 andl $-16,%esp 714 movl %ebp,48(%esp) 715 movdqu (%ebx),%xmm7 716 movdqu (%ecx),%xmm3 717 movl 240(%edx),%ecx 718 movl $202182159,(%esp) 719 movl $134810123,4(%esp) 720 movl $67438087,8(%esp) 721 movl $66051,12(%esp) 722 movl $1,%ebx 723 xorl %ebp,%ebp 724 movl %ebx,16(%esp) 725 movl %ebp,20(%esp) 726 movl %ebp,24(%esp) 727 movl %ebp,28(%esp) 728 movdqa (%esp),%xmm5 729 movdqa %xmm7,%xmm2 730 movl %edx,%ebp 731 movl %ecx,%ebx 732 .byte 102,15,56,0,253 733 movups (%edx),%xmm0 734 movups 16(%edx),%xmm1 735 leal 32(%edx),%edx 736 xorps %xmm0,%xmm2 737 .L032enc1_loop_5: 738 .byte 102,15,56,220,209 739 decl %ecx 740 movups (%edx),%xmm1 741 leal 16(%edx),%edx 742 jnz .L032enc1_loop_5 743 .byte 102,15,56,221,209 744 shll $4,%ebx 745 movl $16,%ecx 746 movups (%esi),%xmm6 747 paddq 16(%esp),%xmm7 748 leal 16(%esi),%esi 749 subl %ebx,%ecx 750 leal 32(%ebp,%ebx,1),%edx 751 movl %ecx,%ebx 752 jmp .L033ccm64_dec_outer 753 .align 16 754 .L033ccm64_dec_outer: 755 xorps %xmm2,%xmm6 756 movdqa %xmm7,%xmm2 757 movups %xmm6,(%edi) 758 leal 16(%edi),%edi 759 .byte 102,15,56,0,213 760 subl $1,%eax 761 jz .L034ccm64_dec_break 762 movups (%ebp),%xmm0 763 movl %ebx,%ecx 764 movups 16(%ebp),%xmm1 765 xorps %xmm0,%xmm6 766 xorps %xmm0,%xmm2 767 xorps %xmm6,%xmm3 768 movups 32(%ebp),%xmm0 769 .L035ccm64_dec2_loop: 770 .byte 102,15,56,220,209 771 .byte 102,15,56,220,217 772 movups (%edx,%ecx,1),%xmm1 773 addl $32,%ecx 774 .byte 102,15,56,220,208 775 .byte 102,15,56,220,216 776 movups -16(%edx,%ecx,1),%xmm0 777 jnz .L035ccm64_dec2_loop 778 movups (%esi),%xmm6 779 paddq 16(%esp),%xmm7 780 .byte 102,15,56,220,209 781 .byte 102,15,56,220,217 782 .byte 102,15,56,221,208 783 .byte 102,15,56,221,216 784 leal 16(%esi),%esi 785 jmp .L033ccm64_dec_outer 786 .align 16 787 .L034ccm64_dec_break: 788 movl 240(%ebp),%ecx 789 movl %ebp,%edx 790 movups (%edx),%xmm0 791 movups 16(%edx),%xmm1 792 xorps %xmm0,%xmm6 793 leal 32(%edx),%edx 794 xorps %xmm6,%xmm3 795 .L036enc1_loop_6: 796 .byte 102,15,56,220,217 797 decl %ecx 798 movups (%edx),%xmm1 799 leal 16(%edx),%edx 800 jnz .L036enc1_loop_6 801 .byte 102,15,56,221,217 802 movl 48(%esp),%esp 803 movl 40(%esp),%edi 804 movups %xmm3,(%edi) 805 pxor %xmm0,%xmm0 806 pxor %xmm1,%xmm1 807 pxor %xmm2,%xmm2 808 pxor %xmm3,%xmm3 809 pxor %xmm4,%xmm4 810 pxor %xmm5,%xmm5 811 pxor %xmm6,%xmm6 812 pxor %xmm7,%xmm7 813 popl %edi 814 popl %esi 815 popl %ebx 816 popl %ebp 817 ret 818 .size aesni_ccm64_decrypt_blocks,.-.L_aesni_ccm64_decrypt_blocks_begin 819 .globl aesni_ctr32_encrypt_blocks 820 .hidden aesni_ctr32_encrypt_blocks 821 .type aesni_ctr32_encrypt_blocks,@function 822 .align 16 823 aesni_ctr32_encrypt_blocks: 824 .L_aesni_ctr32_encrypt_blocks_begin: 825 pushl %ebp 826 pushl %ebx 827 pushl %esi 828 pushl %edi 829 movl 20(%esp),%esi 830 movl 24(%esp),%edi 831 movl 28(%esp),%eax 832 movl 32(%esp),%edx 833 movl 36(%esp),%ebx 834 movl %esp,%ebp 835 subl $88,%esp 836 andl $-16,%esp 837 movl %ebp,80(%esp) 838 cmpl $1,%eax 839 je .L037ctr32_one_shortcut 840 movdqu (%ebx),%xmm7 841 movl $202182159,(%esp) 842 movl $134810123,4(%esp) 843 movl $67438087,8(%esp) 844 movl $66051,12(%esp) 845 movl $6,%ecx 846 xorl %ebp,%ebp 847 movl %ecx,16(%esp) 848 movl %ecx,20(%esp) 849 movl %ecx,24(%esp) 850 movl %ebp,28(%esp) 851 .byte 102,15,58,22,251,3 852 .byte 102,15,58,34,253,3 853 movl 240(%edx),%ecx 854 bswap %ebx 855 pxor %xmm0,%xmm0 856 pxor %xmm1,%xmm1 857 movdqa (%esp),%xmm2 858 .byte 102,15,58,34,195,0 859 leal 3(%ebx),%ebp 860 .byte 102,15,58,34,205,0 861 incl %ebx 862 .byte 102,15,58,34,195,1 863 incl %ebp 864 .byte 102,15,58,34,205,1 865 incl %ebx 866 .byte 102,15,58,34,195,2 867 incl %ebp 868 .byte 102,15,58,34,205,2 869 movdqa %xmm0,48(%esp) 870 .byte 102,15,56,0,194 871 movdqu (%edx),%xmm6 872 movdqa %xmm1,64(%esp) 873 .byte 102,15,56,0,202 874 pshufd $192,%xmm0,%xmm2 875 pshufd $128,%xmm0,%xmm3 876 cmpl $6,%eax 877 jb .L038ctr32_tail 878 pxor %xmm6,%xmm7 879 shll $4,%ecx 880 movl $16,%ebx 881 movdqa %xmm7,32(%esp) 882 movl %edx,%ebp 883 subl %ecx,%ebx 884 leal 32(%edx,%ecx,1),%edx 885 subl $6,%eax 886 jmp .L039ctr32_loop6 887 .align 16 888 .L039ctr32_loop6: 889 pshufd $64,%xmm0,%xmm4 890 movdqa 32(%esp),%xmm0 891 pshufd $192,%xmm1,%xmm5 892 pxor %xmm0,%xmm2 893 pshufd $128,%xmm1,%xmm6 894 pxor %xmm0,%xmm3 895 pshufd $64,%xmm1,%xmm7 896 movups 16(%ebp),%xmm1 897 pxor %xmm0,%xmm4 898 pxor %xmm0,%xmm5 899 .byte 102,15,56,220,209 900 pxor %xmm0,%xmm6 901 pxor %xmm0,%xmm7 902 .byte 102,15,56,220,217 903 movups 32(%ebp),%xmm0 904 movl %ebx,%ecx 905 .byte 102,15,56,220,225 906 .byte 102,15,56,220,233 907 .byte 102,15,56,220,241 908 .byte 102,15,56,220,249 909 call .L_aesni_encrypt6_enter 910 movups (%esi),%xmm1 911 movups 16(%esi),%xmm0 912 xorps %xmm1,%xmm2 913 movups 32(%esi),%xmm1 914 xorps %xmm0,%xmm3 915 movups %xmm2,(%edi) 916 movdqa 16(%esp),%xmm0 917 xorps %xmm1,%xmm4 918 movdqa 64(%esp),%xmm1 919 movups %xmm3,16(%edi) 920 movups %xmm4,32(%edi) 921 paddd %xmm0,%xmm1 922 paddd 48(%esp),%xmm0 923 movdqa (%esp),%xmm2 924 movups 48(%esi),%xmm3 925 movups 64(%esi),%xmm4 926 xorps %xmm3,%xmm5 927 movups 80(%esi),%xmm3 928 leal 96(%esi),%esi 929 movdqa %xmm0,48(%esp) 930 .byte 102,15,56,0,194 931 xorps %xmm4,%xmm6 932 movups %xmm5,48(%edi) 933 xorps %xmm3,%xmm7 934 movdqa %xmm1,64(%esp) 935 .byte 102,15,56,0,202 936 movups %xmm6,64(%edi) 937 pshufd $192,%xmm0,%xmm2 938 movups %xmm7,80(%edi) 939 leal 96(%edi),%edi 940 pshufd $128,%xmm0,%xmm3 941 subl $6,%eax 942 jnc .L039ctr32_loop6 943 addl $6,%eax 944 jz .L040ctr32_ret 945 movdqu (%ebp),%xmm7 946 movl %ebp,%edx 947 pxor 32(%esp),%xmm7 948 movl 240(%ebp),%ecx 949 .L038ctr32_tail: 950 por %xmm7,%xmm2 951 cmpl $2,%eax 952 jb .L041ctr32_one 953 pshufd $64,%xmm0,%xmm4 954 por %xmm7,%xmm3 955 je .L042ctr32_two 956 pshufd $192,%xmm1,%xmm5 957 por %xmm7,%xmm4 958 cmpl $4,%eax 959 jb .L043ctr32_three 960 pshufd $128,%xmm1,%xmm6 961 por %xmm7,%xmm5 962 je .L044ctr32_four 963 por %xmm7,%xmm6 964 call _aesni_encrypt6 965 movups (%esi),%xmm1 966 movups 16(%esi),%xmm0 967 xorps %xmm1,%xmm2 968 movups 32(%esi),%xmm1 969 xorps %xmm0,%xmm3 970 movups 48(%esi),%xmm0 971 xorps %xmm1,%xmm4 972 movups 64(%esi),%xmm1 973 xorps %xmm0,%xmm5 974 movups %xmm2,(%edi) 975 xorps %xmm1,%xmm6 976 movups %xmm3,16(%edi) 977 movups %xmm4,32(%edi) 978 movups %xmm5,48(%edi) 979 movups %xmm6,64(%edi) 980 jmp .L040ctr32_ret 981 .align 16 982 .L037ctr32_one_shortcut: 983 movups (%ebx),%xmm2 984 movl 240(%edx),%ecx 985 .L041ctr32_one: 986 movups (%edx),%xmm0 987 movups 16(%edx),%xmm1 988 leal 32(%edx),%edx 989 xorps %xmm0,%xmm2 990 .L045enc1_loop_7: 991 .byte 102,15,56,220,209 992 decl %ecx 993 movups (%edx),%xmm1 994 leal 16(%edx),%edx 995 jnz .L045enc1_loop_7 996 .byte 102,15,56,221,209 997 movups (%esi),%xmm6 998 xorps %xmm2,%xmm6 999 movups %xmm6,(%edi) 1000 jmp .L040ctr32_ret 1001 .align 16 1002 .L042ctr32_two: 1003 call _aesni_encrypt2 1004 movups (%esi),%xmm5 1005 movups 16(%esi),%xmm6 1006 xorps %xmm5,%xmm2 1007 xorps %xmm6,%xmm3 1008 movups %xmm2,(%edi) 1009 movups %xmm3,16(%edi) 1010 jmp .L040ctr32_ret 1011 .align 16 1012 .L043ctr32_three: 1013 call _aesni_encrypt3 1014 movups (%esi),%xmm5 1015 movups 16(%esi),%xmm6 1016 xorps %xmm5,%xmm2 1017 movups 32(%esi),%xmm7 1018 xorps %xmm6,%xmm3 1019 movups %xmm2,(%edi) 1020 xorps %xmm7,%xmm4 1021 movups %xmm3,16(%edi) 1022 movups %xmm4,32(%edi) 1023 jmp .L040ctr32_ret 1024 .align 16 1025 .L044ctr32_four: 1026 call _aesni_encrypt4 1027 movups (%esi),%xmm6 1028 movups 16(%esi),%xmm7 1029 movups 32(%esi),%xmm1 1030 xorps %xmm6,%xmm2 1031 movups 48(%esi),%xmm0 1032 xorps %xmm7,%xmm3 1033 movups %xmm2,(%edi) 1034 xorps %xmm1,%xmm4 1035 movups %xmm3,16(%edi) 1036 xorps %xmm0,%xmm5 1037 movups %xmm4,32(%edi) 1038 movups %xmm5,48(%edi) 1039 .L040ctr32_ret: 1040 pxor %xmm0,%xmm0 1041 pxor %xmm1,%xmm1 1042 pxor %xmm2,%xmm2 1043 pxor %xmm3,%xmm3 1044 pxor %xmm4,%xmm4 1045 movdqa %xmm0,32(%esp) 1046 pxor %xmm5,%xmm5 1047 movdqa %xmm0,48(%esp) 1048 pxor %xmm6,%xmm6 1049 movdqa %xmm0,64(%esp) 1050 pxor %xmm7,%xmm7 1051 movl 80(%esp),%esp 1052 popl %edi 1053 popl %esi 1054 popl %ebx 1055 popl %ebp 1056 ret 1057 .size aesni_ctr32_encrypt_blocks,.-.L_aesni_ctr32_encrypt_blocks_begin 1058 .globl aesni_xts_encrypt 1059 .hidden aesni_xts_encrypt 1060 .type aesni_xts_encrypt,@function 1061 .align 16 1062 aesni_xts_encrypt: 1063 .L_aesni_xts_encrypt_begin: 1064 pushl %ebp 1065 pushl %ebx 1066 pushl %esi 1067 pushl %edi 1068 movl 36(%esp),%edx 1069 movl 40(%esp),%esi 1070 movl 240(%edx),%ecx 1071 movups (%esi),%xmm2 1072 movups (%edx),%xmm0 1073 movups 16(%edx),%xmm1 1074 leal 32(%edx),%edx 1075 xorps %xmm0,%xmm2 1076 .L046enc1_loop_8: 1077 .byte 102,15,56,220,209 1078 decl %ecx 1079 movups (%edx),%xmm1 1080 leal 16(%edx),%edx 1081 jnz .L046enc1_loop_8 1082 .byte 102,15,56,221,209 1083 movl 20(%esp),%esi 1084 movl 24(%esp),%edi 1085 movl 28(%esp),%eax 1086 movl 32(%esp),%edx 1087 movl %esp,%ebp 1088 subl $120,%esp 1089 movl 240(%edx),%ecx 1090 andl $-16,%esp 1091 movl $135,96(%esp) 1092 movl $0,100(%esp) 1093 movl $1,104(%esp) 1094 movl $0,108(%esp) 1095 movl %eax,112(%esp) 1096 movl %ebp,116(%esp) 1097 movdqa %xmm2,%xmm1 1098 pxor %xmm0,%xmm0 1099 movdqa 96(%esp),%xmm3 1100 pcmpgtd %xmm1,%xmm0 1101 andl $-16,%eax 1102 movl %edx,%ebp 1103 movl %ecx,%ebx 1104 subl $96,%eax 1105 jc .L047xts_enc_short 1106 shll $4,%ecx 1107 movl $16,%ebx 1108 subl %ecx,%ebx 1109 leal 32(%edx,%ecx,1),%edx 1110 jmp .L048xts_enc_loop6 1111 .align 16 1112 .L048xts_enc_loop6: 1113 pshufd $19,%xmm0,%xmm2 1114 pxor %xmm0,%xmm0 1115 movdqa %xmm1,(%esp) 1116 paddq %xmm1,%xmm1 1117 pand %xmm3,%xmm2 1118 pcmpgtd %xmm1,%xmm0 1119 pxor %xmm2,%xmm1 1120 pshufd $19,%xmm0,%xmm2 1121 pxor %xmm0,%xmm0 1122 movdqa %xmm1,16(%esp) 1123 paddq %xmm1,%xmm1 1124 pand %xmm3,%xmm2 1125 pcmpgtd %xmm1,%xmm0 1126 pxor %xmm2,%xmm1 1127 pshufd $19,%xmm0,%xmm2 1128 pxor %xmm0,%xmm0 1129 movdqa %xmm1,32(%esp) 1130 paddq %xmm1,%xmm1 1131 pand %xmm3,%xmm2 1132 pcmpgtd %xmm1,%xmm0 1133 pxor %xmm2,%xmm1 1134 pshufd $19,%xmm0,%xmm2 1135 pxor %xmm0,%xmm0 1136 movdqa %xmm1,48(%esp) 1137 paddq %xmm1,%xmm1 1138 pand %xmm3,%xmm2 1139 pcmpgtd %xmm1,%xmm0 1140 pxor %xmm2,%xmm1 1141 pshufd $19,%xmm0,%xmm7 1142 movdqa %xmm1,64(%esp) 1143 paddq %xmm1,%xmm1 1144 movups (%ebp),%xmm0 1145 pand %xmm3,%xmm7 1146 movups (%esi),%xmm2 1147 pxor %xmm1,%xmm7 1148 movl %ebx,%ecx 1149 movdqu 16(%esi),%xmm3 1150 xorps %xmm0,%xmm2 1151 movdqu 32(%esi),%xmm4 1152 pxor %xmm0,%xmm3 1153 movdqu 48(%esi),%xmm5 1154 pxor %xmm0,%xmm4 1155 movdqu 64(%esi),%xmm6 1156 pxor %xmm0,%xmm5 1157 movdqu 80(%esi),%xmm1 1158 pxor %xmm0,%xmm6 1159 leal 96(%esi),%esi 1160 pxor (%esp),%xmm2 1161 movdqa %xmm7,80(%esp) 1162 pxor %xmm1,%xmm7 1163 movups 16(%ebp),%xmm1 1164 pxor 16(%esp),%xmm3 1165 pxor 32(%esp),%xmm4 1166 .byte 102,15,56,220,209 1167 pxor 48(%esp),%xmm5 1168 pxor 64(%esp),%xmm6 1169 .byte 102,15,56,220,217 1170 pxor %xmm0,%xmm7 1171 movups 32(%ebp),%xmm0 1172 .byte 102,15,56,220,225 1173 .byte 102,15,56,220,233 1174 .byte 102,15,56,220,241 1175 .byte 102,15,56,220,249 1176 call .L_aesni_encrypt6_enter 1177 movdqa 80(%esp),%xmm1 1178 pxor %xmm0,%xmm0 1179 xorps (%esp),%xmm2 1180 pcmpgtd %xmm1,%xmm0 1181 xorps 16(%esp),%xmm3 1182 movups %xmm2,(%edi) 1183 xorps 32(%esp),%xmm4 1184 movups %xmm3,16(%edi) 1185 xorps 48(%esp),%xmm5 1186 movups %xmm4,32(%edi) 1187 xorps 64(%esp),%xmm6 1188 movups %xmm5,48(%edi) 1189 xorps %xmm1,%xmm7 1190 movups %xmm6,64(%edi) 1191 pshufd $19,%xmm0,%xmm2 1192 movups %xmm7,80(%edi) 1193 leal 96(%edi),%edi 1194 movdqa 96(%esp),%xmm3 1195 pxor %xmm0,%xmm0 1196 paddq %xmm1,%xmm1 1197 pand %xmm3,%xmm2 1198 pcmpgtd %xmm1,%xmm0 1199 pxor %xmm2,%xmm1 1200 subl $96,%eax 1201 jnc .L048xts_enc_loop6 1202 movl 240(%ebp),%ecx 1203 movl %ebp,%edx 1204 movl %ecx,%ebx 1205 .L047xts_enc_short: 1206 addl $96,%eax 1207 jz .L049xts_enc_done6x 1208 movdqa %xmm1,%xmm5 1209 cmpl $32,%eax 1210 jb .L050xts_enc_one 1211 pshufd $19,%xmm0,%xmm2 1212 pxor %xmm0,%xmm0 1213 paddq %xmm1,%xmm1 1214 pand %xmm3,%xmm2 1215 pcmpgtd %xmm1,%xmm0 1216 pxor %xmm2,%xmm1 1217 je .L051xts_enc_two 1218 pshufd $19,%xmm0,%xmm2 1219 pxor %xmm0,%xmm0 1220 movdqa %xmm1,%xmm6 1221 paddq %xmm1,%xmm1 1222 pand %xmm3,%xmm2 1223 pcmpgtd %xmm1,%xmm0 1224 pxor %xmm2,%xmm1 1225 cmpl $64,%eax 1226 jb .L052xts_enc_three 1227 pshufd $19,%xmm0,%xmm2 1228 pxor %xmm0,%xmm0 1229 movdqa %xmm1,%xmm7 1230 paddq %xmm1,%xmm1 1231 pand %xmm3,%xmm2 1232 pcmpgtd %xmm1,%xmm0 1233 pxor %xmm2,%xmm1 1234 movdqa %xmm5,(%esp) 1235 movdqa %xmm6,16(%esp) 1236 je .L053xts_enc_four 1237 movdqa %xmm7,32(%esp) 1238 pshufd $19,%xmm0,%xmm7 1239 movdqa %xmm1,48(%esp) 1240 paddq %xmm1,%xmm1 1241 pand %xmm3,%xmm7 1242 pxor %xmm1,%xmm7 1243 movdqu (%esi),%xmm2 1244 movdqu 16(%esi),%xmm3 1245 movdqu 32(%esi),%xmm4 1246 pxor (%esp),%xmm2 1247 movdqu 48(%esi),%xmm5 1248 pxor 16(%esp),%xmm3 1249 movdqu 64(%esi),%xmm6 1250 pxor 32(%esp),%xmm4 1251 leal 80(%esi),%esi 1252 pxor 48(%esp),%xmm5 1253 movdqa %xmm7,64(%esp) 1254 pxor %xmm7,%xmm6 1255 call _aesni_encrypt6 1256 movaps 64(%esp),%xmm1 1257 xorps (%esp),%xmm2 1258 xorps 16(%esp),%xmm3 1259 xorps 32(%esp),%xmm4 1260 movups %xmm2,(%edi) 1261 xorps 48(%esp),%xmm5 1262 movups %xmm3,16(%edi) 1263 xorps %xmm1,%xmm6 1264 movups %xmm4,32(%edi) 1265 movups %xmm5,48(%edi) 1266 movups %xmm6,64(%edi) 1267 leal 80(%edi),%edi 1268 jmp .L054xts_enc_done 1269 .align 16 1270 .L050xts_enc_one: 1271 movups (%esi),%xmm2 1272 leal 16(%esi),%esi 1273 xorps %xmm5,%xmm2 1274 movups (%edx),%xmm0 1275 movups 16(%edx),%xmm1 1276 leal 32(%edx),%edx 1277 xorps %xmm0,%xmm2 1278 .L055enc1_loop_9: 1279 .byte 102,15,56,220,209 1280 decl %ecx 1281 movups (%edx),%xmm1 1282 leal 16(%edx),%edx 1283 jnz .L055enc1_loop_9 1284 .byte 102,15,56,221,209 1285 xorps %xmm5,%xmm2 1286 movups %xmm2,(%edi) 1287 leal 16(%edi),%edi 1288 movdqa %xmm5,%xmm1 1289 jmp .L054xts_enc_done 1290 .align 16 1291 .L051xts_enc_two: 1292 movaps %xmm1,%xmm6 1293 movups (%esi),%xmm2 1294 movups 16(%esi),%xmm3 1295 leal 32(%esi),%esi 1296 xorps %xmm5,%xmm2 1297 xorps %xmm6,%xmm3 1298 call _aesni_encrypt2 1299 xorps %xmm5,%xmm2 1300 xorps %xmm6,%xmm3 1301 movups %xmm2,(%edi) 1302 movups %xmm3,16(%edi) 1303 leal 32(%edi),%edi 1304 movdqa %xmm6,%xmm1 1305 jmp .L054xts_enc_done 1306 .align 16 1307 .L052xts_enc_three: 1308 movaps %xmm1,%xmm7 1309 movups (%esi),%xmm2 1310 movups 16(%esi),%xmm3 1311 movups 32(%esi),%xmm4 1312 leal 48(%esi),%esi 1313 xorps %xmm5,%xmm2 1314 xorps %xmm6,%xmm3 1315 xorps %xmm7,%xmm4 1316 call _aesni_encrypt3 1317 xorps %xmm5,%xmm2 1318 xorps %xmm6,%xmm3 1319 xorps %xmm7,%xmm4 1320 movups %xmm2,(%edi) 1321 movups %xmm3,16(%edi) 1322 movups %xmm4,32(%edi) 1323 leal 48(%edi),%edi 1324 movdqa %xmm7,%xmm1 1325 jmp .L054xts_enc_done 1326 .align 16 1327 .L053xts_enc_four: 1328 movaps %xmm1,%xmm6 1329 movups (%esi),%xmm2 1330 movups 16(%esi),%xmm3 1331 movups 32(%esi),%xmm4 1332 xorps (%esp),%xmm2 1333 movups 48(%esi),%xmm5 1334 leal 64(%esi),%esi 1335 xorps 16(%esp),%xmm3 1336 xorps %xmm7,%xmm4 1337 xorps %xmm6,%xmm5 1338 call _aesni_encrypt4 1339 xorps (%esp),%xmm2 1340 xorps 16(%esp),%xmm3 1341 xorps %xmm7,%xmm4 1342 movups %xmm2,(%edi) 1343 xorps %xmm6,%xmm5 1344 movups %xmm3,16(%edi) 1345 movups %xmm4,32(%edi) 1346 movups %xmm5,48(%edi) 1347 leal 64(%edi),%edi 1348 movdqa %xmm6,%xmm1 1349 jmp .L054xts_enc_done 1350 .align 16 1351 .L049xts_enc_done6x: 1352 movl 112(%esp),%eax 1353 andl $15,%eax 1354 jz .L056xts_enc_ret 1355 movdqa %xmm1,%xmm5 1356 movl %eax,112(%esp) 1357 jmp .L057xts_enc_steal 1358 .align 16 1359 .L054xts_enc_done: 1360 movl 112(%esp),%eax 1361 pxor %xmm0,%xmm0 1362 andl $15,%eax 1363 jz .L056xts_enc_ret 1364 pcmpgtd %xmm1,%xmm0 1365 movl %eax,112(%esp) 1366 pshufd $19,%xmm0,%xmm5 1367 paddq %xmm1,%xmm1 1368 pand 96(%esp),%xmm5 1369 pxor %xmm1,%xmm5 1370 .L057xts_enc_steal: 1371 movzbl (%esi),%ecx 1372 movzbl -16(%edi),%edx 1373 leal 1(%esi),%esi 1374 movb %cl,-16(%edi) 1375 movb %dl,(%edi) 1376 leal 1(%edi),%edi 1377 subl $1,%eax 1378 jnz .L057xts_enc_steal 1379 subl 112(%esp),%edi 1380 movl %ebp,%edx 1381 movl %ebx,%ecx 1382 movups -16(%edi),%xmm2 1383 xorps %xmm5,%xmm2 1384 movups (%edx),%xmm0 1385 movups 16(%edx),%xmm1 1386 leal 32(%edx),%edx 1387 xorps %xmm0,%xmm2 1388 .L058enc1_loop_10: 1389 .byte 102,15,56,220,209 1390 decl %ecx 1391 movups (%edx),%xmm1 1392 leal 16(%edx),%edx 1393 jnz .L058enc1_loop_10 1394 .byte 102,15,56,221,209 1395 xorps %xmm5,%xmm2 1396 movups %xmm2,-16(%edi) 1397 .L056xts_enc_ret: 1398 pxor %xmm0,%xmm0 1399 pxor %xmm1,%xmm1 1400 pxor %xmm2,%xmm2 1401 movdqa %xmm0,(%esp) 1402 pxor %xmm3,%xmm3 1403 movdqa %xmm0,16(%esp) 1404 pxor %xmm4,%xmm4 1405 movdqa %xmm0,32(%esp) 1406 pxor %xmm5,%xmm5 1407 movdqa %xmm0,48(%esp) 1408 pxor %xmm6,%xmm6 1409 movdqa %xmm0,64(%esp) 1410 pxor %xmm7,%xmm7 1411 movdqa %xmm0,80(%esp) 1412 movl 116(%esp),%esp 1413 popl %edi 1414 popl %esi 1415 popl %ebx 1416 popl %ebp 1417 ret 1418 .size aesni_xts_encrypt,.-.L_aesni_xts_encrypt_begin 1419 .globl aesni_xts_decrypt 1420 .hidden aesni_xts_decrypt 1421 .type aesni_xts_decrypt,@function 1422 .align 16 1423 aesni_xts_decrypt: 1424 .L_aesni_xts_decrypt_begin: 1425 pushl %ebp 1426 pushl %ebx 1427 pushl %esi 1428 pushl %edi 1429 movl 36(%esp),%edx 1430 movl 40(%esp),%esi 1431 movl 240(%edx),%ecx 1432 movups (%esi),%xmm2 1433 movups (%edx),%xmm0 1434 movups 16(%edx),%xmm1 1435 leal 32(%edx),%edx 1436 xorps %xmm0,%xmm2 1437 .L059enc1_loop_11: 1438 .byte 102,15,56,220,209 1439 decl %ecx 1440 movups (%edx),%xmm1 1441 leal 16(%edx),%edx 1442 jnz .L059enc1_loop_11 1443 .byte 102,15,56,221,209 1444 movl 20(%esp),%esi 1445 movl 24(%esp),%edi 1446 movl 28(%esp),%eax 1447 movl 32(%esp),%edx 1448 movl %esp,%ebp 1449 subl $120,%esp 1450 andl $-16,%esp 1451 xorl %ebx,%ebx 1452 testl $15,%eax 1453 setnz %bl 1454 shll $4,%ebx 1455 subl %ebx,%eax 1456 movl $135,96(%esp) 1457 movl $0,100(%esp) 1458 movl $1,104(%esp) 1459 movl $0,108(%esp) 1460 movl %eax,112(%esp) 1461 movl %ebp,116(%esp) 1462 movl 240(%edx),%ecx 1463 movl %edx,%ebp 1464 movl %ecx,%ebx 1465 movdqa %xmm2,%xmm1 1466 pxor %xmm0,%xmm0 1467 movdqa 96(%esp),%xmm3 1468 pcmpgtd %xmm1,%xmm0 1469 andl $-16,%eax 1470 subl $96,%eax 1471 jc .L060xts_dec_short 1472 shll $4,%ecx 1473 movl $16,%ebx 1474 subl %ecx,%ebx 1475 leal 32(%edx,%ecx,1),%edx 1476 jmp .L061xts_dec_loop6 1477 .align 16 1478 .L061xts_dec_loop6: 1479 pshufd $19,%xmm0,%xmm2 1480 pxor %xmm0,%xmm0 1481 movdqa %xmm1,(%esp) 1482 paddq %xmm1,%xmm1 1483 pand %xmm3,%xmm2 1484 pcmpgtd %xmm1,%xmm0 1485 pxor %xmm2,%xmm1 1486 pshufd $19,%xmm0,%xmm2 1487 pxor %xmm0,%xmm0 1488 movdqa %xmm1,16(%esp) 1489 paddq %xmm1,%xmm1 1490 pand %xmm3,%xmm2 1491 pcmpgtd %xmm1,%xmm0 1492 pxor %xmm2,%xmm1 1493 pshufd $19,%xmm0,%xmm2 1494 pxor %xmm0,%xmm0 1495 movdqa %xmm1,32(%esp) 1496 paddq %xmm1,%xmm1 1497 pand %xmm3,%xmm2 1498 pcmpgtd %xmm1,%xmm0 1499 pxor %xmm2,%xmm1 1500 pshufd $19,%xmm0,%xmm2 1501 pxor %xmm0,%xmm0 1502 movdqa %xmm1,48(%esp) 1503 paddq %xmm1,%xmm1 1504 pand %xmm3,%xmm2 1505 pcmpgtd %xmm1,%xmm0 1506 pxor %xmm2,%xmm1 1507 pshufd $19,%xmm0,%xmm7 1508 movdqa %xmm1,64(%esp) 1509 paddq %xmm1,%xmm1 1510 movups (%ebp),%xmm0 1511 pand %xmm3,%xmm7 1512 movups (%esi),%xmm2 1513 pxor %xmm1,%xmm7 1514 movl %ebx,%ecx 1515 movdqu 16(%esi),%xmm3 1516 xorps %xmm0,%xmm2 1517 movdqu 32(%esi),%xmm4 1518 pxor %xmm0,%xmm3 1519 movdqu 48(%esi),%xmm5 1520 pxor %xmm0,%xmm4 1521 movdqu 64(%esi),%xmm6 1522 pxor %xmm0,%xmm5 1523 movdqu 80(%esi),%xmm1 1524 pxor %xmm0,%xmm6 1525 leal 96(%esi),%esi 1526 pxor (%esp),%xmm2 1527 movdqa %xmm7,80(%esp) 1528 pxor %xmm1,%xmm7 1529 movups 16(%ebp),%xmm1 1530 pxor 16(%esp),%xmm3 1531 pxor 32(%esp),%xmm4 1532 .byte 102,15,56,222,209 1533 pxor 48(%esp),%xmm5 1534 pxor 64(%esp),%xmm6 1535 .byte 102,15,56,222,217 1536 pxor %xmm0,%xmm7 1537 movups 32(%ebp),%xmm0 1538 .byte 102,15,56,222,225 1539 .byte 102,15,56,222,233 1540 .byte 102,15,56,222,241 1541 .byte 102,15,56,222,249 1542 call .L_aesni_decrypt6_enter 1543 movdqa 80(%esp),%xmm1 1544 pxor %xmm0,%xmm0 1545 xorps (%esp),%xmm2 1546 pcmpgtd %xmm1,%xmm0 1547 xorps 16(%esp),%xmm3 1548 movups %xmm2,(%edi) 1549 xorps 32(%esp),%xmm4 1550 movups %xmm3,16(%edi) 1551 xorps 48(%esp),%xmm5 1552 movups %xmm4,32(%edi) 1553 xorps 64(%esp),%xmm6 1554 movups %xmm5,48(%edi) 1555 xorps %xmm1,%xmm7 1556 movups %xmm6,64(%edi) 1557 pshufd $19,%xmm0,%xmm2 1558 movups %xmm7,80(%edi) 1559 leal 96(%edi),%edi 1560 movdqa 96(%esp),%xmm3 1561 pxor %xmm0,%xmm0 1562 paddq %xmm1,%xmm1 1563 pand %xmm3,%xmm2 1564 pcmpgtd %xmm1,%xmm0 1565 pxor %xmm2,%xmm1 1566 subl $96,%eax 1567 jnc .L061xts_dec_loop6 1568 movl 240(%ebp),%ecx 1569 movl %ebp,%edx 1570 movl %ecx,%ebx 1571 .L060xts_dec_short: 1572 addl $96,%eax 1573 jz .L062xts_dec_done6x 1574 movdqa %xmm1,%xmm5 1575 cmpl $32,%eax 1576 jb .L063xts_dec_one 1577 pshufd $19,%xmm0,%xmm2 1578 pxor %xmm0,%xmm0 1579 paddq %xmm1,%xmm1 1580 pand %xmm3,%xmm2 1581 pcmpgtd %xmm1,%xmm0 1582 pxor %xmm2,%xmm1 1583 je .L064xts_dec_two 1584 pshufd $19,%xmm0,%xmm2 1585 pxor %xmm0,%xmm0 1586 movdqa %xmm1,%xmm6 1587 paddq %xmm1,%xmm1 1588 pand %xmm3,%xmm2 1589 pcmpgtd %xmm1,%xmm0 1590 pxor %xmm2,%xmm1 1591 cmpl $64,%eax 1592 jb .L065xts_dec_three 1593 pshufd $19,%xmm0,%xmm2 1594 pxor %xmm0,%xmm0 1595 movdqa %xmm1,%xmm7 1596 paddq %xmm1,%xmm1 1597 pand %xmm3,%xmm2 1598 pcmpgtd %xmm1,%xmm0 1599 pxor %xmm2,%xmm1 1600 movdqa %xmm5,(%esp) 1601 movdqa %xmm6,16(%esp) 1602 je .L066xts_dec_four 1603 movdqa %xmm7,32(%esp) 1604 pshufd $19,%xmm0,%xmm7 1605 movdqa %xmm1,48(%esp) 1606 paddq %xmm1,%xmm1 1607 pand %xmm3,%xmm7 1608 pxor %xmm1,%xmm7 1609 movdqu (%esi),%xmm2 1610 movdqu 16(%esi),%xmm3 1611 movdqu 32(%esi),%xmm4 1612 pxor (%esp),%xmm2 1613 movdqu 48(%esi),%xmm5 1614 pxor 16(%esp),%xmm3 1615 movdqu 64(%esi),%xmm6 1616 pxor 32(%esp),%xmm4 1617 leal 80(%esi),%esi 1618 pxor 48(%esp),%xmm5 1619 movdqa %xmm7,64(%esp) 1620 pxor %xmm7,%xmm6 1621 call _aesni_decrypt6 1622 movaps 64(%esp),%xmm1 1623 xorps (%esp),%xmm2 1624 xorps 16(%esp),%xmm3 1625 xorps 32(%esp),%xmm4 1626 movups %xmm2,(%edi) 1627 xorps 48(%esp),%xmm5 1628 movups %xmm3,16(%edi) 1629 xorps %xmm1,%xmm6 1630 movups %xmm4,32(%edi) 1631 movups %xmm5,48(%edi) 1632 movups %xmm6,64(%edi) 1633 leal 80(%edi),%edi 1634 jmp .L067xts_dec_done 1635 .align 16 1636 .L063xts_dec_one: 1637 movups (%esi),%xmm2 1638 leal 16(%esi),%esi 1639 xorps %xmm5,%xmm2 1640 movups (%edx),%xmm0 1641 movups 16(%edx),%xmm1 1642 leal 32(%edx),%edx 1643 xorps %xmm0,%xmm2 1644 .L068dec1_loop_12: 1645 .byte 102,15,56,222,209 1646 decl %ecx 1647 movups (%edx),%xmm1 1648 leal 16(%edx),%edx 1649 jnz .L068dec1_loop_12 1650 .byte 102,15,56,223,209 1651 xorps %xmm5,%xmm2 1652 movups %xmm2,(%edi) 1653 leal 16(%edi),%edi 1654 movdqa %xmm5,%xmm1 1655 jmp .L067xts_dec_done 1656 .align 16 1657 .L064xts_dec_two: 1658 movaps %xmm1,%xmm6 1659 movups (%esi),%xmm2 1660 movups 16(%esi),%xmm3 1661 leal 32(%esi),%esi 1662 xorps %xmm5,%xmm2 1663 xorps %xmm6,%xmm3 1664 call _aesni_decrypt2 1665 xorps %xmm5,%xmm2 1666 xorps %xmm6,%xmm3 1667 movups %xmm2,(%edi) 1668 movups %xmm3,16(%edi) 1669 leal 32(%edi),%edi 1670 movdqa %xmm6,%xmm1 1671 jmp .L067xts_dec_done 1672 .align 16 1673 .L065xts_dec_three: 1674 movaps %xmm1,%xmm7 1675 movups (%esi),%xmm2 1676 movups 16(%esi),%xmm3 1677 movups 32(%esi),%xmm4 1678 leal 48(%esi),%esi 1679 xorps %xmm5,%xmm2 1680 xorps %xmm6,%xmm3 1681 xorps %xmm7,%xmm4 1682 call _aesni_decrypt3 1683 xorps %xmm5,%xmm2 1684 xorps %xmm6,%xmm3 1685 xorps %xmm7,%xmm4 1686 movups %xmm2,(%edi) 1687 movups %xmm3,16(%edi) 1688 movups %xmm4,32(%edi) 1689 leal 48(%edi),%edi 1690 movdqa %xmm7,%xmm1 1691 jmp .L067xts_dec_done 1692 .align 16 1693 .L066xts_dec_four: 1694 movaps %xmm1,%xmm6 1695 movups (%esi),%xmm2 1696 movups 16(%esi),%xmm3 1697 movups 32(%esi),%xmm4 1698 xorps (%esp),%xmm2 1699 movups 48(%esi),%xmm5 1700 leal 64(%esi),%esi 1701 xorps 16(%esp),%xmm3 1702 xorps %xmm7,%xmm4 1703 xorps %xmm6,%xmm5 1704 call _aesni_decrypt4 1705 xorps (%esp),%xmm2 1706 xorps 16(%esp),%xmm3 1707 xorps %xmm7,%xmm4 1708 movups %xmm2,(%edi) 1709 xorps %xmm6,%xmm5 1710 movups %xmm3,16(%edi) 1711 movups %xmm4,32(%edi) 1712 movups %xmm5,48(%edi) 1713 leal 64(%edi),%edi 1714 movdqa %xmm6,%xmm1 1715 jmp .L067xts_dec_done 1716 .align 16 1717 .L062xts_dec_done6x: 1718 movl 112(%esp),%eax 1719 andl $15,%eax 1720 jz .L069xts_dec_ret 1721 movl %eax,112(%esp) 1722 jmp .L070xts_dec_only_one_more 1723 .align 16 1724 .L067xts_dec_done: 1725 movl 112(%esp),%eax 1726 pxor %xmm0,%xmm0 1727 andl $15,%eax 1728 jz .L069xts_dec_ret 1729 pcmpgtd %xmm1,%xmm0 1730 movl %eax,112(%esp) 1731 pshufd $19,%xmm0,%xmm2 1732 pxor %xmm0,%xmm0 1733 movdqa 96(%esp),%xmm3 1734 paddq %xmm1,%xmm1 1735 pand %xmm3,%xmm2 1736 pcmpgtd %xmm1,%xmm0 1737 pxor %xmm2,%xmm1 1738 .L070xts_dec_only_one_more: 1739 pshufd $19,%xmm0,%xmm5 1740 movdqa %xmm1,%xmm6 1741 paddq %xmm1,%xmm1 1742 pand %xmm3,%xmm5 1743 pxor %xmm1,%xmm5 1744 movl %ebp,%edx 1745 movl %ebx,%ecx 1746 movups (%esi),%xmm2 1747 xorps %xmm5,%xmm2 1748 movups (%edx),%xmm0 1749 movups 16(%edx),%xmm1 1750 leal 32(%edx),%edx 1751 xorps %xmm0,%xmm2 1752 .L071dec1_loop_13: 1753 .byte 102,15,56,222,209 1754 decl %ecx 1755 movups (%edx),%xmm1 1756 leal 16(%edx),%edx 1757 jnz .L071dec1_loop_13 1758 .byte 102,15,56,223,209 1759 xorps %xmm5,%xmm2 1760 movups %xmm2,(%edi) 1761 .L072xts_dec_steal: 1762 movzbl 16(%esi),%ecx 1763 movzbl (%edi),%edx 1764 leal 1(%esi),%esi 1765 movb %cl,(%edi) 1766 movb %dl,16(%edi) 1767 leal 1(%edi),%edi 1768 subl $1,%eax 1769 jnz .L072xts_dec_steal 1770 subl 112(%esp),%edi 1771 movl %ebp,%edx 1772 movl %ebx,%ecx 1773 movups (%edi),%xmm2 1774 xorps %xmm6,%xmm2 1775 movups (%edx),%xmm0 1776 movups 16(%edx),%xmm1 1777 leal 32(%edx),%edx 1778 xorps %xmm0,%xmm2 1779 .L073dec1_loop_14: 1780 .byte 102,15,56,222,209 1781 decl %ecx 1782 movups (%edx),%xmm1 1783 leal 16(%edx),%edx 1784 jnz .L073dec1_loop_14 1785 .byte 102,15,56,223,209 1786 xorps %xmm6,%xmm2 1787 movups %xmm2,(%edi) 1788 .L069xts_dec_ret: 1789 pxor %xmm0,%xmm0 1790 pxor %xmm1,%xmm1 1791 pxor %xmm2,%xmm2 1792 movdqa %xmm0,(%esp) 1793 pxor %xmm3,%xmm3 1794 movdqa %xmm0,16(%esp) 1795 pxor %xmm4,%xmm4 1796 movdqa %xmm0,32(%esp) 1797 pxor %xmm5,%xmm5 1798 movdqa %xmm0,48(%esp) 1799 pxor %xmm6,%xmm6 1800 movdqa %xmm0,64(%esp) 1801 pxor %xmm7,%xmm7 1802 movdqa %xmm0,80(%esp) 1803 movl 116(%esp),%esp 1804 popl %edi 1805 popl %esi 1806 popl %ebx 1807 popl %ebp 1808 ret 1809 .size aesni_xts_decrypt,.-.L_aesni_xts_decrypt_begin 1810 .globl aesni_cbc_encrypt 1811 .hidden aesni_cbc_encrypt 1812 .type aesni_cbc_encrypt,@function 1813 .align 16 1814 aesni_cbc_encrypt: 1815 .L_aesni_cbc_encrypt_begin: 1816 pushl %ebp 1817 pushl %ebx 1818 pushl %esi 1819 pushl %edi 1820 movl 20(%esp),%esi 1821 movl %esp,%ebx 1822 movl 24(%esp),%edi 1823 subl $24,%ebx 1824 movl 28(%esp),%eax 1825 andl $-16,%ebx 1826 movl 32(%esp),%edx 1827 movl 36(%esp),%ebp 1828 testl %eax,%eax 1829 jz .L074cbc_abort 1830 cmpl $0,40(%esp) 1831 xchgl %esp,%ebx 1832 movups (%ebp),%xmm7 1833 movl 240(%edx),%ecx 1834 movl %edx,%ebp 1835 movl %ebx,16(%esp) 1836 movl %ecx,%ebx 1837 je .L075cbc_decrypt 1838 movaps %xmm7,%xmm2 1839 cmpl $16,%eax 1840 jb .L076cbc_enc_tail 1841 subl $16,%eax 1842 jmp .L077cbc_enc_loop 1843 .align 16 1844 .L077cbc_enc_loop: 1845 movups (%esi),%xmm7 1846 leal 16(%esi),%esi 1847 movups (%edx),%xmm0 1848 movups 16(%edx),%xmm1 1849 xorps %xmm0,%xmm7 1850 leal 32(%edx),%edx 1851 xorps %xmm7,%xmm2 1852 .L078enc1_loop_15: 1853 .byte 102,15,56,220,209 1854 decl %ecx 1855 movups (%edx),%xmm1 1856 leal 16(%edx),%edx 1857 jnz .L078enc1_loop_15 1858 .byte 102,15,56,221,209 1859 movl %ebx,%ecx 1860 movl %ebp,%edx 1861 movups %xmm2,(%edi) 1862 leal 16(%edi),%edi 1863 subl $16,%eax 1864 jnc .L077cbc_enc_loop 1865 addl $16,%eax 1866 jnz .L076cbc_enc_tail 1867 movaps %xmm2,%xmm7 1868 pxor %xmm2,%xmm2 1869 jmp .L079cbc_ret 1870 .L076cbc_enc_tail: 1871 movl %eax,%ecx 1872 .long 2767451785 1873 movl $16,%ecx 1874 subl %eax,%ecx 1875 xorl %eax,%eax 1876 .long 2868115081 1877 leal -16(%edi),%edi 1878 movl %ebx,%ecx 1879 movl %edi,%esi 1880 movl %ebp,%edx 1881 jmp .L077cbc_enc_loop 1882 .align 16 1883 .L075cbc_decrypt: 1884 cmpl $80,%eax 1885 jbe .L080cbc_dec_tail 1886 movaps %xmm7,(%esp) 1887 subl $80,%eax 1888 jmp .L081cbc_dec_loop6_enter 1889 .align 16 1890 .L082cbc_dec_loop6: 1891 movaps %xmm0,(%esp) 1892 movups %xmm7,(%edi) 1893 leal 16(%edi),%edi 1894 .L081cbc_dec_loop6_enter: 1895 movdqu (%esi),%xmm2 1896 movdqu 16(%esi),%xmm3 1897 movdqu 32(%esi),%xmm4 1898 movdqu 48(%esi),%xmm5 1899 movdqu 64(%esi),%xmm6 1900 movdqu 80(%esi),%xmm7 1901 call _aesni_decrypt6 1902 movups (%esi),%xmm1 1903 movups 16(%esi),%xmm0 1904 xorps (%esp),%xmm2 1905 xorps %xmm1,%xmm3 1906 movups 32(%esi),%xmm1 1907 xorps %xmm0,%xmm4 1908 movups 48(%esi),%xmm0 1909 xorps %xmm1,%xmm5 1910 movups 64(%esi),%xmm1 1911 xorps %xmm0,%xmm6 1912 movups 80(%esi),%xmm0 1913 xorps %xmm1,%xmm7 1914 movups %xmm2,(%edi) 1915 movups %xmm3,16(%edi) 1916 leal 96(%esi),%esi 1917 movups %xmm4,32(%edi) 1918 movl %ebx,%ecx 1919 movups %xmm5,48(%edi) 1920 movl %ebp,%edx 1921 movups %xmm6,64(%edi) 1922 leal 80(%edi),%edi 1923 subl $96,%eax 1924 ja .L082cbc_dec_loop6 1925 movaps %xmm7,%xmm2 1926 movaps %xmm0,%xmm7 1927 addl $80,%eax 1928 jle .L083cbc_dec_clear_tail_collected 1929 movups %xmm2,(%edi) 1930 leal 16(%edi),%edi 1931 .L080cbc_dec_tail: 1932 movups (%esi),%xmm2 1933 movaps %xmm2,%xmm6 1934 cmpl $16,%eax 1935 jbe .L084cbc_dec_one 1936 movups 16(%esi),%xmm3 1937 movaps %xmm3,%xmm5 1938 cmpl $32,%eax 1939 jbe .L085cbc_dec_two 1940 movups 32(%esi),%xmm4 1941 cmpl $48,%eax 1942 jbe .L086cbc_dec_three 1943 movups 48(%esi),%xmm5 1944 cmpl $64,%eax 1945 jbe .L087cbc_dec_four 1946 movups 64(%esi),%xmm6 1947 movaps %xmm7,(%esp) 1948 movups (%esi),%xmm2 1949 xorps %xmm7,%xmm7 1950 call _aesni_decrypt6 1951 movups (%esi),%xmm1 1952 movups 16(%esi),%xmm0 1953 xorps (%esp),%xmm2 1954 xorps %xmm1,%xmm3 1955 movups 32(%esi),%xmm1 1956 xorps %xmm0,%xmm4 1957 movups 48(%esi),%xmm0 1958 xorps %xmm1,%xmm5 1959 movups 64(%esi),%xmm7 1960 xorps %xmm0,%xmm6 1961 movups %xmm2,(%edi) 1962 movups %xmm3,16(%edi) 1963 pxor %xmm3,%xmm3 1964 movups %xmm4,32(%edi) 1965 pxor %xmm4,%xmm4 1966 movups %xmm5,48(%edi) 1967 pxor %xmm5,%xmm5 1968 leal 64(%edi),%edi 1969 movaps %xmm6,%xmm2 1970 pxor %xmm6,%xmm6 1971 subl $80,%eax 1972 jmp .L088cbc_dec_tail_collected 1973 .align 16 1974 .L084cbc_dec_one: 1975 movups (%edx),%xmm0 1976 movups 16(%edx),%xmm1 1977 leal 32(%edx),%edx 1978 xorps %xmm0,%xmm2 1979 .L089dec1_loop_16: 1980 .byte 102,15,56,222,209 1981 decl %ecx 1982 movups (%edx),%xmm1 1983 leal 16(%edx),%edx 1984 jnz .L089dec1_loop_16 1985 .byte 102,15,56,223,209 1986 xorps %xmm7,%xmm2 1987 movaps %xmm6,%xmm7 1988 subl $16,%eax 1989 jmp .L088cbc_dec_tail_collected 1990 .align 16 1991 .L085cbc_dec_two: 1992 call _aesni_decrypt2 1993 xorps %xmm7,%xmm2 1994 xorps %xmm6,%xmm3 1995 movups %xmm2,(%edi) 1996 movaps %xmm3,%xmm2 1997 pxor %xmm3,%xmm3 1998 leal 16(%edi),%edi 1999 movaps %xmm5,%xmm7 2000 subl $32,%eax 2001 jmp .L088cbc_dec_tail_collected 2002 .align 16 2003 .L086cbc_dec_three: 2004 call _aesni_decrypt3 2005 xorps %xmm7,%xmm2 2006 xorps %xmm6,%xmm3 2007 xorps %xmm5,%xmm4 2008 movups %xmm2,(%edi) 2009 movaps %xmm4,%xmm2 2010 pxor %xmm4,%xmm4 2011 movups %xmm3,16(%edi) 2012 pxor %xmm3,%xmm3 2013 leal 32(%edi),%edi 2014 movups 32(%esi),%xmm7 2015 subl $48,%eax 2016 jmp .L088cbc_dec_tail_collected 2017 .align 16 2018 .L087cbc_dec_four: 2019 call _aesni_decrypt4 2020 movups 16(%esi),%xmm1 2021 movups 32(%esi),%xmm0 2022 xorps %xmm7,%xmm2 2023 movups 48(%esi),%xmm7 2024 xorps %xmm6,%xmm3 2025 movups %xmm2,(%edi) 2026 xorps %xmm1,%xmm4 2027 movups %xmm3,16(%edi) 2028 pxor %xmm3,%xmm3 2029 xorps %xmm0,%xmm5 2030 movups %xmm4,32(%edi) 2031 pxor %xmm4,%xmm4 2032 leal 48(%edi),%edi 2033 movaps %xmm5,%xmm2 2034 pxor %xmm5,%xmm5 2035 subl $64,%eax 2036 jmp .L088cbc_dec_tail_collected 2037 .align 16 2038 .L083cbc_dec_clear_tail_collected: 2039 pxor %xmm3,%xmm3 2040 pxor %xmm4,%xmm4 2041 pxor %xmm5,%xmm5 2042 pxor %xmm6,%xmm6 2043 .L088cbc_dec_tail_collected: 2044 andl $15,%eax 2045 jnz .L090cbc_dec_tail_partial 2046 movups %xmm2,(%edi) 2047 pxor %xmm0,%xmm0 2048 jmp .L079cbc_ret 2049 .align 16 2050 .L090cbc_dec_tail_partial: 2051 movaps %xmm2,(%esp) 2052 pxor %xmm0,%xmm0 2053 movl $16,%ecx 2054 movl %esp,%esi 2055 subl %eax,%ecx 2056 .long 2767451785 2057 movdqa %xmm2,(%esp) 2058 .L079cbc_ret: 2059 movl 16(%esp),%esp 2060 movl 36(%esp),%ebp 2061 pxor %xmm2,%xmm2 2062 pxor %xmm1,%xmm1 2063 movups %xmm7,(%ebp) 2064 pxor %xmm7,%xmm7 2065 .L074cbc_abort: 2066 popl %edi 2067 popl %esi 2068 popl %ebx 2069 popl %ebp 2070 ret 2071 .size aesni_cbc_encrypt,.-.L_aesni_cbc_encrypt_begin 2072 .hidden _aesni_set_encrypt_key 2073 .type _aesni_set_encrypt_key,@function 2074 .align 16 2075 _aesni_set_encrypt_key: 2076 pushl %ebp 2077 pushl %ebx 2078 testl %eax,%eax 2079 jz .L091bad_pointer 2080 testl %edx,%edx 2081 jz .L091bad_pointer 2082 call .L092pic 2083 .L092pic: 2084 popl %ebx 2085 leal .Lkey_const-.L092pic(%ebx),%ebx 2086 leal OPENSSL_ia32cap_P-.Lkey_const(%ebx),%ebp 2087 movups (%eax),%xmm0 2088 xorps %xmm4,%xmm4 2089 movl 4(%ebp),%ebp 2090 leal 16(%edx),%edx 2091 andl $268437504,%ebp 2092 cmpl $256,%ecx 2093 je .L09314rounds 2094 cmpl $192,%ecx 2095 je .L09412rounds 2096 cmpl $128,%ecx 2097 jne .L095bad_keybits 2098 .align 16 2099 .L09610rounds: 2100 cmpl $268435456,%ebp 2101 je .L09710rounds_alt 2102 movl $9,%ecx 2103 movups %xmm0,-16(%edx) 2104 .byte 102,15,58,223,200,1 2105 call .L098key_128_cold 2106 .byte 102,15,58,223,200,2 2107 call .L099key_128 2108 .byte 102,15,58,223,200,4 2109 call .L099key_128 2110 .byte 102,15,58,223,200,8 2111 call .L099key_128 2112 .byte 102,15,58,223,200,16 2113 call .L099key_128 2114 .byte 102,15,58,223,200,32 2115 call .L099key_128 2116 .byte 102,15,58,223,200,64 2117 call .L099key_128 2118 .byte 102,15,58,223,200,128 2119 call .L099key_128 2120 .byte 102,15,58,223,200,27 2121 call .L099key_128 2122 .byte 102,15,58,223,200,54 2123 call .L099key_128 2124 movups %xmm0,(%edx) 2125 movl %ecx,80(%edx) 2126 jmp .L100good_key 2127 .align 16 2128 .L099key_128: 2129 movups %xmm0,(%edx) 2130 leal 16(%edx),%edx 2131 .L098key_128_cold: 2132 shufps $16,%xmm0,%xmm4 2133 xorps %xmm4,%xmm0 2134 shufps $140,%xmm0,%xmm4 2135 xorps %xmm4,%xmm0 2136 shufps $255,%xmm1,%xmm1 2137 xorps %xmm1,%xmm0 2138 ret 2139 .align 16 2140 .L09710rounds_alt: 2141 movdqa (%ebx),%xmm5 2142 movl $8,%ecx 2143 movdqa 32(%ebx),%xmm4 2144 movdqa %xmm0,%xmm2 2145 movdqu %xmm0,-16(%edx) 2146 .L101loop_key128: 2147 .byte 102,15,56,0,197 2148 .byte 102,15,56,221,196 2149 pslld $1,%xmm4 2150 leal 16(%edx),%edx 2151 movdqa %xmm2,%xmm3 2152 pslldq $4,%xmm2 2153 pxor %xmm2,%xmm3 2154 pslldq $4,%xmm2 2155 pxor %xmm2,%xmm3 2156 pslldq $4,%xmm2 2157 pxor %xmm3,%xmm2 2158 pxor %xmm2,%xmm0 2159 movdqu %xmm0,-16(%edx) 2160 movdqa %xmm0,%xmm2 2161 decl %ecx 2162 jnz .L101loop_key128 2163 movdqa 48(%ebx),%xmm4 2164 .byte 102,15,56,0,197 2165 .byte 102,15,56,221,196 2166 pslld $1,%xmm4 2167 movdqa %xmm2,%xmm3 2168 pslldq $4,%xmm2 2169 pxor %xmm2,%xmm3 2170 pslldq $4,%xmm2 2171 pxor %xmm2,%xmm3 2172 pslldq $4,%xmm2 2173 pxor %xmm3,%xmm2 2174 pxor %xmm2,%xmm0 2175 movdqu %xmm0,(%edx) 2176 movdqa %xmm0,%xmm2 2177 .byte 102,15,56,0,197 2178 .byte 102,15,56,221,196 2179 movdqa %xmm2,%xmm3 2180 pslldq $4,%xmm2 2181 pxor %xmm2,%xmm3 2182 pslldq $4,%xmm2 2183 pxor %xmm2,%xmm3 2184 pslldq $4,%xmm2 2185 pxor %xmm3,%xmm2 2186 pxor %xmm2,%xmm0 2187 movdqu %xmm0,16(%edx) 2188 movl $9,%ecx 2189 movl %ecx,96(%edx) 2190 jmp .L100good_key 2191 .align 16 2192 .L09412rounds: 2193 movq 16(%eax),%xmm2 2194 cmpl $268435456,%ebp 2195 je .L10212rounds_alt 2196 movl $11,%ecx 2197 movups %xmm0,-16(%edx) 2198 .byte 102,15,58,223,202,1 2199 call .L103key_192a_cold 2200 .byte 102,15,58,223,202,2 2201 call .L104key_192b 2202 .byte 102,15,58,223,202,4 2203 call .L105key_192a 2204 .byte 102,15,58,223,202,8 2205 call .L104key_192b 2206 .byte 102,15,58,223,202,16 2207 call .L105key_192a 2208 .byte 102,15,58,223,202,32 2209 call .L104key_192b 2210 .byte 102,15,58,223,202,64 2211 call .L105key_192a 2212 .byte 102,15,58,223,202,128 2213 call .L104key_192b 2214 movups %xmm0,(%edx) 2215 movl %ecx,48(%edx) 2216 jmp .L100good_key 2217 .align 16 2218 .L105key_192a: 2219 movups %xmm0,(%edx) 2220 leal 16(%edx),%edx 2221 .align 16 2222 .L103key_192a_cold: 2223 movaps %xmm2,%xmm5 2224 .L106key_192b_warm: 2225 shufps $16,%xmm0,%xmm4 2226 movdqa %xmm2,%xmm3 2227 xorps %xmm4,%xmm0 2228 shufps $140,%xmm0,%xmm4 2229 pslldq $4,%xmm3 2230 xorps %xmm4,%xmm0 2231 pshufd $85,%xmm1,%xmm1 2232 pxor %xmm3,%xmm2 2233 pxor %xmm1,%xmm0 2234 pshufd $255,%xmm0,%xmm3 2235 pxor %xmm3,%xmm2 2236 ret 2237 .align 16 2238 .L104key_192b: 2239 movaps %xmm0,%xmm3 2240 shufps $68,%xmm0,%xmm5 2241 movups %xmm5,(%edx) 2242 shufps $78,%xmm2,%xmm3 2243 movups %xmm3,16(%edx) 2244 leal 32(%edx),%edx 2245 jmp .L106key_192b_warm 2246 .align 16 2247 .L10212rounds_alt: 2248 movdqa 16(%ebx),%xmm5 2249 movdqa 32(%ebx),%xmm4 2250 movl $8,%ecx 2251 movdqu %xmm0,-16(%edx) 2252 .L107loop_key192: 2253 movq %xmm2,(%edx) 2254 movdqa %xmm2,%xmm1 2255 .byte 102,15,56,0,213 2256 .byte 102,15,56,221,212 2257 pslld $1,%xmm4 2258 leal 24(%edx),%edx 2259 movdqa %xmm0,%xmm3 2260 pslldq $4,%xmm0 2261 pxor %xmm0,%xmm3 2262 pslldq $4,%xmm0 2263 pxor %xmm0,%xmm3 2264 pslldq $4,%xmm0 2265 pxor %xmm3,%xmm0 2266 pshufd $255,%xmm0,%xmm3 2267 pxor %xmm1,%xmm3 2268 pslldq $4,%xmm1 2269 pxor %xmm1,%xmm3 2270 pxor %xmm2,%xmm0 2271 pxor %xmm3,%xmm2 2272 movdqu %xmm0,-16(%edx) 2273 decl %ecx 2274 jnz .L107loop_key192 2275 movl $11,%ecx 2276 movl %ecx,32(%edx) 2277 jmp .L100good_key 2278 .align 16 2279 .L09314rounds: 2280 movups 16(%eax),%xmm2 2281 leal 16(%edx),%edx 2282 cmpl $268435456,%ebp 2283 je .L10814rounds_alt 2284 movl $13,%ecx 2285 movups %xmm0,-32(%edx) 2286 movups %xmm2,-16(%edx) 2287 .byte 102,15,58,223,202,1 2288 call .L109key_256a_cold 2289 .byte 102,15,58,223,200,1 2290 call .L110key_256b 2291 .byte 102,15,58,223,202,2 2292 call .L111key_256a 2293 .byte 102,15,58,223,200,2 2294 call .L110key_256b 2295 .byte 102,15,58,223,202,4 2296 call .L111key_256a 2297 .byte 102,15,58,223,200,4 2298 call .L110key_256b 2299 .byte 102,15,58,223,202,8 2300 call .L111key_256a 2301 .byte 102,15,58,223,200,8 2302 call .L110key_256b 2303 .byte 102,15,58,223,202,16 2304 call .L111key_256a 2305 .byte 102,15,58,223,200,16 2306 call .L110key_256b 2307 .byte 102,15,58,223,202,32 2308 call .L111key_256a 2309 .byte 102,15,58,223,200,32 2310 call .L110key_256b 2311 .byte 102,15,58,223,202,64 2312 call .L111key_256a 2313 movups %xmm0,(%edx) 2314 movl %ecx,16(%edx) 2315 xorl %eax,%eax 2316 jmp .L100good_key 2317 .align 16 2318 .L111key_256a: 2319 movups %xmm2,(%edx) 2320 leal 16(%edx),%edx 2321 .L109key_256a_cold: 2322 shufps $16,%xmm0,%xmm4 2323 xorps %xmm4,%xmm0 2324 shufps $140,%xmm0,%xmm4 2325 xorps %xmm4,%xmm0 2326 shufps $255,%xmm1,%xmm1 2327 xorps %xmm1,%xmm0 2328 ret 2329 .align 16 2330 .L110key_256b: 2331 movups %xmm0,(%edx) 2332 leal 16(%edx),%edx 2333 shufps $16,%xmm2,%xmm4 2334 xorps %xmm4,%xmm2 2335 shufps $140,%xmm2,%xmm4 2336 xorps %xmm4,%xmm2 2337 shufps $170,%xmm1,%xmm1 2338 xorps %xmm1,%xmm2 2339 ret 2340 .align 16 2341 .L10814rounds_alt: 2342 movdqa (%ebx),%xmm5 2343 movdqa 32(%ebx),%xmm4 2344 movl $7,%ecx 2345 movdqu %xmm0,-32(%edx) 2346 movdqa %xmm2,%xmm1 2347 movdqu %xmm2,-16(%edx) 2348 .L112loop_key256: 2349 .byte 102,15,56,0,213 2350 .byte 102,15,56,221,212 2351 movdqa %xmm0,%xmm3 2352 pslldq $4,%xmm0 2353 pxor %xmm0,%xmm3 2354 pslldq $4,%xmm0 2355 pxor %xmm0,%xmm3 2356 pslldq $4,%xmm0 2357 pxor %xmm3,%xmm0 2358 pslld $1,%xmm4 2359 pxor %xmm2,%xmm0 2360 movdqu %xmm0,(%edx) 2361 decl %ecx 2362 jz .L113done_key256 2363 pshufd $255,%xmm0,%xmm2 2364 pxor %xmm3,%xmm3 2365 .byte 102,15,56,221,211 2366 movdqa %xmm1,%xmm3 2367 pslldq $4,%xmm1 2368 pxor %xmm1,%xmm3 2369 pslldq $4,%xmm1 2370 pxor %xmm1,%xmm3 2371 pslldq $4,%xmm1 2372 pxor %xmm3,%xmm1 2373 pxor %xmm1,%xmm2 2374 movdqu %xmm2,16(%edx) 2375 leal 32(%edx),%edx 2376 movdqa %xmm2,%xmm1 2377 jmp .L112loop_key256 2378 .L113done_key256: 2379 movl $13,%ecx 2380 movl %ecx,16(%edx) 2381 .L100good_key: 2382 pxor %xmm0,%xmm0 2383 pxor %xmm1,%xmm1 2384 pxor %xmm2,%xmm2 2385 pxor %xmm3,%xmm3 2386 pxor %xmm4,%xmm4 2387 pxor %xmm5,%xmm5 2388 xorl %eax,%eax 2389 popl %ebx 2390 popl %ebp 2391 ret 2392 .align 4 2393 .L091bad_pointer: 2394 movl $-1,%eax 2395 popl %ebx 2396 popl %ebp 2397 ret 2398 .align 4 2399 .L095bad_keybits: 2400 pxor %xmm0,%xmm0 2401 movl $-2,%eax 2402 popl %ebx 2403 popl %ebp 2404 ret 2405 .size _aesni_set_encrypt_key,.-_aesni_set_encrypt_key 2406 .globl aesni_set_encrypt_key 2407 .hidden aesni_set_encrypt_key 2408 .type aesni_set_encrypt_key,@function 2409 .align 16 2410 aesni_set_encrypt_key: 2411 .L_aesni_set_encrypt_key_begin: 2412 movl 4(%esp),%eax 2413 movl 8(%esp),%ecx 2414 movl 12(%esp),%edx 2415 call _aesni_set_encrypt_key 2416 ret 2417 .size aesni_set_encrypt_key,.-.L_aesni_set_encrypt_key_begin 2418 .globl aesni_set_decrypt_key 2419 .hidden aesni_set_decrypt_key 2420 .type aesni_set_decrypt_key,@function 2421 .align 16 2422 aesni_set_decrypt_key: 2423 .L_aesni_set_decrypt_key_begin: 2424 movl 4(%esp),%eax 2425 movl 8(%esp),%ecx 2426 movl 12(%esp),%edx 2427 call _aesni_set_encrypt_key 2428 movl 12(%esp),%edx 2429 shll $4,%ecx 2430 testl %eax,%eax 2431 jnz .L114dec_key_ret 2432 leal 16(%edx,%ecx,1),%eax 2433 movups (%edx),%xmm0 2434 movups (%eax),%xmm1 2435 movups %xmm0,(%eax) 2436 movups %xmm1,(%edx) 2437 leal 16(%edx),%edx 2438 leal -16(%eax),%eax 2439 .L115dec_key_inverse: 2440 movups (%edx),%xmm0 2441 movups (%eax),%xmm1 2442 .byte 102,15,56,219,192 2443 .byte 102,15,56,219,201 2444 leal 16(%edx),%edx 2445 leal -16(%eax),%eax 2446 movups %xmm0,16(%eax) 2447 movups %xmm1,-16(%edx) 2448 cmpl %edx,%eax 2449 ja .L115dec_key_inverse 2450 movups (%edx),%xmm0 2451 .byte 102,15,56,219,192 2452 movups %xmm0,(%edx) 2453 pxor %xmm0,%xmm0 2454 pxor %xmm1,%xmm1 2455 xorl %eax,%eax 2456 .L114dec_key_ret: 2457 ret 2458 .size aesni_set_decrypt_key,.-.L_aesni_set_decrypt_key_begin 2459 .align 64 2460 .Lkey_const: 2461 .long 202313229,202313229,202313229,202313229 2462 .long 67569157,67569157,67569157,67569157 2463 .long 1,1,1,1 2464 .long 27,27,27,27 2465 .byte 65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69 2466 .byte 83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83 2467 .byte 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 2468 .byte 115,108,46,111,114,103,62,0 2469 #endif 2470
