1 /* ==================================================================== 2 * Copyright (c) 2001-2011 The OpenSSL Project. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * 3. All advertising materials mentioning features or use of this 17 * software must display the following acknowledgment: 18 * "This product includes software developed by the OpenSSL Project 19 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" 20 * 21 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 22 * endorse or promote products derived from this software without 23 * prior written permission. For written permission, please contact 24 * openssl-core (at) openssl.org. 25 * 26 * 5. Products derived from this software may not be called "OpenSSL" 27 * nor may "OpenSSL" appear in their names without prior written 28 * permission of the OpenSSL Project. 29 * 30 * 6. Redistributions of any form whatsoever must retain the following 31 * acknowledgment: 32 * "This product includes software developed by the OpenSSL Project 33 * for use in the OpenSSL Toolkit (http://www.openssl.org/)" 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 36 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 38 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 39 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 42 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 44 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 45 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 46 * OF THE POSSIBILITY OF SUCH DAMAGE. 47 * ==================================================================== */ 48 49 #include <string.h> 50 51 #include <openssl/aead.h> 52 #include <openssl/aes.h> 53 #include <openssl/cipher.h> 54 #include <openssl/cpu.h> 55 #include <openssl/err.h> 56 #include <openssl/mem.h> 57 #include <openssl/nid.h> 58 #include <openssl/rand.h> 59 60 #include "internal.h" 61 #include "../../internal.h" 62 #include "../aes/internal.h" 63 #include "../modes/internal.h" 64 #include "../delocate.h" 65 66 #if defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) 67 #include <openssl/arm_arch.h> 68 #endif 69 70 71 OPENSSL_MSVC_PRAGMA(warning(disable: 4702)) /* Unreachable code. */ 72 73 typedef struct { 74 union { 75 double align; 76 AES_KEY ks; 77 } ks; 78 block128_f block; 79 union { 80 cbc128_f cbc; 81 ctr128_f ctr; 82 } stream; 83 } EVP_AES_KEY; 84 85 typedef struct { 86 union { 87 double align; 88 AES_KEY ks; 89 } ks; /* AES key schedule to use */ 90 int key_set; /* Set if key initialised */ 91 int iv_set; /* Set if an iv is set */ 92 GCM128_CONTEXT gcm; 93 uint8_t *iv; /* Temporary IV store */ 94 int ivlen; /* IV length */ 95 int taglen; 96 int iv_gen; /* It is OK to generate IVs */ 97 ctr128_f ctr; 98 } EVP_AES_GCM_CTX; 99 100 #if !defined(OPENSSL_NO_ASM) && \ 101 (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) 102 #define VPAES 103 static char vpaes_capable(void) { 104 return (OPENSSL_ia32cap_P[1] & (1 << (41 - 32))) != 0; 105 } 106 107 #if defined(OPENSSL_X86_64) 108 #define BSAES 109 static char bsaes_capable(void) { 110 return vpaes_capable(); 111 } 112 #endif 113 114 #elif !defined(OPENSSL_NO_ASM) && \ 115 (defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)) 116 117 #if defined(OPENSSL_ARM) && __ARM_MAX_ARCH__ >= 7 118 #define BSAES 119 static char bsaes_capable(void) { 120 return CRYPTO_is_NEON_capable(); 121 } 122 #endif 123 124 #endif 125 126 127 #if defined(BSAES) 128 /* On platforms where BSAES gets defined (just above), then these functions are 129 * provided by asm. */ 130 void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, 131 const AES_KEY *key, uint8_t ivec[16], int enc); 132 void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, 133 const AES_KEY *key, const uint8_t ivec[16]); 134 #else 135 static char bsaes_capable(void) { 136 return 0; 137 } 138 139 /* On other platforms, bsaes_capable() will always return false and so the 140 * following will never be called. */ 141 static void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, 142 const AES_KEY *key, uint8_t ivec[16], int enc) { 143 abort(); 144 } 145 146 static void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, 147 size_t len, const AES_KEY *key, 148 const uint8_t ivec[16]) { 149 abort(); 150 } 151 #endif 152 153 #if defined(VPAES) 154 /* On platforms where VPAES gets defined (just above), then these functions are 155 * provided by asm. */ 156 int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); 157 int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); 158 159 void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); 160 void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); 161 162 void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, 163 const AES_KEY *key, uint8_t *ivec, int enc); 164 #else 165 static char vpaes_capable(void) { 166 return 0; 167 } 168 169 /* On other platforms, vpaes_capable() will always return false and so the 170 * following will never be called. */ 171 static int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, 172 AES_KEY *key) { 173 abort(); 174 } 175 static int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, 176 AES_KEY *key) { 177 abort(); 178 } 179 static void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { 180 abort(); 181 } 182 static void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { 183 abort(); 184 } 185 static void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, 186 const AES_KEY *key, uint8_t *ivec, int enc) { 187 abort(); 188 } 189 #endif 190 191 #if !defined(OPENSSL_NO_ASM) && \ 192 (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) 193 int aesni_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); 194 int aesni_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); 195 196 void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); 197 void aesni_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); 198 199 void aesni_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length, 200 const AES_KEY *key, int enc); 201 void aesni_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, 202 const AES_KEY *key, uint8_t *ivec, int enc); 203 204 #else 205 206 /* On other platforms, aesni_capable() will always return false and so the 207 * following will never be called. */ 208 static void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { 209 abort(); 210 } 211 static int aesni_set_encrypt_key(const uint8_t *userKey, int bits, 212 AES_KEY *key) { 213 abort(); 214 } 215 static void aesni_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, 216 size_t blocks, const void *key, 217 const uint8_t *ivec) { 218 abort(); 219 } 220 221 #endif 222 223 static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, 224 const uint8_t *iv, int enc) { 225 int ret, mode; 226 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 227 228 mode = ctx->cipher->flags & EVP_CIPH_MODE_MASK; 229 if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { 230 if (hwaes_capable()) { 231 ret = aes_hw_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 232 dat->block = (block128_f)aes_hw_decrypt; 233 dat->stream.cbc = NULL; 234 if (mode == EVP_CIPH_CBC_MODE) { 235 dat->stream.cbc = (cbc128_f)aes_hw_cbc_encrypt; 236 } 237 } else if (bsaes_capable() && mode == EVP_CIPH_CBC_MODE) { 238 ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 239 dat->block = (block128_f)AES_decrypt; 240 dat->stream.cbc = (cbc128_f)bsaes_cbc_encrypt; 241 } else if (vpaes_capable()) { 242 ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 243 dat->block = (block128_f)vpaes_decrypt; 244 dat->stream.cbc = 245 mode == EVP_CIPH_CBC_MODE ? (cbc128_f)vpaes_cbc_encrypt : NULL; 246 } else { 247 ret = AES_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 248 dat->block = (block128_f)AES_decrypt; 249 dat->stream.cbc = 250 mode == EVP_CIPH_CBC_MODE ? (cbc128_f)AES_cbc_encrypt : NULL; 251 } 252 } else if (hwaes_capable()) { 253 ret = aes_hw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 254 dat->block = (block128_f)aes_hw_encrypt; 255 dat->stream.cbc = NULL; 256 if (mode == EVP_CIPH_CBC_MODE) { 257 dat->stream.cbc = (cbc128_f)aes_hw_cbc_encrypt; 258 } else if (mode == EVP_CIPH_CTR_MODE) { 259 dat->stream.ctr = (ctr128_f)aes_hw_ctr32_encrypt_blocks; 260 } 261 } else if (bsaes_capable() && mode == EVP_CIPH_CTR_MODE) { 262 ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 263 dat->block = (block128_f)AES_encrypt; 264 dat->stream.ctr = (ctr128_f)bsaes_ctr32_encrypt_blocks; 265 } else if (vpaes_capable()) { 266 ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 267 dat->block = (block128_f)vpaes_encrypt; 268 dat->stream.cbc = 269 mode == EVP_CIPH_CBC_MODE ? (cbc128_f)vpaes_cbc_encrypt : NULL; 270 } else { 271 ret = AES_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); 272 dat->block = (block128_f)AES_encrypt; 273 dat->stream.cbc = 274 mode == EVP_CIPH_CBC_MODE ? (cbc128_f)AES_cbc_encrypt : NULL; 275 } 276 277 if (ret < 0) { 278 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_AES_KEY_SETUP_FAILED); 279 return 0; 280 } 281 282 return 1; 283 } 284 285 static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, 286 size_t len) { 287 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 288 289 if (dat->stream.cbc) { 290 (*dat->stream.cbc)(in, out, len, &dat->ks, ctx->iv, ctx->encrypt); 291 } else if (ctx->encrypt) { 292 CRYPTO_cbc128_encrypt(in, out, len, &dat->ks, ctx->iv, dat->block); 293 } else { 294 CRYPTO_cbc128_decrypt(in, out, len, &dat->ks, ctx->iv, dat->block); 295 } 296 297 return 1; 298 } 299 300 static int aes_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, 301 size_t len) { 302 size_t bl = ctx->cipher->block_size; 303 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 304 305 if (len < bl) { 306 return 1; 307 } 308 309 len -= bl; 310 for (size_t i = 0; i <= len; i += bl) { 311 (*dat->block)(in + i, out + i, &dat->ks); 312 } 313 314 return 1; 315 } 316 317 static int aes_ctr_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, 318 size_t len) { 319 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 320 321 if (dat->stream.ctr) { 322 CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks, ctx->iv, ctx->buf, 323 &ctx->num, dat->stream.ctr); 324 } else { 325 CRYPTO_ctr128_encrypt(in, out, len, &dat->ks, ctx->iv, ctx->buf, &ctx->num, 326 dat->block); 327 } 328 return 1; 329 } 330 331 static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, 332 size_t len) { 333 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 334 335 CRYPTO_ofb128_encrypt(in, out, len, &dat->ks, ctx->iv, &ctx->num, dat->block); 336 return 1; 337 } 338 339 static char aesni_capable(void); 340 341 ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_CONTEXT *gcm_ctx, 342 block128_f *out_block, const uint8_t *key, 343 size_t key_bytes) { 344 if (aesni_capable()) { 345 aesni_set_encrypt_key(key, key_bytes * 8, aes_key); 346 if (gcm_ctx != NULL) { 347 CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aesni_encrypt, 1); 348 } 349 if (out_block) { 350 *out_block = (block128_f) aesni_encrypt; 351 } 352 return (ctr128_f)aesni_ctr32_encrypt_blocks; 353 } 354 355 if (hwaes_capable()) { 356 aes_hw_set_encrypt_key(key, key_bytes * 8, aes_key); 357 if (gcm_ctx != NULL) { 358 CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aes_hw_encrypt, 0); 359 } 360 if (out_block) { 361 *out_block = (block128_f) aes_hw_encrypt; 362 } 363 return (ctr128_f)aes_hw_ctr32_encrypt_blocks; 364 } 365 366 if (bsaes_capable()) { 367 AES_set_encrypt_key(key, key_bytes * 8, aes_key); 368 if (gcm_ctx != NULL) { 369 CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)AES_encrypt, 0); 370 } 371 if (out_block) { 372 *out_block = (block128_f) AES_encrypt; 373 } 374 return (ctr128_f)bsaes_ctr32_encrypt_blocks; 375 } 376 377 if (vpaes_capable()) { 378 vpaes_set_encrypt_key(key, key_bytes * 8, aes_key); 379 if (out_block) { 380 *out_block = (block128_f) vpaes_encrypt; 381 } 382 if (gcm_ctx != NULL) { 383 CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)vpaes_encrypt, 0); 384 } 385 return NULL; 386 } 387 388 AES_set_encrypt_key(key, key_bytes * 8, aes_key); 389 if (gcm_ctx != NULL) { 390 CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)AES_encrypt, 0); 391 } 392 if (out_block) { 393 *out_block = (block128_f) AES_encrypt; 394 } 395 return NULL; 396 } 397 398 static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, 399 const uint8_t *iv, int enc) { 400 EVP_AES_GCM_CTX *gctx = ctx->cipher_data; 401 if (!iv && !key) { 402 return 1; 403 } 404 if (key) { 405 gctx->ctr = 406 aes_ctr_set_key(&gctx->ks.ks, &gctx->gcm, NULL, key, ctx->key_len); 407 /* If we have an iv can set it directly, otherwise use saved IV. */ 408 if (iv == NULL && gctx->iv_set) { 409 iv = gctx->iv; 410 } 411 if (iv) { 412 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); 413 gctx->iv_set = 1; 414 } 415 gctx->key_set = 1; 416 } else { 417 /* If key set use IV, otherwise copy */ 418 if (gctx->key_set) { 419 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); 420 } else { 421 OPENSSL_memcpy(gctx->iv, iv, gctx->ivlen); 422 } 423 gctx->iv_set = 1; 424 gctx->iv_gen = 0; 425 } 426 return 1; 427 } 428 429 static void aes_gcm_cleanup(EVP_CIPHER_CTX *c) { 430 EVP_AES_GCM_CTX *gctx = c->cipher_data; 431 OPENSSL_cleanse(&gctx->gcm, sizeof(gctx->gcm)); 432 if (gctx->iv != c->iv) { 433 OPENSSL_free(gctx->iv); 434 } 435 } 436 437 /* increment counter (64-bit int) by 1 */ 438 static void ctr64_inc(uint8_t *counter) { 439 int n = 8; 440 uint8_t c; 441 442 do { 443 --n; 444 c = counter[n]; 445 ++c; 446 counter[n] = c; 447 if (c) { 448 return; 449 } 450 } while (n); 451 } 452 453 static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr) { 454 EVP_AES_GCM_CTX *gctx = c->cipher_data; 455 switch (type) { 456 case EVP_CTRL_INIT: 457 gctx->key_set = 0; 458 gctx->iv_set = 0; 459 gctx->ivlen = c->cipher->iv_len; 460 gctx->iv = c->iv; 461 gctx->taglen = -1; 462 gctx->iv_gen = 0; 463 return 1; 464 465 case EVP_CTRL_GCM_SET_IVLEN: 466 if (arg <= 0) { 467 return 0; 468 } 469 470 /* Allocate memory for IV if needed */ 471 if (arg > EVP_MAX_IV_LENGTH && arg > gctx->ivlen) { 472 if (gctx->iv != c->iv) { 473 OPENSSL_free(gctx->iv); 474 } 475 gctx->iv = OPENSSL_malloc(arg); 476 if (!gctx->iv) { 477 return 0; 478 } 479 } 480 gctx->ivlen = arg; 481 return 1; 482 483 case EVP_CTRL_GCM_SET_TAG: 484 if (arg <= 0 || arg > 16 || c->encrypt) { 485 return 0; 486 } 487 OPENSSL_memcpy(c->buf, ptr, arg); 488 gctx->taglen = arg; 489 return 1; 490 491 case EVP_CTRL_GCM_GET_TAG: 492 if (arg <= 0 || arg > 16 || !c->encrypt || gctx->taglen < 0) { 493 return 0; 494 } 495 OPENSSL_memcpy(ptr, c->buf, arg); 496 return 1; 497 498 case EVP_CTRL_GCM_SET_IV_FIXED: 499 /* Special case: -1 length restores whole IV */ 500 if (arg == -1) { 501 OPENSSL_memcpy(gctx->iv, ptr, gctx->ivlen); 502 gctx->iv_gen = 1; 503 return 1; 504 } 505 /* Fixed field must be at least 4 bytes and invocation field 506 * at least 8. */ 507 if (arg < 4 || (gctx->ivlen - arg) < 8) { 508 return 0; 509 } 510 if (arg) { 511 OPENSSL_memcpy(gctx->iv, ptr, arg); 512 } 513 if (c->encrypt && !RAND_bytes(gctx->iv + arg, gctx->ivlen - arg)) { 514 return 0; 515 } 516 gctx->iv_gen = 1; 517 return 1; 518 519 case EVP_CTRL_GCM_IV_GEN: 520 if (gctx->iv_gen == 0 || gctx->key_set == 0) { 521 return 0; 522 } 523 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, gctx->iv, gctx->ivlen); 524 if (arg <= 0 || arg > gctx->ivlen) { 525 arg = gctx->ivlen; 526 } 527 OPENSSL_memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg); 528 /* Invocation field will be at least 8 bytes in size and 529 * so no need to check wrap around or increment more than 530 * last 8 bytes. */ 531 ctr64_inc(gctx->iv + gctx->ivlen - 8); 532 gctx->iv_set = 1; 533 return 1; 534 535 case EVP_CTRL_GCM_SET_IV_INV: 536 if (gctx->iv_gen == 0 || gctx->key_set == 0 || c->encrypt) { 537 return 0; 538 } 539 OPENSSL_memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg); 540 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, gctx->iv, gctx->ivlen); 541 gctx->iv_set = 1; 542 return 1; 543 544 case EVP_CTRL_COPY: { 545 EVP_CIPHER_CTX *out = ptr; 546 EVP_AES_GCM_CTX *gctx_out = out->cipher_data; 547 if (gctx->iv == c->iv) { 548 gctx_out->iv = out->iv; 549 } else { 550 gctx_out->iv = OPENSSL_malloc(gctx->ivlen); 551 if (!gctx_out->iv) { 552 return 0; 553 } 554 OPENSSL_memcpy(gctx_out->iv, gctx->iv, gctx->ivlen); 555 } 556 return 1; 557 } 558 559 default: 560 return -1; 561 } 562 } 563 564 static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, 565 size_t len) { 566 EVP_AES_GCM_CTX *gctx = ctx->cipher_data; 567 568 /* If not set up, return error */ 569 if (!gctx->key_set) { 570 return -1; 571 } 572 if (!gctx->iv_set) { 573 return -1; 574 } 575 576 if (in) { 577 if (out == NULL) { 578 if (!CRYPTO_gcm128_aad(&gctx->gcm, in, len)) { 579 return -1; 580 } 581 } else if (ctx->encrypt) { 582 if (gctx->ctr) { 583 if (!CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm, &gctx->ks.ks, in, out, len, 584 gctx->ctr)) { 585 return -1; 586 } 587 } else { 588 if (!CRYPTO_gcm128_encrypt(&gctx->gcm, &gctx->ks.ks, in, out, len)) { 589 return -1; 590 } 591 } 592 } else { 593 if (gctx->ctr) { 594 if (!CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm, &gctx->ks.ks, in, out, len, 595 gctx->ctr)) { 596 return -1; 597 } 598 } else { 599 if (!CRYPTO_gcm128_decrypt(&gctx->gcm, &gctx->ks.ks, in, out, len)) { 600 return -1; 601 } 602 } 603 } 604 return len; 605 } else { 606 if (!ctx->encrypt) { 607 if (gctx->taglen < 0 || 608 !CRYPTO_gcm128_finish(&gctx->gcm, ctx->buf, gctx->taglen)) { 609 return -1; 610 } 611 gctx->iv_set = 0; 612 return 0; 613 } 614 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16); 615 gctx->taglen = 16; 616 /* Don't reuse the IV */ 617 gctx->iv_set = 0; 618 return 0; 619 } 620 } 621 622 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_cbc_generic) { 623 memset(out, 0, sizeof(EVP_CIPHER)); 624 625 out->nid = NID_aes_128_cbc; 626 out->block_size = 16; 627 out->key_len = 16; 628 out->iv_len = 16; 629 out->ctx_size = sizeof(EVP_AES_KEY); 630 out->flags = EVP_CIPH_CBC_MODE; 631 out->init = aes_init_key; 632 out->cipher = aes_cbc_cipher; 633 } 634 635 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_ctr_generic) { 636 memset(out, 0, sizeof(EVP_CIPHER)); 637 638 out->nid = NID_aes_128_ctr; 639 out->block_size = 1; 640 out->key_len = 16; 641 out->iv_len = 16; 642 out->ctx_size = sizeof(EVP_AES_KEY); 643 out->flags = EVP_CIPH_CTR_MODE; 644 out->init = aes_init_key; 645 out->cipher = aes_ctr_cipher; 646 } 647 648 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_ecb_generic) { 649 memset(out, 0, sizeof(EVP_CIPHER)); 650 651 out->nid = NID_aes_128_ecb; 652 out->block_size = 16; 653 out->key_len = 16; 654 out->ctx_size = sizeof(EVP_AES_KEY); 655 out->flags = EVP_CIPH_ECB_MODE; 656 out->init = aes_init_key; 657 out->cipher = aes_ecb_cipher; 658 } 659 660 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_ofb_generic) { 661 memset(out, 0, sizeof(EVP_CIPHER)); 662 663 out->nid = NID_aes_128_ofb128; 664 out->block_size = 1; 665 out->key_len = 16; 666 out->iv_len = 16; 667 out->ctx_size = sizeof(EVP_AES_KEY); 668 out->flags = EVP_CIPH_OFB_MODE; 669 out->init = aes_init_key; 670 out->cipher = aes_ofb_cipher; 671 } 672 673 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_128_gcm_generic) { 674 memset(out, 0, sizeof(EVP_CIPHER)); 675 676 out->nid = NID_aes_128_gcm; 677 out->block_size = 1; 678 out->key_len = 16; 679 out->iv_len = 12; 680 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 681 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 682 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 683 EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; 684 out->init = aes_gcm_init_key; 685 out->cipher = aes_gcm_cipher; 686 out->cleanup = aes_gcm_cleanup; 687 out->ctrl = aes_gcm_ctrl; 688 } 689 690 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_cbc_generic) { 691 memset(out, 0, sizeof(EVP_CIPHER)); 692 693 out->nid = NID_aes_192_cbc; 694 out->block_size = 16; 695 out->key_len = 24; 696 out->iv_len = 16; 697 out->ctx_size = sizeof(EVP_AES_KEY); 698 out->flags = EVP_CIPH_CBC_MODE; 699 out->init = aes_init_key; 700 out->cipher = aes_cbc_cipher; 701 } 702 703 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_ctr_generic) { 704 memset(out, 0, sizeof(EVP_CIPHER)); 705 706 out->nid = NID_aes_192_ctr; 707 out->block_size = 1; 708 out->key_len = 24; 709 out->iv_len = 16; 710 out->ctx_size = sizeof(EVP_AES_KEY); 711 out->flags = EVP_CIPH_CTR_MODE; 712 out->init = aes_init_key; 713 out->cipher = aes_ctr_cipher; 714 } 715 716 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_ecb_generic) { 717 memset(out, 0, sizeof(EVP_CIPHER)); 718 719 out->nid = NID_aes_192_ecb; 720 out->block_size = 16; 721 out->key_len = 24; 722 out->ctx_size = sizeof(EVP_AES_KEY); 723 out->flags = EVP_CIPH_ECB_MODE; 724 out->init = aes_init_key; 725 out->cipher = aes_ecb_cipher; 726 } 727 728 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_192_gcm_generic) { 729 memset(out, 0, sizeof(EVP_CIPHER)); 730 731 out->nid = NID_aes_192_gcm; 732 out->block_size = 1; 733 out->key_len = 24; 734 out->iv_len = 12; 735 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 736 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 737 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 738 EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; 739 out->init = aes_gcm_init_key; 740 out->cipher = aes_gcm_cipher; 741 out->cleanup = aes_gcm_cleanup; 742 out->ctrl = aes_gcm_ctrl; 743 } 744 745 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_cbc_generic) { 746 memset(out, 0, sizeof(EVP_CIPHER)); 747 748 out->nid = NID_aes_256_cbc; 749 out->block_size = 16; 750 out->key_len = 32; 751 out->iv_len = 16; 752 out->ctx_size = sizeof(EVP_AES_KEY); 753 out->flags = EVP_CIPH_CBC_MODE; 754 out->init = aes_init_key; 755 out->cipher = aes_cbc_cipher; 756 } 757 758 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_ctr_generic) { 759 memset(out, 0, sizeof(EVP_CIPHER)); 760 761 out->nid = NID_aes_256_ctr; 762 out->block_size = 1; 763 out->key_len = 32; 764 out->iv_len = 16; 765 out->ctx_size = sizeof(EVP_AES_KEY); 766 out->flags = EVP_CIPH_CTR_MODE; 767 out->init = aes_init_key; 768 out->cipher = aes_ctr_cipher; 769 } 770 771 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_ecb_generic) { 772 memset(out, 0, sizeof(EVP_CIPHER)); 773 774 out->nid = NID_aes_256_ecb; 775 out->block_size = 16; 776 out->key_len = 32; 777 out->ctx_size = sizeof(EVP_AES_KEY); 778 out->flags = EVP_CIPH_ECB_MODE; 779 out->init = aes_init_key; 780 out->cipher = aes_ecb_cipher; 781 } 782 783 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_ofb_generic) { 784 memset(out, 0, sizeof(EVP_CIPHER)); 785 786 out->nid = NID_aes_256_ofb128; 787 out->block_size = 1; 788 out->key_len = 32; 789 out->iv_len = 16; 790 out->ctx_size = sizeof(EVP_AES_KEY); 791 out->flags = EVP_CIPH_OFB_MODE; 792 out->init = aes_init_key; 793 out->cipher = aes_ofb_cipher; 794 } 795 796 DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_gcm_generic) { 797 memset(out, 0, sizeof(EVP_CIPHER)); 798 799 out->nid = NID_aes_256_gcm; 800 out->block_size = 1; 801 out->key_len = 32; 802 out->iv_len = 12; 803 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 804 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 805 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 806 EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; 807 out->init = aes_gcm_init_key; 808 out->cipher = aes_gcm_cipher; 809 out->cleanup = aes_gcm_cleanup; 810 out->ctrl = aes_gcm_ctrl; 811 } 812 813 #if !defined(OPENSSL_NO_ASM) && \ 814 (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) 815 816 /* AES-NI section. */ 817 818 static char aesni_capable(void) { 819 return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0; 820 } 821 822 static int aesni_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, 823 const uint8_t *iv, int enc) { 824 int ret, mode; 825 EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; 826 827 mode = ctx->cipher->flags & EVP_CIPH_MODE_MASK; 828 if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { 829 ret = aesni_set_decrypt_key(key, ctx->key_len * 8, ctx->cipher_data); 830 dat->block = (block128_f)aesni_decrypt; 831 dat->stream.cbc = 832 mode == EVP_CIPH_CBC_MODE ? (cbc128_f)aesni_cbc_encrypt : NULL; 833 } else { 834 ret = aesni_set_encrypt_key(key, ctx->key_len * 8, ctx->cipher_data); 835 dat->block = (block128_f)aesni_encrypt; 836 if (mode == EVP_CIPH_CBC_MODE) { 837 dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt; 838 } else if (mode == EVP_CIPH_CTR_MODE) { 839 dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks; 840 } else { 841 dat->stream.cbc = NULL; 842 } 843 } 844 845 if (ret < 0) { 846 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_AES_KEY_SETUP_FAILED); 847 return 0; 848 } 849 850 return 1; 851 } 852 853 static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, 854 const uint8_t *in, size_t len) { 855 aesni_cbc_encrypt(in, out, len, ctx->cipher_data, ctx->iv, ctx->encrypt); 856 857 return 1; 858 } 859 860 static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, 861 const uint8_t *in, size_t len) { 862 size_t bl = ctx->cipher->block_size; 863 864 if (len < bl) { 865 return 1; 866 } 867 868 aesni_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt); 869 870 return 1; 871 } 872 873 static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, 874 const uint8_t *iv, int enc) { 875 EVP_AES_GCM_CTX *gctx = ctx->cipher_data; 876 if (!iv && !key) { 877 return 1; 878 } 879 if (key) { 880 aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); 881 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)aesni_encrypt, 1); 882 gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks; 883 /* If we have an iv can set it directly, otherwise use 884 * saved IV. */ 885 if (iv == NULL && gctx->iv_set) { 886 iv = gctx->iv; 887 } 888 if (iv) { 889 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); 890 gctx->iv_set = 1; 891 } 892 gctx->key_set = 1; 893 } else { 894 /* If key set use IV, otherwise copy */ 895 if (gctx->key_set) { 896 CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); 897 } else { 898 OPENSSL_memcpy(gctx->iv, iv, gctx->ivlen); 899 } 900 gctx->iv_set = 1; 901 gctx->iv_gen = 0; 902 } 903 return 1; 904 } 905 906 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_cbc) { 907 memset(out, 0, sizeof(EVP_CIPHER)); 908 909 out->nid = NID_aes_128_cbc; 910 out->block_size = 16; 911 out->key_len = 16; 912 out->iv_len = 16; 913 out->ctx_size = sizeof(EVP_AES_KEY); 914 out->flags = EVP_CIPH_CBC_MODE; 915 out->init = aesni_init_key; 916 out->cipher = aesni_cbc_cipher; 917 } 918 919 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ctr) { 920 memset(out, 0, sizeof(EVP_CIPHER)); 921 922 out->nid = NID_aes_128_ctr; 923 out->block_size = 1; 924 out->key_len = 16; 925 out->iv_len = 16; 926 out->ctx_size = sizeof(EVP_AES_KEY); 927 out->flags = EVP_CIPH_CTR_MODE; 928 out->init = aesni_init_key; 929 out->cipher = aes_ctr_cipher; 930 } 931 932 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) { 933 memset(out, 0, sizeof(EVP_CIPHER)); 934 935 out->nid = NID_aes_128_ecb; 936 out->block_size = 16; 937 out->key_len = 16; 938 out->ctx_size = sizeof(EVP_AES_KEY); 939 out->flags = EVP_CIPH_ECB_MODE; 940 out->init = aesni_init_key; 941 out->cipher = aesni_ecb_cipher; 942 } 943 944 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ofb) { 945 memset(out, 0, sizeof(EVP_CIPHER)); 946 947 out->nid = NID_aes_128_ofb128; 948 out->block_size = 1; 949 out->key_len = 16; 950 out->iv_len = 16; 951 out->ctx_size = sizeof(EVP_AES_KEY); 952 out->flags = EVP_CIPH_OFB_MODE; 953 out->init = aesni_init_key; 954 out->cipher = aes_ofb_cipher; 955 } 956 957 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_gcm) { 958 memset(out, 0, sizeof(EVP_CIPHER)); 959 960 out->nid = NID_aes_128_gcm; 961 out->block_size = 1; 962 out->key_len = 16; 963 out->iv_len = 12; 964 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 965 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 966 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 967 EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; 968 out->init = aesni_gcm_init_key; 969 out->cipher = aes_gcm_cipher; 970 out->cleanup = aes_gcm_cleanup; 971 out->ctrl = aes_gcm_ctrl; 972 } 973 974 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_cbc) { 975 memset(out, 0, sizeof(EVP_CIPHER)); 976 977 out->nid = NID_aes_192_cbc; 978 out->block_size = 16; 979 out->key_len = 24; 980 out->iv_len = 16; 981 out->ctx_size = sizeof(EVP_AES_KEY); 982 out->flags = EVP_CIPH_CBC_MODE; 983 out->init = aesni_init_key; 984 out->cipher = aesni_cbc_cipher; 985 } 986 987 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ctr) { 988 memset(out, 0, sizeof(EVP_CIPHER)); 989 990 out->nid = NID_aes_192_ctr; 991 out->block_size = 1; 992 out->key_len = 24; 993 out->iv_len = 16; 994 out->ctx_size = sizeof(EVP_AES_KEY); 995 out->flags = EVP_CIPH_CTR_MODE; 996 out->init = aesni_init_key; 997 out->cipher = aes_ctr_cipher; 998 } 999 1000 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) { 1001 memset(out, 0, sizeof(EVP_CIPHER)); 1002 1003 out->nid = NID_aes_192_ecb; 1004 out->block_size = 16; 1005 out->key_len = 24; 1006 out->ctx_size = sizeof(EVP_AES_KEY); 1007 out->flags = EVP_CIPH_ECB_MODE; 1008 out->init = aesni_init_key; 1009 out->cipher = aesni_ecb_cipher; 1010 } 1011 1012 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_gcm) { 1013 memset(out, 0, sizeof(EVP_CIPHER)); 1014 1015 out->nid = NID_aes_192_gcm; 1016 out->block_size = 1; 1017 out->key_len = 24; 1018 out->iv_len = 12; 1019 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 1020 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 1021 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 1022 EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; 1023 out->init = aesni_gcm_init_key; 1024 out->cipher = aes_gcm_cipher; 1025 out->cleanup = aes_gcm_cleanup; 1026 out->ctrl = aes_gcm_ctrl; 1027 } 1028 1029 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_cbc) { 1030 memset(out, 0, sizeof(EVP_CIPHER)); 1031 1032 out->nid = NID_aes_256_cbc; 1033 out->block_size = 16; 1034 out->key_len = 32; 1035 out->iv_len = 16; 1036 out->ctx_size = sizeof(EVP_AES_KEY); 1037 out->flags = EVP_CIPH_CBC_MODE; 1038 out->init = aesni_init_key; 1039 out->cipher = aesni_cbc_cipher; 1040 } 1041 1042 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ctr) { 1043 memset(out, 0, sizeof(EVP_CIPHER)); 1044 1045 out->nid = NID_aes_256_ctr; 1046 out->block_size = 1; 1047 out->key_len = 32; 1048 out->iv_len = 16; 1049 out->ctx_size = sizeof(EVP_AES_KEY); 1050 out->flags = EVP_CIPH_CTR_MODE; 1051 out->init = aesni_init_key; 1052 out->cipher = aes_ctr_cipher; 1053 } 1054 1055 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) { 1056 memset(out, 0, sizeof(EVP_CIPHER)); 1057 1058 out->nid = NID_aes_256_ecb; 1059 out->block_size = 16; 1060 out->key_len = 32; 1061 out->ctx_size = sizeof(EVP_AES_KEY); 1062 out->flags = EVP_CIPH_ECB_MODE; 1063 out->init = aesni_init_key; 1064 out->cipher = aesni_ecb_cipher; 1065 } 1066 1067 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ofb) { 1068 memset(out, 0, sizeof(EVP_CIPHER)); 1069 1070 out->nid = NID_aes_256_ofb128; 1071 out->block_size = 1; 1072 out->key_len = 32; 1073 out->iv_len = 16; 1074 out->ctx_size = sizeof(EVP_AES_KEY); 1075 out->flags = EVP_CIPH_OFB_MODE; 1076 out->init = aesni_init_key; 1077 out->cipher = aes_ofb_cipher; 1078 } 1079 1080 DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_gcm) { 1081 memset(out, 0, sizeof(EVP_CIPHER)); 1082 1083 out->nid = NID_aes_256_gcm; 1084 out->block_size = 1; 1085 out->key_len = 32; 1086 out->iv_len = 12; 1087 out->ctx_size = sizeof(EVP_AES_GCM_CTX); 1088 out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | 1089 EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | 1090 EVP_CIPH_CTRL_INIT | EVP_CIPH_CUSTOM_COPY | 1091 EVP_CIPH_FLAG_AEAD_CIPHER; 1092 out->init = aesni_gcm_init_key; 1093 out->cipher = aes_gcm_cipher; 1094 out->cleanup = aes_gcm_cleanup; 1095 out->ctrl = aes_gcm_ctrl; 1096 } 1097 1098 #define EVP_CIPHER_FUNCTION(keybits, mode) \ 1099 const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \ 1100 if (aesni_capable()) { \ 1101 return aesni_##keybits##_##mode(); \ 1102 } else { \ 1103 return aes_##keybits##_##mode##_generic(); \ 1104 } \ 1105 } 1106 1107 #else /* ^^^ OPENSSL_X86_64 || OPENSSL_X86 */ 1108 1109 static char aesni_capable(void) { 1110 return 0; 1111 } 1112 1113 #define EVP_CIPHER_FUNCTION(keybits, mode) \ 1114 const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \ 1115 return aes_##keybits##_##mode##_generic(); \ 1116 } 1117 1118 #endif 1119 1120 EVP_CIPHER_FUNCTION(128, cbc) 1121 EVP_CIPHER_FUNCTION(128, ctr) 1122 EVP_CIPHER_FUNCTION(128, ecb) 1123 EVP_CIPHER_FUNCTION(128, ofb) 1124 EVP_CIPHER_FUNCTION(128, gcm) 1125 1126 EVP_CIPHER_FUNCTION(192, cbc) 1127 EVP_CIPHER_FUNCTION(192, ctr) 1128 EVP_CIPHER_FUNCTION(192, ecb) 1129 EVP_CIPHER_FUNCTION(192, gcm) 1130 1131 EVP_CIPHER_FUNCTION(256, cbc) 1132 EVP_CIPHER_FUNCTION(256, ctr) 1133 EVP_CIPHER_FUNCTION(256, ecb) 1134 EVP_CIPHER_FUNCTION(256, ofb) 1135 EVP_CIPHER_FUNCTION(256, gcm) 1136 1137 1138 #define EVP_AEAD_AES_GCM_TAG_LEN 16 1139 1140 struct aead_aes_gcm_ctx { 1141 union { 1142 double align; 1143 AES_KEY ks; 1144 } ks; 1145 GCM128_CONTEXT gcm; 1146 ctr128_f ctr; 1147 }; 1148 1149 struct aead_aes_gcm_tls12_ctx { 1150 struct aead_aes_gcm_ctx gcm_ctx; 1151 uint64_t counter; 1152 }; 1153 1154 static int aead_aes_gcm_init_impl(struct aead_aes_gcm_ctx *gcm_ctx, 1155 size_t *out_tag_len, const uint8_t *key, 1156 size_t key_len, size_t tag_len) { 1157 const size_t key_bits = key_len * 8; 1158 1159 if (key_bits != 128 && key_bits != 256) { 1160 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_KEY_LENGTH); 1161 return 0; /* EVP_AEAD_CTX_init should catch this. */ 1162 } 1163 1164 if (tag_len == EVP_AEAD_DEFAULT_TAG_LENGTH) { 1165 tag_len = EVP_AEAD_AES_GCM_TAG_LEN; 1166 } 1167 1168 if (tag_len > EVP_AEAD_AES_GCM_TAG_LEN) { 1169 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TAG_TOO_LARGE); 1170 return 0; 1171 } 1172 1173 gcm_ctx->ctr = 1174 aes_ctr_set_key(&gcm_ctx->ks.ks, &gcm_ctx->gcm, NULL, key, key_len); 1175 *out_tag_len = tag_len; 1176 return 1; 1177 } 1178 1179 static int aead_aes_gcm_init(EVP_AEAD_CTX *ctx, const uint8_t *key, 1180 size_t key_len, size_t requested_tag_len) { 1181 struct aead_aes_gcm_ctx *gcm_ctx; 1182 gcm_ctx = OPENSSL_malloc(sizeof(struct aead_aes_gcm_ctx)); 1183 if (gcm_ctx == NULL) { 1184 return 0; 1185 } 1186 1187 size_t actual_tag_len; 1188 if (!aead_aes_gcm_init_impl(gcm_ctx, &actual_tag_len, key, key_len, 1189 requested_tag_len)) { 1190 OPENSSL_free(gcm_ctx); 1191 return 0; 1192 } 1193 1194 ctx->aead_state = gcm_ctx; 1195 ctx->tag_len = actual_tag_len; 1196 return 1; 1197 } 1198 1199 static void aead_aes_gcm_cleanup(EVP_AEAD_CTX *ctx) { 1200 struct aead_aes_gcm_ctx *gcm_ctx = ctx->aead_state; 1201 OPENSSL_cleanse(gcm_ctx, sizeof(struct aead_aes_gcm_ctx)); 1202 OPENSSL_free(gcm_ctx); 1203 } 1204 1205 static int aead_aes_gcm_seal_scatter(const EVP_AEAD_CTX *ctx, uint8_t *out, 1206 uint8_t *out_tag, size_t *out_tag_len, 1207 size_t max_out_tag_len, 1208 const uint8_t *nonce, size_t nonce_len, 1209 const uint8_t *in, size_t in_len, 1210 const uint8_t *extra_in, 1211 size_t extra_in_len, 1212 const uint8_t *ad, size_t ad_len) { 1213 const struct aead_aes_gcm_ctx *gcm_ctx = ctx->aead_state; 1214 GCM128_CONTEXT gcm; 1215 1216 if (extra_in_len + ctx->tag_len < ctx->tag_len) { 1217 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_TOO_LARGE); 1218 return 0; 1219 } 1220 if (max_out_tag_len < ctx->tag_len + extra_in_len) { 1221 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BUFFER_TOO_SMALL); 1222 return 0; 1223 } 1224 if (nonce_len == 0) { 1225 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE_SIZE); 1226 return 0; 1227 } 1228 1229 if (max_out_tag_len < ctx->tag_len) { 1230 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BUFFER_TOO_SMALL); 1231 return 0; 1232 } 1233 1234 const AES_KEY *key = &gcm_ctx->ks.ks; 1235 1236 OPENSSL_memcpy(&gcm, &gcm_ctx->gcm, sizeof(gcm)); 1237 CRYPTO_gcm128_setiv(&gcm, key, nonce, nonce_len); 1238 1239 if (ad_len > 0 && !CRYPTO_gcm128_aad(&gcm, ad, ad_len)) { 1240 return 0; 1241 } 1242 1243 if (gcm_ctx->ctr) { 1244 if (!CRYPTO_gcm128_encrypt_ctr32(&gcm, key, in, out, in_len, 1245 gcm_ctx->ctr)) { 1246 return 0; 1247 } 1248 } else { 1249 if (!CRYPTO_gcm128_encrypt(&gcm, key, in, out, in_len)) { 1250 return 0; 1251 } 1252 } 1253 1254 if (extra_in_len) { 1255 if (gcm_ctx->ctr) { 1256 if (!CRYPTO_gcm128_encrypt_ctr32(&gcm, key, extra_in, out_tag, 1257 extra_in_len, gcm_ctx->ctr)) { 1258 return 0; 1259 } 1260 } else { 1261 if (!CRYPTO_gcm128_encrypt(&gcm, key, extra_in, out_tag, extra_in_len)) { 1262 return 0; 1263 } 1264 } 1265 } 1266 1267 CRYPTO_gcm128_tag(&gcm, out_tag + extra_in_len, ctx->tag_len); 1268 *out_tag_len = ctx->tag_len + extra_in_len; 1269 1270 return 1; 1271 } 1272 1273 static int aead_aes_gcm_open_gather(const EVP_AEAD_CTX *ctx, uint8_t *out, 1274 const uint8_t *nonce, size_t nonce_len, 1275 const uint8_t *in, size_t in_len, 1276 const uint8_t *in_tag, size_t in_tag_len, 1277 const uint8_t *ad, size_t ad_len) { 1278 const struct aead_aes_gcm_ctx *gcm_ctx = ctx->aead_state; 1279 uint8_t tag[EVP_AEAD_AES_GCM_TAG_LEN]; 1280 GCM128_CONTEXT gcm; 1281 1282 if (nonce_len == 0) { 1283 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE_SIZE); 1284 return 0; 1285 } 1286 1287 if (in_tag_len != ctx->tag_len) { 1288 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT); 1289 return 0; 1290 } 1291 1292 const AES_KEY *key = &gcm_ctx->ks.ks; 1293 1294 OPENSSL_memcpy(&gcm, &gcm_ctx->gcm, sizeof(gcm)); 1295 CRYPTO_gcm128_setiv(&gcm, key, nonce, nonce_len); 1296 1297 if (!CRYPTO_gcm128_aad(&gcm, ad, ad_len)) { 1298 return 0; 1299 } 1300 1301 if (gcm_ctx->ctr) { 1302 if (!CRYPTO_gcm128_decrypt_ctr32(&gcm, key, in, out, in_len, 1303 gcm_ctx->ctr)) { 1304 return 0; 1305 } 1306 } else { 1307 if (!CRYPTO_gcm128_decrypt(&gcm, key, in, out, in_len)) { 1308 return 0; 1309 } 1310 } 1311 1312 CRYPTO_gcm128_tag(&gcm, tag, ctx->tag_len); 1313 if (CRYPTO_memcmp(tag, in_tag, ctx->tag_len) != 0) { 1314 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_BAD_DECRYPT); 1315 return 0; 1316 } 1317 1318 return 1; 1319 } 1320 1321 DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_128_gcm) { 1322 memset(out, 0, sizeof(EVP_AEAD)); 1323 1324 out->key_len = 16; 1325 out->nonce_len = 12; 1326 out->overhead = EVP_AEAD_AES_GCM_TAG_LEN; 1327 out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN; 1328 out->seal_scatter_supports_extra_in = 1; 1329 1330 out->init = aead_aes_gcm_init; 1331 out->cleanup = aead_aes_gcm_cleanup; 1332 out->seal_scatter = aead_aes_gcm_seal_scatter; 1333 out->open_gather = aead_aes_gcm_open_gather; 1334 } 1335 1336 DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm) { 1337 memset(out, 0, sizeof(EVP_AEAD)); 1338 1339 out->key_len = 32; 1340 out->nonce_len = 12; 1341 out->overhead = EVP_AEAD_AES_GCM_TAG_LEN; 1342 out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN; 1343 out->seal_scatter_supports_extra_in = 1; 1344 1345 out->init = aead_aes_gcm_init; 1346 out->cleanup = aead_aes_gcm_cleanup; 1347 out->seal_scatter = aead_aes_gcm_seal_scatter; 1348 out->open_gather = aead_aes_gcm_open_gather; 1349 } 1350 1351 static int aead_aes_gcm_tls12_init(EVP_AEAD_CTX *ctx, const uint8_t *key, 1352 size_t key_len, size_t requested_tag_len) { 1353 struct aead_aes_gcm_tls12_ctx *gcm_ctx; 1354 gcm_ctx = OPENSSL_malloc(sizeof(struct aead_aes_gcm_tls12_ctx)); 1355 if (gcm_ctx == NULL) { 1356 return 0; 1357 } 1358 1359 gcm_ctx->counter = 0; 1360 1361 size_t actual_tag_len; 1362 if (!aead_aes_gcm_init_impl(&gcm_ctx->gcm_ctx, &actual_tag_len, key, key_len, 1363 requested_tag_len)) { 1364 OPENSSL_free(gcm_ctx); 1365 return 0; 1366 } 1367 1368 ctx->aead_state = gcm_ctx; 1369 ctx->tag_len = actual_tag_len; 1370 return 1; 1371 } 1372 1373 static void aead_aes_gcm_tls12_cleanup(EVP_AEAD_CTX *ctx) { 1374 struct aead_aes_gcm_tls12_ctx *gcm_ctx = ctx->aead_state; 1375 OPENSSL_cleanse(gcm_ctx, sizeof(struct aead_aes_gcm_tls12_ctx)); 1376 OPENSSL_free(gcm_ctx); 1377 } 1378 1379 static int aead_aes_gcm_tls12_seal_scatter( 1380 const EVP_AEAD_CTX *ctx, uint8_t *out, uint8_t *out_tag, 1381 size_t *out_tag_len, size_t max_out_tag_len, const uint8_t *nonce, 1382 size_t nonce_len, const uint8_t *in, size_t in_len, const uint8_t *extra_in, 1383 size_t extra_in_len, const uint8_t *ad, size_t ad_len) { 1384 struct aead_aes_gcm_tls12_ctx *gcm_ctx = ctx->aead_state; 1385 if (gcm_ctx->counter == UINT64_MAX) { 1386 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE); 1387 return 0; 1388 } 1389 1390 if (nonce_len != 12) { 1391 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_UNSUPPORTED_NONCE_SIZE); 1392 return 0; 1393 } 1394 1395 const uint64_t be_counter = CRYPTO_bswap8(gcm_ctx->counter); 1396 if (OPENSSL_memcmp((uint8_t *)&be_counter, nonce + nonce_len - 8, 8) != 0) { 1397 OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_INVALID_NONCE); 1398 return 0; 1399 } 1400 1401 gcm_ctx->counter++; 1402 1403 return aead_aes_gcm_seal_scatter(ctx, out, out_tag, out_tag_len, 1404 max_out_tag_len, nonce, nonce_len, in, 1405 in_len, extra_in, extra_in_len, ad, ad_len); 1406 } 1407 1408 DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_128_gcm_tls12) { 1409 memset(out, 0, sizeof(EVP_AEAD)); 1410 1411 out->key_len = 16; 1412 out->nonce_len = 12; 1413 out->overhead = EVP_AEAD_AES_GCM_TAG_LEN; 1414 out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN; 1415 out->seal_scatter_supports_extra_in = 1; 1416 1417 out->init = aead_aes_gcm_tls12_init; 1418 out->cleanup = aead_aes_gcm_tls12_cleanup; 1419 out->seal_scatter = aead_aes_gcm_tls12_seal_scatter; 1420 out->open_gather = aead_aes_gcm_open_gather; 1421 } 1422 1423 DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm_tls12) { 1424 memset(out, 0, sizeof(EVP_AEAD)); 1425 1426 out->key_len = 32; 1427 out->nonce_len = 12; 1428 out->overhead = EVP_AEAD_AES_GCM_TAG_LEN; 1429 out->max_tag_len = EVP_AEAD_AES_GCM_TAG_LEN; 1430 out->seal_scatter_supports_extra_in = 1; 1431 1432 out->init = aead_aes_gcm_tls12_init; 1433 out->cleanup = aead_aes_gcm_tls12_cleanup; 1434 out->seal_scatter = aead_aes_gcm_tls12_seal_scatter; 1435 out->open_gather = aead_aes_gcm_open_gather; 1436 } 1437 1438 int EVP_has_aes_hardware(void) { 1439 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) 1440 return aesni_capable() && crypto_gcm_clmul_enabled(); 1441 #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) 1442 return hwaes_capable() && CRYPTO_is_ARMv8_PMULL_capable(); 1443 #else 1444 return 0; 1445 #endif 1446 } 1447
