1 /* Copyright (c) 2014, Google Inc. 2 * 3 * Permission to use, copy, modify, and/or distribute this software for any 4 * purpose with or without fee is hereby granted, provided that the above 5 * copyright notice and this permission notice appear in all copies. 6 * 7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ 14 15 /* This implementation of poly1305 is by Andrew Moon 16 * (https://github.com/floodyberry/poly1305-donna) and released as public 17 * domain. */ 18 19 #include <openssl/poly1305.h> 20 21 #include <string.h> 22 23 #include <openssl/cpu.h> 24 25 26 #if defined(OPENSSL_WINDOWS) || !defined(OPENSSL_X86_64) 27 28 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || defined(OPENSSL_ARM) 29 /* We can assume little-endian. */ 30 static uint32_t U8TO32_LE(const uint8_t *m) { 31 uint32_t r; 32 memcpy(&r, m, sizeof(r)); 33 return r; 34 } 35 36 static void U32TO8_LE(uint8_t *m, uint32_t v) { memcpy(m, &v, sizeof(v)); } 37 #else 38 static uint32_t U8TO32_LE(const uint8_t *m) { 39 return (uint32_t)m[0] | (uint32_t)m[1] << 8 | (uint32_t)m[2] << 16 | 40 (uint32_t)m[3] << 24; 41 } 42 43 static void U32TO8_LE(uint8_t *m, uint32_t v) { 44 m[0] = v; 45 m[1] = v >> 8; 46 m[2] = v >> 16; 47 m[3] = v >> 24; 48 } 49 #endif 50 51 #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 52 void CRYPTO_poly1305_init_neon(poly1305_state *state, const uint8_t key[32]); 53 54 void CRYPTO_poly1305_update_neon(poly1305_state *state, const uint8_t *in, 55 size_t in_len); 56 57 void CRYPTO_poly1305_finish_neon(poly1305_state *state, uint8_t mac[16]); 58 #endif 59 60 static uint64_t mul32x32_64(uint32_t a, uint32_t b) { return (uint64_t)a * b; } 61 62 struct poly1305_state_st { 63 uint32_t r0, r1, r2, r3, r4; 64 uint32_t s1, s2, s3, s4; 65 uint32_t h0, h1, h2, h3, h4; 66 uint8_t buf[16]; 67 unsigned int buf_used; 68 uint8_t key[16]; 69 }; 70 71 /* poly1305_blocks updates |state| given some amount of input data. This 72 * function may only be called with a |len| that is not a multiple of 16 at the 73 * end of the data. Otherwise the input must be buffered into 16 byte blocks. */ 74 static void poly1305_update(struct poly1305_state_st *state, const uint8_t *in, 75 size_t len) { 76 uint32_t t0, t1, t2, t3; 77 uint64_t t[5]; 78 uint32_t b; 79 uint64_t c; 80 size_t j; 81 uint8_t mp[16]; 82 83 if (len < 16) { 84 goto poly1305_donna_atmost15bytes; 85 } 86 87 poly1305_donna_16bytes: 88 t0 = U8TO32_LE(in); 89 t1 = U8TO32_LE(in + 4); 90 t2 = U8TO32_LE(in + 8); 91 t3 = U8TO32_LE(in + 12); 92 93 in += 16; 94 len -= 16; 95 96 state->h0 += t0 & 0x3ffffff; 97 state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; 98 state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; 99 state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; 100 state->h4 += (t3 >> 8) | (1 << 24); 101 102 poly1305_donna_mul: 103 t[0] = mul32x32_64(state->h0, state->r0) + mul32x32_64(state->h1, state->s4) + 104 mul32x32_64(state->h2, state->s3) + mul32x32_64(state->h3, state->s2) + 105 mul32x32_64(state->h4, state->s1); 106 t[1] = mul32x32_64(state->h0, state->r1) + mul32x32_64(state->h1, state->r0) + 107 mul32x32_64(state->h2, state->s4) + mul32x32_64(state->h3, state->s3) + 108 mul32x32_64(state->h4, state->s2); 109 t[2] = mul32x32_64(state->h0, state->r2) + mul32x32_64(state->h1, state->r1) + 110 mul32x32_64(state->h2, state->r0) + mul32x32_64(state->h3, state->s4) + 111 mul32x32_64(state->h4, state->s3); 112 t[3] = mul32x32_64(state->h0, state->r3) + mul32x32_64(state->h1, state->r2) + 113 mul32x32_64(state->h2, state->r1) + mul32x32_64(state->h3, state->r0) + 114 mul32x32_64(state->h4, state->s4); 115 t[4] = mul32x32_64(state->h0, state->r4) + mul32x32_64(state->h1, state->r3) + 116 mul32x32_64(state->h2, state->r2) + mul32x32_64(state->h3, state->r1) + 117 mul32x32_64(state->h4, state->r0); 118 119 state->h0 = (uint32_t)t[0] & 0x3ffffff; 120 c = (t[0] >> 26); 121 t[1] += c; 122 state->h1 = (uint32_t)t[1] & 0x3ffffff; 123 b = (uint32_t)(t[1] >> 26); 124 t[2] += b; 125 state->h2 = (uint32_t)t[2] & 0x3ffffff; 126 b = (uint32_t)(t[2] >> 26); 127 t[3] += b; 128 state->h3 = (uint32_t)t[3] & 0x3ffffff; 129 b = (uint32_t)(t[3] >> 26); 130 t[4] += b; 131 state->h4 = (uint32_t)t[4] & 0x3ffffff; 132 b = (uint32_t)(t[4] >> 26); 133 state->h0 += b * 5; 134 135 if (len >= 16) { 136 goto poly1305_donna_16bytes; 137 } 138 139 /* final bytes */ 140 poly1305_donna_atmost15bytes: 141 if (!len) { 142 return; 143 } 144 145 for (j = 0; j < len; j++) { 146 mp[j] = in[j]; 147 } 148 mp[j++] = 1; 149 for (; j < 16; j++) { 150 mp[j] = 0; 151 } 152 len = 0; 153 154 t0 = U8TO32_LE(mp + 0); 155 t1 = U8TO32_LE(mp + 4); 156 t2 = U8TO32_LE(mp + 8); 157 t3 = U8TO32_LE(mp + 12); 158 159 state->h0 += t0 & 0x3ffffff; 160 state->h1 += ((((uint64_t)t1 << 32) | t0) >> 26) & 0x3ffffff; 161 state->h2 += ((((uint64_t)t2 << 32) | t1) >> 20) & 0x3ffffff; 162 state->h3 += ((((uint64_t)t3 << 32) | t2) >> 14) & 0x3ffffff; 163 state->h4 += (t3 >> 8); 164 165 goto poly1305_donna_mul; 166 } 167 168 void CRYPTO_poly1305_init(poly1305_state *statep, const uint8_t key[32]) { 169 struct poly1305_state_st *state = (struct poly1305_state_st *)statep; 170 uint32_t t0, t1, t2, t3; 171 172 #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 173 if (CRYPTO_is_NEON_functional()) { 174 CRYPTO_poly1305_init_neon(statep, key); 175 return; 176 } 177 #endif 178 179 t0 = U8TO32_LE(key + 0); 180 t1 = U8TO32_LE(key + 4); 181 t2 = U8TO32_LE(key + 8); 182 t3 = U8TO32_LE(key + 12); 183 184 /* precompute multipliers */ 185 state->r0 = t0 & 0x3ffffff; 186 t0 >>= 26; 187 t0 |= t1 << 6; 188 state->r1 = t0 & 0x3ffff03; 189 t1 >>= 20; 190 t1 |= t2 << 12; 191 state->r2 = t1 & 0x3ffc0ff; 192 t2 >>= 14; 193 t2 |= t3 << 18; 194 state->r3 = t2 & 0x3f03fff; 195 t3 >>= 8; 196 state->r4 = t3 & 0x00fffff; 197 198 state->s1 = state->r1 * 5; 199 state->s2 = state->r2 * 5; 200 state->s3 = state->r3 * 5; 201 state->s4 = state->r4 * 5; 202 203 /* init state */ 204 state->h0 = 0; 205 state->h1 = 0; 206 state->h2 = 0; 207 state->h3 = 0; 208 state->h4 = 0; 209 210 state->buf_used = 0; 211 memcpy(state->key, key + 16, sizeof(state->key)); 212 } 213 214 void CRYPTO_poly1305_update(poly1305_state *statep, const uint8_t *in, 215 size_t in_len) { 216 unsigned int i; 217 struct poly1305_state_st *state = (struct poly1305_state_st *)statep; 218 219 #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 220 if (CRYPTO_is_NEON_functional()) { 221 CRYPTO_poly1305_update_neon(statep, in, in_len); 222 return; 223 } 224 #endif 225 226 if (state->buf_used) { 227 unsigned int todo = 16 - state->buf_used; 228 if (todo > in_len) { 229 todo = in_len; 230 } 231 for (i = 0; i < todo; i++) { 232 state->buf[state->buf_used + i] = in[i]; 233 } 234 state->buf_used += todo; 235 in_len -= todo; 236 in += todo; 237 238 if (state->buf_used == 16) { 239 poly1305_update(state, state->buf, 16); 240 state->buf_used = 0; 241 } 242 } 243 244 if (in_len >= 16) { 245 size_t todo = in_len & ~0xf; 246 poly1305_update(state, in, todo); 247 in += todo; 248 in_len &= 0xf; 249 } 250 251 if (in_len) { 252 for (i = 0; i < in_len; i++) { 253 state->buf[i] = in[i]; 254 } 255 state->buf_used = in_len; 256 } 257 } 258 259 void CRYPTO_poly1305_finish(poly1305_state *statep, uint8_t mac[16]) { 260 struct poly1305_state_st *state = (struct poly1305_state_st *)statep; 261 uint64_t f0, f1, f2, f3; 262 uint32_t g0, g1, g2, g3, g4; 263 uint32_t b, nb; 264 265 #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) 266 if (CRYPTO_is_NEON_functional()) { 267 CRYPTO_poly1305_finish_neon(statep, mac); 268 return; 269 } 270 #endif 271 272 if (state->buf_used) { 273 poly1305_update(state, state->buf, state->buf_used); 274 } 275 276 b = state->h0 >> 26; 277 state->h0 = state->h0 & 0x3ffffff; 278 state->h1 += b; 279 b = state->h1 >> 26; 280 state->h1 = state->h1 & 0x3ffffff; 281 state->h2 += b; 282 b = state->h2 >> 26; 283 state->h2 = state->h2 & 0x3ffffff; 284 state->h3 += b; 285 b = state->h3 >> 26; 286 state->h3 = state->h3 & 0x3ffffff; 287 state->h4 += b; 288 b = state->h4 >> 26; 289 state->h4 = state->h4 & 0x3ffffff; 290 state->h0 += b * 5; 291 292 g0 = state->h0 + 5; 293 b = g0 >> 26; 294 g0 &= 0x3ffffff; 295 g1 = state->h1 + b; 296 b = g1 >> 26; 297 g1 &= 0x3ffffff; 298 g2 = state->h2 + b; 299 b = g2 >> 26; 300 g2 &= 0x3ffffff; 301 g3 = state->h3 + b; 302 b = g3 >> 26; 303 g3 &= 0x3ffffff; 304 g4 = state->h4 + b - (1 << 26); 305 306 b = (g4 >> 31) - 1; 307 nb = ~b; 308 state->h0 = (state->h0 & nb) | (g0 & b); 309 state->h1 = (state->h1 & nb) | (g1 & b); 310 state->h2 = (state->h2 & nb) | (g2 & b); 311 state->h3 = (state->h3 & nb) | (g3 & b); 312 state->h4 = (state->h4 & nb) | (g4 & b); 313 314 f0 = ((state->h0) | (state->h1 << 26)) + (uint64_t)U8TO32_LE(&state->key[0]); 315 f1 = ((state->h1 >> 6) | (state->h2 << 20)) + 316 (uint64_t)U8TO32_LE(&state->key[4]); 317 f2 = ((state->h2 >> 12) | (state->h3 << 14)) + 318 (uint64_t)U8TO32_LE(&state->key[8]); 319 f3 = ((state->h3 >> 18) | (state->h4 << 8)) + 320 (uint64_t)U8TO32_LE(&state->key[12]); 321 322 U32TO8_LE(&mac[0], f0); 323 f1 += (f0 >> 32); 324 U32TO8_LE(&mac[4], f1); 325 f2 += (f1 >> 32); 326 U32TO8_LE(&mac[8], f2); 327 f3 += (f2 >> 32); 328 U32TO8_LE(&mac[12], f3); 329 } 330 331 #endif /* OPENSSL_WINDOWS || !OPENSSL_X86_64 */ 332
