1 // Copyright 2016 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package aes 6 7 import ( 8 "crypto/cipher" 9 "crypto/subtle" 10 "errors" 11 ) 12 13 // This file contains two implementations of AES-GCM. The first implementation 14 // (gcmAsm) uses the KMCTR instruction to encrypt using AES in counter mode and 15 // the KIMD instruction for GHASH. The second implementation (gcmKMA) uses the 16 // newer KMA instruction which performs both operations. 17 18 // gcmCount represents a 16-byte big-endian count value. 19 type gcmCount [16]byte 20 21 // inc increments the rightmost 32-bits of the count value by 1. 22 func (x *gcmCount) inc() { 23 // The compiler should optimize this to a 32-bit addition. 24 n := uint32(x[15]) | uint32(x[14])<<8 | uint32(x[13])<<16 | uint32(x[12])<<24 25 n += 1 26 x[12] = byte(n >> 24) 27 x[13] = byte(n >> 16) 28 x[14] = byte(n >> 8) 29 x[15] = byte(n) 30 } 31 32 // gcmLengths writes len0 || len1 as big-endian values to a 16-byte array. 33 func gcmLengths(len0, len1 uint64) [16]byte { 34 return [16]byte{ 35 byte(len0 >> 56), 36 byte(len0 >> 48), 37 byte(len0 >> 40), 38 byte(len0 >> 32), 39 byte(len0 >> 24), 40 byte(len0 >> 16), 41 byte(len0 >> 8), 42 byte(len0), 43 byte(len1 >> 56), 44 byte(len1 >> 48), 45 byte(len1 >> 40), 46 byte(len1 >> 32), 47 byte(len1 >> 24), 48 byte(len1 >> 16), 49 byte(len1 >> 8), 50 byte(len1), 51 } 52 } 53 54 // gcmHashKey represents the 16-byte hash key required by the GHASH algorithm. 55 type gcmHashKey [16]byte 56 57 type gcmAsm struct { 58 block *aesCipherAsm 59 hashKey gcmHashKey 60 nonceSize int 61 } 62 63 const ( 64 gcmBlockSize = 16 65 gcmTagSize = 16 66 gcmStandardNonceSize = 12 67 ) 68 69 var errOpen = errors.New("cipher: message authentication failed") 70 71 // Assert that aesCipherAsm implements the gcmAble interface. 72 var _ gcmAble = (*aesCipherAsm)(nil) 73 74 // NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only 75 // called by crypto/cipher.NewGCM via the gcmAble interface. 76 func (c *aesCipherAsm) NewGCM(nonceSize int) (cipher.AEAD, error) { 77 var hk gcmHashKey 78 c.Encrypt(hk[:], hk[:]) 79 g := gcmAsm{ 80 block: c, 81 hashKey: hk, 82 nonceSize: nonceSize, 83 } 84 if hasKMA { 85 g := gcmKMA{g} 86 return &g, nil 87 } 88 return &g, nil 89 } 90 91 func (g *gcmAsm) NonceSize() int { 92 return g.nonceSize 93 } 94 95 func (*gcmAsm) Overhead() int { 96 return gcmTagSize 97 } 98 99 // sliceForAppend takes a slice and a requested number of bytes. It returns a 100 // slice with the contents of the given slice followed by that many bytes and a 101 // second slice that aliases into it and contains only the extra bytes. If the 102 // original slice has sufficient capacity then no allocation is performed. 103 func sliceForAppend(in []byte, n int) (head, tail []byte) { 104 if total := len(in) + n; cap(in) >= total { 105 head = in[:total] 106 } else { 107 head = make([]byte, total) 108 copy(head, in) 109 } 110 tail = head[len(in):] 111 return 112 } 113 114 // ghash uses the GHASH algorithm to hash data with the given key. The initial 115 // hash value is given by hash which will be updated with the new hash value. 116 // The length of data must be a multiple of 16-bytes. 117 //go:noescape 118 func ghash(key *gcmHashKey, hash *[16]byte, data []byte) 119 120 // paddedGHASH pads data with zeroes until its length is a multiple of 121 // 16-bytes. It then calculates a new value for hash using the GHASH algorithm. 122 func (g *gcmAsm) paddedGHASH(hash *[16]byte, data []byte) { 123 siz := len(data) &^ 0xf // align size to 16-bytes 124 if siz > 0 { 125 ghash(&g.hashKey, hash, data[:siz]) 126 data = data[siz:] 127 } 128 if len(data) > 0 { 129 var s [16]byte 130 copy(s[:], data) 131 ghash(&g.hashKey, hash, s[:]) 132 } 133 } 134 135 // cryptBlocksGCM encrypts src using AES in counter mode using the given 136 // function code and key. The rightmost 32-bits of the counter are incremented 137 // between each block as required by the GCM spec. The initial counter value 138 // is given by cnt, which is updated with the value of the next counter value 139 // to use. 140 // 141 // The lengths of both dst and buf must be greater than or equal to the length 142 // of src. buf may be partially or completely overwritten during the execution 143 // of the function. 144 //go:noescape 145 func cryptBlocksGCM(fn code, key, dst, src, buf []byte, cnt *gcmCount) 146 147 // counterCrypt encrypts src using AES in counter mode and places the result 148 // into dst. cnt is the initial count value and will be updated with the next 149 // count value. The length of dst must be greater than or equal to the length 150 // of src. 151 func (g *gcmAsm) counterCrypt(dst, src []byte, cnt *gcmCount) { 152 // Copying src into a buffer improves performance on some models when 153 // src and dst point to the same underlying array. We also need a 154 // buffer for counter values. 155 var ctrbuf, srcbuf [2048]byte 156 for len(src) >= 16 { 157 siz := len(src) 158 if len(src) > len(ctrbuf) { 159 siz = len(ctrbuf) 160 } 161 siz &^= 0xf // align siz to 16-bytes 162 copy(srcbuf[:], src[:siz]) 163 cryptBlocksGCM(g.block.function, g.block.key, dst[:siz], srcbuf[:siz], ctrbuf[:], cnt) 164 src = src[siz:] 165 dst = dst[siz:] 166 } 167 if len(src) > 0 { 168 var x [16]byte 169 g.block.Encrypt(x[:], cnt[:]) 170 for i := range src { 171 dst[i] = src[i] ^ x[i] 172 } 173 cnt.inc() 174 } 175 } 176 177 // deriveCounter computes the initial GCM counter state from the given nonce. 178 // See NIST SP 800-38D, section 7.1. 179 func (g *gcmAsm) deriveCounter(nonce []byte) gcmCount { 180 // GCM has two modes of operation with respect to the initial counter 181 // state: a "fast path" for 96-bit (12-byte) nonces, and a "slow path" 182 // for nonces of other lengths. For a 96-bit nonce, the nonce, along 183 // with a four-byte big-endian counter starting at one, is used 184 // directly as the starting counter. For other nonce sizes, the counter 185 // is computed by passing it through the GHASH function. 186 var counter gcmCount 187 if len(nonce) == gcmStandardNonceSize { 188 copy(counter[:], nonce) 189 counter[gcmBlockSize-1] = 1 190 } else { 191 var hash [16]byte 192 g.paddedGHASH(&hash, nonce) 193 lens := gcmLengths(0, uint64(len(nonce))*8) 194 g.paddedGHASH(&hash, lens[:]) 195 copy(counter[:], hash[:]) 196 } 197 return counter 198 } 199 200 // auth calculates GHASH(ciphertext, additionalData), masks the result with 201 // tagMask and writes the result to out. 202 func (g *gcmAsm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize]byte) { 203 var hash [16]byte 204 g.paddedGHASH(&hash, additionalData) 205 g.paddedGHASH(&hash, ciphertext) 206 lens := gcmLengths(uint64(len(additionalData))*8, uint64(len(ciphertext))*8) 207 g.paddedGHASH(&hash, lens[:]) 208 209 copy(out, hash[:]) 210 for i := range out { 211 out[i] ^= tagMask[i] 212 } 213 } 214 215 // Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for 216 // details. 217 func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte { 218 if len(nonce) != g.nonceSize { 219 panic("cipher: incorrect nonce length given to GCM") 220 } 221 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 222 panic("cipher: message too large for GCM") 223 } 224 225 ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize) 226 227 counter := g.deriveCounter(nonce) 228 229 var tagMask [gcmBlockSize]byte 230 g.block.Encrypt(tagMask[:], counter[:]) 231 counter.inc() 232 233 g.counterCrypt(out, plaintext, &counter) 234 g.auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask) 235 236 return ret 237 } 238 239 // Open authenticates and decrypts ciphertext. See the cipher.AEAD interface 240 // for details. 241 func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 242 if len(nonce) != g.nonceSize { 243 panic("cipher: incorrect nonce length given to GCM") 244 } 245 if len(ciphertext) < gcmTagSize { 246 return nil, errOpen 247 } 248 if uint64(len(ciphertext)) > ((1<<32)-2)*BlockSize+gcmTagSize { 249 return nil, errOpen 250 } 251 252 tag := ciphertext[len(ciphertext)-gcmTagSize:] 253 ciphertext = ciphertext[:len(ciphertext)-gcmTagSize] 254 255 counter := g.deriveCounter(nonce) 256 257 var tagMask [gcmBlockSize]byte 258 g.block.Encrypt(tagMask[:], counter[:]) 259 counter.inc() 260 261 var expectedTag [gcmTagSize]byte 262 g.auth(expectedTag[:], ciphertext, data, &tagMask) 263 264 ret, out := sliceForAppend(dst, len(ciphertext)) 265 266 if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 { 267 // The AESNI code decrypts and authenticates concurrently, and 268 // so overwrites dst in the event of a tag mismatch. That 269 // behavior is mimicked here in order to be consistent across 270 // platforms. 271 for i := range out { 272 out[i] = 0 273 } 274 return nil, errOpen 275 } 276 277 g.counterCrypt(out, ciphertext, &counter) 278 return ret, nil 279 } 280 281 // supportsKMA reports whether the message-security-assist 8 facility is available. 282 // This function call may be expensive so hasKMA should be queried instead. 283 func supportsKMA() bool 284 285 // hasKMA contains the result of supportsKMA. 286 var hasKMA = supportsKMA() 287 288 // gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should 289 // only be used if hasKMA is true. 290 type gcmKMA struct { 291 gcmAsm 292 } 293 294 // flags for the KMA instruction 295 const ( 296 kmaHS = 1 << 10 // hash subkey supplied 297 kmaLAAD = 1 << 9 // last series of additional authenticated data 298 kmaLPC = 1 << 8 // last series of plaintext or ciphertext blocks 299 kmaDecrypt = 1 << 7 // decrypt 300 ) 301 302 // kmaGCM executes the encryption or decryption operation given by fn. The tag 303 // will be calculated and written to tag. cnt should contain the current 304 // counter state and will be overwritten with the updated counter state. 305 // TODO(mundaym): could pass in hash subkey 306 //go:noescape 307 func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount) 308 309 // Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for 310 // details. 311 func (g *gcmKMA) Seal(dst, nonce, plaintext, data []byte) []byte { 312 if len(nonce) != g.nonceSize { 313 panic("cipher: incorrect nonce length given to GCM") 314 } 315 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 316 panic("cipher: message too large for GCM") 317 } 318 319 ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize) 320 321 counter := g.deriveCounter(nonce) 322 fc := g.block.function | kmaLAAD | kmaLPC 323 324 var tag [gcmTagSize]byte 325 kmaGCM(fc, g.block.key, out[:len(plaintext)], plaintext, data, &tag, &counter) 326 copy(out[len(plaintext):], tag[:]) 327 328 return ret 329 } 330 331 // Open authenticates and decrypts ciphertext. See the cipher.AEAD interface 332 // for details. 333 func (g *gcmKMA) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 334 if len(nonce) != g.nonceSize { 335 panic("cipher: incorrect nonce length given to GCM") 336 } 337 if len(ciphertext) < gcmTagSize { 338 return nil, errOpen 339 } 340 if uint64(len(ciphertext)) > ((1<<32)-2)*BlockSize+gcmTagSize { 341 return nil, errOpen 342 } 343 344 tag := ciphertext[len(ciphertext)-gcmTagSize:] 345 ciphertext = ciphertext[:len(ciphertext)-gcmTagSize] 346 ret, out := sliceForAppend(dst, len(ciphertext)) 347 348 counter := g.deriveCounter(nonce) 349 fc := g.block.function | kmaLAAD | kmaLPC | kmaDecrypt 350 351 var expectedTag [gcmTagSize]byte 352 kmaGCM(fc, g.block.key, out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter) 353 354 if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 { 355 // The AESNI code decrypts and authenticates concurrently, and 356 // so overwrites dst in the event of a tag mismatch. That 357 // behavior is mimicked here in order to be consistent across 358 // platforms. 359 for i := range out { 360 out[i] = 0 361 } 362 return nil, errOpen 363 } 364 365 return ret, nil 366 } 367
