summaryrefslogtreecommitdiff
path: root/crypto/gcm.c (plain)
blob: 398f048c452af2a1148edda5cb9abac02301d836
1/*
2 * GCM: Galois/Counter Mode.
3 *
4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <crypto/gf128mul.h>
12#include <crypto/internal/aead.h>
13#include <crypto/internal/skcipher.h>
14#include <crypto/internal/hash.h>
15#include <crypto/null.h>
16#include <crypto/scatterwalk.h>
17#include <crypto/hash.h>
18#include "internal.h"
19#include <linux/completion.h>
20#include <linux/err.h>
21#include <linux/init.h>
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/slab.h>
25
26struct gcm_instance_ctx {
27 struct crypto_skcipher_spawn ctr;
28 struct crypto_ahash_spawn ghash;
29};
30
31struct crypto_gcm_ctx {
32 struct crypto_skcipher *ctr;
33 struct crypto_ahash *ghash;
34};
35
36struct crypto_rfc4106_ctx {
37 struct crypto_aead *child;
38 u8 nonce[4];
39};
40
41struct crypto_rfc4106_req_ctx {
42 struct scatterlist src[3];
43 struct scatterlist dst[3];
44 struct aead_request subreq;
45};
46
47struct crypto_rfc4543_instance_ctx {
48 struct crypto_aead_spawn aead;
49};
50
51struct crypto_rfc4543_ctx {
52 struct crypto_aead *child;
53 struct crypto_skcipher *null;
54 u8 nonce[4];
55};
56
57struct crypto_rfc4543_req_ctx {
58 struct aead_request subreq;
59};
60
61struct crypto_gcm_ghash_ctx {
62 unsigned int cryptlen;
63 struct scatterlist *src;
64 int (*complete)(struct aead_request *req, u32 flags);
65};
66
67struct crypto_gcm_req_priv_ctx {
68 u8 iv[16];
69 u8 auth_tag[16];
70 u8 iauth_tag[16];
71 struct scatterlist src[3];
72 struct scatterlist dst[3];
73 struct scatterlist sg;
74 struct crypto_gcm_ghash_ctx ghash_ctx;
75 union {
76 struct ahash_request ahreq;
77 struct skcipher_request skreq;
78 } u;
79};
80
81struct crypto_gcm_setkey_result {
82 int err;
83 struct completion completion;
84};
85
86static struct {
87 u8 buf[16];
88 struct scatterlist sg;
89} *gcm_zeroes;
90
91static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
92
93static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
94 struct aead_request *req)
95{
96 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
97
98 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
99}
100
101static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
102{
103 struct crypto_gcm_setkey_result *result = req->data;
104
105 if (err == -EINPROGRESS)
106 return;
107
108 result->err = err;
109 complete(&result->completion);
110}
111
112static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
113 unsigned int keylen)
114{
115 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
116 struct crypto_ahash *ghash = ctx->ghash;
117 struct crypto_skcipher *ctr = ctx->ctr;
118 struct {
119 be128 hash;
120 u8 iv[16];
121
122 struct crypto_gcm_setkey_result result;
123
124 struct scatterlist sg[1];
125 struct skcipher_request req;
126 } *data;
127 int err;
128
129 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
130 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
131 CRYPTO_TFM_REQ_MASK);
132 err = crypto_skcipher_setkey(ctr, key, keylen);
133 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
134 CRYPTO_TFM_RES_MASK);
135 if (err)
136 return err;
137
138 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
139 GFP_KERNEL);
140 if (!data)
141 return -ENOMEM;
142
143 init_completion(&data->result.completion);
144 sg_init_one(data->sg, &data->hash, sizeof(data->hash));
145 skcipher_request_set_tfm(&data->req, ctr);
146 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
147 CRYPTO_TFM_REQ_MAY_BACKLOG,
148 crypto_gcm_setkey_done,
149 &data->result);
150 skcipher_request_set_crypt(&data->req, data->sg, data->sg,
151 sizeof(data->hash), data->iv);
152
153 err = crypto_skcipher_encrypt(&data->req);
154 if (err == -EINPROGRESS || err == -EBUSY) {
155 wait_for_completion(&data->result.completion);
156 err = data->result.err;
157 }
158
159 if (err)
160 goto out;
161
162 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
163 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
164 CRYPTO_TFM_REQ_MASK);
165 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
166 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
167 CRYPTO_TFM_RES_MASK);
168
169out:
170 kzfree(data);
171 return err;
172}
173
174static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
175 unsigned int authsize)
176{
177 switch (authsize) {
178 case 4:
179 case 8:
180 case 12:
181 case 13:
182 case 14:
183 case 15:
184 case 16:
185 break;
186 default:
187 return -EINVAL;
188 }
189
190 return 0;
191}
192
193static void crypto_gcm_init_common(struct aead_request *req)
194{
195 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
196 __be32 counter = cpu_to_be32(1);
197 struct scatterlist *sg;
198
199 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
200 memcpy(pctx->iv, req->iv, 12);
201 memcpy(pctx->iv + 12, &counter, 4);
202
203 sg_init_table(pctx->src, 3);
204 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
205 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);
206 if (sg != pctx->src + 1)
207 sg_chain(pctx->src, 2, sg);
208
209 if (req->src != req->dst) {
210 sg_init_table(pctx->dst, 3);
211 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
212 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);
213 if (sg != pctx->dst + 1)
214 sg_chain(pctx->dst, 2, sg);
215 }
216}
217
218static void crypto_gcm_init_crypt(struct aead_request *req,
219 unsigned int cryptlen)
220{
221 struct crypto_aead *aead = crypto_aead_reqtfm(req);
222 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
223 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
224 struct skcipher_request *skreq = &pctx->u.skreq;
225 struct scatterlist *dst;
226
227 dst = req->src == req->dst ? pctx->src : pctx->dst;
228
229 skcipher_request_set_tfm(skreq, ctx->ctr);
230 skcipher_request_set_crypt(skreq, pctx->src, dst,
231 cryptlen + sizeof(pctx->auth_tag),
232 pctx->iv);
233}
234
235static inline unsigned int gcm_remain(unsigned int len)
236{
237 len &= 0xfU;
238 return len ? 16 - len : 0;
239}
240
241static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
242
243static int gcm_hash_update(struct aead_request *req,
244 crypto_completion_t compl,
245 struct scatterlist *src,
246 unsigned int len, u32 flags)
247{
248 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
249 struct ahash_request *ahreq = &pctx->u.ahreq;
250
251 ahash_request_set_callback(ahreq, flags, compl, req);
252 ahash_request_set_crypt(ahreq, src, NULL, len);
253
254 return crypto_ahash_update(ahreq);
255}
256
257static int gcm_hash_remain(struct aead_request *req,
258 unsigned int remain,
259 crypto_completion_t compl, u32 flags)
260{
261 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);
262}
263
264static int gcm_hash_len(struct aead_request *req, u32 flags)
265{
266 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
267 struct ahash_request *ahreq = &pctx->u.ahreq;
268 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
269 u128 lengths;
270
271 lengths.a = cpu_to_be64(req->assoclen * 8);
272 lengths.b = cpu_to_be64(gctx->cryptlen * 8);
273 memcpy(pctx->iauth_tag, &lengths, 16);
274 sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
275 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);
276 ahash_request_set_crypt(ahreq, &pctx->sg,
277 pctx->iauth_tag, sizeof(lengths));
278
279 return crypto_ahash_finup(ahreq);
280}
281
282static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
283{
284 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
285 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
286
287 return gctx->complete(req, flags);
288}
289
290static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
291{
292 struct aead_request *req = areq->data;
293
294 if (err)
295 goto out;
296
297 err = gcm_hash_len_continue(req, 0);
298 if (err == -EINPROGRESS)
299 return;
300
301out:
302 aead_request_complete(req, err);
303}
304
305static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
306{
307 return gcm_hash_len(req, flags) ?:
308 gcm_hash_len_continue(req, flags);
309}
310
311static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
312 int err)
313{
314 struct aead_request *req = areq->data;
315
316 if (err)
317 goto out;
318
319 err = gcm_hash_crypt_remain_continue(req, 0);
320 if (err == -EINPROGRESS)
321 return;
322
323out:
324 aead_request_complete(req, err);
325}
326
327static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
328{
329 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
330 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
331 unsigned int remain;
332
333 remain = gcm_remain(gctx->cryptlen);
334 if (remain)
335 return gcm_hash_remain(req, remain,
336 gcm_hash_crypt_remain_done, flags) ?:
337 gcm_hash_crypt_remain_continue(req, flags);
338
339 return gcm_hash_crypt_remain_continue(req, flags);
340}
341
342static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
343{
344 struct aead_request *req = areq->data;
345
346 if (err)
347 goto out;
348
349 err = gcm_hash_crypt_continue(req, 0);
350 if (err == -EINPROGRESS)
351 return;
352
353out:
354 aead_request_complete(req, err);
355}
356
357static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
358{
359 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
360 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
361
362 if (gctx->cryptlen)
363 return gcm_hash_update(req, gcm_hash_crypt_done,
364 gctx->src, gctx->cryptlen, flags) ?:
365 gcm_hash_crypt_continue(req, flags);
366
367 return gcm_hash_crypt_remain_continue(req, flags);
368}
369
370static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
371 int err)
372{
373 struct aead_request *req = areq->data;
374
375 if (err)
376 goto out;
377
378 err = gcm_hash_assoc_remain_continue(req, 0);
379 if (err == -EINPROGRESS)
380 return;
381
382out:
383 aead_request_complete(req, err);
384}
385
386static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
387{
388 unsigned int remain;
389
390 remain = gcm_remain(req->assoclen);
391 if (remain)
392 return gcm_hash_remain(req, remain,
393 gcm_hash_assoc_remain_done, flags) ?:
394 gcm_hash_assoc_remain_continue(req, flags);
395
396 return gcm_hash_assoc_remain_continue(req, flags);
397}
398
399static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
400{
401 struct aead_request *req = areq->data;
402
403 if (err)
404 goto out;
405
406 err = gcm_hash_assoc_continue(req, 0);
407 if (err == -EINPROGRESS)
408 return;
409
410out:
411 aead_request_complete(req, err);
412}
413
414static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
415{
416 if (req->assoclen)
417 return gcm_hash_update(req, gcm_hash_assoc_done,
418 req->src, req->assoclen, flags) ?:
419 gcm_hash_assoc_continue(req, flags);
420
421 return gcm_hash_assoc_remain_continue(req, flags);
422}
423
424static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
425{
426 struct aead_request *req = areq->data;
427
428 if (err)
429 goto out;
430
431 err = gcm_hash_init_continue(req, 0);
432 if (err == -EINPROGRESS)
433 return;
434
435out:
436 aead_request_complete(req, err);
437}
438
439static int gcm_hash(struct aead_request *req, u32 flags)
440{
441 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
442 struct ahash_request *ahreq = &pctx->u.ahreq;
443 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
444
445 ahash_request_set_tfm(ahreq, ctx->ghash);
446
447 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);
448 return crypto_ahash_init(ahreq) ?:
449 gcm_hash_init_continue(req, flags);
450}
451
452static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
453{
454 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
455 struct crypto_aead *aead = crypto_aead_reqtfm(req);
456 u8 *auth_tag = pctx->auth_tag;
457
458 crypto_xor(auth_tag, pctx->iauth_tag, 16);
459 scatterwalk_map_and_copy(auth_tag, req->dst,
460 req->assoclen + req->cryptlen,
461 crypto_aead_authsize(aead), 1);
462 return 0;
463}
464
465static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
466{
467 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
468 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
469
470 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);
471 gctx->cryptlen = req->cryptlen;
472 gctx->complete = gcm_enc_copy_hash;
473
474 return gcm_hash(req, flags);
475}
476
477static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
478{
479 struct aead_request *req = areq->data;
480
481 if (err)
482 goto out;
483
484 err = gcm_encrypt_continue(req, 0);
485 if (err == -EINPROGRESS)
486 return;
487
488out:
489 aead_request_complete(req, err);
490}
491
492static int crypto_gcm_encrypt(struct aead_request *req)
493{
494 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
495 struct skcipher_request *skreq = &pctx->u.skreq;
496 u32 flags = aead_request_flags(req);
497
498 crypto_gcm_init_common(req);
499 crypto_gcm_init_crypt(req, req->cryptlen);
500 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
501
502 return crypto_skcipher_encrypt(skreq) ?:
503 gcm_encrypt_continue(req, flags);
504}
505
506static int crypto_gcm_verify(struct aead_request *req)
507{
508 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
509 struct crypto_aead *aead = crypto_aead_reqtfm(req);
510 u8 *auth_tag = pctx->auth_tag;
511 u8 *iauth_tag = pctx->iauth_tag;
512 unsigned int authsize = crypto_aead_authsize(aead);
513 unsigned int cryptlen = req->cryptlen - authsize;
514
515 crypto_xor(auth_tag, iauth_tag, 16);
516 scatterwalk_map_and_copy(iauth_tag, req->src,
517 req->assoclen + cryptlen, authsize, 0);
518 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
519}
520
521static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
522{
523 struct aead_request *req = areq->data;
524
525 if (!err)
526 err = crypto_gcm_verify(req);
527
528 aead_request_complete(req, err);
529}
530
531static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
532{
533 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
534 struct skcipher_request *skreq = &pctx->u.skreq;
535 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
536
537 crypto_gcm_init_crypt(req, gctx->cryptlen);
538 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
539 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
540}
541
542static int crypto_gcm_decrypt(struct aead_request *req)
543{
544 struct crypto_aead *aead = crypto_aead_reqtfm(req);
545 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
546 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
547 unsigned int authsize = crypto_aead_authsize(aead);
548 unsigned int cryptlen = req->cryptlen;
549 u32 flags = aead_request_flags(req);
550
551 cryptlen -= authsize;
552
553 crypto_gcm_init_common(req);
554
555 gctx->src = sg_next(pctx->src);
556 gctx->cryptlen = cryptlen;
557 gctx->complete = gcm_dec_hash_continue;
558
559 return gcm_hash(req, flags);
560}
561
562static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
563{
564 struct aead_instance *inst = aead_alg_instance(tfm);
565 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
566 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
567 struct crypto_skcipher *ctr;
568 struct crypto_ahash *ghash;
569 unsigned long align;
570 int err;
571
572 ghash = crypto_spawn_ahash(&ictx->ghash);
573 if (IS_ERR(ghash))
574 return PTR_ERR(ghash);
575
576 ctr = crypto_spawn_skcipher2(&ictx->ctr);
577 err = PTR_ERR(ctr);
578 if (IS_ERR(ctr))
579 goto err_free_hash;
580
581 ctx->ctr = ctr;
582 ctx->ghash = ghash;
583
584 align = crypto_aead_alignmask(tfm);
585 align &= ~(crypto_tfm_ctx_alignment() - 1);
586 crypto_aead_set_reqsize(tfm,
587 align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
588 max(sizeof(struct skcipher_request) +
589 crypto_skcipher_reqsize(ctr),
590 sizeof(struct ahash_request) +
591 crypto_ahash_reqsize(ghash)));
592
593 return 0;
594
595err_free_hash:
596 crypto_free_ahash(ghash);
597 return err;
598}
599
600static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
601{
602 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
603
604 crypto_free_ahash(ctx->ghash);
605 crypto_free_skcipher(ctx->ctr);
606}
607
608static void crypto_gcm_free(struct aead_instance *inst)
609{
610 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);
611
612 crypto_drop_skcipher(&ctx->ctr);
613 crypto_drop_ahash(&ctx->ghash);
614 kfree(inst);
615}
616
617static int crypto_gcm_create_common(struct crypto_template *tmpl,
618 struct rtattr **tb,
619 const char *ctr_name,
620 const char *ghash_name)
621{
622 struct crypto_attr_type *algt;
623 struct aead_instance *inst;
624 struct skcipher_alg *ctr;
625 struct crypto_alg *ghash_alg;
626 struct hash_alg_common *ghash;
627 struct gcm_instance_ctx *ctx;
628 int err;
629
630 algt = crypto_get_attr_type(tb);
631 if (IS_ERR(algt))
632 return PTR_ERR(algt);
633
634 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
635 return -EINVAL;
636
637 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
638 CRYPTO_ALG_TYPE_HASH,
639 CRYPTO_ALG_TYPE_AHASH_MASK |
640 crypto_requires_sync(algt->type,
641 algt->mask));
642 if (IS_ERR(ghash_alg))
643 return PTR_ERR(ghash_alg);
644
645 ghash = __crypto_hash_alg_common(ghash_alg);
646
647 err = -ENOMEM;
648 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
649 if (!inst)
650 goto out_put_ghash;
651
652 ctx = aead_instance_ctx(inst);
653 err = crypto_init_ahash_spawn(&ctx->ghash, ghash,
654 aead_crypto_instance(inst));
655 if (err)
656 goto err_free_inst;
657
658 err = -EINVAL;
659 if (strcmp(ghash->base.cra_name, "ghash") != 0 ||
660 ghash->digestsize != 16)
661 goto err_drop_ghash;
662
663 crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
664 err = crypto_grab_skcipher2(&ctx->ctr, ctr_name, 0,
665 crypto_requires_sync(algt->type,
666 algt->mask));
667 if (err)
668 goto err_drop_ghash;
669
670 ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
671
672 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */
673 err = -EINVAL;
674 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 ||
675 crypto_skcipher_alg_ivsize(ctr) != 16 ||
676 ctr->base.cra_blocksize != 1)
677 goto out_put_ctr;
678
679 err = -ENAMETOOLONG;
680 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
681 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME)
682 goto out_put_ctr;
683
684 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
685 "gcm_base(%s,%s)", ctr->base.cra_driver_name,
686 ghash_alg->cra_driver_name) >=
687 CRYPTO_MAX_ALG_NAME)
688 goto out_put_ctr;
689
690 inst->alg.base.cra_flags = (ghash->base.cra_flags |
691 ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
692 inst->alg.base.cra_priority = (ghash->base.cra_priority +
693 ctr->base.cra_priority) / 2;
694 inst->alg.base.cra_blocksize = 1;
695 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
696 ctr->base.cra_alignmask;
697 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
698 inst->alg.ivsize = 12;
699 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
700 inst->alg.maxauthsize = 16;
701 inst->alg.init = crypto_gcm_init_tfm;
702 inst->alg.exit = crypto_gcm_exit_tfm;
703 inst->alg.setkey = crypto_gcm_setkey;
704 inst->alg.setauthsize = crypto_gcm_setauthsize;
705 inst->alg.encrypt = crypto_gcm_encrypt;
706 inst->alg.decrypt = crypto_gcm_decrypt;
707
708 inst->free = crypto_gcm_free;
709
710 err = aead_register_instance(tmpl, inst);
711 if (err)
712 goto out_put_ctr;
713
714out_put_ghash:
715 crypto_mod_put(ghash_alg);
716 return err;
717
718out_put_ctr:
719 crypto_drop_skcipher(&ctx->ctr);
720err_drop_ghash:
721 crypto_drop_ahash(&ctx->ghash);
722err_free_inst:
723 kfree(inst);
724 goto out_put_ghash;
725}
726
727static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
728{
729 const char *cipher_name;
730 char ctr_name[CRYPTO_MAX_ALG_NAME];
731
732 cipher_name = crypto_attr_alg_name(tb[1]);
733 if (IS_ERR(cipher_name))
734 return PTR_ERR(cipher_name);
735
736 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
737 CRYPTO_MAX_ALG_NAME)
738 return -ENAMETOOLONG;
739
740 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash");
741}
742
743static struct crypto_template crypto_gcm_tmpl = {
744 .name = "gcm",
745 .create = crypto_gcm_create,
746 .module = THIS_MODULE,
747};
748
749static int crypto_gcm_base_create(struct crypto_template *tmpl,
750 struct rtattr **tb)
751{
752 const char *ctr_name;
753 const char *ghash_name;
754
755 ctr_name = crypto_attr_alg_name(tb[1]);
756 if (IS_ERR(ctr_name))
757 return PTR_ERR(ctr_name);
758
759 ghash_name = crypto_attr_alg_name(tb[2]);
760 if (IS_ERR(ghash_name))
761 return PTR_ERR(ghash_name);
762
763 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name);
764}
765
766static struct crypto_template crypto_gcm_base_tmpl = {
767 .name = "gcm_base",
768 .create = crypto_gcm_base_create,
769 .module = THIS_MODULE,
770};
771
772static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
773 unsigned int keylen)
774{
775 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
776 struct crypto_aead *child = ctx->child;
777 int err;
778
779 if (keylen < 4)
780 return -EINVAL;
781
782 keylen -= 4;
783 memcpy(ctx->nonce, key + keylen, 4);
784
785 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
786 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
787 CRYPTO_TFM_REQ_MASK);
788 err = crypto_aead_setkey(child, key, keylen);
789 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
790 CRYPTO_TFM_RES_MASK);
791
792 return err;
793}
794
795static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
796 unsigned int authsize)
797{
798 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
799
800 switch (authsize) {
801 case 8:
802 case 12:
803 case 16:
804 break;
805 default:
806 return -EINVAL;
807 }
808
809 return crypto_aead_setauthsize(ctx->child, authsize);
810}
811
812static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
813{
814 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);
815 struct crypto_aead *aead = crypto_aead_reqtfm(req);
816 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
817 struct aead_request *subreq = &rctx->subreq;
818 struct crypto_aead *child = ctx->child;
819 struct scatterlist *sg;
820 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
821 crypto_aead_alignmask(child) + 1);
822
823 scatterwalk_map_and_copy(iv + 12, req->src, 0, req->assoclen - 8, 0);
824
825 memcpy(iv, ctx->nonce, 4);
826 memcpy(iv + 4, req->iv, 8);
827
828 sg_init_table(rctx->src, 3);
829 sg_set_buf(rctx->src, iv + 12, req->assoclen - 8);
830 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);
831 if (sg != rctx->src + 1)
832 sg_chain(rctx->src, 2, sg);
833
834 if (req->src != req->dst) {
835 sg_init_table(rctx->dst, 3);
836 sg_set_buf(rctx->dst, iv + 12, req->assoclen - 8);
837 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);
838 if (sg != rctx->dst + 1)
839 sg_chain(rctx->dst, 2, sg);
840 }
841
842 aead_request_set_tfm(subreq, child);
843 aead_request_set_callback(subreq, req->base.flags, req->base.complete,
844 req->base.data);
845 aead_request_set_crypt(subreq, rctx->src,
846 req->src == req->dst ? rctx->src : rctx->dst,
847 req->cryptlen, iv);
848 aead_request_set_ad(subreq, req->assoclen - 8);
849
850 return subreq;
851}
852
853static int crypto_rfc4106_encrypt(struct aead_request *req)
854{
855 if (req->assoclen != 16 && req->assoclen != 20)
856 return -EINVAL;
857
858 req = crypto_rfc4106_crypt(req);
859
860 return crypto_aead_encrypt(req);
861}
862
863static int crypto_rfc4106_decrypt(struct aead_request *req)
864{
865 if (req->assoclen != 16 && req->assoclen != 20)
866 return -EINVAL;
867
868 req = crypto_rfc4106_crypt(req);
869
870 return crypto_aead_decrypt(req);
871}
872
873static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
874{
875 struct aead_instance *inst = aead_alg_instance(tfm);
876 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
877 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
878 struct crypto_aead *aead;
879 unsigned long align;
880
881 aead = crypto_spawn_aead(spawn);
882 if (IS_ERR(aead))
883 return PTR_ERR(aead);
884
885 ctx->child = aead;
886
887 align = crypto_aead_alignmask(aead);
888 align &= ~(crypto_tfm_ctx_alignment() - 1);
889 crypto_aead_set_reqsize(
890 tfm,
891 sizeof(struct crypto_rfc4106_req_ctx) +
892 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
893 align + 24);
894
895 return 0;
896}
897
898static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
899{
900 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
901
902 crypto_free_aead(ctx->child);
903}
904
905static void crypto_rfc4106_free(struct aead_instance *inst)
906{
907 crypto_drop_aead(aead_instance_ctx(inst));
908 kfree(inst);
909}
910
911static int crypto_rfc4106_create(struct crypto_template *tmpl,
912 struct rtattr **tb)
913{
914 struct crypto_attr_type *algt;
915 struct aead_instance *inst;
916 struct crypto_aead_spawn *spawn;
917 struct aead_alg *alg;
918 const char *ccm_name;
919 int err;
920
921 algt = crypto_get_attr_type(tb);
922 if (IS_ERR(algt))
923 return PTR_ERR(algt);
924
925 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
926 return -EINVAL;
927
928 ccm_name = crypto_attr_alg_name(tb[1]);
929 if (IS_ERR(ccm_name))
930 return PTR_ERR(ccm_name);
931
932 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
933 if (!inst)
934 return -ENOMEM;
935
936 spawn = aead_instance_ctx(inst);
937 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
938 err = crypto_grab_aead(spawn, ccm_name, 0,
939 crypto_requires_sync(algt->type, algt->mask));
940 if (err)
941 goto out_free_inst;
942
943 alg = crypto_spawn_aead_alg(spawn);
944
945 err = -EINVAL;
946
947 /* Underlying IV size must be 12. */
948 if (crypto_aead_alg_ivsize(alg) != 12)
949 goto out_drop_alg;
950
951 /* Not a stream cipher? */
952 if (alg->base.cra_blocksize != 1)
953 goto out_drop_alg;
954
955 err = -ENAMETOOLONG;
956 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
957 "rfc4106(%s)", alg->base.cra_name) >=
958 CRYPTO_MAX_ALG_NAME ||
959 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
960 "rfc4106(%s)", alg->base.cra_driver_name) >=
961 CRYPTO_MAX_ALG_NAME)
962 goto out_drop_alg;
963
964 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
965 inst->alg.base.cra_priority = alg->base.cra_priority;
966 inst->alg.base.cra_blocksize = 1;
967 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
968
969 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
970
971 inst->alg.ivsize = 8;
972 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
973 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
974
975 inst->alg.init = crypto_rfc4106_init_tfm;
976 inst->alg.exit = crypto_rfc4106_exit_tfm;
977
978 inst->alg.setkey = crypto_rfc4106_setkey;
979 inst->alg.setauthsize = crypto_rfc4106_setauthsize;
980 inst->alg.encrypt = crypto_rfc4106_encrypt;
981 inst->alg.decrypt = crypto_rfc4106_decrypt;
982
983 inst->free = crypto_rfc4106_free;
984
985 err = aead_register_instance(tmpl, inst);
986 if (err)
987 goto out_drop_alg;
988
989out:
990 return err;
991
992out_drop_alg:
993 crypto_drop_aead(spawn);
994out_free_inst:
995 kfree(inst);
996 goto out;
997}
998
999static struct crypto_template crypto_rfc4106_tmpl = {
1000 .name = "rfc4106",
1001 .create = crypto_rfc4106_create,
1002 .module = THIS_MODULE,
1003};
1004
1005static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
1006 unsigned int keylen)
1007{
1008 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
1009 struct crypto_aead *child = ctx->child;
1010 int err;
1011
1012 if (keylen < 4)
1013 return -EINVAL;
1014
1015 keylen -= 4;
1016 memcpy(ctx->nonce, key + keylen, 4);
1017
1018 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
1019 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
1020 CRYPTO_TFM_REQ_MASK);
1021 err = crypto_aead_setkey(child, key, keylen);
1022 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
1023 CRYPTO_TFM_RES_MASK);
1024
1025 return err;
1026}
1027
1028static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
1029 unsigned int authsize)
1030{
1031 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
1032
1033 if (authsize != 16)
1034 return -EINVAL;
1035
1036 return crypto_aead_setauthsize(ctx->child, authsize);
1037}
1038
1039static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
1040{
1041 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1042 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1043 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
1044 struct aead_request *subreq = &rctx->subreq;
1045 unsigned int authsize = crypto_aead_authsize(aead);
1046 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
1047 crypto_aead_alignmask(ctx->child) + 1);
1048 int err;
1049
1050 if (req->src != req->dst) {
1051 err = crypto_rfc4543_copy_src_to_dst(req, enc);
1052 if (err)
1053 return err;
1054 }
1055
1056 memcpy(iv, ctx->nonce, 4);
1057 memcpy(iv + 4, req->iv, 8);
1058
1059 aead_request_set_tfm(subreq, ctx->child);
1060 aead_request_set_callback(subreq, req->base.flags,
1061 req->base.complete, req->base.data);
1062 aead_request_set_crypt(subreq, req->src, req->dst,
1063 enc ? 0 : authsize, iv);
1064 aead_request_set_ad(subreq, req->assoclen + req->cryptlen -
1065 subreq->cryptlen);
1066
1067 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
1068}
1069
1070static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
1071{
1072 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1073 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1074 unsigned int authsize = crypto_aead_authsize(aead);
1075 unsigned int nbytes = req->assoclen + req->cryptlen -
1076 (enc ? 0 : authsize);
1077 SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
1078
1079 skcipher_request_set_tfm(nreq, ctx->null);
1080 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
1081 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
1082
1083 return crypto_skcipher_encrypt(nreq);
1084}
1085
1086static int crypto_rfc4543_encrypt(struct aead_request *req)
1087{
1088 return crypto_rfc4543_crypt(req, true);
1089}
1090
1091static int crypto_rfc4543_decrypt(struct aead_request *req)
1092{
1093 return crypto_rfc4543_crypt(req, false);
1094}
1095
1096static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
1097{
1098 struct aead_instance *inst = aead_alg_instance(tfm);
1099 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
1100 struct crypto_aead_spawn *spawn = &ictx->aead;
1101 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1102 struct crypto_aead *aead;
1103 struct crypto_skcipher *null;
1104 unsigned long align;
1105 int err = 0;
1106
1107 aead = crypto_spawn_aead(spawn);
1108 if (IS_ERR(aead))
1109 return PTR_ERR(aead);
1110
1111 null = crypto_get_default_null_skcipher2();
1112 err = PTR_ERR(null);
1113 if (IS_ERR(null))
1114 goto err_free_aead;
1115
1116 ctx->child = aead;
1117 ctx->null = null;
1118
1119 align = crypto_aead_alignmask(aead);
1120 align &= ~(crypto_tfm_ctx_alignment() - 1);
1121 crypto_aead_set_reqsize(
1122 tfm,
1123 sizeof(struct crypto_rfc4543_req_ctx) +
1124 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
1125 align + 12);
1126
1127 return 0;
1128
1129err_free_aead:
1130 crypto_free_aead(aead);
1131 return err;
1132}
1133
1134static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
1135{
1136 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1137
1138 crypto_free_aead(ctx->child);
1139 crypto_put_default_null_skcipher2();
1140}
1141
1142static void crypto_rfc4543_free(struct aead_instance *inst)
1143{
1144 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);
1145
1146 crypto_drop_aead(&ctx->aead);
1147
1148 kfree(inst);
1149}
1150
1151static int crypto_rfc4543_create(struct crypto_template *tmpl,
1152 struct rtattr **tb)
1153{
1154 struct crypto_attr_type *algt;
1155 struct aead_instance *inst;
1156 struct crypto_aead_spawn *spawn;
1157 struct aead_alg *alg;
1158 struct crypto_rfc4543_instance_ctx *ctx;
1159 const char *ccm_name;
1160 int err;
1161
1162 algt = crypto_get_attr_type(tb);
1163 if (IS_ERR(algt))
1164 return PTR_ERR(algt);
1165
1166 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1167 return -EINVAL;
1168
1169 ccm_name = crypto_attr_alg_name(tb[1]);
1170 if (IS_ERR(ccm_name))
1171 return PTR_ERR(ccm_name);
1172
1173 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1174 if (!inst)
1175 return -ENOMEM;
1176
1177 ctx = aead_instance_ctx(inst);
1178 spawn = &ctx->aead;
1179 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
1180 err = crypto_grab_aead(spawn, ccm_name, 0,
1181 crypto_requires_sync(algt->type, algt->mask));
1182 if (err)
1183 goto out_free_inst;
1184
1185 alg = crypto_spawn_aead_alg(spawn);
1186
1187 err = -EINVAL;
1188
1189 /* Underlying IV size must be 12. */
1190 if (crypto_aead_alg_ivsize(alg) != 12)
1191 goto out_drop_alg;
1192
1193 /* Not a stream cipher? */
1194 if (alg->base.cra_blocksize != 1)
1195 goto out_drop_alg;
1196
1197 err = -ENAMETOOLONG;
1198 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
1199 "rfc4543(%s)", alg->base.cra_name) >=
1200 CRYPTO_MAX_ALG_NAME ||
1201 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1202 "rfc4543(%s)", alg->base.cra_driver_name) >=
1203 CRYPTO_MAX_ALG_NAME)
1204 goto out_drop_alg;
1205
1206 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
1207 inst->alg.base.cra_priority = alg->base.cra_priority;
1208 inst->alg.base.cra_blocksize = 1;
1209 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
1210
1211 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
1212
1213 inst->alg.ivsize = 8;
1214 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
1215 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1216
1217 inst->alg.init = crypto_rfc4543_init_tfm;
1218 inst->alg.exit = crypto_rfc4543_exit_tfm;
1219
1220 inst->alg.setkey = crypto_rfc4543_setkey;
1221 inst->alg.setauthsize = crypto_rfc4543_setauthsize;
1222 inst->alg.encrypt = crypto_rfc4543_encrypt;
1223 inst->alg.decrypt = crypto_rfc4543_decrypt;
1224
1225 inst->free = crypto_rfc4543_free,
1226
1227 err = aead_register_instance(tmpl, inst);
1228 if (err)
1229 goto out_drop_alg;
1230
1231out:
1232 return err;
1233
1234out_drop_alg:
1235 crypto_drop_aead(spawn);
1236out_free_inst:
1237 kfree(inst);
1238 goto out;
1239}
1240
1241static struct crypto_template crypto_rfc4543_tmpl = {
1242 .name = "rfc4543",
1243 .create = crypto_rfc4543_create,
1244 .module = THIS_MODULE,
1245};
1246
1247static int __init crypto_gcm_module_init(void)
1248{
1249 int err;
1250
1251 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);
1252 if (!gcm_zeroes)
1253 return -ENOMEM;
1254
1255 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));
1256
1257 err = crypto_register_template(&crypto_gcm_base_tmpl);
1258 if (err)
1259 goto out;
1260
1261 err = crypto_register_template(&crypto_gcm_tmpl);
1262 if (err)
1263 goto out_undo_base;
1264
1265 err = crypto_register_template(&crypto_rfc4106_tmpl);
1266 if (err)
1267 goto out_undo_gcm;
1268
1269 err = crypto_register_template(&crypto_rfc4543_tmpl);
1270 if (err)
1271 goto out_undo_rfc4106;
1272
1273 return 0;
1274
1275out_undo_rfc4106:
1276 crypto_unregister_template(&crypto_rfc4106_tmpl);
1277out_undo_gcm:
1278 crypto_unregister_template(&crypto_gcm_tmpl);
1279out_undo_base:
1280 crypto_unregister_template(&crypto_gcm_base_tmpl);
1281out:
1282 kfree(gcm_zeroes);
1283 return err;
1284}
1285
1286static void __exit crypto_gcm_module_exit(void)
1287{
1288 kfree(gcm_zeroes);
1289 crypto_unregister_template(&crypto_rfc4543_tmpl);
1290 crypto_unregister_template(&crypto_rfc4106_tmpl);
1291 crypto_unregister_template(&crypto_gcm_tmpl);
1292 crypto_unregister_template(&crypto_gcm_base_tmpl);
1293}
1294
1295module_init(crypto_gcm_module_init);
1296module_exit(crypto_gcm_module_exit);
1297
1298MODULE_LICENSE("GPL");
1299MODULE_DESCRIPTION("Galois/Counter Mode");
1300MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1301MODULE_ALIAS_CRYPTO("gcm_base");
1302MODULE_ALIAS_CRYPTO("rfc4106");
1303MODULE_ALIAS_CRYPTO("rfc4543");
1304MODULE_ALIAS_CRYPTO("gcm");
1305
generated by cgit at 2024-05-18 23:30:05 (GMT)