168810Sarchie// SPDX-License-Identifier: GPL-2.0-or-later 268810Sarchie/* 3139823Simp * Software async crypto daemon. 4139823Simp * 5139823Simp * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 668810Sarchie * 768810Sarchie * Added AEAD support to cryptd. 868810Sarchie * Authors: Tadeusz Struk (tadeusz.struk@intel.com) 968810Sarchie * Adrian Hoban <adrian.hoban@intel.com> 1068810Sarchie * Gabriele Paoloni <gabriele.paoloni@intel.com> 1168810Sarchie * Aidan O'Mahony (aidan.o.mahony@intel.com) 1268810Sarchie * Copyright (c) 2010, Intel Corporation. 1368810Sarchie */ 1468810Sarchie 1568810Sarchie#include <crypto/internal/hash.h> 1668810Sarchie#include <crypto/internal/aead.h> 1768810Sarchie#include <crypto/internal/skcipher.h> 1868810Sarchie#include <crypto/cryptd.h> 1968810Sarchie#include <linux/refcount.h> 2068810Sarchie#include <linux/err.h> 2168810Sarchie#include <linux/init.h> 2268810Sarchie#include <linux/kernel.h> 2368810Sarchie#include <linux/list.h> 2468810Sarchie#include <linux/module.h> 2568810Sarchie#include <linux/scatterlist.h> 2668810Sarchie#include <linux/sched.h> 2768810Sarchie#include <linux/slab.h> 2868810Sarchie#include <linux/workqueue.h> 2968810Sarchie 3068810Sarchiestatic unsigned int cryptd_max_cpu_qlen = 1000; 3168810Sarchiemodule_param(cryptd_max_cpu_qlen, uint, 0); 3268810SarchieMODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); 3368810Sarchie 3468810Sarchiestatic struct workqueue_struct *cryptd_wq; 3568810Sarchie 3668810Sarchiestruct cryptd_cpu_queue { 3768810Sarchie struct crypto_queue queue; 3868810Sarchie struct work_struct work; 3968810Sarchie}; 4068810Sarchie 4168810Sarchiestruct cryptd_queue { 4268810Sarchie /* 4368810Sarchie * Protected by disabling BH to allow enqueueing from softinterrupt and 4468810Sarchie * dequeuing from kworker (cryptd_queue_worker()). 4568810Sarchie */ 4668810Sarchie struct cryptd_cpu_queue __percpu *cpu_queue; 47298813Spfg}; 4868810Sarchie 4968810Sarchiestruct cryptd_instance_ctx { 5068810Sarchie struct crypto_spawn spawn; 5168810Sarchie struct cryptd_queue *queue; 5268810Sarchie}; 5368810Sarchie 5468810Sarchiestruct skcipherd_instance_ctx { 5568810Sarchie struct crypto_skcipher_spawn spawn; 5668810Sarchie struct cryptd_queue *queue; 5768810Sarchie}; 5868810Sarchie 5968810Sarchiestruct hashd_instance_ctx { 6068810Sarchie struct crypto_shash_spawn spawn; 6168810Sarchie struct cryptd_queue *queue; 6268810Sarchie}; 6368810Sarchie 6468810Sarchiestruct aead_instance_ctx { 6568810Sarchie struct crypto_aead_spawn aead_spawn; 6668810Sarchie struct cryptd_queue *queue; 6768810Sarchie}; 6868810Sarchie 6968810Sarchiestruct cryptd_skcipher_ctx { 7068810Sarchie refcount_t refcnt; 7168810Sarchie struct crypto_skcipher *child; 72138010Sglebius}; 7368810Sarchie 7468810Sarchiestruct cryptd_skcipher_request_ctx { 7568810Sarchie struct skcipher_request req; 7668810Sarchie}; 7768810Sarchie 7868810Sarchiestruct cryptd_hash_ctx { 7968810Sarchie refcount_t refcnt; 8068810Sarchie struct crypto_shash *child; 8168810Sarchie}; 8268810Sarchie 8368810Sarchiestruct cryptd_hash_request_ctx { 8468810Sarchie crypto_completion_t complete; 8570700Sjulian void *data; 8668810Sarchie struct shash_desc desc; 8768810Sarchie}; 8868810Sarchie 8968810Sarchiestruct cryptd_aead_ctx { 9068810Sarchie refcount_t refcnt; 9168810Sarchie struct crypto_aead *child; 92138010Sglebius}; 9368810Sarchie 9468810Sarchiestruct cryptd_aead_request_ctx { 9568810Sarchie struct aead_request req; 9668810Sarchie}; 9768810Sarchie 9868810Sarchiestatic void cryptd_queue_worker(struct work_struct *work); 9968810Sarchie 10068810Sarchiestatic int cryptd_init_queue(struct cryptd_queue *queue, 10168810Sarchie unsigned int max_cpu_qlen) 10268810Sarchie{ 10368810Sarchie int cpu; 10468810Sarchie struct cryptd_cpu_queue *cpu_queue; 10568810Sarchie 10668810Sarchie queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); 10768810Sarchie if (!queue->cpu_queue) 10897685Sarchie return -ENOMEM; 10968810Sarchie for_each_possible_cpu(cpu) { 11068810Sarchie cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 11168810Sarchie crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); 11297685Sarchie INIT_WORK(&cpu_queue->work, cryptd_queue_worker); 11368810Sarchie } 11468810Sarchie pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); 11568810Sarchie return 0; 11697685Sarchie} 11797685Sarchie 11868810Sarchiestatic void cryptd_fini_queue(struct cryptd_queue *queue) 11968810Sarchie{ 12097685Sarchie int cpu; 12168810Sarchie struct cryptd_cpu_queue *cpu_queue; 12268810Sarchie 12368810Sarchie for_each_possible_cpu(cpu) { 12468810Sarchie cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 12568810Sarchie BUG_ON(cpu_queue->queue.qlen); 12668810Sarchie } 12768810Sarchie free_percpu(queue->cpu_queue); 12868810Sarchie} 12968810Sarchie 13068810Sarchiestatic int cryptd_enqueue_request(struct cryptd_queue *queue, 13168810Sarchie struct crypto_async_request *request) 13268810Sarchie{ 13368810Sarchie int err; 13468810Sarchie struct cryptd_cpu_queue *cpu_queue; 13568810Sarchie refcount_t *refcnt; 13668810Sarchie 13768810Sarchie local_bh_disable(); 13868810Sarchie cpu_queue = this_cpu_ptr(queue->cpu_queue); 13968810Sarchie err = crypto_enqueue_request(&cpu_queue->queue, request); 14068810Sarchie 14168810Sarchie refcnt = crypto_tfm_ctx(request->tfm); 14268810Sarchie 14368810Sarchie if (err == -ENOSPC) 14468810Sarchie goto out; 14568810Sarchie 14668810Sarchie queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work); 14768810Sarchie 14868810Sarchie if (!refcount_read(refcnt)) 14968810Sarchie goto out; 15068810Sarchie 15168810Sarchie refcount_inc(refcnt); 15268810Sarchie 15368810Sarchieout: 15468810Sarchie local_bh_enable(); 15568810Sarchie 15668810Sarchie return err; 15768810Sarchie} 15868810Sarchie 15968810Sarchie/* Called in workqueue context, do one real cryption work (via 16068810Sarchie * req->complete) and reschedule itself if there are more work to 16168810Sarchie * do. */ 16268810Sarchiestatic void cryptd_queue_worker(struct work_struct *work) 16368810Sarchie{ 16468810Sarchie struct cryptd_cpu_queue *cpu_queue; 165129823Sjulian struct crypto_async_request *req, *backlog; 166129823Sjulian 167129823Sjulian cpu_queue = container_of(work, struct cryptd_cpu_queue, work); 168129823Sjulian /* 169129823Sjulian * Only handle one request at a time to avoid hogging crypto workqueue. 170129823Sjulian */ 171129823Sjulian local_bh_disable(); 172129823Sjulian backlog = crypto_get_backlog(&cpu_queue->queue); 173129823Sjulian req = crypto_dequeue_request(&cpu_queue->queue); 17468810Sarchie local_bh_enable(); 17568810Sarchie 17668810Sarchie if (!req) 17768810Sarchie return; 17868810Sarchie 17968810Sarchie if (backlog) 18068810Sarchie crypto_request_complete(backlog, -EINPROGRESS); 18168810Sarchie crypto_request_complete(req, 0); 18268810Sarchie 18368810Sarchie if (cpu_queue->queue.qlen) 18468810Sarchie queue_work(cryptd_wq, &cpu_queue->work); 18570700Sjulian} 18668810Sarchie 18768810Sarchiestatic inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) 18868810Sarchie{ 18968810Sarchie struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 190220768Sglebius struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); 19168810Sarchie return ictx->queue; 19268810Sarchie} 19368810Sarchie 194138010Sglebiusstatic void cryptd_type_and_mask(struct crypto_attr_type *algt, 19570784Sjulian u32 *type, u32 *mask) 196138010Sglebius{ 19768810Sarchie /* 19868810Sarchie * cryptd is allowed to wrap internal algorithms, but in that case the 19968810Sarchie * resulting cryptd instance will be marked as internal as well. 20068810Sarchie */ 20168810Sarchie *type = algt->type & CRYPTO_ALG_INTERNAL; 20268810Sarchie *mask = algt->mask & CRYPTO_ALG_INTERNAL; 20368810Sarchie 20468810Sarchie /* No point in cryptd wrapping an algorithm that's already async. */ 20568810Sarchie *mask |= CRYPTO_ALG_ASYNC; 20668810Sarchie 20768810Sarchie *mask |= crypto_algt_inherited_mask(algt); 20870784Sjulian} 20968810Sarchie 21068810Sarchiestatic int cryptd_init_instance(struct crypto_instance *inst, 21168810Sarchie struct crypto_alg *alg) 21268810Sarchie{ 21368810Sarchie if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 21468810Sarchie "cryptd(%s)", 21568810Sarchie alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 21668810Sarchie return -ENAMETOOLONG; 21768810Sarchie 21868810Sarchie memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 21968810Sarchie 22068810Sarchie inst->alg.cra_priority = alg->cra_priority + 50; 22168810Sarchie inst->alg.cra_blocksize = alg->cra_blocksize; 22268810Sarchie inst->alg.cra_alignmask = alg->cra_alignmask; 223257174Sglebius 22468810Sarchie return 0; 22568810Sarchie} 22668810Sarchie 22768810Sarchiestatic int cryptd_skcipher_setkey(struct crypto_skcipher *parent, 22868810Sarchie const u8 *key, unsigned int keylen) 22968810Sarchie{ 23068810Sarchie struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); 23168810Sarchie struct crypto_skcipher *child = ctx->child; 23268810Sarchie 23368810Sarchie crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 23468810Sarchie crypto_skcipher_set_flags(child, 23568810Sarchie crypto_skcipher_get_flags(parent) & 23668810Sarchie CRYPTO_TFM_REQ_MASK); 23768810Sarchie return crypto_skcipher_setkey(child, key, keylen); 238106665Sjhb} 23968810Sarchie 24068810Sarchiestatic struct skcipher_request *cryptd_skcipher_prepare( 24168810Sarchie struct skcipher_request *req, int err) 24268810Sarchie{ 24368810Sarchie struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 24468810Sarchie struct skcipher_request *subreq = &rctx->req; 24568810Sarchie struct cryptd_skcipher_ctx *ctx; 24668810Sarchie struct crypto_skcipher *child; 24768810Sarchie 24868810Sarchie req->base.complete = subreq->base.complete; 24968810Sarchie req->base.data = subreq->base.data; 25068810Sarchie 25168810Sarchie if (unlikely(err == -EINPROGRESS)) 25268810Sarchie return NULL; 25368810Sarchie 25470700Sjulian ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 25568810Sarchie child = ctx->child; 25670784Sjulian 25768810Sarchie skcipher_request_set_tfm(subreq, child); 25868810Sarchie skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, 25970700Sjulian NULL, NULL); 26068810Sarchie skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 26170700Sjulian req->iv); 26268810Sarchie 26368810Sarchie return subreq; 26468810Sarchie} 26568810Sarchie 26668810Sarchiestatic void cryptd_skcipher_complete(struct skcipher_request *req, int err, 26768810Sarchie crypto_completion_t complete) 26868810Sarchie{ 26968810Sarchie struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 27068810Sarchie struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 27168810Sarchie struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 27268810Sarchie struct skcipher_request *subreq = &rctx->req; 27368810Sarchie int refcnt = refcount_read(&ctx->refcnt); 27468810Sarchie 27568810Sarchie local_bh_disable(); 27668810Sarchie skcipher_request_complete(req, err); 27768810Sarchie local_bh_enable(); 27871738Sjulian 279219127Sae if (unlikely(err == -EINPROGRESS)) { 28068810Sarchie subreq->base.complete = req->base.complete; 28168810Sarchie subreq->base.data = req->base.data; 28268810Sarchie req->base.complete = complete; 28368810Sarchie req->base.data = req; 28468810Sarchie } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 28568810Sarchie crypto_free_skcipher(tfm); 28668810Sarchie} 287138010Sglebius 28868810Sarchiestatic void cryptd_skcipher_encrypt(void *data, int err) 28968810Sarchie{ 29068810Sarchie struct skcipher_request *req = data; 29168810Sarchie struct skcipher_request *subreq; 29268810Sarchie 29368810Sarchie subreq = cryptd_skcipher_prepare(req, err); 29468810Sarchie if (likely(subreq)) 29568810Sarchie err = crypto_skcipher_encrypt(subreq); 29668810Sarchie 29768810Sarchie cryptd_skcipher_complete(req, err, cryptd_skcipher_encrypt); 29868810Sarchie} 29968810Sarchie 30068810Sarchiestatic void cryptd_skcipher_decrypt(void *data, int err) 30168810Sarchie{ 30268810Sarchie struct skcipher_request *req = data; 30368810Sarchie struct skcipher_request *subreq; 30468810Sarchie 30568810Sarchie subreq = cryptd_skcipher_prepare(req, err); 30668810Sarchie if (likely(subreq)) 30768810Sarchie err = crypto_skcipher_decrypt(subreq); 30868810Sarchie 30968810Sarchie cryptd_skcipher_complete(req, err, cryptd_skcipher_decrypt); 31068810Sarchie} 31168810Sarchie 31268810Sarchiestatic int cryptd_skcipher_enqueue(struct skcipher_request *req, 31368810Sarchie crypto_completion_t compl) 31468810Sarchie{ 31568810Sarchie struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); 31668810Sarchie struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 31768810Sarchie struct skcipher_request *subreq = &rctx->req; 31868810Sarchie struct cryptd_queue *queue; 31968810Sarchie 32068810Sarchie queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); 32168810Sarchie subreq->base.complete = req->base.complete; 32268810Sarchie subreq->base.data = req->base.data; 32368810Sarchie req->base.complete = compl; 32468810Sarchie req->base.data = req; 32568810Sarchie 32668810Sarchie return cryptd_enqueue_request(queue, &req->base); 32768810Sarchie} 32868810Sarchie 32968810Sarchiestatic int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) 33068810Sarchie{ 33168810Sarchie return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); 33268810Sarchie} 333143404Sglebius 33468810Sarchiestatic int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) 33568810Sarchie{ 33668810Sarchie return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); 33768810Sarchie} 33868810Sarchie 33968810Sarchiestatic int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) 34068810Sarchie{ 34168810Sarchie struct skcipher_instance *inst = skcipher_alg_instance(tfm); 34268810Sarchie struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); 34368810Sarchie struct crypto_skcipher_spawn *spawn = &ictx->spawn; 34468810Sarchie struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 34568810Sarchie struct crypto_skcipher *cipher; 34668810Sarchie 34768810Sarchie cipher = crypto_spawn_skcipher(spawn); 34868810Sarchie if (IS_ERR(cipher)) 34968810Sarchie return PTR_ERR(cipher); 35068810Sarchie 35168810Sarchie ctx->child = cipher; 35268810Sarchie crypto_skcipher_set_reqsize( 35368810Sarchie tfm, sizeof(struct cryptd_skcipher_request_ctx) + 35468810Sarchie crypto_skcipher_reqsize(cipher)); 35568810Sarchie return 0; 35668810Sarchie} 35768810Sarchie 35868810Sarchiestatic void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) 35968810Sarchie{ 36068810Sarchie struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 361138010Sglebius 362138010Sglebius crypto_free_skcipher(ctx->child); 363138010Sglebius} 364138010Sglebius 365138010Sglebiusstatic void cryptd_skcipher_free(struct skcipher_instance *inst) 366138010Sglebius{ 367138010Sglebius struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); 368138010Sglebius 369138010Sglebius crypto_drop_skcipher(&ctx->spawn); 370138010Sglebius kfree(inst); 371138010Sglebius} 372138010Sglebius 373138010Sglebiusstatic int cryptd_create_skcipher(struct crypto_template *tmpl, 374138010Sglebius struct rtattr **tb, 375138010Sglebius struct crypto_attr_type *algt, 376138010Sglebius struct cryptd_queue *queue) 377138010Sglebius{ 378138010Sglebius struct skcipherd_instance_ctx *ctx; 379138010Sglebius struct skcipher_instance *inst; 380138010Sglebius struct skcipher_alg_common *alg; 381138010Sglebius u32 type; 382138010Sglebius u32 mask; 383138010Sglebius int err; 384138010Sglebius 385138010Sglebius cryptd_type_and_mask(algt, &type, &mask); 386138010Sglebius 387138010Sglebius inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 388138010Sglebius if (!inst) 389138010Sglebius return -ENOMEM; 390138010Sglebius 391138010Sglebius ctx = skcipher_instance_ctx(inst); 392138010Sglebius ctx->queue = queue; 393138010Sglebius 394138010Sglebius err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), 395138010Sglebius crypto_attr_alg_name(tb[1]), type, mask); 396138010Sglebius if (err) 39768810Sarchie goto err_free_inst; 39868810Sarchie 39968810Sarchie alg = crypto_spawn_skcipher_alg_common(&ctx->spawn); 40068810Sarchie err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); 40168810Sarchie if (err) 40268810Sarchie goto err_free_inst; 40370700Sjulian 40470700Sjulian inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | 40568810Sarchie (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); 40668810Sarchie inst->alg.ivsize = alg->ivsize; 40768810Sarchie inst->alg.chunksize = alg->chunksize; 40868810Sarchie inst->alg.min_keysize = alg->min_keysize; 40968810Sarchie inst->alg.max_keysize = alg->max_keysize; 41068810Sarchie 41168810Sarchie inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); 41270700Sjulian 41368810Sarchie inst->alg.init = cryptd_skcipher_init_tfm; 41470784Sjulian inst->alg.exit = cryptd_skcipher_exit_tfm; 41570784Sjulian 41668810Sarchie inst->alg.setkey = cryptd_skcipher_setkey; 41771849Sjulian inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; 41868810Sarchie inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; 41968810Sarchie 42071738Sjulian inst->free = cryptd_skcipher_free; 42170700Sjulian 42268810Sarchie err = skcipher_register_instance(tmpl, inst); 42370700Sjulian if (err) { 42468810Sarchieerr_free_inst: 425106665Sjhb cryptd_skcipher_free(inst); 42668810Sarchie } 42768810Sarchie return err; 42887599Sobrien} 42968810Sarchie 43068810Sarchiestatic int cryptd_hash_init_tfm(struct crypto_ahash *tfm) 43168810Sarchie{ 43268810Sarchie struct ahash_instance *inst = ahash_alg_instance(tfm); 43387599Sobrien struct hashd_instance_ctx *ictx = ahash_instance_ctx(inst); 43468810Sarchie struct crypto_shash_spawn *spawn = &ictx->spawn; 43568810Sarchie struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 43668810Sarchie struct crypto_shash *hash; 43768810Sarchie 43868810Sarchie hash = crypto_spawn_shash(spawn); 43968810Sarchie if (IS_ERR(hash)) 44068810Sarchie return PTR_ERR(hash); 44168810Sarchie 44270700Sjulian ctx->child = hash; 44368810Sarchie crypto_ahash_set_reqsize(tfm, 44468810Sarchie sizeof(struct cryptd_hash_request_ctx) + 44571738Sjulian crypto_shash_descsize(hash)); 44671738Sjulian return 0; 44771738Sjulian} 44871738Sjulian 44971738Sjulianstatic int cryptd_hash_clone_tfm(struct crypto_ahash *ntfm, 45071738Sjulian struct crypto_ahash *tfm) 45171738Sjulian{ 45271738Sjulian struct cryptd_hash_ctx *nctx = crypto_ahash_ctx(ntfm); 45371738Sjulian struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 45471738Sjulian struct crypto_shash *hash; 45571738Sjulian 45671738Sjulian hash = crypto_clone_shash(ctx->child); 45771738Sjulian if (IS_ERR(hash)) 45871738Sjulian return PTR_ERR(hash); 45971738Sjulian 46071738Sjulian nctx->child = hash; 46171738Sjulian return 0; 462243882Sglebius} 46371738Sjulian 46471738Sjulianstatic void cryptd_hash_exit_tfm(struct crypto_ahash *tfm) 46571738Sjulian{ 46671738Sjulian struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 46771738Sjulian 46871738Sjulian crypto_free_shash(ctx->child); 46971738Sjulian} 47071738Sjulian 47171738Sjulianstatic int cryptd_hash_setkey(struct crypto_ahash *parent, 472131155Sjulian const u8 *key, unsigned int keylen) 47371738Sjulian{ 47471738Sjulian struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); 475219127Sae struct crypto_shash *child = ctx->child; 476219127Sae 477219127Sae crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 47871738Sjulian crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & 47971738Sjulian CRYPTO_TFM_REQ_MASK); 48087599Sobrien return crypto_shash_setkey(child, key, keylen); 48171738Sjulian} 48271738Sjulian 48371849Sjulianstatic int cryptd_hash_enqueue(struct ahash_request *req, 48468810Sarchie crypto_completion_t compl) 48571849Sjulian{ 48668810Sarchie struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 48768810Sarchie struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 48868810Sarchie struct cryptd_queue *queue = 48968810Sarchie cryptd_get_queue(crypto_ahash_tfm(tfm)); 49068810Sarchie 49168810Sarchie rctx->complete = req->base.complete; 49270784Sjulian rctx->data = req->base.data; 49368810Sarchie req->base.complete = compl; 49468810Sarchie req->base.data = req; 49568810Sarchie 49668810Sarchie return cryptd_enqueue_request(queue, &req->base); 49768810Sarchie} 49868810Sarchie 49968810Sarchiestatic struct shash_desc *cryptd_hash_prepare(struct ahash_request *req, 50070700Sjulian int err) 50168810Sarchie{ 50270784Sjulian struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 50368810Sarchie 50468810Sarchie req->base.complete = rctx->complete; 50587599Sobrien req->base.data = rctx->data; 506184205Sdes 50770784Sjulian if (unlikely(err == -EINPROGRESS)) 50870784Sjulian return NULL; 50968810Sarchie 51068810Sarchie return &rctx->desc; 51168810Sarchie} 51268810Sarchie 51368810Sarchiestatic void cryptd_hash_complete(struct ahash_request *req, int err, 51468810Sarchie crypto_completion_t complete) 51568810Sarchie{ 51668810Sarchie struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 51768810Sarchie struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 51870784Sjulian int refcnt = refcount_read(&ctx->refcnt); 51968810Sarchie 52068810Sarchie local_bh_disable(); 52168810Sarchie ahash_request_complete(req, err); 522106665Sjhb local_bh_enable(); 52368810Sarchie 52468810Sarchie if (err == -EINPROGRESS) { 52587599Sobrien req->base.complete = complete; 52668810Sarchie req->base.data = req; 52768810Sarchie } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 52868810Sarchie crypto_free_ahash(tfm); 52968810Sarchie} 53068810Sarchie 53168810Sarchiestatic void cryptd_hash_init(void *data, int err) 53268810Sarchie{ 53368810Sarchie struct ahash_request *req = data; 53468810Sarchie struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 53568810Sarchie struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 53668810Sarchie struct crypto_shash *child = ctx->child; 53770784Sjulian struct shash_desc *desc; 53870784Sjulian 53970784Sjulian desc = cryptd_hash_prepare(req, err); 54068810Sarchie if (unlikely(!desc)) 54168810Sarchie goto out; 54268810Sarchie 54368810Sarchie desc->tfm = child; 54468810Sarchie 54568810Sarchie err = crypto_shash_init(desc); 54668810Sarchie 54768810Sarchieout: 54868810Sarchie cryptd_hash_complete(req, err, cryptd_hash_init); 54968810Sarchie} 55068810Sarchie 55168810Sarchiestatic int cryptd_hash_init_enqueue(struct ahash_request *req) 55268810Sarchie{ 553138010Sglebius return cryptd_hash_enqueue(req, cryptd_hash_init); 55468810Sarchie} 55568810Sarchie 55668810Sarchiestatic void cryptd_hash_update(void *data, int err) 55768810Sarchie{ 55868810Sarchie struct ahash_request *req = data; 55968810Sarchie struct shash_desc *desc; 56068810Sarchie 561138010Sglebius desc = cryptd_hash_prepare(req, err); 56268810Sarchie if (likely(desc)) 56368810Sarchie err = shash_ahash_update(req, desc); 56468810Sarchie 56568810Sarchie cryptd_hash_complete(req, err, cryptd_hash_update); 56668810Sarchie} 56768810Sarchie 56868810Sarchiestatic int cryptd_hash_update_enqueue(struct ahash_request *req) 56968810Sarchie{ 57087599Sobrien return cryptd_hash_enqueue(req, cryptd_hash_update); 57168810Sarchie} 57268810Sarchie 57368810Sarchiestatic void cryptd_hash_final(void *data, int err) 57468810Sarchie{ 575138010Sglebius struct ahash_request *req = data; 576138010Sglebius struct shash_desc *desc; 577138010Sglebius 578138010Sglebius desc = cryptd_hash_prepare(req, err); 579138010Sglebius if (likely(desc)) 580138010Sglebius err = crypto_shash_final(desc, req->result); 58168810Sarchie 58268810Sarchie cryptd_hash_complete(req, err, cryptd_hash_final); 58368810Sarchie} 58468810Sarchie 58568810Sarchiestatic int cryptd_hash_final_enqueue(struct ahash_request *req) 58668810Sarchie{ 58771738Sjulian return cryptd_hash_enqueue(req, cryptd_hash_final); 588219127Sae} 58971738Sjulian 59068810Sarchiestatic void cryptd_hash_finup(void *data, int err) 59168810Sarchie{ 59287599Sobrien struct ahash_request *req = data; 59368810Sarchie struct shash_desc *desc; 59468810Sarchie 59568810Sarchie desc = cryptd_hash_prepare(req, err); 59668810Sarchie if (likely(desc)) 597138010Sglebius err = shash_ahash_finup(req, desc); 598138010Sglebius 599138010Sglebius cryptd_hash_complete(req, err, cryptd_hash_finup); 600138010Sglebius} 601138010Sglebius 602138010Sglebiusstatic int cryptd_hash_finup_enqueue(struct ahash_request *req) 603138010Sglebius{ 604138010Sglebius return cryptd_hash_enqueue(req, cryptd_hash_finup); 60568810Sarchie} 606138010Sglebius 607138010Sglebiusstatic void cryptd_hash_digest(void *data, int err) 608138010Sglebius{ 609138010Sglebius struct ahash_request *req = data; 610138010Sglebius struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 611138010Sglebius struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 612138010Sglebius struct crypto_shash *child = ctx->child; 613 struct shash_desc *desc; 614 615 desc = cryptd_hash_prepare(req, err); 616 if (unlikely(!desc)) 617 goto out; 618 619 desc->tfm = child; 620 621 err = shash_ahash_digest(req, desc); 622 623out: 624 cryptd_hash_complete(req, err, cryptd_hash_digest); 625} 626 627static int cryptd_hash_digest_enqueue(struct ahash_request *req) 628{ 629 return cryptd_hash_enqueue(req, cryptd_hash_digest); 630} 631 632static int cryptd_hash_export(struct ahash_request *req, void *out) 633{ 634 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 635 636 return crypto_shash_export(&rctx->desc, out); 637} 638 639static int cryptd_hash_import(struct ahash_request *req, const void *in) 640{ 641 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 642 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); 643 struct shash_desc *desc = cryptd_shash_desc(req); 644 645 desc->tfm = ctx->child; 646 647 return crypto_shash_import(desc, in); 648} 649 650static void cryptd_hash_free(struct ahash_instance *inst) 651{ 652 struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); 653 654 crypto_drop_shash(&ctx->spawn); 655 kfree(inst); 656} 657 658static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, 659 struct crypto_attr_type *algt, 660 struct cryptd_queue *queue) 661{ 662 struct hashd_instance_ctx *ctx; 663 struct ahash_instance *inst; 664 struct shash_alg *alg; 665 u32 type; 666 u32 mask; 667 int err; 668 669 cryptd_type_and_mask(algt, &type, &mask); 670 671 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 672 if (!inst) 673 return -ENOMEM; 674 675 ctx = ahash_instance_ctx(inst); 676 ctx->queue = queue; 677 678 err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), 679 crypto_attr_alg_name(tb[1]), type, mask); 680 if (err) 681 goto err_free_inst; 682 alg = crypto_spawn_shash_alg(&ctx->spawn); 683 684 err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); 685 if (err) 686 goto err_free_inst; 687 688 inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC | 689 (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL| 690 CRYPTO_ALG_OPTIONAL_KEY)); 691 inst->alg.halg.digestsize = alg->digestsize; 692 inst->alg.halg.statesize = alg->statesize; 693 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); 694 695 inst->alg.init_tfm = cryptd_hash_init_tfm; 696 inst->alg.clone_tfm = cryptd_hash_clone_tfm; 697 inst->alg.exit_tfm = cryptd_hash_exit_tfm; 698 699 inst->alg.init = cryptd_hash_init_enqueue; 700 inst->alg.update = cryptd_hash_update_enqueue; 701 inst->alg.final = cryptd_hash_final_enqueue; 702 inst->alg.finup = cryptd_hash_finup_enqueue; 703 inst->alg.export = cryptd_hash_export; 704 inst->alg.import = cryptd_hash_import; 705 if (crypto_shash_alg_has_setkey(alg)) 706 inst->alg.setkey = cryptd_hash_setkey; 707 inst->alg.digest = cryptd_hash_digest_enqueue; 708 709 inst->free = cryptd_hash_free; 710 711 err = ahash_register_instance(tmpl, inst); 712 if (err) { 713err_free_inst: 714 cryptd_hash_free(inst); 715 } 716 return err; 717} 718 719static int cryptd_aead_setkey(struct crypto_aead *parent, 720 const u8 *key, unsigned int keylen) 721{ 722 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 723 struct crypto_aead *child = ctx->child; 724 725 return crypto_aead_setkey(child, key, keylen); 726} 727 728static int cryptd_aead_setauthsize(struct crypto_aead *parent, 729 unsigned int authsize) 730{ 731 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 732 struct crypto_aead *child = ctx->child; 733 734 return crypto_aead_setauthsize(child, authsize); 735} 736 737static void cryptd_aead_crypt(struct aead_request *req, 738 struct crypto_aead *child, int err, 739 int (*crypt)(struct aead_request *req), 740 crypto_completion_t compl) 741{ 742 struct cryptd_aead_request_ctx *rctx; 743 struct aead_request *subreq; 744 struct cryptd_aead_ctx *ctx; 745 struct crypto_aead *tfm; 746 int refcnt; 747 748 rctx = aead_request_ctx(req); 749 subreq = &rctx->req; 750 req->base.complete = subreq->base.complete; 751 req->base.data = subreq->base.data; 752 753 tfm = crypto_aead_reqtfm(req); 754 755 if (unlikely(err == -EINPROGRESS)) 756 goto out; 757 758 aead_request_set_tfm(subreq, child); 759 aead_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, 760 NULL, NULL); 761 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 762 req->iv); 763 aead_request_set_ad(subreq, req->assoclen); 764 765 err = crypt(subreq); 766 767out: 768 ctx = crypto_aead_ctx(tfm); 769 refcnt = refcount_read(&ctx->refcnt); 770 771 local_bh_disable(); 772 aead_request_complete(req, err); 773 local_bh_enable(); 774 775 if (err == -EINPROGRESS) { 776 subreq->base.complete = req->base.complete; 777 subreq->base.data = req->base.data; 778 req->base.complete = compl; 779 req->base.data = req; 780 } else if (refcnt && refcount_dec_and_test(&ctx->refcnt)) 781 crypto_free_aead(tfm); 782} 783 784static void cryptd_aead_encrypt(void *data, int err) 785{ 786 struct aead_request *req = data; 787 struct cryptd_aead_ctx *ctx; 788 struct crypto_aead *child; 789 790 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 791 child = ctx->child; 792 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt, 793 cryptd_aead_encrypt); 794} 795 796static void cryptd_aead_decrypt(void *data, int err) 797{ 798 struct aead_request *req = data; 799 struct cryptd_aead_ctx *ctx; 800 struct crypto_aead *child; 801 802 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 803 child = ctx->child; 804 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt, 805 cryptd_aead_decrypt); 806} 807 808static int cryptd_aead_enqueue(struct aead_request *req, 809 crypto_completion_t compl) 810{ 811 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); 812 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 813 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); 814 struct aead_request *subreq = &rctx->req; 815 816 subreq->base.complete = req->base.complete; 817 subreq->base.data = req->base.data; 818 req->base.complete = compl; 819 req->base.data = req; 820 return cryptd_enqueue_request(queue, &req->base); 821} 822 823static int cryptd_aead_encrypt_enqueue(struct aead_request *req) 824{ 825 return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); 826} 827 828static int cryptd_aead_decrypt_enqueue(struct aead_request *req) 829{ 830 return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); 831} 832 833static int cryptd_aead_init_tfm(struct crypto_aead *tfm) 834{ 835 struct aead_instance *inst = aead_alg_instance(tfm); 836 struct aead_instance_ctx *ictx = aead_instance_ctx(inst); 837 struct crypto_aead_spawn *spawn = &ictx->aead_spawn; 838 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 839 struct crypto_aead *cipher; 840 841 cipher = crypto_spawn_aead(spawn); 842 if (IS_ERR(cipher)) 843 return PTR_ERR(cipher); 844 845 ctx->child = cipher; 846 crypto_aead_set_reqsize( 847 tfm, sizeof(struct cryptd_aead_request_ctx) + 848 crypto_aead_reqsize(cipher)); 849 return 0; 850} 851 852static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) 853{ 854 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 855 crypto_free_aead(ctx->child); 856} 857 858static void cryptd_aead_free(struct aead_instance *inst) 859{ 860 struct aead_instance_ctx *ctx = aead_instance_ctx(inst); 861 862 crypto_drop_aead(&ctx->aead_spawn); 863 kfree(inst); 864} 865 866static int cryptd_create_aead(struct crypto_template *tmpl, 867 struct rtattr **tb, 868 struct crypto_attr_type *algt, 869 struct cryptd_queue *queue) 870{ 871 struct aead_instance_ctx *ctx; 872 struct aead_instance *inst; 873 struct aead_alg *alg; 874 u32 type; 875 u32 mask; 876 int err; 877 878 cryptd_type_and_mask(algt, &type, &mask); 879 880 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 881 if (!inst) 882 return -ENOMEM; 883 884 ctx = aead_instance_ctx(inst); 885 ctx->queue = queue; 886 887 err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), 888 crypto_attr_alg_name(tb[1]), type, mask); 889 if (err) 890 goto err_free_inst; 891 892 alg = crypto_spawn_aead_alg(&ctx->aead_spawn); 893 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); 894 if (err) 895 goto err_free_inst; 896 897 inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | 898 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); 899 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); 900 901 inst->alg.ivsize = crypto_aead_alg_ivsize(alg); 902 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 903 904 inst->alg.init = cryptd_aead_init_tfm; 905 inst->alg.exit = cryptd_aead_exit_tfm; 906 inst->alg.setkey = cryptd_aead_setkey; 907 inst->alg.setauthsize = cryptd_aead_setauthsize; 908 inst->alg.encrypt = cryptd_aead_encrypt_enqueue; 909 inst->alg.decrypt = cryptd_aead_decrypt_enqueue; 910 911 inst->free = cryptd_aead_free; 912 913 err = aead_register_instance(tmpl, inst); 914 if (err) { 915err_free_inst: 916 cryptd_aead_free(inst); 917 } 918 return err; 919} 920 921static struct cryptd_queue queue; 922 923static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) 924{ 925 struct crypto_attr_type *algt; 926 927 algt = crypto_get_attr_type(tb); 928 if (IS_ERR(algt)) 929 return PTR_ERR(algt); 930 931 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 932 case CRYPTO_ALG_TYPE_LSKCIPHER: 933 return cryptd_create_skcipher(tmpl, tb, algt, &queue); 934 case CRYPTO_ALG_TYPE_HASH: 935 return cryptd_create_hash(tmpl, tb, algt, &queue); 936 case CRYPTO_ALG_TYPE_AEAD: 937 return cryptd_create_aead(tmpl, tb, algt, &queue); 938 } 939 940 return -EINVAL; 941} 942 943static struct crypto_template cryptd_tmpl = { 944 .name = "cryptd", 945 .create = cryptd_create, 946 .module = THIS_MODULE, 947}; 948 949struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, 950 u32 type, u32 mask) 951{ 952 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 953 struct cryptd_skcipher_ctx *ctx; 954 struct crypto_skcipher *tfm; 955 956 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 957 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 958 return ERR_PTR(-EINVAL); 959 960 tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); 961 if (IS_ERR(tfm)) 962 return ERR_CAST(tfm); 963 964 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 965 crypto_free_skcipher(tfm); 966 return ERR_PTR(-EINVAL); 967 } 968 969 ctx = crypto_skcipher_ctx(tfm); 970 refcount_set(&ctx->refcnt, 1); 971 972 return container_of(tfm, struct cryptd_skcipher, base); 973} 974EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); 975 976struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) 977{ 978 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 979 980 return ctx->child; 981} 982EXPORT_SYMBOL_GPL(cryptd_skcipher_child); 983 984bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) 985{ 986 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 987 988 return refcount_read(&ctx->refcnt) - 1; 989} 990EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); 991 992void cryptd_free_skcipher(struct cryptd_skcipher *tfm) 993{ 994 struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); 995 996 if (refcount_dec_and_test(&ctx->refcnt)) 997 crypto_free_skcipher(&tfm->base); 998} 999EXPORT_SYMBOL_GPL(cryptd_free_skcipher); 1000 1001struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, 1002 u32 type, u32 mask) 1003{ 1004 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 1005 struct cryptd_hash_ctx *ctx; 1006 struct crypto_ahash *tfm; 1007 1008 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 1009 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 1010 return ERR_PTR(-EINVAL); 1011 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); 1012 if (IS_ERR(tfm)) 1013 return ERR_CAST(tfm); 1014 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 1015 crypto_free_ahash(tfm); 1016 return ERR_PTR(-EINVAL); 1017 } 1018 1019 ctx = crypto_ahash_ctx(tfm); 1020 refcount_set(&ctx->refcnt, 1); 1021 1022 return __cryptd_ahash_cast(tfm); 1023} 1024EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); 1025 1026struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) 1027{ 1028 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1029 1030 return ctx->child; 1031} 1032EXPORT_SYMBOL_GPL(cryptd_ahash_child); 1033 1034struct shash_desc *cryptd_shash_desc(struct ahash_request *req) 1035{ 1036 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 1037 return &rctx->desc; 1038} 1039EXPORT_SYMBOL_GPL(cryptd_shash_desc); 1040 1041bool cryptd_ahash_queued(struct cryptd_ahash *tfm) 1042{ 1043 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1044 1045 return refcount_read(&ctx->refcnt) - 1; 1046} 1047EXPORT_SYMBOL_GPL(cryptd_ahash_queued); 1048 1049void cryptd_free_ahash(struct cryptd_ahash *tfm) 1050{ 1051 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 1052 1053 if (refcount_dec_and_test(&ctx->refcnt)) 1054 crypto_free_ahash(&tfm->base); 1055} 1056EXPORT_SYMBOL_GPL(cryptd_free_ahash); 1057 1058struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, 1059 u32 type, u32 mask) 1060{ 1061 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 1062 struct cryptd_aead_ctx *ctx; 1063 struct crypto_aead *tfm; 1064 1065 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 1066 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 1067 return ERR_PTR(-EINVAL); 1068 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); 1069 if (IS_ERR(tfm)) 1070 return ERR_CAST(tfm); 1071 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 1072 crypto_free_aead(tfm); 1073 return ERR_PTR(-EINVAL); 1074 } 1075 1076 ctx = crypto_aead_ctx(tfm); 1077 refcount_set(&ctx->refcnt, 1); 1078 1079 return __cryptd_aead_cast(tfm); 1080} 1081EXPORT_SYMBOL_GPL(cryptd_alloc_aead); 1082 1083struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) 1084{ 1085 struct cryptd_aead_ctx *ctx; 1086 ctx = crypto_aead_ctx(&tfm->base); 1087 return ctx->child; 1088} 1089EXPORT_SYMBOL_GPL(cryptd_aead_child); 1090 1091bool cryptd_aead_queued(struct cryptd_aead *tfm) 1092{ 1093 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); 1094 1095 return refcount_read(&ctx->refcnt) - 1; 1096} 1097EXPORT_SYMBOL_GPL(cryptd_aead_queued); 1098 1099void cryptd_free_aead(struct cryptd_aead *tfm) 1100{ 1101 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); 1102 1103 if (refcount_dec_and_test(&ctx->refcnt)) 1104 crypto_free_aead(&tfm->base); 1105} 1106EXPORT_SYMBOL_GPL(cryptd_free_aead); 1107 1108static int __init cryptd_init(void) 1109{ 1110 int err; 1111 1112 cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1113 1); 1114 if (!cryptd_wq) 1115 return -ENOMEM; 1116 1117 err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); 1118 if (err) 1119 goto err_destroy_wq; 1120 1121 err = crypto_register_template(&cryptd_tmpl); 1122 if (err) 1123 goto err_fini_queue; 1124 1125 return 0; 1126 1127err_fini_queue: 1128 cryptd_fini_queue(&queue); 1129err_destroy_wq: 1130 destroy_workqueue(cryptd_wq); 1131 return err; 1132} 1133 1134static void __exit cryptd_exit(void) 1135{ 1136 destroy_workqueue(cryptd_wq); 1137 cryptd_fini_queue(&queue); 1138 crypto_unregister_template(&cryptd_tmpl); 1139} 1140 1141subsys_initcall(cryptd_init); 1142module_exit(cryptd_exit); 1143 1144MODULE_LICENSE("GPL"); 1145MODULE_DESCRIPTION("Software async crypto daemon"); 1146MODULE_ALIAS_CRYPTO("cryptd"); 1147