crypto.c revision 158827
12061Sjkh/* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 228531Sbde/*- 32061Sjkh * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 42061Sjkh * 515603Smarkm * This code was written by Angelos D. Keromytis in Athens, Greece, in 63197Scsgr * February 2000. Network Security Technologies Inc. (NSTI) kindly 720710Sasami * supported the development of this code. 820710Sasami * 93197Scsgr * Copyright (c) 2000, 2001 Angelos D. Keromytis 102061Sjkh * 1112483Speter * Permission to use, copy, and modify this software with or without fee 122160Scsgr * is hereby granted, provided that this entire notice is included in 132834Swollman * all source code copies of any software which is or includes a copy or 142061Sjkh * modification of this software. 152061Sjkh * 162160Scsgr * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 1717308Speter * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 1819320Sadam * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 1927788Sasami * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 2027788Sasami * PURPOSE. 2125980Sasami */ 221594Srgrimes 2317308Speter#include <sys/cdefs.h> 2417308Speter__FBSDID("$FreeBSD: head/sys/opencrypto/crypto.c 158827 2006-05-22 10:05:23Z pjd $"); 2527910Sasami 2627910Sasami#define CRYPTO_TIMING /* enable timing support */ 2727910Sasami 2817308Speter#include <sys/param.h> 2917308Speter#include <sys/systm.h> 3017308Speter#include <sys/eventhandler.h> 3119175Sbde#include <sys/kernel.h> 3219175Sbde#include <sys/kthread.h> 3319175Sbde#include <sys/lock.h> 3419175Sbde#include <sys/module.h> 3517308Speter#include <sys/mutex.h> 3627910Sasami#include <sys/malloc.h> 3725647Sbde#include <sys/proc.h> 3827910Sasami#include <sys/sysctl.h> 3917308Speter 402061Sjkh#include <vm/uma.h> 412061Sjkh#include <opencrypto/cryptodev.h> 421594Srgrimes#include <opencrypto/xform.h> /* XXX for M_XDATA */ 4328531Sbde 4428531Sbde/* 4528531Sbde * Crypto drivers register themselves by allocating a slot in the 4628531Sbde * crypto_drivers table with crypto_get_driverid() and then registering 4728531Sbde * each algorithm they support with crypto_register() and crypto_kregister(). 4828531Sbde */ 4928531Sbdestatic struct mtx crypto_drivers_mtx; /* lock on driver table */ 5028531Sbde#define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 517407Srgrimes#define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 527108Sphkstatic struct cryptocap *crypto_drivers = NULL; 537108Sphkstatic int crypto_drivers_num = 0; 547108Sphk 557407Srgrimes/* 567407Srgrimes * There are two queues for crypto requests; one for symmetric (e.g. 577407Srgrimes * cipher) operations and one for asymmetric (e.g. MOD)operations. 587108Sphk * A single mutex is used to lock access to both queues. We could 592061Sjkh * have one per-queue but having one simplifies handling of block/unblock 602061Sjkh * operations. 612061Sjkh */ 6217308Speterstatic int crp_sleep = 0; 632061Sjkhstatic TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 642061Sjkhstatic TAILQ_HEAD(,cryptkop) crp_kq; 652061Sjkhstatic struct mtx crypto_q_mtx; 662061Sjkh#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 672061Sjkh#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 683197Scsgr 692626Scsgr/* 702626Scsgr * There are two queues for processing completed crypto requests; one 712061Sjkh * for the symmetric and one for the asymmetric ops. We only need one 722061Sjkh * but have two to avoid type futzing (cryptop vs. cryptkop). A single 732061Sjkh * mutex is used to lock access to both queues. Note that this lock 742061Sjkh * must be separate from the lock on request queues to insure driver 752061Sjkh * callbacks don't generate lock order reversals. 762061Sjkh */ 7719320Sadamstatic TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 782061Sjkhstatic TAILQ_HEAD(,cryptkop) crp_ret_kq; 792061Sjkhstatic struct mtx crypto_ret_q_mtx; 802061Sjkh#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 812061Sjkh#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 822061Sjkh#define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) 832061Sjkh 842061Sjkhstatic uma_zone_t cryptop_zone; 852061Sjkhstatic uma_zone_t cryptodesc_zone; 862061Sjkh 872061Sjkhint crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 882061SjkhSYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 892834Swollman &crypto_userasymcrypto, 0, 902834Swollman "Enable/disable user-mode access to asymmetric crypto support"); 912834Swollmanint crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 922834SwollmanSYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 932834Swollman &crypto_devallowsoft, 0, 942834Swollman "Enable/disable use of software asym crypto support"); 951594Srgrimes 964486SphkMALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 974486Sphk 984486Sphkstatic void crypto_proc(void); 994486Sphkstatic struct proc *cryptoproc; 1004486Sphkstatic void crypto_ret_proc(void); 1012061Sjkhstatic struct proc *cryptoretproc; 1022061Sjkhstatic void crypto_destroy(void); 10325979Sjkhstatic int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 10425979Sjkhstatic int crypto_kinvoke(struct cryptkop *krp); 10525979Sjkh 10625979Sjkhstatic struct cryptostats cryptostats; 1072061SjkhSYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 10825979Sjkh cryptostats, "Crypto system statistics"); 1092061Sjkh 1102061Sjkh#ifdef CRYPTO_TIMING 11117308Speterstatic int crypto_timing = 0; 1122061SjkhSYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 1132061Sjkh &crypto_timing, 0, "Enable/disable crypto timing support"); 1142061Sjkh#endif 1152061Sjkh 1162061Sjkhstatic int 11712483Spetercrypto_init(void) 11812483Speter{ 11912483Speter int error; 12012483Speter 1212061Sjkh mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 1222061Sjkh MTX_DEF|MTX_QUIET); 1238854Srgrimes 1242061Sjkh TAILQ_INIT(&crp_q); 1252061Sjkh TAILQ_INIT(&crp_kq); 12612483Speter mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 1272061Sjkh 12827910Sasami TAILQ_INIT(&crp_ret_q); 12927910Sasami TAILQ_INIT(&crp_ret_kq); 13018714Sache mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 13117308Speter 13217308Speter cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 13317308Speter 0, 0, 0, 0, 13417308Speter UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 13521536Sjmacd cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 13615603Smarkm 0, 0, 0, 0, 13717308Speter UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 13817308Speter if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 13917308Speter printf("crypto_init: cannot setup crypto zones\n"); 14017308Speter error = ENOMEM; 14117308Speter goto bad; 14217308Speter } 14317308Speter 14417308Speter crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 14517308Speter crypto_drivers = malloc(crypto_drivers_num * 14618362Sjkh sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 14719966Sache if (crypto_drivers == NULL) { 14818362Sjkh printf("crypto_init: cannot setup crypto drivers\n"); 14917308Speter error = ENOMEM; 15027910Sasami goto bad; 15117308Speter } 15217308Speter 15317308Speter error = kthread_create((void (*)(void *)) crypto_proc, NULL, 15428358Speter &cryptoproc, 0, 0, "crypto"); 15527910Sasami if (error) { 15628358Speter printf("crypto_init: cannot start crypto thread; error %d", 15728358Speter error); 15827910Sasami goto bad; 15917308Speter } 1602061Sjkh 16127910Sasami error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 1622061Sjkh &cryptoretproc, 0, 0, "crypto returns"); 16328358Speter if (error) { 16427910Sasami printf("crypto_init: cannot start cryptoret thread; error %d", 1652061Sjkh error); 16617308Speter goto bad; 16727910Sasami } 16817308Speter return 0; 16927910Sasamibad: 17027910Sasami crypto_destroy(); 17127910Sasami return error; 17217308Speter} 17327910Sasami 17417308Speter/* 17527910Sasami * Signal a crypto thread to terminate. We use the driver 17627910Sasami * table lock to synchronize the sleep/wakeups so that we 17727910Sasami * are sure the threads have terminated before we release 17827910Sasami * the data structures they use. See crypto_finis below 17927910Sasami * for the other half of this song-and-dance. 18027910Sasami */ 18127910Sasamistatic void 18227910Sasamicrypto_terminate(struct proc **pp, void *q) 18327910Sasami{ 18427910Sasami struct proc *p; 18527910Sasami 18627910Sasami mtx_assert(&crypto_drivers_mtx, MA_OWNED); 18727910Sasami p = *pp; 18827910Sasami *pp = NULL; 18927910Sasami if (p) { 19027910Sasami wakeup_one(q); 19127910Sasami PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 19227910Sasami CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 19327910Sasami msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 19427910Sasami PROC_UNLOCK(p); 19527910Sasami CRYPTO_DRIVER_LOCK(); 19627910Sasami } 19727910Sasami} 19827910Sasami 19927910Sasamistatic void 20027910Sasamicrypto_destroy(void) 20127910Sasami{ 20227910Sasami /* 20327910Sasami * Terminate any crypto threads. 20427910Sasami */ 20527910Sasami CRYPTO_DRIVER_LOCK(); 20627910Sasami crypto_terminate(&cryptoproc, &crp_q); 20727910Sasami crypto_terminate(&cryptoretproc, &crp_ret_q); 20827910Sasami CRYPTO_DRIVER_UNLOCK(); 20927910Sasami 21027910Sasami /* XXX flush queues??? */ 21127910Sasami 21217308Speter /* 21317308Speter * Reclaim dynamically allocated resources. 21427910Sasami */ 21517308Speter if (crypto_drivers != NULL) 21627910Sasami free(crypto_drivers, M_CRYPTO_DATA); 21727910Sasami 21827910Sasami if (cryptodesc_zone != NULL) 21927910Sasami uma_zdestroy(cryptodesc_zone); 22017466Speter if (cryptop_zone != NULL) 22117308Speter uma_zdestroy(cryptop_zone); 22227910Sasami mtx_destroy(&crypto_q_mtx); 22317308Speter mtx_destroy(&crypto_ret_q_mtx); 22427910Sasami mtx_destroy(&crypto_drivers_mtx); 22527910Sasami} 22628531Sbde 22728531Sbde/* 22828531Sbde * Initialization code, both for static and dynamic loading. 22928531Sbde */ 23017308Speterstatic int 23117308Spetercrypto_modevent(module_t mod, int type, void *unused) 23227910Sasami{ 23317308Speter int error = EINVAL; 23427910Sasami 23527910Sasami switch (type) { 23617308Speter case MOD_LOAD: 23717308Speter error = crypto_init(); 23827910Sasami if (error == 0 && bootverbose) 23917308Speter printf("crypto: <crypto core>\n"); 24027910Sasami break; 24127910Sasami case MOD_UNLOAD: 24227910Sasami /*XXX disallow if active sessions */ 24317308Speter error = 0; 24417308Speter crypto_destroy(); 24527910Sasami return 0; 24617308Speter } 24727910Sasami return error; 24827910Sasami} 24917308Speter 25017308Speterstatic moduledata_t crypto_mod = { 25127910Sasami "crypto", 25217308Speter crypto_modevent, 25327910Sasami 0 25417308Speter}; 25517308SpeterMODULE_VERSION(crypto, 1); 25627910SasamiDECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 25717308SpeterMODULE_DEPEND(crypto, zlib, 1, 1, 1); 25827910Sasami 25917308Speter/* 26017308Speter * Create a new session. 26127910Sasami */ 26217308Speterint 26327910Sasamicrypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 26417962Speter{ 26517962Speter struct cryptoini *cr; 26627910Sasami u_int32_t hid, lid; 26717962Speter int err = EINVAL; 26827910Sasami 26917962Speter CRYPTO_DRIVER_LOCK(); 27017962Speter 27127910Sasami if (crypto_drivers == NULL) 27217962Speter goto done; 27327910Sasami 27417962Speter /* 27517962Speter * The algorithm we use here is pretty stupid; just use the 27627910Sasami * first driver that supports all the algorithms we need. 27717962Speter * 27827910Sasami * XXX We need more smarts here (in real life too, but that's 27917308Speter * XXX another story altogether). 28017308Speter */ 28127910Sasami 28217308Speter for (hid = 0; hid < crypto_drivers_num; hid++) { 28327910Sasami struct cryptocap *cap = &crypto_drivers[hid]; 28417308Speter /* 28517308Speter * If it's not initialized or has remaining sessions 28627910Sasami * referencing it, skip. 28717962Speter */ 28827910Sasami if (cap->cc_newsession == NULL || 2892061Sjkh (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) 29017308Speter continue; 29127910Sasami 29227910Sasami /* Hardware required -- ignore software drivers. */ 29327910Sasami if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) 29427910Sasami continue; 29527910Sasami /* Software required -- ignore hardware drivers. */ 29627910Sasami if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 29727910Sasami continue; 29827910Sasami 29917308Speter /* See if all the algorithms are supported. */ 30017308Speter for (cr = cri; cr; cr = cr->cri_next) 30117308Speter if (cap->cc_alg[cr->cri_alg] == 0) 30217308Speter break; 30317308Speter 30417308Speter if (cr == NULL) { 30517308Speter /* Ok, all algorithms are supported. */ 30612483Speter 30717308Speter /* 30812483Speter * Can't do everything in one session. 30917308Speter * 31012483Speter * XXX Fix this. We need to inject a "virtual" session layer right 3112061Sjkh * XXX about here. 31217962Speter */ 31317962Speter 31417308Speter /* Call the driver initialization routine. */ 31517962Speter lid = hid; /* Pass the driver ID. */ 31617962Speter err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri); 31717962Speter if (err == 0) { 31817962Speter /* XXX assert (hid &~ 0xffffff) == 0 */ 31917308Speter /* XXX assert (cap->cc_flags &~ 0xff) == 0 */ 3202061Sjkh (*sid) = ((cap->cc_flags & 0xff) << 24) | hid; 32117308Speter (*sid) <<= 32; 32217308Speter (*sid) |= (lid & 0xffffffff); 32317308Speter cap->cc_sessions++; 32417308Speter } 32517308Speter break; 32617308Speter } 3272302Spaul } 3282302Spauldone: 3292302Spaul CRYPTO_DRIVER_UNLOCK(); 3302302Spaul return err; 3312302Spaul} 33218714Sache 33310760Sachestatic void 33418714Sachecrypto_remove(struct cryptocap *cap) 3352302Spaul{ 33610760Sache 33718714Sache mtx_assert(&crypto_drivers_mtx, MA_OWNED); 33810760Sache if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 33910760Sache bzero(cap, sizeof(*cap)); 3402302Spaul} 3412302Spaul 3422302Spaul/* 3432302Spaul * Delete an existing session (or a reserved session on an unregistered 34427790Sasami * driver). 3452302Spaul */ 3462302Spaulint 34717308Spetercrypto_freesession(u_int64_t sid) 34817308Speter{ 34917308Speter struct cryptocap *cap; 35017308Speter u_int32_t hid; 35117308Speter int err; 35217308Speter 3532061Sjkh CRYPTO_DRIVER_LOCK(); 35417308Speter 3552061Sjkh if (crypto_drivers == NULL) { 35617308Speter err = EINVAL; 35717308Speter goto done; 35817308Speter } 35917308Speter 36017308Speter /* Determine two IDs. */ 36117308Speter hid = CRYPTO_SESID2HID(sid); 36217308Speter 36317308Speter if (hid >= crypto_drivers_num) { 36417308Speter err = ENOENT; 36517308Speter goto done; 36617308Speter } 36717308Speter cap = &crypto_drivers[hid]; 36817308Speter 36917308Speter if (cap->cc_sessions) 3702061Sjkh cap->cc_sessions--; 37117308Speter 37217308Speter /* Call the driver cleanup routine, if available. */ 37317308Speter if (cap->cc_freesession) 37417308Speter err = cap->cc_freesession(cap->cc_arg, sid); 37517308Speter else 37617308Speter err = 0; 3773626Swollman 3783626Swollman if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 3793626Swollman crypto_remove(cap); 3803626Swollman 3813626Swollmandone: 3823626Swollman CRYPTO_DRIVER_UNLOCK(); 3833626Swollman return err; 3843626Swollman} 3853626Swollman 3863626Swollman/* 3873626Swollman * Return an unused driver id. Used by drivers prior to registering 3887059Sroberto * support for the algorithms they handle. 3893626Swollman */ 3903626Swollmanint32_t 3913626Swollmancrypto_get_driverid(u_int32_t flags) 3923626Swollman{ 3933626Swollman struct cryptocap *newdrv; 3943626Swollman int i; 3953626Swollman 39617308Speter CRYPTO_DRIVER_LOCK(); 39717308Speter 39817308Speter for (i = 0; i < crypto_drivers_num; i++) { 39917308Speter if (crypto_drivers[i].cc_process == NULL && 40017308Speter (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 40117308Speter break; 40217308Speter } 40317308Speter } 40417308Speter 40517308Speter /* Out of entries, allocate some more. */ 4063626Swollman if (i == crypto_drivers_num) { 40717308Speter /* Be careful about wrap-around. */ 40817308Speter if (2 * crypto_drivers_num <= crypto_drivers_num) { 40917308Speter CRYPTO_DRIVER_UNLOCK(); 41017308Speter printf("crypto: driver count wraparound!\n"); 41117308Speter return -1; 41217308Speter } 41317308Speter 41417308Speter newdrv = malloc(2 * crypto_drivers_num * 41517308Speter sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 41617308Speter if (newdrv == NULL) { 41717308Speter CRYPTO_DRIVER_UNLOCK(); 41817308Speter printf("crypto: no space to expand driver table!\n"); 41927910Sasami return -1; 42027910Sasami } 42127910Sasami 42227910Sasami bcopy(crypto_drivers, newdrv, 42327910Sasami crypto_drivers_num * sizeof(struct cryptocap)); 42427910Sasami 42517820Sjkh crypto_drivers_num *= 2; 42617308Speter 42717820Sjkh free(crypto_drivers, M_CRYPTO_DATA); 42817308Speter crypto_drivers = newdrv; 42917820Sjkh } 43017467Speter 43117308Speter /* NB: state is zero'd on free */ 43217308Speter crypto_drivers[i].cc_sessions = 1; /* Mark */ 43317308Speter crypto_drivers[i].cc_flags = flags; 43417308Speter if (bootverbose) 43517308Speter printf("crypto: assign driver %u, flags %u\n", i, flags); 43627910Sasami 43727910Sasami CRYPTO_DRIVER_UNLOCK(); 43827910Sasami 43917308Speter return i; 44017308Speter} 44127910Sasami 44227910Sasamistatic struct cryptocap * 44317308Spetercrypto_checkdriver(u_int32_t hid) 44417308Speter{ 44517308Speter if (crypto_drivers == NULL) 44617308Speter return NULL; 44714119Speter return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 4482061Sjkh} 4497130Srgrimes 4507130Srgrimes/* 4517130Srgrimes * Register support for a key-related algorithm. This routine 4522061Sjkh * is called once for each algorithm supported a driver. 45328357Speter */ 45417308Speterint 4552685Srgrimescrypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 4566927Snate int (*kprocess)(void*, struct cryptkop *, int), 45727790Sasami void *karg) 45827790Sasami{ 45927790Sasami struct cryptocap *cap; 46028329Sandreas int err; 4613197Scsgr 4623197Scsgr CRYPTO_DRIVER_LOCK(); 46312166Sjkh 46412485Sjkh cap = crypto_checkdriver(driverid); 4653197Scsgr if (cap != NULL && 46625313Sbde (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 46726152Speter /* 4682061Sjkh * XXX Do some performance testing to determine placing. 4692061Sjkh * XXX We probably need an auxiliary data structure that 4702061Sjkh * XXX describes relative performances. 47116786Snate */ 4722883Sphk 47327790Sasami cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 47417308Speter if (bootverbose) 4757281Srgrimes printf("crypto: driver %u registers key alg %u flags %u\n" 4763242Spaul , driverid 4773242Spaul , kalg 4787171Sats , flags 4792061Sjkh ); 4803213Spst 48128104Sasami if (cap->cc_kprocess == NULL) { 48228104Sasami cap->cc_karg = karg; 48317308Speter cap->cc_kprocess = kprocess; 48428104Sasami } 48517308Speter err = 0; 4865749Swollman } else 4875772Swollman err = EINVAL; 48817308Speter 48917308Speter CRYPTO_DRIVER_UNLOCK(); 49026504Sjhay return err; 4912061Sjkh} 49217308Speter 49317308Speter/* 49417308Speter * Register support for a non-key-related algorithm. This routine 49527910Sasami * is called once for each such algorithm supported by a driver. 49627910Sasami */ 4975366Snateint 49827910Sasamicrypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 49927910Sasami u_int32_t flags, 50027910Sasami int (*newses)(void*, u_int32_t*, struct cryptoini*), 50127910Sasami int (*freeses)(void*, u_int64_t), 50227910Sasami int (*process)(void*, struct cryptop *, int), 50327910Sasami void *arg) 50427910Sasami{ 50527910Sasami struct cryptocap *cap; 50627910Sasami int err; 50727910Sasami 50827910Sasami CRYPTO_DRIVER_LOCK(); 50927910Sasami 51027910Sasami cap = crypto_checkdriver(driverid); 51127910Sasami /* NB: algorithms are in the range [1..max] */ 51227910Sasami if (cap != NULL && 51317467Speter (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 51427910Sasami /* 5155366Snate * XXX Do some performance testing to determine placing. 51617308Speter * XXX We probably need an auxiliary data structure that 51717308Speter * XXX describes relative performances. 51817308Speter */ 5192061Sjkh 52024754Sjdp cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 52124754Sjdp cap->cc_max_op_len[alg] = maxoplen; 52224754Sjdp if (bootverbose) 52324754Sjdp printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 5248295Srgrimes , driverid 52517820Sjkh , alg 52617308Speter , flags 5278295Srgrimes , maxoplen 5288489Srgrimes ); 52917820Sjkh 53017308Speter if (cap->cc_process == NULL) { 5318489Srgrimes cap->cc_arg = arg; 5328489Srgrimes cap->cc_newsession = newses; 53317820Sjkh cap->cc_process = process; 53417308Speter cap->cc_freesession = freeses; 5358489Srgrimes cap->cc_sessions = 0; /* Unmark */ 5368295Srgrimes } 53717820Sjkh err = 0; 53817308Speter } else 5398295Srgrimes err = EINVAL; 5402160Scsgr 54117820Sjkh CRYPTO_DRIVER_UNLOCK(); 54217308Speter return err; 5432160Scsgr} 5442279Spaul 54517820Sjkh/* 54617308Speter * Unregister a crypto driver. If there are pending sessions using it, 5472279Spaul * leave enough information around so that subsequent calls using those 54817234Sjraynard * sessions will correctly detect the driver has been unregistered and 54917820Sjkh * reroute requests. 55017308Speter */ 55111772Snateint 5523197Scsgrcrypto_unregister(u_int32_t driverid, int alg) 55317820Sjkh{ 55417308Speter struct cryptocap *cap; 5552626Scsgr u_int32_t ses, kops; 5568304Srgrimes int i, err; 55717820Sjkh 55817308Speter CRYPTO_DRIVER_LOCK(); 5598304Srgrimes 5602061Sjkh cap = crypto_checkdriver(driverid); 56117308Speter if (cap != NULL && 56217308Speter (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 56317308Speter cap->cc_alg[alg] != 0) { 56427910Sasami cap->cc_alg[alg] = 0; 56527910Sasami cap->cc_max_op_len[alg] = 0; 56627910Sasami 56727910Sasami /* Was this the last algorithm ? */ 56827910Sasami for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 56927910Sasami if (cap->cc_alg[i] != 0) 57017308Speter break; 57111806Sphk 57219175Sbde if (i == CRYPTO_ALGORITHM_MAX + 1) { 57327910Sasami ses = cap->cc_sessions; 57427910Sasami kops = cap->cc_koperations; 57527910Sasami bzero(cap, sizeof(*cap)); 57627910Sasami if (ses != 0 || kops != 0) { 57727910Sasami /* 57827910Sasami * If there are pending sessions, just mark as invalid. 57927910Sasami */ 58027910Sasami cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 58127910Sasami cap->cc_sessions = ses; 58227910Sasami cap->cc_koperations = kops; 58327910Sasami } 58427910Sasami } 58527910Sasami err = 0; 58627910Sasami } else 58727910Sasami err = EINVAL; 58827910Sasami 58927910Sasami CRYPTO_DRIVER_UNLOCK(); 59027910Sasami return err; 59127910Sasami} 59227910Sasami 59327910Sasami/* 59427910Sasami * Unregister all algorithms associated with a crypto driver. 59527910Sasami * If there are pending sessions using it, leave enough information 59621673Sjkh * around so that subsequent calls using those sessions will 59727910Sasami * correctly detect the driver has been unregistered and reroute 59827910Sasami * requests. 59927910Sasami */ 60027910Sasamiint 60127910Sasamicrypto_unregister_all(u_int32_t driverid) 60227910Sasami{ 60327910Sasami struct cryptocap *cap; 60427910Sasami u_int32_t ses, kops; 60527910Sasami int i, err; 60627910Sasami 60727910Sasami CRYPTO_DRIVER_LOCK(); 60827910Sasami 60927910Sasami cap = crypto_checkdriver(driverid); 61027910Sasami if (cap != NULL) { 61127910Sasami for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 61227910Sasami cap->cc_alg[i] = 0; 61327910Sasami cap->cc_max_op_len[i] = 0; 61427910Sasami } 61519175Sbde ses = cap->cc_sessions; 61627910Sasami kops = cap->cc_koperations; 61727910Sasami bzero(cap, sizeof(*cap)); 61827910Sasami if (ses != 0 || kops != 0) { 61927910Sasami /* 62027910Sasami * If there are pending sessions, just mark as invalid. 62127910Sasami */ 62227910Sasami cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 62327910Sasami cap->cc_sessions = ses; 62427910Sasami cap->cc_koperations = kops; 62527910Sasami } 62627910Sasami err = 0; 62727910Sasami } else 62819175Sbde err = EINVAL; 62917308Speter 63019175Sbde CRYPTO_DRIVER_UNLOCK(); 6312061Sjkh return err; 6321594Srgrimes} 633 634/* 635 * Clear blockage on a driver. The what parameter indicates whether 636 * the driver is now ready for cryptop's and/or cryptokop's. 637 */ 638int 639crypto_unblock(u_int32_t driverid, int what) 640{ 641 struct cryptocap *cap; 642 int err; 643 644 CRYPTO_Q_LOCK(); 645 cap = crypto_checkdriver(driverid); 646 if (cap != NULL) { 647 if (what & CRYPTO_SYMQ) 648 cap->cc_qblocked = 0; 649 if (what & CRYPTO_ASYMQ) 650 cap->cc_kqblocked = 0; 651 if (crp_sleep) 652 wakeup_one(&crp_q); 653 err = 0; 654 } else 655 err = EINVAL; 656 CRYPTO_Q_UNLOCK(); 657 658 return err; 659} 660 661/* 662 * Add a crypto request to a queue, to be processed by the kernel thread. 663 */ 664int 665crypto_dispatch(struct cryptop *crp) 666{ 667 struct cryptocap *cap; 668 u_int32_t hid; 669 int result; 670 671 cryptostats.cs_ops++; 672 673#ifdef CRYPTO_TIMING 674 if (crypto_timing) 675 binuptime(&crp->crp_tstamp); 676#endif 677 678 hid = CRYPTO_SESID2HID(crp->crp_sid); 679 680 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 681 /* 682 * Caller marked the request to be processed 683 * immediately; dispatch it directly to the 684 * driver unless the driver is currently blocked. 685 */ 686 cap = crypto_checkdriver(hid); 687 /* Driver cannot disappeared when there is an active session. */ 688 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 689 if (!cap->cc_qblocked) { 690 result = crypto_invoke(cap, crp, 0); 691 if (result != ERESTART) 692 return (result); 693 /* 694 * The driver ran out of resources, put the request on 695 * the queue. 696 */ 697 } 698 } 699 CRYPTO_Q_LOCK(); 700 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 701 if (crp_sleep) 702 wakeup_one(&crp_q); 703 CRYPTO_Q_UNLOCK(); 704 return 0; 705} 706 707/* 708 * Add an asymetric crypto request to a queue, 709 * to be processed by the kernel thread. 710 */ 711int 712crypto_kdispatch(struct cryptkop *krp) 713{ 714 int result; 715 716 cryptostats.cs_kops++; 717 718 result = crypto_kinvoke(krp); 719 if (result != ERESTART) 720 return (result); 721 CRYPTO_Q_LOCK(); 722 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 723 if (crp_sleep) 724 wakeup_one(&crp_q); 725 CRYPTO_Q_UNLOCK(); 726 727 return 0; 728} 729 730/* 731 * Dispatch an assymetric crypto request to the appropriate crypto devices. 732 */ 733static int 734crypto_kinvoke(struct cryptkop *krp) 735{ 736 struct cryptocap *cap = NULL; 737 u_int32_t hid; 738 int error = 0; 739 740 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 741 KASSERT(krp->krp_callback != NULL, 742 ("%s: krp->crp_callback == NULL", __func__)); 743 744 CRYPTO_DRIVER_LOCK(); 745 for (hid = 0; hid < crypto_drivers_num; hid++) { 746 cap = &crypto_drivers[hid]; 747 if (cap == NULL) 748 continue; 749 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 750 !crypto_devallowsoft) { 751 continue; 752 } 753 if (cap->cc_kprocess == NULL) 754 continue; 755 if (!(cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED)) 756 continue; 757 if (cap->cc_kqblocked) { 758 error = ERESTART; 759 continue; 760 } 761 error = 0; 762 break; 763 } 764 krp->krp_hid = hid; 765 if (hid < crypto_drivers_num) { 766 cap->cc_koperations++; 767 CRYPTO_DRIVER_UNLOCK(); 768 error = cap->cc_kprocess(cap->cc_karg, krp, 0); 769 CRYPTO_DRIVER_LOCK(); 770 if (error == ERESTART) { 771 cap->cc_koperations--; 772 CRYPTO_DRIVER_UNLOCK(); 773 return (error); 774 } 775 } else { 776 error = ENODEV; 777 } 778 CRYPTO_DRIVER_UNLOCK(); 779 780 if (error) { 781 krp->krp_status = error; 782 crypto_kdone(krp); 783 } 784 return 0; 785} 786 787#ifdef CRYPTO_TIMING 788static void 789crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 790{ 791 struct bintime now, delta; 792 struct timespec t; 793 uint64_t u; 794 795 binuptime(&now); 796 u = now.frac; 797 delta.frac = now.frac - bt->frac; 798 delta.sec = now.sec - bt->sec; 799 if (u < delta.frac) 800 delta.sec--; 801 bintime2timespec(&delta, &t); 802 timespecadd(&ts->acc, &t); 803 if (timespeccmp(&t, &ts->min, <)) 804 ts->min = t; 805 if (timespeccmp(&t, &ts->max, >)) 806 ts->max = t; 807 ts->count++; 808 809 *bt = now; 810} 811#endif 812 813/* 814 * Dispatch a crypto request to the appropriate crypto devices. 815 */ 816static int 817crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 818{ 819 820 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 821 KASSERT(crp->crp_callback != NULL, 822 ("%s: crp->crp_callback == NULL", __func__)); 823 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 824 825#ifdef CRYPTO_TIMING 826 if (crypto_timing) 827 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 828#endif 829 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 830 struct cryptodesc *crd; 831 u_int64_t nid; 832 833 /* 834 * Driver has unregistered; migrate the session and return 835 * an error to the caller so they'll resubmit the op. 836 * 837 * XXX: What if there are more already queued requests for this 838 * session? 839 */ 840 crypto_freesession(crp->crp_sid); 841 842 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 843 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 844 845 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 846 crp->crp_sid = nid; 847 848 crp->crp_etype = EAGAIN; 849 crypto_done(crp); 850 return 0; 851 } else { 852 /* 853 * Invoke the driver to process the request. 854 */ 855 return cap->cc_process(cap->cc_arg, crp, hint); 856 } 857} 858 859/* 860 * Release a set of crypto descriptors. 861 */ 862void 863crypto_freereq(struct cryptop *crp) 864{ 865 struct cryptodesc *crd; 866 867 if (crp == NULL) 868 return; 869 870 while ((crd = crp->crp_desc) != NULL) { 871 crp->crp_desc = crd->crd_next; 872 uma_zfree(cryptodesc_zone, crd); 873 } 874 875 uma_zfree(cryptop_zone, crp); 876} 877 878/* 879 * Acquire a set of crypto descriptors. 880 */ 881struct cryptop * 882crypto_getreq(int num) 883{ 884 struct cryptodesc *crd; 885 struct cryptop *crp; 886 887 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 888 if (crp != NULL) { 889 while (num--) { 890 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 891 if (crd == NULL) { 892 crypto_freereq(crp); 893 return NULL; 894 } 895 896 crd->crd_next = crp->crp_desc; 897 crp->crp_desc = crd; 898 } 899 } 900 return crp; 901} 902 903/* 904 * Invoke the callback on behalf of the driver. 905 */ 906void 907crypto_done(struct cryptop *crp) 908{ 909 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 910 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 911 crp->crp_flags |= CRYPTO_F_DONE; 912 if (crp->crp_etype != 0) 913 cryptostats.cs_errs++; 914#ifdef CRYPTO_TIMING 915 if (crypto_timing) 916 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 917#endif 918 /* 919 * CBIMM means unconditionally do the callback immediately; 920 * CBIFSYNC means do the callback immediately only if the 921 * operation was done synchronously. Both are used to avoid 922 * doing extraneous context switches; the latter is mostly 923 * used with the software crypto driver. 924 */ 925 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 926 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 927 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 928 /* 929 * Do the callback directly. This is ok when the 930 * callback routine does very little (e.g. the 931 * /dev/crypto callback method just does a wakeup). 932 */ 933#ifdef CRYPTO_TIMING 934 if (crypto_timing) { 935 /* 936 * NB: We must copy the timestamp before 937 * doing the callback as the cryptop is 938 * likely to be reclaimed. 939 */ 940 struct bintime t = crp->crp_tstamp; 941 crypto_tstat(&cryptostats.cs_cb, &t); 942 crp->crp_callback(crp); 943 crypto_tstat(&cryptostats.cs_finis, &t); 944 } else 945#endif 946 crp->crp_callback(crp); 947 } else { 948 /* 949 * Normal case; queue the callback for the thread. 950 */ 951 CRYPTO_RETQ_LOCK(); 952 if (CRYPTO_RETQ_EMPTY()) 953 wakeup_one(&crp_ret_q); /* shared wait channel */ 954 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 955 CRYPTO_RETQ_UNLOCK(); 956 } 957} 958 959/* 960 * Invoke the callback on behalf of the driver. 961 */ 962void 963crypto_kdone(struct cryptkop *krp) 964{ 965 struct cryptocap *cap; 966 967 if (krp->krp_status != 0) 968 cryptostats.cs_kerrs++; 969 CRYPTO_DRIVER_LOCK(); 970 /* XXX: What if driver is loaded in the meantime? */ 971 if (krp->krp_hid < crypto_drivers_num) { 972 cap = &crypto_drivers[krp->krp_hid]; 973 cap->cc_koperations--; 974 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 975 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 976 crypto_remove(cap); 977 } 978 CRYPTO_DRIVER_UNLOCK(); 979 CRYPTO_RETQ_LOCK(); 980 if (CRYPTO_RETQ_EMPTY()) 981 wakeup_one(&crp_ret_q); /* shared wait channel */ 982 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 983 CRYPTO_RETQ_UNLOCK(); 984} 985 986int 987crypto_getfeat(int *featp) 988{ 989 int hid, kalg, feat = 0; 990 991 if (!crypto_userasymcrypto) 992 goto out; 993 994 CRYPTO_DRIVER_LOCK(); 995 for (hid = 0; hid < crypto_drivers_num; hid++) { 996 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 997 !crypto_devallowsoft) { 998 continue; 999 } 1000 if (crypto_drivers[hid].cc_kprocess == NULL) 1001 continue; 1002 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1003 if ((crypto_drivers[hid].cc_kalg[kalg] & 1004 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1005 feat |= 1 << kalg; 1006 } 1007 CRYPTO_DRIVER_UNLOCK(); 1008out: 1009 *featp = feat; 1010 return (0); 1011} 1012 1013/* 1014 * Terminate a thread at module unload. The process that 1015 * initiated this is waiting for us to signal that we're gone; 1016 * wake it up and exit. We use the driver table lock to insure 1017 * we don't do the wakeup before they're waiting. There is no 1018 * race here because the waiter sleeps on the proc lock for the 1019 * thread so it gets notified at the right time because of an 1020 * extra wakeup that's done in exit1(). 1021 */ 1022static void 1023crypto_finis(void *chan) 1024{ 1025 CRYPTO_DRIVER_LOCK(); 1026 wakeup_one(chan); 1027 CRYPTO_DRIVER_UNLOCK(); 1028 kthread_exit(0); 1029} 1030 1031/* 1032 * Crypto thread, dispatches crypto requests. 1033 */ 1034static void 1035crypto_proc(void) 1036{ 1037 struct cryptop *crp, *submit; 1038 struct cryptkop *krp; 1039 struct cryptocap *cap; 1040 u_int32_t hid; 1041 int result, hint; 1042 1043 CRYPTO_Q_LOCK(); 1044 for (;;) { 1045 /* 1046 * Find the first element in the queue that can be 1047 * processed and look-ahead to see if multiple ops 1048 * are ready for the same driver. 1049 */ 1050 submit = NULL; 1051 hint = 0; 1052 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1053 hid = CRYPTO_SESID2HID(crp->crp_sid); 1054 cap = crypto_checkdriver(hid); 1055 /* 1056 * Driver cannot disappeared when there is an active 1057 * session. 1058 */ 1059 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1060 __func__, __LINE__)); 1061 if (cap == NULL || cap->cc_process == NULL) { 1062 /* Op needs to be migrated, process it. */ 1063 if (submit == NULL) 1064 submit = crp; 1065 break; 1066 } 1067 if (!cap->cc_qblocked) { 1068 if (submit != NULL) { 1069 /* 1070 * We stop on finding another op, 1071 * regardless whether its for the same 1072 * driver or not. We could keep 1073 * searching the queue but it might be 1074 * better to just use a per-driver 1075 * queue instead. 1076 */ 1077 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1078 hint = CRYPTO_HINT_MORE; 1079 break; 1080 } else { 1081 submit = crp; 1082 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1083 break; 1084 /* keep scanning for more are q'd */ 1085 } 1086 } 1087 } 1088 if (submit != NULL) { 1089 TAILQ_REMOVE(&crp_q, submit, crp_next); 1090 hid = CRYPTO_SESID2HID(submit->crp_sid); 1091 cap = crypto_checkdriver(hid); 1092 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1093 __func__, __LINE__)); 1094 result = crypto_invoke(cap, submit, hint); 1095 if (result == ERESTART) { 1096 /* 1097 * The driver ran out of resources, mark the 1098 * driver ``blocked'' for cryptop's and put 1099 * the request back in the queue. It would 1100 * best to put the request back where we got 1101 * it but that's hard so for now we put it 1102 * at the front. This should be ok; putting 1103 * it at the end does not work. 1104 */ 1105 /* XXX validate sid again? */ 1106 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1107 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1108 cryptostats.cs_blocks++; 1109 } 1110 } 1111 1112 /* As above, but for key ops */ 1113 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1114 cap = crypto_checkdriver(krp->krp_hid); 1115 if (cap == NULL || cap->cc_kprocess == NULL) { 1116 /* Op needs to be migrated, process it. */ 1117 break; 1118 } 1119 if (!cap->cc_kqblocked) 1120 break; 1121 } 1122 if (krp != NULL) { 1123 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1124 result = crypto_kinvoke(krp); 1125 if (result == ERESTART) { 1126 /* 1127 * The driver ran out of resources, mark the 1128 * driver ``blocked'' for cryptkop's and put 1129 * the request back in the queue. It would 1130 * best to put the request back where we got 1131 * it but that's hard so for now we put it 1132 * at the front. This should be ok; putting 1133 * it at the end does not work. 1134 */ 1135 /* XXX validate sid again? */ 1136 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1137 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1138 cryptostats.cs_kblocks++; 1139 } 1140 } 1141 1142 if (submit == NULL && krp == NULL) { 1143 /* 1144 * Nothing more to be processed. Sleep until we're 1145 * woken because there are more ops to process. 1146 * This happens either by submission or by a driver 1147 * becoming unblocked and notifying us through 1148 * crypto_unblock. Note that when we wakeup we 1149 * start processing each queue again from the 1150 * front. It's not clear that it's important to 1151 * preserve this ordering since ops may finish 1152 * out of order if dispatched to different devices 1153 * and some become blocked while others do not. 1154 */ 1155 crp_sleep = 1; 1156 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1157 crp_sleep = 0; 1158 if (cryptoproc == NULL) 1159 break; 1160 cryptostats.cs_intrs++; 1161 } 1162 } 1163 CRYPTO_Q_UNLOCK(); 1164 1165 crypto_finis(&crp_q); 1166} 1167 1168/* 1169 * Crypto returns thread, does callbacks for processed crypto requests. 1170 * Callbacks are done here, rather than in the crypto drivers, because 1171 * callbacks typically are expensive and would slow interrupt handling. 1172 */ 1173static void 1174crypto_ret_proc(void) 1175{ 1176 struct cryptop *crpt; 1177 struct cryptkop *krpt; 1178 1179 CRYPTO_RETQ_LOCK(); 1180 for (;;) { 1181 /* Harvest return q's for completed ops */ 1182 crpt = TAILQ_FIRST(&crp_ret_q); 1183 if (crpt != NULL) 1184 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1185 1186 krpt = TAILQ_FIRST(&crp_ret_kq); 1187 if (krpt != NULL) 1188 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1189 1190 if (crpt != NULL || krpt != NULL) { 1191 CRYPTO_RETQ_UNLOCK(); 1192 /* 1193 * Run callbacks unlocked. 1194 */ 1195 if (crpt != NULL) { 1196#ifdef CRYPTO_TIMING 1197 if (crypto_timing) { 1198 /* 1199 * NB: We must copy the timestamp before 1200 * doing the callback as the cryptop is 1201 * likely to be reclaimed. 1202 */ 1203 struct bintime t = crpt->crp_tstamp; 1204 crypto_tstat(&cryptostats.cs_cb, &t); 1205 crpt->crp_callback(crpt); 1206 crypto_tstat(&cryptostats.cs_finis, &t); 1207 } else 1208#endif 1209 crpt->crp_callback(crpt); 1210 } 1211 if (krpt != NULL) 1212 krpt->krp_callback(krpt); 1213 CRYPTO_RETQ_LOCK(); 1214 } else { 1215 /* 1216 * Nothing more to be processed. Sleep until we're 1217 * woken because there are more returns to process. 1218 */ 1219 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1220 "crypto_ret_wait", 0); 1221 if (cryptoretproc == NULL) 1222 break; 1223 cryptostats.cs_rets++; 1224 } 1225 } 1226 CRYPTO_RETQ_UNLOCK(); 1227 1228 crypto_finis(&crp_ret_q); 1229} 1230