crypto.c revision 108587
1104476Ssam/* $FreeBSD: head/sys/opencrypto/crypto.c 108587 2003-01-03 06:16:59Z sam $ */ 2104476Ssam/* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 3104476Ssam/* 4104476Ssam * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 5104476Ssam * 6104476Ssam * This code was written by Angelos D. Keromytis in Athens, Greece, in 7104476Ssam * February 2000. Network Security Technologies Inc. (NSTI) kindly 8104476Ssam * supported the development of this code. 9104476Ssam * 10104476Ssam * Copyright (c) 2000, 2001 Angelos D. Keromytis 11104476Ssam * 12104476Ssam * Permission to use, copy, and modify this software with or without fee 13104476Ssam * is hereby granted, provided that this entire notice is included in 14104476Ssam * all source code copies of any software which is or includes a copy or 15104476Ssam * modification of this software. 16104476Ssam * 17104476Ssam * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 18104476Ssam * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 19104476Ssam * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 20104476Ssam * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 21104476Ssam * PURPOSE. 22104476Ssam */ 23108587Ssam#define CRYPTO_TIMING /* enable timing support */ 24104476Ssam 25104476Ssam#include <sys/param.h> 26104476Ssam#include <sys/systm.h> 27104476Ssam#include <sys/eventhandler.h> 28104476Ssam#include <sys/kernel.h> 29104476Ssam#include <sys/kthread.h> 30104476Ssam#include <sys/lock.h> 31104476Ssam#include <sys/mutex.h> 32104476Ssam#include <sys/malloc.h> 33104476Ssam#include <sys/proc.h> 34104476Ssam#include <sys/sysctl.h> 35104476Ssam 36104476Ssam#include <vm/uma.h> 37104476Ssam#include <opencrypto/cryptodev.h> 38104628Ssam#include <opencrypto/xform.h> /* XXX for M_XDATA */ 39104476Ssam 40104476Ssam#define SESID2HID(sid) (((sid) >> 32) & 0xffffffff) 41104476Ssam 42104476Ssam/* 43104476Ssam * Crypto drivers register themselves by allocating a slot in the 44104476Ssam * crypto_drivers table with crypto_get_driverid() and then registering 45104476Ssam * each algorithm they support with crypto_register() and crypto_kregister(). 46104476Ssam */ 47104476Ssamstatic struct mtx crypto_drivers_mtx; /* lock on driver table */ 48104476Ssam#define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 49104476Ssam#define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 50104476Ssamstatic struct cryptocap *crypto_drivers = NULL; 51104476Ssamstatic int crypto_drivers_num = 0; 52104476Ssam 53104476Ssam/* 54104476Ssam * There are two queues for crypto requests; one for symmetric (e.g. 55104476Ssam * cipher) operations and one for asymmetric (e.g. MOD)operations. 56104476Ssam * A single mutex is used to lock access to both queues. We could 57104476Ssam * have one per-queue but having one simplifies handling of block/unblock 58104476Ssam * operations. 59104476Ssam */ 60104476Ssamstatic TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 61104476Ssamstatic TAILQ_HEAD(,cryptkop) crp_kq; 62104476Ssamstatic struct mtx crypto_q_mtx; 63104476Ssam#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 64104476Ssam#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 65104476Ssam 66104476Ssam/* 67104476Ssam * There are two queues for processing completed crypto requests; one 68104476Ssam * for the symmetric and one for the asymmetric ops. We only need one 69104476Ssam * but have two to avoid type futzing (cryptop vs. cryptkop). A single 70104476Ssam * mutex is used to lock access to both queues. Note that this lock 71104476Ssam * must be separate from the lock on request queues to insure driver 72104476Ssam * callbacks don't generate lock order reversals. 73104476Ssam */ 74104476Ssamstatic TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 75104476Ssamstatic TAILQ_HEAD(,cryptkop) crp_ret_kq; 76104476Ssamstatic struct mtx crypto_ret_q_mtx; 77104476Ssam#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 78104476Ssam#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 79104476Ssam 80104476Ssamstatic uma_zone_t cryptop_zone; 81104476Ssamstatic uma_zone_t cryptodesc_zone; 82104476Ssam 83104476Ssamint crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 84104476SsamSYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 85104476Ssam &crypto_userasymcrypto, 0, 86104476Ssam "Enable/disable user-mode access to asymmetric crypto support"); 87104476Ssamint crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 88104476SsamSYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 89104476Ssam &crypto_devallowsoft, 0, 90104476Ssam "Enable/disable use of software asym crypto support"); 91104476Ssam 92104476SsamMALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 93104476Ssam 94108587Ssamstatic struct cryptostats cryptostats; 95108587SsamSYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 96108587Ssam cryptostats, "Crypto system statistics"); 97108587Ssam 98108587Ssam#ifdef CRYPTO_TIMING 99108587Ssamstatic int crypto_timing = 0; 100108587SsamSYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 101108587Ssam &crypto_timing, 0, "Enable/disable crypto timing support"); 102108587Ssam#endif 103108587Ssam 104104476Ssamstatic void 105104476Ssamcrypto_init(void) 106104476Ssam{ 107104476Ssam cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 108104476Ssam 0, 0, 0, 0, 109104476Ssam UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 110104476Ssam cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 111104476Ssam 0, 0, 0, 0, 112104476Ssam UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 113104476Ssam if (cryptodesc_zone == NULL || cryptop_zone == NULL) 114104476Ssam panic("cannot setup crypto zones"); 115104476Ssam 116104476Ssam mtx_init(&crypto_drivers_mtx, "crypto driver table", 117104476Ssam NULL, MTX_DEF|MTX_QUIET); 118104476Ssam 119104476Ssam crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 120104476Ssam crypto_drivers = malloc(crypto_drivers_num * 121104476Ssam sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 122104476Ssam if (crypto_drivers == NULL) 123104476Ssam panic("cannot setup crypto drivers"); 124104476Ssam 125104476Ssam TAILQ_INIT(&crp_q); 126104476Ssam TAILQ_INIT(&crp_kq); 127104476Ssam mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF); 128104476Ssam 129104476Ssam TAILQ_INIT(&crp_ret_q); 130104476Ssam TAILQ_INIT(&crp_ret_kq); 131104476Ssam mtx_init(&crypto_ret_q_mtx, "crypto return queues", NULL, MTX_DEF); 132104476Ssam} 133104476Ssam 134104476Ssam/* 135105251Smarkm * Initialization code, both for static and dynamic loading. 136105251Smarkm */ 137105251Smarkmstatic int 138105251Smarkmcrypto_modevent(module_t mod, int type, void *unused) 139105251Smarkm{ 140105251Smarkm switch (type) { 141105251Smarkm case MOD_LOAD: 142105251Smarkm crypto_init(); 143105251Smarkm if (bootverbose) 144105251Smarkm printf("crypto: <crypto core>\n"); 145105251Smarkm return 0; 146105251Smarkm case MOD_UNLOAD: 147105251Smarkm /*XXX disallow if active sessions */ 148105251Smarkm /*XXX kill kthreads */ 149105251Smarkm return 0; 150105251Smarkm } 151105251Smarkm return EINVAL; 152105251Smarkm} 153105251Smarkm 154105251Smarkmstatic moduledata_t crypto_mod = { 155105251Smarkm "crypto", 156105251Smarkm crypto_modevent, 157105251Smarkm 0 158105251Smarkm}; 159105251SmarkmMODULE_VERSION(crypto, 1); 160106676SsamDECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 161105251Smarkm 162105251Smarkm/* 163104476Ssam * Create a new session. 164104476Ssam */ 165104476Ssamint 166104476Ssamcrypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 167104476Ssam{ 168104476Ssam struct cryptoini *cr; 169104476Ssam u_int32_t hid, lid; 170104476Ssam int err = EINVAL; 171104476Ssam 172104476Ssam CRYPTO_DRIVER_LOCK(); 173104476Ssam 174104476Ssam if (crypto_drivers == NULL) 175104476Ssam goto done; 176104476Ssam 177104476Ssam /* 178104476Ssam * The algorithm we use here is pretty stupid; just use the 179104476Ssam * first driver that supports all the algorithms we need. 180104476Ssam * 181104476Ssam * XXX We need more smarts here (in real life too, but that's 182104476Ssam * XXX another story altogether). 183104476Ssam */ 184104476Ssam 185104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 186104476Ssam /* 187104476Ssam * If it's not initialized or has remaining sessions 188104476Ssam * referencing it, skip. 189104476Ssam */ 190104476Ssam if (crypto_drivers[hid].cc_newsession == NULL || 191104476Ssam (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)) 192104476Ssam continue; 193104476Ssam 194104476Ssam /* Hardware required -- ignore software drivers. */ 195104476Ssam if (hard > 0 && 196104476Ssam (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)) 197104476Ssam continue; 198104476Ssam /* Software required -- ignore hardware drivers. */ 199104476Ssam if (hard < 0 && 200104476Ssam (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 201104476Ssam continue; 202104476Ssam 203104476Ssam /* See if all the algorithms are supported. */ 204104476Ssam for (cr = cri; cr; cr = cr->cri_next) 205104476Ssam if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) 206104476Ssam break; 207104476Ssam 208104476Ssam if (cr == NULL) { 209104476Ssam /* Ok, all algorithms are supported. */ 210104476Ssam 211104476Ssam /* 212104476Ssam * Can't do everything in one session. 213104476Ssam * 214104476Ssam * XXX Fix this. We need to inject a "virtual" session layer right 215104476Ssam * XXX about here. 216104476Ssam */ 217104476Ssam 218104476Ssam /* Call the driver initialization routine. */ 219104476Ssam lid = hid; /* Pass the driver ID. */ 220104476Ssam err = crypto_drivers[hid].cc_newsession( 221104476Ssam crypto_drivers[hid].cc_arg, &lid, cri); 222104476Ssam if (err == 0) { 223104476Ssam (*sid) = hid; 224104476Ssam (*sid) <<= 32; 225104476Ssam (*sid) |= (lid & 0xffffffff); 226104476Ssam crypto_drivers[hid].cc_sessions++; 227104476Ssam } 228104476Ssam break; 229104476Ssam } 230104476Ssam } 231104476Ssamdone: 232104476Ssam CRYPTO_DRIVER_UNLOCK(); 233104476Ssam return err; 234104476Ssam} 235104476Ssam 236104476Ssam/* 237104476Ssam * Delete an existing session (or a reserved session on an unregistered 238104476Ssam * driver). 239104476Ssam */ 240104476Ssamint 241104476Ssamcrypto_freesession(u_int64_t sid) 242104476Ssam{ 243104476Ssam u_int32_t hid; 244104476Ssam int err; 245104476Ssam 246104476Ssam CRYPTO_DRIVER_LOCK(); 247104476Ssam 248104476Ssam if (crypto_drivers == NULL) { 249104476Ssam err = EINVAL; 250104476Ssam goto done; 251104476Ssam } 252104476Ssam 253104476Ssam /* Determine two IDs. */ 254104476Ssam hid = SESID2HID(sid); 255104476Ssam 256104476Ssam if (hid >= crypto_drivers_num) { 257104476Ssam err = ENOENT; 258104476Ssam goto done; 259104476Ssam } 260104476Ssam 261104476Ssam if (crypto_drivers[hid].cc_sessions) 262104476Ssam crypto_drivers[hid].cc_sessions--; 263104476Ssam 264104476Ssam /* Call the driver cleanup routine, if available. */ 265104476Ssam if (crypto_drivers[hid].cc_freesession) 266104476Ssam err = crypto_drivers[hid].cc_freesession( 267104476Ssam crypto_drivers[hid].cc_arg, sid); 268104476Ssam else 269104476Ssam err = 0; 270104476Ssam 271104476Ssam /* 272104476Ssam * If this was the last session of a driver marked as invalid, 273104476Ssam * make the entry available for reuse. 274104476Ssam */ 275104476Ssam if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 276104476Ssam crypto_drivers[hid].cc_sessions == 0) 277104476Ssam bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 278104476Ssam 279104476Ssamdone: 280104476Ssam CRYPTO_DRIVER_UNLOCK(); 281104476Ssam return err; 282104476Ssam} 283104476Ssam 284104476Ssam/* 285104476Ssam * Return an unused driver id. Used by drivers prior to registering 286104476Ssam * support for the algorithms they handle. 287104476Ssam */ 288104476Ssamint32_t 289104476Ssamcrypto_get_driverid(u_int32_t flags) 290104476Ssam{ 291104476Ssam struct cryptocap *newdrv; 292104476Ssam int i; 293104476Ssam 294104476Ssam CRYPTO_DRIVER_LOCK(); 295104476Ssam 296104476Ssam for (i = 0; i < crypto_drivers_num; i++) 297104476Ssam if (crypto_drivers[i].cc_process == NULL && 298104476Ssam (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 299104476Ssam crypto_drivers[i].cc_sessions == 0) 300104476Ssam break; 301104476Ssam 302104476Ssam /* Out of entries, allocate some more. */ 303104476Ssam if (i == crypto_drivers_num) { 304104476Ssam /* Be careful about wrap-around. */ 305104476Ssam if (2 * crypto_drivers_num <= crypto_drivers_num) { 306104476Ssam CRYPTO_DRIVER_UNLOCK(); 307104476Ssam printf("crypto: driver count wraparound!\n"); 308104476Ssam return -1; 309104476Ssam } 310104476Ssam 311104476Ssam newdrv = malloc(2 * crypto_drivers_num * 312104476Ssam sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 313104476Ssam if (newdrv == NULL) { 314104476Ssam CRYPTO_DRIVER_UNLOCK(); 315104476Ssam printf("crypto: no space to expand driver table!\n"); 316104476Ssam return -1; 317104476Ssam } 318104476Ssam 319104476Ssam bcopy(crypto_drivers, newdrv, 320104476Ssam crypto_drivers_num * sizeof(struct cryptocap)); 321104476Ssam 322104476Ssam crypto_drivers_num *= 2; 323104476Ssam 324104476Ssam free(crypto_drivers, M_CRYPTO_DATA); 325104476Ssam crypto_drivers = newdrv; 326104476Ssam } 327104476Ssam 328104476Ssam /* NB: state is zero'd on free */ 329104476Ssam crypto_drivers[i].cc_sessions = 1; /* Mark */ 330104476Ssam crypto_drivers[i].cc_flags = flags; 331104476Ssam if (bootverbose) 332104476Ssam printf("crypto: assign driver %u, flags %u\n", i, flags); 333104476Ssam 334104476Ssam CRYPTO_DRIVER_UNLOCK(); 335104476Ssam 336104476Ssam return i; 337104476Ssam} 338104476Ssam 339104476Ssamstatic struct cryptocap * 340104476Ssamcrypto_checkdriver(u_int32_t hid) 341104476Ssam{ 342104476Ssam if (crypto_drivers == NULL) 343104476Ssam return NULL; 344104476Ssam return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 345104476Ssam} 346104476Ssam 347104476Ssam/* 348104476Ssam * Register support for a key-related algorithm. This routine 349104476Ssam * is called once for each algorithm supported a driver. 350104476Ssam */ 351104476Ssamint 352104476Ssamcrypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 353104476Ssam int (*kprocess)(void*, struct cryptkop *, int), 354104476Ssam void *karg) 355104476Ssam{ 356104476Ssam struct cryptocap *cap; 357104476Ssam int err; 358104476Ssam 359104476Ssam CRYPTO_DRIVER_LOCK(); 360104476Ssam 361104476Ssam cap = crypto_checkdriver(driverid); 362104476Ssam if (cap != NULL && 363104476Ssam (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 364104476Ssam /* 365104476Ssam * XXX Do some performance testing to determine placing. 366104476Ssam * XXX We probably need an auxiliary data structure that 367104476Ssam * XXX describes relative performances. 368104476Ssam */ 369104476Ssam 370104476Ssam cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 371104476Ssam if (bootverbose) 372104476Ssam printf("crypto: driver %u registers key alg %u flags %u\n" 373104476Ssam , driverid 374104476Ssam , kalg 375104476Ssam , flags 376104476Ssam ); 377104476Ssam 378104476Ssam if (cap->cc_kprocess == NULL) { 379104476Ssam cap->cc_karg = karg; 380104476Ssam cap->cc_kprocess = kprocess; 381104476Ssam } 382104476Ssam err = 0; 383104476Ssam } else 384104476Ssam err = EINVAL; 385104476Ssam 386104476Ssam CRYPTO_DRIVER_UNLOCK(); 387104476Ssam return err; 388104476Ssam} 389104476Ssam 390104476Ssam/* 391104476Ssam * Register support for a non-key-related algorithm. This routine 392104476Ssam * is called once for each such algorithm supported by a driver. 393104476Ssam */ 394104476Ssamint 395104476Ssamcrypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 396104476Ssam u_int32_t flags, 397104476Ssam int (*newses)(void*, u_int32_t*, struct cryptoini*), 398104476Ssam int (*freeses)(void*, u_int64_t), 399104476Ssam int (*process)(void*, struct cryptop *, int), 400104476Ssam void *arg) 401104476Ssam{ 402104476Ssam struct cryptocap *cap; 403104476Ssam int err; 404104476Ssam 405104476Ssam CRYPTO_DRIVER_LOCK(); 406104476Ssam 407104476Ssam cap = crypto_checkdriver(driverid); 408104476Ssam /* NB: algorithms are in the range [1..max] */ 409104476Ssam if (cap != NULL && 410104476Ssam (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 411104476Ssam /* 412104476Ssam * XXX Do some performance testing to determine placing. 413104476Ssam * XXX We probably need an auxiliary data structure that 414104476Ssam * XXX describes relative performances. 415104476Ssam */ 416104476Ssam 417104476Ssam cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 418104476Ssam cap->cc_max_op_len[alg] = maxoplen; 419104476Ssam if (bootverbose) 420104476Ssam printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 421104476Ssam , driverid 422104476Ssam , alg 423104476Ssam , flags 424104476Ssam , maxoplen 425104476Ssam ); 426104476Ssam 427104476Ssam if (cap->cc_process == NULL) { 428104476Ssam cap->cc_arg = arg; 429104476Ssam cap->cc_newsession = newses; 430104476Ssam cap->cc_process = process; 431104476Ssam cap->cc_freesession = freeses; 432104476Ssam cap->cc_sessions = 0; /* Unmark */ 433104476Ssam } 434104476Ssam err = 0; 435104476Ssam } else 436104476Ssam err = EINVAL; 437104476Ssam 438104476Ssam CRYPTO_DRIVER_UNLOCK(); 439104476Ssam return err; 440104476Ssam} 441104476Ssam 442104476Ssam/* 443104476Ssam * Unregister a crypto driver. If there are pending sessions using it, 444104476Ssam * leave enough information around so that subsequent calls using those 445104476Ssam * sessions will correctly detect the driver has been unregistered and 446104476Ssam * reroute requests. 447104476Ssam */ 448104476Ssamint 449104476Ssamcrypto_unregister(u_int32_t driverid, int alg) 450104476Ssam{ 451104476Ssam int i, err; 452104476Ssam u_int32_t ses; 453104476Ssam struct cryptocap *cap; 454104476Ssam 455104476Ssam CRYPTO_DRIVER_LOCK(); 456104476Ssam 457104476Ssam cap = crypto_checkdriver(driverid); 458104476Ssam if (cap != NULL && 459104476Ssam (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 460104476Ssam cap->cc_alg[alg] != 0) { 461104476Ssam cap->cc_alg[alg] = 0; 462104476Ssam cap->cc_max_op_len[alg] = 0; 463104476Ssam 464104476Ssam /* Was this the last algorithm ? */ 465104476Ssam for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 466104476Ssam if (cap->cc_alg[i] != 0) 467104476Ssam break; 468104476Ssam 469104476Ssam if (i == CRYPTO_ALGORITHM_MAX + 1) { 470104476Ssam ses = cap->cc_sessions; 471104476Ssam bzero(cap, sizeof(struct cryptocap)); 472104476Ssam if (ses != 0) { 473104476Ssam /* 474104476Ssam * If there are pending sessions, just mark as invalid. 475104476Ssam */ 476104476Ssam cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 477104476Ssam cap->cc_sessions = ses; 478104476Ssam } 479104476Ssam } 480104476Ssam err = 0; 481104476Ssam } else 482104476Ssam err = EINVAL; 483104476Ssam 484104476Ssam CRYPTO_DRIVER_UNLOCK(); 485104476Ssam return err; 486104476Ssam} 487104476Ssam 488104476Ssam/* 489104476Ssam * Unregister all algorithms associated with a crypto driver. 490104476Ssam * If there are pending sessions using it, leave enough information 491104476Ssam * around so that subsequent calls using those sessions will 492104476Ssam * correctly detect the driver has been unregistered and reroute 493104476Ssam * requests. 494104476Ssam */ 495104476Ssamint 496104476Ssamcrypto_unregister_all(u_int32_t driverid) 497104476Ssam{ 498104476Ssam int i, err; 499104476Ssam u_int32_t ses; 500104476Ssam struct cryptocap *cap; 501104476Ssam 502104476Ssam CRYPTO_DRIVER_LOCK(); 503104476Ssam 504104476Ssam cap = crypto_checkdriver(driverid); 505104476Ssam if (cap != NULL) { 506104476Ssam for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 507104476Ssam cap->cc_alg[i] = 0; 508104476Ssam cap->cc_max_op_len[i] = 0; 509104476Ssam } 510104476Ssam ses = cap->cc_sessions; 511104476Ssam bzero(cap, sizeof(struct cryptocap)); 512104476Ssam if (ses != 0) { 513104476Ssam /* 514104476Ssam * If there are pending sessions, just mark as invalid. 515104476Ssam */ 516104476Ssam cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 517104476Ssam cap->cc_sessions = ses; 518104476Ssam } 519104476Ssam err = 0; 520104476Ssam } else 521104476Ssam err = EINVAL; 522104476Ssam 523104476Ssam CRYPTO_DRIVER_UNLOCK(); 524104476Ssam return err; 525104476Ssam} 526104476Ssam 527104476Ssam/* 528104476Ssam * Clear blockage on a driver. The what parameter indicates whether 529104476Ssam * the driver is now ready for cryptop's and/or cryptokop's. 530104476Ssam */ 531104476Ssamint 532104476Ssamcrypto_unblock(u_int32_t driverid, int what) 533104476Ssam{ 534104476Ssam struct cryptocap *cap; 535104476Ssam int needwakeup, err; 536104476Ssam 537104476Ssam CRYPTO_Q_LOCK(); 538104476Ssam cap = crypto_checkdriver(driverid); 539104476Ssam if (cap != NULL) { 540104628Ssam needwakeup = 0; 541104476Ssam if (what & CRYPTO_SYMQ) { 542104476Ssam needwakeup |= cap->cc_qblocked; 543104476Ssam cap->cc_qblocked = 0; 544104476Ssam } 545104476Ssam if (what & CRYPTO_ASYMQ) { 546104476Ssam needwakeup |= cap->cc_kqblocked; 547104476Ssam cap->cc_kqblocked = 0; 548104476Ssam } 549104628Ssam if (needwakeup) 550104628Ssam wakeup_one(&crp_q); 551104476Ssam err = 0; 552104476Ssam } else 553104476Ssam err = EINVAL; 554104476Ssam CRYPTO_Q_UNLOCK(); 555104476Ssam 556104476Ssam return err; 557104476Ssam} 558104476Ssam 559104476Ssam/* 560104476Ssam * Add a crypto request to a queue, to be processed by the kernel thread. 561104476Ssam */ 562104476Ssamint 563104476Ssamcrypto_dispatch(struct cryptop *crp) 564104476Ssam{ 565104476Ssam struct cryptocap *cap; 566104476Ssam int wasempty; 567104476Ssam 568108587Ssam cryptostats.cs_ops++; 569108587Ssam 570108587Ssam#ifdef CRYPTO_TIMING 571108587Ssam if (crypto_timing) 572108587Ssam binuptime(&crp->crp_tstamp); 573108587Ssam#endif 574108587Ssam 575104476Ssam CRYPTO_Q_LOCK(); 576104476Ssam wasempty = TAILQ_EMPTY(&crp_q); 577104476Ssam TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 578104476Ssam 579104476Ssam /* 580104476Ssam * Wakeup processing thread if driver is not blocked. 581104476Ssam */ 582104476Ssam cap = crypto_checkdriver(SESID2HID(crp->crp_sid)); 583104476Ssam if (cap && !cap->cc_qblocked && wasempty) 584104476Ssam wakeup_one(&crp_q); 585104476Ssam CRYPTO_Q_UNLOCK(); 586104476Ssam 587104476Ssam return 0; 588104476Ssam} 589104476Ssam 590104476Ssam/* 591104476Ssam * Add an asymetric crypto request to a queue, 592104476Ssam * to be processed by the kernel thread. 593104476Ssam */ 594104476Ssamint 595104476Ssamcrypto_kdispatch(struct cryptkop *krp) 596104476Ssam{ 597104476Ssam struct cryptocap *cap; 598104476Ssam int wasempty; 599104476Ssam 600108587Ssam cryptostats.cs_kops++; 601108587Ssam 602104476Ssam CRYPTO_Q_LOCK(); 603104476Ssam wasempty = TAILQ_EMPTY(&crp_kq); 604104476Ssam TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 605104476Ssam 606104476Ssam /* 607104476Ssam * Wakeup processing thread if driver is not blocked. 608104476Ssam */ 609104476Ssam cap = crypto_checkdriver(krp->krp_hid); 610104476Ssam if (cap && !cap->cc_kqblocked && wasempty) 611104476Ssam wakeup_one(&crp_q); /* NB: shared wait channel */ 612104476Ssam CRYPTO_Q_UNLOCK(); 613104476Ssam 614104476Ssam return 0; 615104476Ssam} 616104476Ssam 617104476Ssam/* 618104476Ssam * Dispatch an assymetric crypto request to the appropriate crypto devices. 619104476Ssam */ 620104476Ssamstatic int 621104476Ssamcrypto_kinvoke(struct cryptkop *krp, int hint) 622104476Ssam{ 623104476Ssam u_int32_t hid; 624104476Ssam int error; 625104476Ssam 626104476Ssam mtx_assert(&crypto_q_mtx, MA_OWNED); 627104476Ssam 628104476Ssam /* Sanity checks. */ 629104628Ssam if (krp == NULL) 630104476Ssam return EINVAL; 631104628Ssam if (krp->krp_callback == NULL) { 632104628Ssam free(krp, M_XDATA); /* XXX allocated in cryptodev */ 633104628Ssam return EINVAL; 634104628Ssam } 635104476Ssam 636104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 637104476Ssam if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 638104476Ssam !crypto_devallowsoft) 639104476Ssam continue; 640104476Ssam if (crypto_drivers[hid].cc_kprocess == NULL) 641104476Ssam continue; 642104476Ssam if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 643104476Ssam CRYPTO_ALG_FLAG_SUPPORTED) == 0) 644104476Ssam continue; 645104476Ssam break; 646104476Ssam } 647104476Ssam if (hid < crypto_drivers_num) { 648104476Ssam krp->krp_hid = hid; 649104476Ssam error = crypto_drivers[hid].cc_kprocess( 650104476Ssam crypto_drivers[hid].cc_karg, krp, hint); 651104476Ssam } else 652104476Ssam error = ENODEV; 653104476Ssam 654104476Ssam if (error) { 655104476Ssam krp->krp_status = error; 656104628Ssam crypto_kdone(krp); 657104476Ssam } 658104476Ssam return 0; 659104476Ssam} 660104476Ssam 661108587Ssam#ifdef CRYPTO_TIMING 662108587Ssamstatic void 663108587Ssamcrypto_tstat(struct cryptotstat *ts, struct bintime *bt) 664108587Ssam{ 665108587Ssam struct bintime now, delta; 666108587Ssam struct timespec t; 667108587Ssam uint64_t u; 668108587Ssam 669108587Ssam binuptime(&now); 670108587Ssam u = now.frac; 671108587Ssam delta.frac = now.frac - bt->frac; 672108587Ssam delta.sec = now.sec - bt->sec; 673108587Ssam if (u < delta.frac) 674108587Ssam delta.sec--; 675108587Ssam bintime2timespec(&delta, &t); 676108587Ssam timespecadd(&ts->acc, &t); 677108587Ssam if (timespeccmp(&t, &ts->min, <)) 678108587Ssam ts->min = t; 679108587Ssam if (timespeccmp(&t, &ts->max, >)) 680108587Ssam ts->max = t; 681108587Ssam ts->count++; 682108587Ssam 683108587Ssam *bt = now; 684108587Ssam} 685108587Ssam#endif 686108587Ssam 687104476Ssam/* 688104476Ssam * Dispatch a crypto request to the appropriate crypto devices. 689104476Ssam */ 690104476Ssamstatic int 691104476Ssamcrypto_invoke(struct cryptop *crp, int hint) 692104476Ssam{ 693104476Ssam u_int32_t hid; 694104476Ssam int (*process)(void*, struct cryptop *, int); 695104476Ssam 696108587Ssam#ifdef CRYPTO_TIMING 697108587Ssam if (crypto_timing) 698108587Ssam crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 699108587Ssam#endif 700104476Ssam mtx_assert(&crypto_q_mtx, MA_OWNED); 701104476Ssam 702104476Ssam /* Sanity checks. */ 703104628Ssam if (crp == NULL) 704104476Ssam return EINVAL; 705104628Ssam if (crp->crp_callback == NULL) { 706104628Ssam crypto_freereq(crp); 707104628Ssam return EINVAL; 708104628Ssam } 709104476Ssam if (crp->crp_desc == NULL) { 710104476Ssam crp->crp_etype = EINVAL; 711104628Ssam crypto_done(crp); 712104476Ssam return 0; 713104476Ssam } 714104476Ssam 715104476Ssam hid = SESID2HID(crp->crp_sid); 716104476Ssam if (hid < crypto_drivers_num) { 717104476Ssam if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 718104476Ssam crypto_freesession(crp->crp_sid); 719104476Ssam process = crypto_drivers[hid].cc_process; 720104476Ssam } else { 721104476Ssam process = NULL; 722104476Ssam } 723104476Ssam 724104476Ssam if (process == NULL) { 725104476Ssam struct cryptodesc *crd; 726104476Ssam u_int64_t nid; 727104476Ssam 728104476Ssam /* 729104476Ssam * Driver has unregistered; migrate the session and return 730104476Ssam * an error to the caller so they'll resubmit the op. 731104476Ssam */ 732104476Ssam for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 733104476Ssam crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 734104476Ssam 735104476Ssam if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 736104476Ssam crp->crp_sid = nid; 737104476Ssam 738104476Ssam crp->crp_etype = EAGAIN; 739104628Ssam crypto_done(crp); 740104476Ssam return 0; 741104476Ssam } else { 742104476Ssam /* 743104476Ssam * Invoke the driver to process the request. 744104476Ssam */ 745104476Ssam return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 746104476Ssam } 747104476Ssam} 748104476Ssam 749104476Ssam/* 750104476Ssam * Release a set of crypto descriptors. 751104476Ssam */ 752104476Ssamvoid 753104476Ssamcrypto_freereq(struct cryptop *crp) 754104476Ssam{ 755104476Ssam struct cryptodesc *crd; 756104476Ssam 757104476Ssam if (crp == NULL) 758104476Ssam return; 759104476Ssam 760104476Ssam while ((crd = crp->crp_desc) != NULL) { 761104476Ssam crp->crp_desc = crd->crd_next; 762104476Ssam uma_zfree(cryptodesc_zone, crd); 763104476Ssam } 764104476Ssam 765104476Ssam uma_zfree(cryptop_zone, crp); 766104476Ssam} 767104476Ssam 768104476Ssam/* 769104476Ssam * Acquire a set of crypto descriptors. 770104476Ssam */ 771104476Ssamstruct cryptop * 772104476Ssamcrypto_getreq(int num) 773104476Ssam{ 774104476Ssam struct cryptodesc *crd; 775104476Ssam struct cryptop *crp; 776104476Ssam 777104476Ssam crp = uma_zalloc(cryptop_zone, 0); 778104476Ssam if (crp != NULL) { 779104476Ssam while (num--) { 780104476Ssam crd = uma_zalloc(cryptodesc_zone, 0); 781104476Ssam if (crd == NULL) { 782104476Ssam crypto_freereq(crp); 783104476Ssam return NULL; 784104476Ssam } 785104476Ssam 786104476Ssam crd->crd_next = crp->crp_desc; 787104476Ssam crp->crp_desc = crd; 788104476Ssam } 789104476Ssam } 790104476Ssam return crp; 791104476Ssam} 792104476Ssam 793104476Ssam/* 794104476Ssam * Invoke the callback on behalf of the driver. 795104476Ssam */ 796104476Ssamvoid 797104476Ssamcrypto_done(struct cryptop *crp) 798104476Ssam{ 799104476Ssam int wasempty; 800104476Ssam 801108587Ssam if (crp->crp_etype != 0) 802108587Ssam cryptostats.cs_errs++; 803108587Ssam#ifdef CRYPTO_TIMING 804108587Ssam if (crypto_timing) 805108587Ssam crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 806108587Ssam#endif 807104476Ssam CRYPTO_RETQ_LOCK(); 808104476Ssam wasempty = TAILQ_EMPTY(&crp_ret_q); 809104476Ssam TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 810104476Ssam 811104476Ssam if (wasempty) 812104628Ssam wakeup_one(&crp_ret_q); /* shared wait channel */ 813104628Ssam CRYPTO_RETQ_UNLOCK(); 814104476Ssam} 815104476Ssam 816104476Ssam/* 817104476Ssam * Invoke the callback on behalf of the driver. 818104476Ssam */ 819104476Ssamvoid 820104476Ssamcrypto_kdone(struct cryptkop *krp) 821104476Ssam{ 822104476Ssam int wasempty; 823104476Ssam 824108587Ssam if (krp->krp_status != 0) 825108587Ssam cryptostats.cs_kerrs++; 826104476Ssam CRYPTO_RETQ_LOCK(); 827104476Ssam wasempty = TAILQ_EMPTY(&crp_ret_kq); 828104476Ssam TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 829104476Ssam 830104476Ssam if (wasempty) 831104628Ssam wakeup_one(&crp_ret_q); /* shared wait channel */ 832104628Ssam CRYPTO_RETQ_UNLOCK(); 833104476Ssam} 834104476Ssam 835104476Ssamint 836104476Ssamcrypto_getfeat(int *featp) 837104476Ssam{ 838104476Ssam int hid, kalg, feat = 0; 839104476Ssam 840104476Ssam if (!crypto_userasymcrypto) 841104476Ssam goto out; 842104476Ssam 843104476Ssam CRYPTO_DRIVER_LOCK(); 844104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 845104476Ssam if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 846104476Ssam !crypto_devallowsoft) { 847104476Ssam continue; 848104476Ssam } 849104476Ssam if (crypto_drivers[hid].cc_kprocess == NULL) 850104476Ssam continue; 851104476Ssam for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 852104476Ssam if ((crypto_drivers[hid].cc_kalg[kalg] & 853104476Ssam CRYPTO_ALG_FLAG_SUPPORTED) != 0) 854104476Ssam feat |= 1 << kalg; 855104476Ssam } 856104476Ssam CRYPTO_DRIVER_UNLOCK(); 857104476Ssamout: 858104476Ssam *featp = feat; 859104476Ssam return (0); 860104476Ssam} 861104476Ssam 862104476Ssamstatic struct proc *cryptoproc; 863104476Ssam 864104476Ssamstatic void 865104476Ssamcrypto_shutdown(void *arg, int howto) 866104476Ssam{ 867104476Ssam /* XXX flush queues */ 868104476Ssam} 869104476Ssam 870104476Ssam/* 871104628Ssam * Crypto thread, dispatches crypto requests. 872104476Ssam */ 873104476Ssamstatic void 874104476Ssamcrypto_proc(void) 875104476Ssam{ 876104628Ssam struct cryptop *crp, *submit; 877104628Ssam struct cryptkop *krp; 878104476Ssam struct cryptocap *cap; 879104476Ssam int result, hint; 880104476Ssam 881104476Ssam EVENTHANDLER_REGISTER(shutdown_pre_sync, crypto_shutdown, NULL, 882104476Ssam SHUTDOWN_PRI_FIRST); 883104476Ssam 884104628Ssam CRYPTO_Q_LOCK(); 885104628Ssam 886104476Ssam for (;;) { 887104476Ssam /* 888104476Ssam * Find the first element in the queue that can be 889104476Ssam * processed and look-ahead to see if multiple ops 890104476Ssam * are ready for the same driver. 891104476Ssam */ 892104476Ssam submit = NULL; 893104476Ssam hint = 0; 894104476Ssam TAILQ_FOREACH(crp, &crp_q, crp_next) { 895104476Ssam u_int32_t hid = SESID2HID(crp->crp_sid); 896104476Ssam cap = crypto_checkdriver(hid); 897104476Ssam if (cap == NULL || cap->cc_process == NULL) { 898104476Ssam /* Op needs to be migrated, process it. */ 899104476Ssam if (submit == NULL) 900104476Ssam submit = crp; 901104476Ssam break; 902104476Ssam } 903104476Ssam if (!cap->cc_qblocked) { 904104476Ssam if (submit != NULL) { 905104476Ssam /* 906104476Ssam * We stop on finding another op, 907104476Ssam * regardless whether its for the same 908104476Ssam * driver or not. We could keep 909104476Ssam * searching the queue but it might be 910104476Ssam * better to just use a per-driver 911104476Ssam * queue instead. 912104476Ssam */ 913104476Ssam if (SESID2HID(submit->crp_sid) == hid) 914104476Ssam hint = CRYPTO_HINT_MORE; 915104476Ssam break; 916104476Ssam } else { 917104476Ssam submit = crp; 918104476Ssam if (submit->crp_flags & CRYPTO_F_NODELAY) 919104476Ssam break; 920104476Ssam /* keep scanning for more are q'd */ 921104476Ssam } 922104476Ssam } 923104476Ssam } 924104476Ssam if (submit != NULL) { 925104476Ssam TAILQ_REMOVE(&crp_q, submit, crp_next); 926104476Ssam result = crypto_invoke(submit, hint); 927104476Ssam if (result == ERESTART) { 928104476Ssam /* 929104476Ssam * The driver ran out of resources, mark the 930104476Ssam * driver ``blocked'' for cryptop's and put 931104476Ssam * the request back in the queue. It would 932104476Ssam * best to put the request back where we got 933104476Ssam * it but that's hard so for now we put it 934104476Ssam * at the front. This should be ok; putting 935104476Ssam * it at the end does not work. 936104476Ssam */ 937104476Ssam /* XXX validate sid again? */ 938104476Ssam crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1; 939104476Ssam TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 940108587Ssam cryptostats.cs_blocks++; 941104476Ssam } 942104476Ssam } 943104476Ssam 944104476Ssam /* As above, but for key ops */ 945104476Ssam TAILQ_FOREACH(krp, &crp_kq, krp_next) { 946104476Ssam cap = crypto_checkdriver(krp->krp_hid); 947104476Ssam if (cap == NULL || cap->cc_kprocess == NULL) { 948104476Ssam /* Op needs to be migrated, process it. */ 949104476Ssam break; 950104476Ssam } 951104476Ssam if (!cap->cc_kqblocked) 952104476Ssam break; 953104476Ssam } 954104476Ssam if (krp != NULL) { 955104476Ssam TAILQ_REMOVE(&crp_kq, krp, krp_next); 956104476Ssam result = crypto_kinvoke(krp, 0); 957104476Ssam if (result == ERESTART) { 958104476Ssam /* 959104476Ssam * The driver ran out of resources, mark the 960104476Ssam * driver ``blocked'' for cryptkop's and put 961104476Ssam * the request back in the queue. It would 962104476Ssam * best to put the request back where we got 963104476Ssam * it but that's hard so for now we put it 964104476Ssam * at the front. This should be ok; putting 965104476Ssam * it at the end does not work. 966104476Ssam */ 967104476Ssam /* XXX validate sid again? */ 968104476Ssam crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 969104476Ssam TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 970108587Ssam cryptostats.cs_kblocks++; 971104476Ssam } 972104476Ssam } 973104476Ssam 974104628Ssam if (submit == NULL && krp == NULL) { 975104476Ssam /* 976104476Ssam * Nothing more to be processed. Sleep until we're 977104476Ssam * woken because there are more ops to process. 978104476Ssam * This happens either by submission or by a driver 979104476Ssam * becoming unblocked and notifying us through 980104476Ssam * crypto_unblock. Note that when we wakeup we 981104476Ssam * start processing each queue again from the 982104476Ssam * front. It's not clear that it's important to 983104476Ssam * preserve this ordering since ops may finish 984104476Ssam * out of order if dispatched to different devices 985104476Ssam * and some become blocked while others do not. 986104476Ssam */ 987104628Ssam msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 988108587Ssam cryptostats.cs_intrs++; 989104476Ssam } 990104476Ssam } 991104476Ssam} 992104476Ssamstatic struct kproc_desc crypto_kp = { 993104476Ssam "crypto", 994104476Ssam crypto_proc, 995104476Ssam &cryptoproc 996104476Ssam}; 997105251SmarkmSYSINIT(crypto_proc, SI_SUB_KTHREAD_IDLE, SI_ORDER_THIRD, 998104628Ssam kproc_start, &crypto_kp) 999104628Ssam 1000104628Ssamstatic struct proc *cryptoretproc; 1001104628Ssam 1002104628Ssamstatic void 1003104628Ssamcrypto_ret_shutdown(void *arg, int howto) 1004104628Ssam{ 1005104628Ssam /* XXX flush queues */ 1006104628Ssam} 1007104628Ssam 1008104628Ssam/* 1009104628Ssam * Crypto returns thread, does callbacks for processed crypto requests. 1010104628Ssam * Callbacks are done here, rather than in the crypto drivers, because 1011104628Ssam * callbacks typically are expensive and would slow interrupt handling. 1012104628Ssam */ 1013104628Ssamstatic void 1014104628Ssamcrypto_ret_proc(void) 1015104628Ssam{ 1016104628Ssam struct cryptop *crpt; 1017104628Ssam struct cryptkop *krpt; 1018104628Ssam 1019104628Ssam EVENTHANDLER_REGISTER(shutdown_pre_sync, crypto_ret_shutdown, NULL, 1020104628Ssam SHUTDOWN_PRI_FIRST); 1021104628Ssam 1022104628Ssam CRYPTO_RETQ_LOCK(); 1023104628Ssam 1024104628Ssam for (;;) { 1025104628Ssam /* Harvest return q's for completed ops */ 1026104628Ssam crpt = TAILQ_FIRST(&crp_ret_q); 1027104628Ssam if (crpt != NULL) 1028104628Ssam TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1029104628Ssam 1030104628Ssam krpt = TAILQ_FIRST(&crp_ret_kq); 1031104628Ssam if (krpt != NULL) 1032104628Ssam TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1033104628Ssam 1034104628Ssam if (crpt != NULL || krpt != NULL) { 1035104628Ssam CRYPTO_RETQ_UNLOCK(); 1036104628Ssam /* 1037104628Ssam * Run callbacks unlocked. 1038104628Ssam */ 1039108587Ssam if (crpt != NULL) { 1040108587Ssam#ifdef CRYPTO_TIMING 1041108587Ssam if (crypto_timing) { 1042108587Ssam /* 1043108587Ssam * NB: We must copy the timestamp before 1044108587Ssam * doing the callback as the cryptop is 1045108587Ssam * likely to be reclaimed. 1046108587Ssam */ 1047108587Ssam struct bintime t = crpt->crp_tstamp; 1048108587Ssam crypto_tstat(&cryptostats.cs_cb, &t); 1049108587Ssam crpt->crp_callback(crpt); 1050108587Ssam crypto_tstat(&cryptostats.cs_finis, &t); 1051108587Ssam } else 1052108587Ssam#endif 1053108587Ssam crpt->crp_callback(crpt); 1054108587Ssam } 1055104628Ssam if (krpt != NULL) 1056104628Ssam krpt->krp_callback(krpt); 1057104628Ssam CRYPTO_RETQ_LOCK(); 1058104628Ssam } else { 1059104628Ssam /* 1060104628Ssam * Nothing more to be processed. Sleep until we're 1061104628Ssam * woken because there are more returns to process. 1062104628Ssam */ 1063104628Ssam msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1064104628Ssam "crypto_ret_wait", 0); 1065108587Ssam cryptostats.cs_rets++; 1066104628Ssam } 1067104628Ssam } 1068104628Ssam} 1069104628Ssamstatic struct kproc_desc crypto_ret_kp = { 1070104628Ssam "crypto returns", 1071104628Ssam crypto_ret_proc, 1072104628Ssam &cryptoretproc 1073104628Ssam}; 1074105251SmarkmSYSINIT(crypto_ret_proc, SI_SUB_KTHREAD_IDLE, SI_ORDER_THIRD, 1075104628Ssam kproc_start, &crypto_ret_kp) 1076