crypto.c revision 158716
1104476Ssam/* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 2139825Simp/*- 3104476Ssam * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 4104476Ssam * 5104476Ssam * This code was written by Angelos D. Keromytis in Athens, Greece, in 6104476Ssam * February 2000. Network Security Technologies Inc. (NSTI) kindly 7104476Ssam * supported the development of this code. 8104476Ssam * 9104476Ssam * Copyright (c) 2000, 2001 Angelos D. Keromytis 10104476Ssam * 11104476Ssam * Permission to use, copy, and modify this software with or without fee 12104476Ssam * is hereby granted, provided that this entire notice is included in 13104476Ssam * all source code copies of any software which is or includes a copy or 14104476Ssam * modification of this software. 15104476Ssam * 16104476Ssam * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 17104476Ssam * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 18104476Ssam * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 19104476Ssam * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 20104476Ssam * PURPOSE. 21104476Ssam */ 22116191Sobrien 23116191Sobrien#include <sys/cdefs.h> 24116191Sobrien__FBSDID("$FreeBSD: head/sys/opencrypto/crypto.c 158716 2006-05-18 06:28:39Z pjd $"); 25116191Sobrien 26108587Ssam#define CRYPTO_TIMING /* enable timing support */ 27104476Ssam 28104476Ssam#include <sys/param.h> 29104476Ssam#include <sys/systm.h> 30104476Ssam#include <sys/eventhandler.h> 31104476Ssam#include <sys/kernel.h> 32104476Ssam#include <sys/kthread.h> 33104476Ssam#include <sys/lock.h> 34129880Sphk#include <sys/module.h> 35104476Ssam#include <sys/mutex.h> 36104476Ssam#include <sys/malloc.h> 37104476Ssam#include <sys/proc.h> 38104476Ssam#include <sys/sysctl.h> 39104476Ssam 40104476Ssam#include <vm/uma.h> 41104476Ssam#include <opencrypto/cryptodev.h> 42104628Ssam#include <opencrypto/xform.h> /* XXX for M_XDATA */ 43104476Ssam 44104476Ssam/* 45104476Ssam * Crypto drivers register themselves by allocating a slot in the 46104476Ssam * crypto_drivers table with crypto_get_driverid() and then registering 47104476Ssam * each algorithm they support with crypto_register() and crypto_kregister(). 48104476Ssam */ 49104476Ssamstatic struct mtx crypto_drivers_mtx; /* lock on driver table */ 50104476Ssam#define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 51104476Ssam#define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 52104476Ssamstatic struct cryptocap *crypto_drivers = NULL; 53104476Ssamstatic int crypto_drivers_num = 0; 54104476Ssam 55104476Ssam/* 56104476Ssam * There are two queues for crypto requests; one for symmetric (e.g. 57104476Ssam * cipher) operations and one for asymmetric (e.g. MOD)operations. 58104476Ssam * A single mutex is used to lock access to both queues. We could 59104476Ssam * have one per-queue but having one simplifies handling of block/unblock 60104476Ssam * operations. 61104476Ssam */ 62104476Ssamstatic TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 63104476Ssamstatic TAILQ_HEAD(,cryptkop) crp_kq; 64104476Ssamstatic struct mtx crypto_q_mtx; 65104476Ssam#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 66104476Ssam#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 67158702Spjd#define CRYPTO_Q_EMPTY() (TAILQ_EMPTY(&crp_q) && TAILQ_EMPTY(&crp_kq)) 68104476Ssam 69104476Ssam/* 70104476Ssam * There are two queues for processing completed crypto requests; one 71104476Ssam * for the symmetric and one for the asymmetric ops. We only need one 72104476Ssam * but have two to avoid type futzing (cryptop vs. cryptkop). A single 73104476Ssam * mutex is used to lock access to both queues. Note that this lock 74104476Ssam * must be separate from the lock on request queues to insure driver 75104476Ssam * callbacks don't generate lock order reversals. 76104476Ssam */ 77104476Ssamstatic TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 78104476Ssamstatic TAILQ_HEAD(,cryptkop) crp_ret_kq; 79104476Ssamstatic struct mtx crypto_ret_q_mtx; 80104476Ssam#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 81104476Ssam#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 82104476Ssam 83104476Ssamstatic uma_zone_t cryptop_zone; 84104476Ssamstatic uma_zone_t cryptodesc_zone; 85104476Ssam 86104476Ssamint crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 87104476SsamSYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 88104476Ssam &crypto_userasymcrypto, 0, 89104476Ssam "Enable/disable user-mode access to asymmetric crypto support"); 90104476Ssamint crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 91104476SsamSYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 92104476Ssam &crypto_devallowsoft, 0, 93104476Ssam "Enable/disable use of software asym crypto support"); 94104476Ssam 95104476SsamMALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 96104476Ssam 97108588Ssamstatic void crypto_proc(void); 98108588Ssamstatic struct proc *cryptoproc; 99108588Ssamstatic void crypto_ret_proc(void); 100108588Ssamstatic struct proc *cryptoretproc; 101108588Ssamstatic void crypto_destroy(void); 102158702Spjdstatic int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 103158702Spjdstatic int crypto_kinvoke(struct cryptkop *krp); 104108588Ssam 105108587Ssamstatic struct cryptostats cryptostats; 106108587SsamSYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 107108587Ssam cryptostats, "Crypto system statistics"); 108108587Ssam 109108587Ssam#ifdef CRYPTO_TIMING 110108587Ssamstatic int crypto_timing = 0; 111108587SsamSYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 112108587Ssam &crypto_timing, 0, "Enable/disable crypto timing support"); 113108587Ssam#endif 114108587Ssam 115108588Ssamstatic int 116104476Ssamcrypto_init(void) 117104476Ssam{ 118108588Ssam int error; 119108588Ssam 120115746Ssam mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 121115746Ssam MTX_DEF|MTX_QUIET); 122108588Ssam 123108588Ssam TAILQ_INIT(&crp_q); 124108588Ssam TAILQ_INIT(&crp_kq); 125115746Ssam mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 126108588Ssam 127108588Ssam TAILQ_INIT(&crp_ret_q); 128108588Ssam TAILQ_INIT(&crp_ret_kq); 129115746Ssam mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 130108588Ssam 131104476Ssam cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 132104476Ssam 0, 0, 0, 0, 133104476Ssam UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 134104476Ssam cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 135104476Ssam 0, 0, 0, 0, 136104476Ssam UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 137108588Ssam if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 138108588Ssam printf("crypto_init: cannot setup crypto zones\n"); 139108588Ssam error = ENOMEM; 140108588Ssam goto bad; 141108588Ssam } 142104476Ssam 143104476Ssam crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 144104476Ssam crypto_drivers = malloc(crypto_drivers_num * 145104476Ssam sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 146108588Ssam if (crypto_drivers == NULL) { 147108588Ssam printf("crypto_init: cannot setup crypto drivers\n"); 148108588Ssam error = ENOMEM; 149108588Ssam goto bad; 150108588Ssam } 151104476Ssam 152108588Ssam error = kthread_create((void (*)(void *)) crypto_proc, NULL, 153108588Ssam &cryptoproc, 0, 0, "crypto"); 154108588Ssam if (error) { 155108588Ssam printf("crypto_init: cannot start crypto thread; error %d", 156108588Ssam error); 157108588Ssam goto bad; 158108588Ssam } 159104476Ssam 160108588Ssam error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 161108588Ssam &cryptoretproc, 0, 0, "crypto returns"); 162108588Ssam if (error) { 163108588Ssam printf("crypto_init: cannot start cryptoret thread; error %d", 164108588Ssam error); 165108588Ssam goto bad; 166108588Ssam } 167108588Ssam return 0; 168108588Ssambad: 169108588Ssam crypto_destroy(); 170108588Ssam return error; 171104476Ssam} 172104476Ssam 173104476Ssam/* 174108588Ssam * Signal a crypto thread to terminate. We use the driver 175108588Ssam * table lock to synchronize the sleep/wakeups so that we 176108588Ssam * are sure the threads have terminated before we release 177108588Ssam * the data structures they use. See crypto_finis below 178108588Ssam * for the other half of this song-and-dance. 179108588Ssam */ 180108588Ssamstatic void 181108588Ssamcrypto_terminate(struct proc **pp, void *q) 182108588Ssam{ 183108588Ssam struct proc *p; 184108588Ssam 185108588Ssam mtx_assert(&crypto_drivers_mtx, MA_OWNED); 186108588Ssam p = *pp; 187108588Ssam *pp = NULL; 188108588Ssam if (p) { 189108588Ssam wakeup_one(q); 190108588Ssam PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 191108588Ssam CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 192108588Ssam msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 193108588Ssam PROC_UNLOCK(p); 194108588Ssam CRYPTO_DRIVER_LOCK(); 195108588Ssam } 196108588Ssam} 197108588Ssam 198108588Ssamstatic void 199108588Ssamcrypto_destroy(void) 200108588Ssam{ 201108588Ssam /* 202108588Ssam * Terminate any crypto threads. 203108588Ssam */ 204108588Ssam CRYPTO_DRIVER_LOCK(); 205108588Ssam crypto_terminate(&cryptoproc, &crp_q); 206108588Ssam crypto_terminate(&cryptoretproc, &crp_ret_q); 207108588Ssam CRYPTO_DRIVER_UNLOCK(); 208108588Ssam 209108588Ssam /* XXX flush queues??? */ 210108588Ssam 211108588Ssam /* 212108588Ssam * Reclaim dynamically allocated resources. 213108588Ssam */ 214108588Ssam if (crypto_drivers != NULL) 215108588Ssam free(crypto_drivers, M_CRYPTO_DATA); 216108588Ssam 217108588Ssam if (cryptodesc_zone != NULL) 218108588Ssam uma_zdestroy(cryptodesc_zone); 219108588Ssam if (cryptop_zone != NULL) 220108588Ssam uma_zdestroy(cryptop_zone); 221108588Ssam mtx_destroy(&crypto_q_mtx); 222108588Ssam mtx_destroy(&crypto_ret_q_mtx); 223108588Ssam mtx_destroy(&crypto_drivers_mtx); 224108588Ssam} 225108588Ssam 226108588Ssam/* 227105251Smarkm * Initialization code, both for static and dynamic loading. 228105251Smarkm */ 229105251Smarkmstatic int 230105251Smarkmcrypto_modevent(module_t mod, int type, void *unused) 231105251Smarkm{ 232108588Ssam int error = EINVAL; 233108588Ssam 234105251Smarkm switch (type) { 235105251Smarkm case MOD_LOAD: 236108588Ssam error = crypto_init(); 237108588Ssam if (error == 0 && bootverbose) 238105251Smarkm printf("crypto: <crypto core>\n"); 239108588Ssam break; 240105251Smarkm case MOD_UNLOAD: 241105251Smarkm /*XXX disallow if active sessions */ 242108588Ssam error = 0; 243108588Ssam crypto_destroy(); 244105251Smarkm return 0; 245105251Smarkm } 246108588Ssam return error; 247105251Smarkm} 248105251Smarkm 249105251Smarkmstatic moduledata_t crypto_mod = { 250105251Smarkm "crypto", 251105251Smarkm crypto_modevent, 252105251Smarkm 0 253105251Smarkm}; 254105251SmarkmMODULE_VERSION(crypto, 1); 255106676SsamDECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 256156281SwkoszekMODULE_DEPEND(crypto, zlib, 1, 1, 1); 257105251Smarkm 258105251Smarkm/* 259104476Ssam * Create a new session. 260104476Ssam */ 261104476Ssamint 262104476Ssamcrypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 263104476Ssam{ 264104476Ssam struct cryptoini *cr; 265104476Ssam u_int32_t hid, lid; 266104476Ssam int err = EINVAL; 267104476Ssam 268104476Ssam CRYPTO_DRIVER_LOCK(); 269104476Ssam 270104476Ssam if (crypto_drivers == NULL) 271104476Ssam goto done; 272104476Ssam 273104476Ssam /* 274104476Ssam * The algorithm we use here is pretty stupid; just use the 275104476Ssam * first driver that supports all the algorithms we need. 276104476Ssam * 277104476Ssam * XXX We need more smarts here (in real life too, but that's 278104476Ssam * XXX another story altogether). 279104476Ssam */ 280104476Ssam 281104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 282116924Ssam struct cryptocap *cap = &crypto_drivers[hid]; 283104476Ssam /* 284104476Ssam * If it's not initialized or has remaining sessions 285104476Ssam * referencing it, skip. 286104476Ssam */ 287116924Ssam if (cap->cc_newsession == NULL || 288116924Ssam (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) 289104476Ssam continue; 290104476Ssam 291104476Ssam /* Hardware required -- ignore software drivers. */ 292116924Ssam if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) 293104476Ssam continue; 294104476Ssam /* Software required -- ignore hardware drivers. */ 295116924Ssam if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 296104476Ssam continue; 297104476Ssam 298104476Ssam /* See if all the algorithms are supported. */ 299104476Ssam for (cr = cri; cr; cr = cr->cri_next) 300116924Ssam if (cap->cc_alg[cr->cri_alg] == 0) 301104476Ssam break; 302104476Ssam 303104476Ssam if (cr == NULL) { 304104476Ssam /* Ok, all algorithms are supported. */ 305104476Ssam 306104476Ssam /* 307104476Ssam * Can't do everything in one session. 308104476Ssam * 309104476Ssam * XXX Fix this. We need to inject a "virtual" session layer right 310104476Ssam * XXX about here. 311104476Ssam */ 312104476Ssam 313104476Ssam /* Call the driver initialization routine. */ 314104476Ssam lid = hid; /* Pass the driver ID. */ 315116924Ssam err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri); 316104476Ssam if (err == 0) { 317116924Ssam /* XXX assert (hid &~ 0xffffff) == 0 */ 318116924Ssam /* XXX assert (cap->cc_flags &~ 0xff) == 0 */ 319116924Ssam (*sid) = ((cap->cc_flags & 0xff) << 24) | hid; 320104476Ssam (*sid) <<= 32; 321104476Ssam (*sid) |= (lid & 0xffffffff); 322116924Ssam cap->cc_sessions++; 323104476Ssam } 324104476Ssam break; 325104476Ssam } 326104476Ssam } 327104476Ssamdone: 328104476Ssam CRYPTO_DRIVER_UNLOCK(); 329104476Ssam return err; 330104476Ssam} 331104476Ssam 332158702Spjdstatic void 333158702Spjdcrypto_remove(struct cryptocap *cap) 334158702Spjd{ 335158702Spjd 336158702Spjd mtx_assert(&crypto_drivers_mtx, MA_OWNED); 337158702Spjd if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 338158702Spjd bzero(cap, sizeof(*cap)); 339158702Spjd} 340158702Spjd 341104476Ssam/* 342104476Ssam * Delete an existing session (or a reserved session on an unregistered 343104476Ssam * driver). 344104476Ssam */ 345104476Ssamint 346104476Ssamcrypto_freesession(u_int64_t sid) 347104476Ssam{ 348158702Spjd struct cryptocap *cap; 349104476Ssam u_int32_t hid; 350104476Ssam int err; 351104476Ssam 352104476Ssam CRYPTO_DRIVER_LOCK(); 353104476Ssam 354104476Ssam if (crypto_drivers == NULL) { 355104476Ssam err = EINVAL; 356104476Ssam goto done; 357104476Ssam } 358104476Ssam 359104476Ssam /* Determine two IDs. */ 360116924Ssam hid = CRYPTO_SESID2HID(sid); 361104476Ssam 362104476Ssam if (hid >= crypto_drivers_num) { 363104476Ssam err = ENOENT; 364104476Ssam goto done; 365104476Ssam } 366158702Spjd cap = &crypto_drivers[hid]; 367104476Ssam 368158702Spjd if (cap->cc_sessions) 369158702Spjd cap->cc_sessions--; 370104476Ssam 371104476Ssam /* Call the driver cleanup routine, if available. */ 372158702Spjd if (cap->cc_freesession) 373158702Spjd err = cap->cc_freesession(cap->cc_arg, sid); 374104476Ssam else 375104476Ssam err = 0; 376104476Ssam 377158702Spjd if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 378158702Spjd crypto_remove(cap); 379104476Ssam 380104476Ssamdone: 381104476Ssam CRYPTO_DRIVER_UNLOCK(); 382104476Ssam return err; 383104476Ssam} 384104476Ssam 385104476Ssam/* 386104476Ssam * Return an unused driver id. Used by drivers prior to registering 387104476Ssam * support for the algorithms they handle. 388104476Ssam */ 389104476Ssamint32_t 390104476Ssamcrypto_get_driverid(u_int32_t flags) 391104476Ssam{ 392104476Ssam struct cryptocap *newdrv; 393104476Ssam int i; 394104476Ssam 395104476Ssam CRYPTO_DRIVER_LOCK(); 396104476Ssam 397158702Spjd for (i = 0; i < crypto_drivers_num; i++) { 398104476Ssam if (crypto_drivers[i].cc_process == NULL && 399158702Spjd (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 400104476Ssam break; 401158702Spjd } 402158702Spjd } 403104476Ssam 404104476Ssam /* Out of entries, allocate some more. */ 405104476Ssam if (i == crypto_drivers_num) { 406104476Ssam /* Be careful about wrap-around. */ 407104476Ssam if (2 * crypto_drivers_num <= crypto_drivers_num) { 408104476Ssam CRYPTO_DRIVER_UNLOCK(); 409104476Ssam printf("crypto: driver count wraparound!\n"); 410104476Ssam return -1; 411104476Ssam } 412104476Ssam 413104476Ssam newdrv = malloc(2 * crypto_drivers_num * 414104476Ssam sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 415104476Ssam if (newdrv == NULL) { 416104476Ssam CRYPTO_DRIVER_UNLOCK(); 417104476Ssam printf("crypto: no space to expand driver table!\n"); 418104476Ssam return -1; 419104476Ssam } 420104476Ssam 421104476Ssam bcopy(crypto_drivers, newdrv, 422104476Ssam crypto_drivers_num * sizeof(struct cryptocap)); 423104476Ssam 424104476Ssam crypto_drivers_num *= 2; 425104476Ssam 426104476Ssam free(crypto_drivers, M_CRYPTO_DATA); 427104476Ssam crypto_drivers = newdrv; 428104476Ssam } 429104476Ssam 430104476Ssam /* NB: state is zero'd on free */ 431104476Ssam crypto_drivers[i].cc_sessions = 1; /* Mark */ 432104476Ssam crypto_drivers[i].cc_flags = flags; 433104476Ssam if (bootverbose) 434104476Ssam printf("crypto: assign driver %u, flags %u\n", i, flags); 435104476Ssam 436104476Ssam CRYPTO_DRIVER_UNLOCK(); 437104476Ssam 438104476Ssam return i; 439104476Ssam} 440104476Ssam 441104476Ssamstatic struct cryptocap * 442104476Ssamcrypto_checkdriver(u_int32_t hid) 443104476Ssam{ 444104476Ssam if (crypto_drivers == NULL) 445104476Ssam return NULL; 446104476Ssam return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 447104476Ssam} 448104476Ssam 449104476Ssam/* 450104476Ssam * Register support for a key-related algorithm. This routine 451104476Ssam * is called once for each algorithm supported a driver. 452104476Ssam */ 453104476Ssamint 454104476Ssamcrypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 455104476Ssam int (*kprocess)(void*, struct cryptkop *, int), 456104476Ssam void *karg) 457104476Ssam{ 458104476Ssam struct cryptocap *cap; 459104476Ssam int err; 460104476Ssam 461104476Ssam CRYPTO_DRIVER_LOCK(); 462104476Ssam 463104476Ssam cap = crypto_checkdriver(driverid); 464104476Ssam if (cap != NULL && 465104476Ssam (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 466104476Ssam /* 467104476Ssam * XXX Do some performance testing to determine placing. 468104476Ssam * XXX We probably need an auxiliary data structure that 469104476Ssam * XXX describes relative performances. 470104476Ssam */ 471104476Ssam 472104476Ssam cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 473104476Ssam if (bootverbose) 474104476Ssam printf("crypto: driver %u registers key alg %u flags %u\n" 475104476Ssam , driverid 476104476Ssam , kalg 477104476Ssam , flags 478104476Ssam ); 479104476Ssam 480104476Ssam if (cap->cc_kprocess == NULL) { 481104476Ssam cap->cc_karg = karg; 482104476Ssam cap->cc_kprocess = kprocess; 483104476Ssam } 484104476Ssam err = 0; 485104476Ssam } else 486104476Ssam err = EINVAL; 487104476Ssam 488104476Ssam CRYPTO_DRIVER_UNLOCK(); 489104476Ssam return err; 490104476Ssam} 491104476Ssam 492104476Ssam/* 493104476Ssam * Register support for a non-key-related algorithm. This routine 494104476Ssam * is called once for each such algorithm supported by a driver. 495104476Ssam */ 496104476Ssamint 497104476Ssamcrypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 498104476Ssam u_int32_t flags, 499104476Ssam int (*newses)(void*, u_int32_t*, struct cryptoini*), 500104476Ssam int (*freeses)(void*, u_int64_t), 501104476Ssam int (*process)(void*, struct cryptop *, int), 502104476Ssam void *arg) 503104476Ssam{ 504104476Ssam struct cryptocap *cap; 505104476Ssam int err; 506104476Ssam 507104476Ssam CRYPTO_DRIVER_LOCK(); 508104476Ssam 509104476Ssam cap = crypto_checkdriver(driverid); 510104476Ssam /* NB: algorithms are in the range [1..max] */ 511104476Ssam if (cap != NULL && 512104476Ssam (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 513104476Ssam /* 514104476Ssam * XXX Do some performance testing to determine placing. 515104476Ssam * XXX We probably need an auxiliary data structure that 516104476Ssam * XXX describes relative performances. 517104476Ssam */ 518104476Ssam 519104476Ssam cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 520104476Ssam cap->cc_max_op_len[alg] = maxoplen; 521104476Ssam if (bootverbose) 522104476Ssam printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 523104476Ssam , driverid 524104476Ssam , alg 525104476Ssam , flags 526104476Ssam , maxoplen 527104476Ssam ); 528104476Ssam 529104476Ssam if (cap->cc_process == NULL) { 530104476Ssam cap->cc_arg = arg; 531104476Ssam cap->cc_newsession = newses; 532104476Ssam cap->cc_process = process; 533104476Ssam cap->cc_freesession = freeses; 534104476Ssam cap->cc_sessions = 0; /* Unmark */ 535104476Ssam } 536104476Ssam err = 0; 537104476Ssam } else 538104476Ssam err = EINVAL; 539104476Ssam 540104476Ssam CRYPTO_DRIVER_UNLOCK(); 541104476Ssam return err; 542104476Ssam} 543104476Ssam 544104476Ssam/* 545104476Ssam * Unregister a crypto driver. If there are pending sessions using it, 546104476Ssam * leave enough information around so that subsequent calls using those 547104476Ssam * sessions will correctly detect the driver has been unregistered and 548104476Ssam * reroute requests. 549104476Ssam */ 550104476Ssamint 551104476Ssamcrypto_unregister(u_int32_t driverid, int alg) 552104476Ssam{ 553158702Spjd struct cryptocap *cap; 554158702Spjd u_int32_t ses, kops; 555104476Ssam int i, err; 556104476Ssam 557104476Ssam CRYPTO_DRIVER_LOCK(); 558104476Ssam 559104476Ssam cap = crypto_checkdriver(driverid); 560104476Ssam if (cap != NULL && 561104476Ssam (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 562104476Ssam cap->cc_alg[alg] != 0) { 563104476Ssam cap->cc_alg[alg] = 0; 564104476Ssam cap->cc_max_op_len[alg] = 0; 565104476Ssam 566104476Ssam /* Was this the last algorithm ? */ 567104476Ssam for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 568104476Ssam if (cap->cc_alg[i] != 0) 569104476Ssam break; 570104476Ssam 571104476Ssam if (i == CRYPTO_ALGORITHM_MAX + 1) { 572104476Ssam ses = cap->cc_sessions; 573158702Spjd kops = cap->cc_koperations; 574158702Spjd bzero(cap, sizeof(*cap)); 575158702Spjd if (ses != 0 || kops != 0) { 576104476Ssam /* 577104476Ssam * If there are pending sessions, just mark as invalid. 578104476Ssam */ 579104476Ssam cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 580104476Ssam cap->cc_sessions = ses; 581158702Spjd cap->cc_koperations = kops; 582104476Ssam } 583104476Ssam } 584104476Ssam err = 0; 585104476Ssam } else 586104476Ssam err = EINVAL; 587104476Ssam 588104476Ssam CRYPTO_DRIVER_UNLOCK(); 589104476Ssam return err; 590104476Ssam} 591104476Ssam 592104476Ssam/* 593104476Ssam * Unregister all algorithms associated with a crypto driver. 594104476Ssam * If there are pending sessions using it, leave enough information 595104476Ssam * around so that subsequent calls using those sessions will 596104476Ssam * correctly detect the driver has been unregistered and reroute 597104476Ssam * requests. 598104476Ssam */ 599104476Ssamint 600104476Ssamcrypto_unregister_all(u_int32_t driverid) 601104476Ssam{ 602158702Spjd struct cryptocap *cap; 603158702Spjd u_int32_t ses, kops; 604104476Ssam int i, err; 605104476Ssam 606104476Ssam CRYPTO_DRIVER_LOCK(); 607104476Ssam 608104476Ssam cap = crypto_checkdriver(driverid); 609104476Ssam if (cap != NULL) { 610104476Ssam for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 611104476Ssam cap->cc_alg[i] = 0; 612104476Ssam cap->cc_max_op_len[i] = 0; 613104476Ssam } 614104476Ssam ses = cap->cc_sessions; 615158702Spjd kops = cap->cc_koperations; 616158702Spjd bzero(cap, sizeof(*cap)); 617158702Spjd if (ses != 0 || kops != 0) { 618104476Ssam /* 619104476Ssam * If there are pending sessions, just mark as invalid. 620104476Ssam */ 621104476Ssam cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 622104476Ssam cap->cc_sessions = ses; 623158702Spjd cap->cc_koperations = kops; 624104476Ssam } 625104476Ssam err = 0; 626104476Ssam } else 627104476Ssam err = EINVAL; 628104476Ssam 629104476Ssam CRYPTO_DRIVER_UNLOCK(); 630104476Ssam return err; 631104476Ssam} 632104476Ssam 633104476Ssam/* 634104476Ssam * Clear blockage on a driver. The what parameter indicates whether 635104476Ssam * the driver is now ready for cryptop's and/or cryptokop's. 636104476Ssam */ 637104476Ssamint 638104476Ssamcrypto_unblock(u_int32_t driverid, int what) 639104476Ssam{ 640104476Ssam struct cryptocap *cap; 641104476Ssam int needwakeup, err; 642104476Ssam 643104476Ssam CRYPTO_Q_LOCK(); 644104476Ssam cap = crypto_checkdriver(driverid); 645104476Ssam if (cap != NULL) { 646104628Ssam needwakeup = 0; 647104476Ssam if (what & CRYPTO_SYMQ) { 648104476Ssam needwakeup |= cap->cc_qblocked; 649104476Ssam cap->cc_qblocked = 0; 650104476Ssam } 651104476Ssam if (what & CRYPTO_ASYMQ) { 652104476Ssam needwakeup |= cap->cc_kqblocked; 653104476Ssam cap->cc_kqblocked = 0; 654104476Ssam } 655104628Ssam if (needwakeup) 656104628Ssam wakeup_one(&crp_q); 657104476Ssam err = 0; 658104476Ssam } else 659104476Ssam err = EINVAL; 660104476Ssam CRYPTO_Q_UNLOCK(); 661104476Ssam 662104476Ssam return err; 663104476Ssam} 664104476Ssam 665104476Ssam/* 666104476Ssam * Add a crypto request to a queue, to be processed by the kernel thread. 667104476Ssam */ 668104476Ssamint 669104476Ssamcrypto_dispatch(struct cryptop *crp) 670104476Ssam{ 671158702Spjd struct cryptocap *cap; 672158702Spjd u_int32_t hid; 673158702Spjd int result; 674104476Ssam 675108587Ssam cryptostats.cs_ops++; 676108587Ssam 677108587Ssam#ifdef CRYPTO_TIMING 678108587Ssam if (crypto_timing) 679108587Ssam binuptime(&crp->crp_tstamp); 680108587Ssam#endif 681108587Ssam 682158702Spjd hid = CRYPTO_SESID2HID(crp->crp_sid); 683158702Spjd 684111297Ssam if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 685111297Ssam /* 686111297Ssam * Caller marked the request to be processed 687111297Ssam * immediately; dispatch it directly to the 688111297Ssam * driver unless the driver is currently blocked. 689111297Ssam */ 690111297Ssam cap = crypto_checkdriver(hid); 691158702Spjd /* Driver cannot disappeared when there is an active session. */ 692158702Spjd KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 693158702Spjd if (!cap->cc_qblocked) { 694158702Spjd result = crypto_invoke(cap, crp, 0); 695158702Spjd if (result != ERESTART) 696158702Spjd return (result); 697158702Spjd else { 698111297Ssam /* 699111297Ssam * The driver ran out of resources, mark the 700111297Ssam * driver ``blocked'' for cryptop's and put 701111297Ssam * the request on the queue. 702115746Ssam * 703115746Ssam * XXX ops are placed at the tail so their 704115746Ssam * order is preserved but this can place them 705115746Ssam * behind batch'd ops. 706111297Ssam */ 707158702Spjd cap->cc_qblocked = 1; 708111297Ssam cryptostats.cs_blocks++; 709111297Ssam } 710108990Ssam } 711108990Ssam } 712158702Spjd CRYPTO_Q_LOCK(); 713158702Spjd if (CRYPTO_Q_EMPTY()) 714157665Spjd wakeup_one(&crp_q); 715158702Spjd TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 716115746Ssam CRYPTO_Q_UNLOCK(); 717158702Spjd return 0; 718104476Ssam} 719104476Ssam 720104476Ssam/* 721104476Ssam * Add an asymetric crypto request to a queue, 722104476Ssam * to be processed by the kernel thread. 723104476Ssam */ 724104476Ssamint 725104476Ssamcrypto_kdispatch(struct cryptkop *krp) 726104476Ssam{ 727158702Spjd int result; 728104476Ssam 729108587Ssam cryptostats.cs_kops++; 730108587Ssam 731158702Spjd result = crypto_kinvoke(krp); 732158702Spjd if (result != ERESTART) 733158702Spjd return (result); 734104476Ssam CRYPTO_Q_LOCK(); 735158702Spjd if (CRYPTO_Q_EMPTY()) 736157665Spjd wakeup_one(&crp_q); 737158702Spjd TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 738104476Ssam CRYPTO_Q_UNLOCK(); 739104476Ssam 740158702Spjd return 0; 741104476Ssam} 742104476Ssam 743104476Ssam/* 744104476Ssam * Dispatch an assymetric crypto request to the appropriate crypto devices. 745104476Ssam */ 746104476Ssamstatic int 747158702Spjdcrypto_kinvoke(struct cryptkop *krp) 748104476Ssam{ 749158702Spjd struct cryptocap *cap = NULL; 750104476Ssam u_int32_t hid; 751158702Spjd int error = 0; 752104476Ssam 753158702Spjd KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 754158702Spjd KASSERT(krp->krp_callback != NULL, 755158702Spjd ("%s: krp->crp_callback == NULL", __func__)); 756104476Ssam 757158702Spjd CRYPTO_DRIVER_LOCK(); 758104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 759158702Spjd cap = &crypto_drivers[hid]; 760158702Spjd if (cap == NULL) 761104476Ssam continue; 762158702Spjd if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 763158702Spjd !crypto_devallowsoft) { 764104476Ssam continue; 765158702Spjd } 766158702Spjd if (cap->cc_kprocess == NULL) 767104476Ssam continue; 768158702Spjd if (!(cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED)) 769158702Spjd continue; 770158702Spjd if (cap->cc_kqblocked) { 771158702Spjd error = ERESTART; 772158702Spjd continue; 773158702Spjd } 774158702Spjd error = 0; 775104476Ssam break; 776104476Ssam } 777158702Spjd krp->krp_hid = hid; 778104476Ssam if (hid < crypto_drivers_num) { 779158702Spjd cap->cc_koperations++; 780158702Spjd CRYPTO_DRIVER_UNLOCK(); 781158702Spjd error = cap->cc_kprocess(cap->cc_karg, krp, 0); 782158702Spjd CRYPTO_DRIVER_LOCK(); 783158702Spjd if (error == ERESTART) { 784158702Spjd cap->cc_koperations--; 785158702Spjd cap->cc_kqblocked = 1; 786158702Spjd CRYPTO_DRIVER_UNLOCK(); 787158702Spjd cryptostats.cs_kblocks++; 788158702Spjd return (error); 789158702Spjd } 790158702Spjd } else { 791104476Ssam error = ENODEV; 792158702Spjd } 793158702Spjd CRYPTO_DRIVER_UNLOCK(); 794104476Ssam 795104476Ssam if (error) { 796104476Ssam krp->krp_status = error; 797104628Ssam crypto_kdone(krp); 798104476Ssam } 799104476Ssam return 0; 800104476Ssam} 801104476Ssam 802108587Ssam#ifdef CRYPTO_TIMING 803108587Ssamstatic void 804108587Ssamcrypto_tstat(struct cryptotstat *ts, struct bintime *bt) 805108587Ssam{ 806108587Ssam struct bintime now, delta; 807108587Ssam struct timespec t; 808108587Ssam uint64_t u; 809108587Ssam 810108587Ssam binuptime(&now); 811108587Ssam u = now.frac; 812108587Ssam delta.frac = now.frac - bt->frac; 813108587Ssam delta.sec = now.sec - bt->sec; 814108587Ssam if (u < delta.frac) 815108587Ssam delta.sec--; 816108587Ssam bintime2timespec(&delta, &t); 817108587Ssam timespecadd(&ts->acc, &t); 818108587Ssam if (timespeccmp(&t, &ts->min, <)) 819108587Ssam ts->min = t; 820108587Ssam if (timespeccmp(&t, &ts->max, >)) 821108587Ssam ts->max = t; 822108587Ssam ts->count++; 823108587Ssam 824108587Ssam *bt = now; 825108587Ssam} 826108587Ssam#endif 827108587Ssam 828104476Ssam/* 829104476Ssam * Dispatch a crypto request to the appropriate crypto devices. 830104476Ssam */ 831104476Ssamstatic int 832158702Spjdcrypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 833104476Ssam{ 834104476Ssam 835158702Spjd KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 836158702Spjd KASSERT(crp->crp_callback != NULL, 837158702Spjd ("%s: crp->crp_callback == NULL", __func__)); 838158702Spjd KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 839158702Spjd 840108587Ssam#ifdef CRYPTO_TIMING 841108587Ssam if (crypto_timing) 842108587Ssam crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 843108587Ssam#endif 844158702Spjd if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 845104476Ssam struct cryptodesc *crd; 846104476Ssam u_int64_t nid; 847104476Ssam 848104476Ssam /* 849104476Ssam * Driver has unregistered; migrate the session and return 850104476Ssam * an error to the caller so they'll resubmit the op. 851158702Spjd * 852158702Spjd * XXX: What if there are more already queued requests for this 853158702Spjd * session? 854104476Ssam */ 855158702Spjd crypto_freesession(crp->crp_sid); 856158702Spjd 857104476Ssam for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 858104476Ssam crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 859104476Ssam 860104476Ssam if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 861104476Ssam crp->crp_sid = nid; 862104476Ssam 863104476Ssam crp->crp_etype = EAGAIN; 864104628Ssam crypto_done(crp); 865104476Ssam return 0; 866104476Ssam } else { 867104476Ssam /* 868104476Ssam * Invoke the driver to process the request. 869104476Ssam */ 870158702Spjd return cap->cc_process(cap->cc_arg, crp, hint); 871104476Ssam } 872104476Ssam} 873104476Ssam 874104476Ssam/* 875104476Ssam * Release a set of crypto descriptors. 876104476Ssam */ 877104476Ssamvoid 878104476Ssamcrypto_freereq(struct cryptop *crp) 879104476Ssam{ 880104476Ssam struct cryptodesc *crd; 881104476Ssam 882104476Ssam if (crp == NULL) 883104476Ssam return; 884104476Ssam 885104476Ssam while ((crd = crp->crp_desc) != NULL) { 886104476Ssam crp->crp_desc = crd->crd_next; 887104476Ssam uma_zfree(cryptodesc_zone, crd); 888104476Ssam } 889104476Ssam 890104476Ssam uma_zfree(cryptop_zone, crp); 891104476Ssam} 892104476Ssam 893104476Ssam/* 894104476Ssam * Acquire a set of crypto descriptors. 895104476Ssam */ 896104476Ssamstruct cryptop * 897104476Ssamcrypto_getreq(int num) 898104476Ssam{ 899104476Ssam struct cryptodesc *crd; 900104476Ssam struct cryptop *crp; 901104476Ssam 902108813Ssam crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 903104476Ssam if (crp != NULL) { 904104476Ssam while (num--) { 905108813Ssam crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 906104476Ssam if (crd == NULL) { 907104476Ssam crypto_freereq(crp); 908104476Ssam return NULL; 909104476Ssam } 910104476Ssam 911104476Ssam crd->crd_next = crp->crp_desc; 912104476Ssam crp->crp_desc = crd; 913104476Ssam } 914104476Ssam } 915104476Ssam return crp; 916104476Ssam} 917104476Ssam 918104476Ssam/* 919104476Ssam * Invoke the callback on behalf of the driver. 920104476Ssam */ 921104476Ssamvoid 922104476Ssamcrypto_done(struct cryptop *crp) 923104476Ssam{ 924115746Ssam KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 925115746Ssam ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 926115746Ssam crp->crp_flags |= CRYPTO_F_DONE; 927108587Ssam if (crp->crp_etype != 0) 928108587Ssam cryptostats.cs_errs++; 929108587Ssam#ifdef CRYPTO_TIMING 930108587Ssam if (crypto_timing) 931108587Ssam crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 932108587Ssam#endif 933117058Ssam /* 934117058Ssam * CBIMM means unconditionally do the callback immediately; 935117058Ssam * CBIFSYNC means do the callback immediately only if the 936117058Ssam * operation was done synchronously. Both are used to avoid 937117058Ssam * doing extraneous context switches; the latter is mostly 938117058Ssam * used with the software crypto driver. 939117058Ssam */ 940117058Ssam if ((crp->crp_flags & CRYPTO_F_CBIMM) || 941117058Ssam ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 942117058Ssam (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 943111297Ssam /* 944111297Ssam * Do the callback directly. This is ok when the 945111297Ssam * callback routine does very little (e.g. the 946111297Ssam * /dev/crypto callback method just does a wakeup). 947111297Ssam */ 948111297Ssam#ifdef CRYPTO_TIMING 949111297Ssam if (crypto_timing) { 950111297Ssam /* 951111297Ssam * NB: We must copy the timestamp before 952111297Ssam * doing the callback as the cryptop is 953111297Ssam * likely to be reclaimed. 954111297Ssam */ 955111297Ssam struct bintime t = crp->crp_tstamp; 956111297Ssam crypto_tstat(&cryptostats.cs_cb, &t); 957111297Ssam crp->crp_callback(crp); 958111297Ssam crypto_tstat(&cryptostats.cs_finis, &t); 959111297Ssam } else 960111297Ssam#endif 961111297Ssam crp->crp_callback(crp); 962111297Ssam } else { 963111297Ssam /* 964111297Ssam * Normal case; queue the callback for the thread. 965111297Ssam */ 966111297Ssam CRYPTO_RETQ_LOCK(); 967158702Spjd if (TAILQ_EMPTY(&crp_ret_q)) 968158702Spjd wakeup_one(&crp_ret_q); /* shared wait channel */ 969111297Ssam TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 970111297Ssam CRYPTO_RETQ_UNLOCK(); 971111297Ssam } 972104476Ssam} 973104476Ssam 974104476Ssam/* 975104476Ssam * Invoke the callback on behalf of the driver. 976104476Ssam */ 977104476Ssamvoid 978104476Ssamcrypto_kdone(struct cryptkop *krp) 979104476Ssam{ 980158702Spjd struct cryptocap *cap; 981104476Ssam 982108587Ssam if (krp->krp_status != 0) 983108587Ssam cryptostats.cs_kerrs++; 984158702Spjd CRYPTO_DRIVER_LOCK(); 985158702Spjd /* XXX: What if driver is loaded in the meantime? */ 986158702Spjd if (krp->krp_hid < crypto_drivers_num) { 987158702Spjd cap = &crypto_drivers[krp->krp_hid]; 988158702Spjd cap->cc_koperations--; 989158702Spjd KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 990158702Spjd if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 991158702Spjd crypto_remove(cap); 992158702Spjd } 993158702Spjd CRYPTO_DRIVER_UNLOCK(); 994104476Ssam CRYPTO_RETQ_LOCK(); 995158702Spjd if (TAILQ_EMPTY(&crp_ret_kq)) 996158702Spjd wakeup_one(&crp_ret_q); /* shared wait channel */ 997104476Ssam TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 998104628Ssam CRYPTO_RETQ_UNLOCK(); 999104476Ssam} 1000104476Ssam 1001104476Ssamint 1002104476Ssamcrypto_getfeat(int *featp) 1003104476Ssam{ 1004104476Ssam int hid, kalg, feat = 0; 1005104476Ssam 1006104476Ssam if (!crypto_userasymcrypto) 1007104476Ssam goto out; 1008104476Ssam 1009104476Ssam CRYPTO_DRIVER_LOCK(); 1010104476Ssam for (hid = 0; hid < crypto_drivers_num; hid++) { 1011104476Ssam if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 1012104476Ssam !crypto_devallowsoft) { 1013104476Ssam continue; 1014104476Ssam } 1015104476Ssam if (crypto_drivers[hid].cc_kprocess == NULL) 1016104476Ssam continue; 1017104476Ssam for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1018104476Ssam if ((crypto_drivers[hid].cc_kalg[kalg] & 1019104476Ssam CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1020104476Ssam feat |= 1 << kalg; 1021104476Ssam } 1022104476Ssam CRYPTO_DRIVER_UNLOCK(); 1023104476Ssamout: 1024104476Ssam *featp = feat; 1025104476Ssam return (0); 1026104476Ssam} 1027104476Ssam 1028108588Ssam/* 1029108588Ssam * Terminate a thread at module unload. The process that 1030108588Ssam * initiated this is waiting for us to signal that we're gone; 1031108588Ssam * wake it up and exit. We use the driver table lock to insure 1032108588Ssam * we don't do the wakeup before they're waiting. There is no 1033108588Ssam * race here because the waiter sleeps on the proc lock for the 1034108588Ssam * thread so it gets notified at the right time because of an 1035108588Ssam * extra wakeup that's done in exit1(). 1036108588Ssam */ 1037104476Ssamstatic void 1038108588Ssamcrypto_finis(void *chan) 1039104476Ssam{ 1040108588Ssam CRYPTO_DRIVER_LOCK(); 1041108588Ssam wakeup_one(chan); 1042108588Ssam CRYPTO_DRIVER_UNLOCK(); 1043108588Ssam kthread_exit(0); 1044104476Ssam} 1045104476Ssam 1046104476Ssam/* 1047104628Ssam * Crypto thread, dispatches crypto requests. 1048104476Ssam */ 1049104476Ssamstatic void 1050104476Ssamcrypto_proc(void) 1051104476Ssam{ 1052104628Ssam struct cryptop *crp, *submit; 1053104628Ssam struct cryptkop *krp; 1054104476Ssam struct cryptocap *cap; 1055158702Spjd u_int32_t hid; 1056104476Ssam int result, hint; 1057104476Ssam 1058104628Ssam CRYPTO_Q_LOCK(); 1059104476Ssam for (;;) { 1060104476Ssam /* 1061104476Ssam * Find the first element in the queue that can be 1062104476Ssam * processed and look-ahead to see if multiple ops 1063104476Ssam * are ready for the same driver. 1064104476Ssam */ 1065104476Ssam submit = NULL; 1066104476Ssam hint = 0; 1067104476Ssam TAILQ_FOREACH(crp, &crp_q, crp_next) { 1068158702Spjd hid = CRYPTO_SESID2HID(crp->crp_sid); 1069104476Ssam cap = crypto_checkdriver(hid); 1070158702Spjd /* 1071158702Spjd * Driver cannot disappeared when there is an active 1072158702Spjd * session. 1073158702Spjd */ 1074158716Spjd KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1075158716Spjd __func__, __LINE__)); 1076104476Ssam if (cap == NULL || cap->cc_process == NULL) { 1077104476Ssam /* Op needs to be migrated, process it. */ 1078104476Ssam if (submit == NULL) 1079104476Ssam submit = crp; 1080104476Ssam break; 1081104476Ssam } 1082104476Ssam if (!cap->cc_qblocked) { 1083104476Ssam if (submit != NULL) { 1084104476Ssam /* 1085104476Ssam * We stop on finding another op, 1086104476Ssam * regardless whether its for the same 1087104476Ssam * driver or not. We could keep 1088104476Ssam * searching the queue but it might be 1089104476Ssam * better to just use a per-driver 1090104476Ssam * queue instead. 1091104476Ssam */ 1092116924Ssam if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1093104476Ssam hint = CRYPTO_HINT_MORE; 1094104476Ssam break; 1095104476Ssam } else { 1096104476Ssam submit = crp; 1097111297Ssam if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1098104476Ssam break; 1099104476Ssam /* keep scanning for more are q'd */ 1100104476Ssam } 1101104476Ssam } 1102104476Ssam } 1103104476Ssam if (submit != NULL) { 1104104476Ssam TAILQ_REMOVE(&crp_q, submit, crp_next); 1105158702Spjd CRYPTO_Q_UNLOCK(); 1106158702Spjd hid = CRYPTO_SESID2HID(submit->crp_sid); 1107158702Spjd cap = crypto_checkdriver(hid); 1108158716Spjd KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1109158716Spjd __func__, __LINE__)); 1110158702Spjd result = crypto_invoke(cap, submit, hint); 1111158702Spjd CRYPTO_Q_LOCK(); 1112104476Ssam if (result == ERESTART) { 1113104476Ssam /* 1114104476Ssam * The driver ran out of resources, mark the 1115104476Ssam * driver ``blocked'' for cryptop's and put 1116104476Ssam * the request back in the queue. It would 1117104476Ssam * best to put the request back where we got 1118104476Ssam * it but that's hard so for now we put it 1119104476Ssam * at the front. This should be ok; putting 1120104476Ssam * it at the end does not work. 1121104476Ssam */ 1122104476Ssam /* XXX validate sid again? */ 1123116924Ssam crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1124104476Ssam TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1125108587Ssam cryptostats.cs_blocks++; 1126104476Ssam } 1127104476Ssam } 1128104476Ssam 1129104476Ssam /* As above, but for key ops */ 1130104476Ssam TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1131104476Ssam cap = crypto_checkdriver(krp->krp_hid); 1132104476Ssam if (cap == NULL || cap->cc_kprocess == NULL) { 1133104476Ssam /* Op needs to be migrated, process it. */ 1134104476Ssam break; 1135104476Ssam } 1136104476Ssam if (!cap->cc_kqblocked) 1137104476Ssam break; 1138104476Ssam } 1139104476Ssam if (krp != NULL) { 1140104476Ssam TAILQ_REMOVE(&crp_kq, krp, krp_next); 1141158702Spjd CRYPTO_Q_UNLOCK(); 1142158702Spjd result = crypto_kinvoke(krp); 1143158702Spjd CRYPTO_Q_LOCK(); 1144104476Ssam if (result == ERESTART) { 1145104476Ssam /* 1146104476Ssam * The driver ran out of resources, mark the 1147104476Ssam * driver ``blocked'' for cryptkop's and put 1148104476Ssam * the request back in the queue. It would 1149104476Ssam * best to put the request back where we got 1150104476Ssam * it but that's hard so for now we put it 1151104476Ssam * at the front. This should be ok; putting 1152104476Ssam * it at the end does not work. 1153104476Ssam */ 1154104476Ssam /* XXX validate sid again? */ 1155104476Ssam crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1156104476Ssam TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1157108587Ssam cryptostats.cs_kblocks++; 1158104476Ssam } 1159104476Ssam } 1160104476Ssam 1161104628Ssam if (submit == NULL && krp == NULL) { 1162104476Ssam /* 1163104476Ssam * Nothing more to be processed. Sleep until we're 1164104476Ssam * woken because there are more ops to process. 1165104476Ssam * This happens either by submission or by a driver 1166104476Ssam * becoming unblocked and notifying us through 1167104476Ssam * crypto_unblock. Note that when we wakeup we 1168104476Ssam * start processing each queue again from the 1169104476Ssam * front. It's not clear that it's important to 1170104476Ssam * preserve this ordering since ops may finish 1171104476Ssam * out of order if dispatched to different devices 1172104476Ssam * and some become blocked while others do not. 1173104476Ssam */ 1174104628Ssam msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1175108588Ssam if (cryptoproc == NULL) 1176108588Ssam break; 1177108587Ssam cryptostats.cs_intrs++; 1178104476Ssam } 1179104476Ssam } 1180108588Ssam CRYPTO_Q_UNLOCK(); 1181104628Ssam 1182108588Ssam crypto_finis(&crp_q); 1183104628Ssam} 1184104628Ssam 1185104628Ssam/* 1186104628Ssam * Crypto returns thread, does callbacks for processed crypto requests. 1187104628Ssam * Callbacks are done here, rather than in the crypto drivers, because 1188104628Ssam * callbacks typically are expensive and would slow interrupt handling. 1189104628Ssam */ 1190104628Ssamstatic void 1191104628Ssamcrypto_ret_proc(void) 1192104628Ssam{ 1193104628Ssam struct cryptop *crpt; 1194104628Ssam struct cryptkop *krpt; 1195104628Ssam 1196104628Ssam CRYPTO_RETQ_LOCK(); 1197104628Ssam for (;;) { 1198104628Ssam /* Harvest return q's for completed ops */ 1199104628Ssam crpt = TAILQ_FIRST(&crp_ret_q); 1200104628Ssam if (crpt != NULL) 1201104628Ssam TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1202104628Ssam 1203104628Ssam krpt = TAILQ_FIRST(&crp_ret_kq); 1204104628Ssam if (krpt != NULL) 1205104628Ssam TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1206104628Ssam 1207104628Ssam if (crpt != NULL || krpt != NULL) { 1208104628Ssam CRYPTO_RETQ_UNLOCK(); 1209104628Ssam /* 1210104628Ssam * Run callbacks unlocked. 1211104628Ssam */ 1212108587Ssam if (crpt != NULL) { 1213108587Ssam#ifdef CRYPTO_TIMING 1214108587Ssam if (crypto_timing) { 1215108587Ssam /* 1216108587Ssam * NB: We must copy the timestamp before 1217108587Ssam * doing the callback as the cryptop is 1218108587Ssam * likely to be reclaimed. 1219108587Ssam */ 1220108587Ssam struct bintime t = crpt->crp_tstamp; 1221108587Ssam crypto_tstat(&cryptostats.cs_cb, &t); 1222108587Ssam crpt->crp_callback(crpt); 1223108587Ssam crypto_tstat(&cryptostats.cs_finis, &t); 1224108587Ssam } else 1225108587Ssam#endif 1226108587Ssam crpt->crp_callback(crpt); 1227108587Ssam } 1228104628Ssam if (krpt != NULL) 1229104628Ssam krpt->krp_callback(krpt); 1230104628Ssam CRYPTO_RETQ_LOCK(); 1231104628Ssam } else { 1232104628Ssam /* 1233104628Ssam * Nothing more to be processed. Sleep until we're 1234104628Ssam * woken because there are more returns to process. 1235104628Ssam */ 1236104628Ssam msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1237104628Ssam "crypto_ret_wait", 0); 1238108588Ssam if (cryptoretproc == NULL) 1239108588Ssam break; 1240108587Ssam cryptostats.cs_rets++; 1241104628Ssam } 1242104628Ssam } 1243108588Ssam CRYPTO_RETQ_UNLOCK(); 1244108588Ssam 1245108588Ssam crypto_finis(&crp_ret_q); 1246104628Ssam} 1247