cam_xpt.c revision 302376 
1/*-
2 * Implementation of the Common Access Method Transport (XPT) layer.
3 *
4 * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions, and the following disclaimer,
13 *    without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/10/sys/cam/cam_xpt.c 302376 2016-07-06 17:42:09Z truckman $");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/systm.h>
36#include <sys/types.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/time.h>
40#include <sys/conf.h>
41#include <sys/fcntl.h>
42#include <sys/interrupt.h>
43#include <sys/proc.h>
44#include <sys/sbuf.h>
45#include <sys/smp.h>
46#include <sys/taskqueue.h>
47
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/sysctl.h>
51#include <sys/kthread.h>
52
53#include <cam/cam.h>
54#include <cam/cam_ccb.h>
55#include <cam/cam_periph.h>
56#include <cam/cam_queue.h>
57#include <cam/cam_sim.h>
58#include <cam/cam_xpt.h>
59#include <cam/cam_xpt_sim.h>
60#include <cam/cam_xpt_periph.h>
61#include <cam/cam_xpt_internal.h>
62#include <cam/cam_debug.h>
63#include <cam/cam_compat.h>
64
65#include <cam/scsi/scsi_all.h>
66#include <cam/scsi/scsi_message.h>
67#include <cam/scsi/scsi_pass.h>
68
69#include <machine/md_var.h>	/* geometry translation */
70#include <machine/stdarg.h>	/* for xpt_print below */
71
72#include "opt_cam.h"
73
74/*
75 * This is the maximum number of high powered commands (e.g. start unit)
76 * that can be outstanding at a particular time.
77 */
78#ifndef CAM_MAX_HIGHPOWER
79#define CAM_MAX_HIGHPOWER  4
80#endif
81
82/* Datastructures internal to the xpt layer */
83MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87
88/* Object for defering XPT actions to a taskqueue */
89struct xpt_task {
90	struct task	task;
91	void		*data1;
92	uintptr_t	data2;
93};
94
95struct xpt_softc {
96	uint32_t		xpt_generation;
97
98	/* number of high powered commands that can go through right now */
99	struct mtx		xpt_highpower_lock;
100	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
101	int			num_highpower;
102
103	/* queue for handling async rescan requests. */
104	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
105	int buses_to_config;
106	int buses_config_done;
107
108	/* Registered busses */
109	TAILQ_HEAD(,cam_eb)	xpt_busses;
110	u_int			bus_generation;
111
112	struct intr_config_hook	*xpt_config_hook;
113
114	int			boot_delay;
115	struct callout 		boot_callout;
116
117	struct mtx		xpt_topo_lock;
118	struct mtx		xpt_lock;
119	struct taskqueue	*xpt_taskq;
120};
121
122typedef enum {
123	DM_RET_COPY		= 0x01,
124	DM_RET_FLAG_MASK	= 0x0f,
125	DM_RET_NONE		= 0x00,
126	DM_RET_STOP		= 0x10,
127	DM_RET_DESCEND		= 0x20,
128	DM_RET_ERROR		= 0x30,
129	DM_RET_ACTION_MASK	= 0xf0
130} dev_match_ret;
131
132typedef enum {
133	XPT_DEPTH_BUS,
134	XPT_DEPTH_TARGET,
135	XPT_DEPTH_DEVICE,
136	XPT_DEPTH_PERIPH
137} xpt_traverse_depth;
138
139struct xpt_traverse_config {
140	xpt_traverse_depth	depth;
141	void			*tr_func;
142	void			*tr_arg;
143};
144
145typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
146typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
147typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
148typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
149typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
150
151/* Transport layer configuration information */
152static struct xpt_softc xsoftc;
153
154MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
155
156TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
157SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
158           &xsoftc.boot_delay, 0, "Bus registration wait time");
159SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
160	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
161
162struct cam_doneq {
163	struct mtx_padalign	cam_doneq_mtx;
164	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
165	int			cam_doneq_sleep;
166};
167
168static struct cam_doneq cam_doneqs[MAXCPU];
169static int cam_num_doneqs;
170static struct proc *cam_proc;
171
172TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
173SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
174           &cam_num_doneqs, 0, "Number of completion queues/threads");
175
176struct cam_periph *xpt_periph;
177
178static periph_init_t xpt_periph_init;
179
180static struct periph_driver xpt_driver =
181{
182	xpt_periph_init, "xpt",
183	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
184	CAM_PERIPH_DRV_EARLY
185};
186
187PERIPHDRIVER_DECLARE(xpt, xpt_driver);
188
189static d_open_t xptopen;
190static d_close_t xptclose;
191static d_ioctl_t xptioctl;
192static d_ioctl_t xptdoioctl;
193
194static struct cdevsw xpt_cdevsw = {
195	.d_version =	D_VERSION,
196	.d_flags =	0,
197	.d_open =	xptopen,
198	.d_close =	xptclose,
199	.d_ioctl =	xptioctl,
200	.d_name =	"xpt",
201};
202
203/* Storage for debugging datastructures */
204struct cam_path *cam_dpath;
205u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
206TUNABLE_INT("kern.cam.dflags", &cam_dflags);
207SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
208	&cam_dflags, 0, "Enabled debug flags");
209u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
210TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
211SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
212	&cam_debug_delay, 0, "Delay in us after each debug message");
213
214/* Our boot-time initialization hook */
215static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
216
217static moduledata_t cam_moduledata = {
218	"cam",
219	cam_module_event_handler,
220	NULL
221};
222
223static int	xpt_init(void *);
224
225DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
226MODULE_VERSION(cam, 1);
227
228
229static void		xpt_async_bcast(struct async_list *async_head,
230					u_int32_t async_code,
231					struct cam_path *path,
232					void *async_arg);
233static path_id_t xptnextfreepathid(void);
234static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
235static union ccb *xpt_get_ccb(struct cam_periph *periph);
236static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
237static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
238static void	 xpt_run_allocq_task(void *context, int pending);
239static void	 xpt_run_devq(struct cam_devq *devq);
240static timeout_t xpt_release_devq_timeout;
241static void	 xpt_release_simq_timeout(void *arg) __unused;
242static void	 xpt_acquire_bus(struct cam_eb *bus);
243static void	 xpt_release_bus(struct cam_eb *bus);
244static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
245static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
246		    int run_queue);
247static struct cam_et*
248		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
249static void	 xpt_acquire_target(struct cam_et *target);
250static void	 xpt_release_target(struct cam_et *target);
251static struct cam_eb*
252		 xpt_find_bus(path_id_t path_id);
253static struct cam_et*
254		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
255static struct cam_ed*
256		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
257static void	 xpt_config(void *arg);
258static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
259				 u_int32_t new_priority);
260static xpt_devicefunc_t xptpassannouncefunc;
261static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
262static void	 xptpoll(struct cam_sim *sim);
263static void	 camisr_runqueue(void);
264static void	 xpt_done_process(struct ccb_hdr *ccb_h);
265static void	 xpt_done_td(void *);
266static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
267				    u_int num_patterns, struct cam_eb *bus);
268static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
269				       u_int num_patterns,
270				       struct cam_ed *device);
271static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
272				       u_int num_patterns,
273				       struct cam_periph *periph);
274static xpt_busfunc_t	xptedtbusfunc;
275static xpt_targetfunc_t	xptedttargetfunc;
276static xpt_devicefunc_t	xptedtdevicefunc;
277static xpt_periphfunc_t	xptedtperiphfunc;
278static xpt_pdrvfunc_t	xptplistpdrvfunc;
279static xpt_periphfunc_t	xptplistperiphfunc;
280static int		xptedtmatch(struct ccb_dev_match *cdm);
281static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
282static int		xptbustraverse(struct cam_eb *start_bus,
283				       xpt_busfunc_t *tr_func, void *arg);
284static int		xpttargettraverse(struct cam_eb *bus,
285					  struct cam_et *start_target,
286					  xpt_targetfunc_t *tr_func, void *arg);
287static int		xptdevicetraverse(struct cam_et *target,
288					  struct cam_ed *start_device,
289					  xpt_devicefunc_t *tr_func, void *arg);
290static int		xptperiphtraverse(struct cam_ed *device,
291					  struct cam_periph *start_periph,
292					  xpt_periphfunc_t *tr_func, void *arg);
293static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
294					xpt_pdrvfunc_t *tr_func, void *arg);
295static int		xptpdperiphtraverse(struct periph_driver **pdrv,
296					    struct cam_periph *start_periph,
297					    xpt_periphfunc_t *tr_func,
298					    void *arg);
299static xpt_busfunc_t	xptdefbusfunc;
300static xpt_targetfunc_t	xptdeftargetfunc;
301static xpt_devicefunc_t	xptdefdevicefunc;
302static xpt_periphfunc_t	xptdefperiphfunc;
303static void		xpt_finishconfig_task(void *context, int pending);
304static void		xpt_dev_async_default(u_int32_t async_code,
305					      struct cam_eb *bus,
306					      struct cam_et *target,
307					      struct cam_ed *device,
308					      void *async_arg);
309static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
310						 struct cam_et *target,
311						 lun_id_t lun_id);
312static xpt_devicefunc_t	xptsetasyncfunc;
313static xpt_busfunc_t	xptsetasyncbusfunc;
314static cam_status	xptregister(struct cam_periph *periph,
315				    void *arg);
316static __inline int device_is_queued(struct cam_ed *device);
317
318static __inline int
319xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
320{
321	int	retval;
322
323	mtx_assert(&devq->send_mtx, MA_OWNED);
324	if ((dev->ccbq.queue.entries > 0) &&
325	    (dev->ccbq.dev_openings > 0) &&
326	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
327		/*
328		 * The priority of a device waiting for controller
329		 * resources is that of the highest priority CCB
330		 * enqueued.
331		 */
332		retval =
333		    xpt_schedule_dev(&devq->send_queue,
334				     &dev->devq_entry,
335				     CAMQ_GET_PRIO(&dev->ccbq.queue));
336	} else {
337		retval = 0;
338	}
339	return (retval);
340}
341
342static __inline int
343device_is_queued(struct cam_ed *device)
344{
345	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
346}
347
348static void
349xpt_periph_init()
350{
351	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
352}
353
354static int
355xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
356{
357
358	/*
359	 * Only allow read-write access.
360	 */
361	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
362		return(EPERM);
363
364	/*
365	 * We don't allow nonblocking access.
366	 */
367	if ((flags & O_NONBLOCK) != 0) {
368		printf("%s: can't do nonblocking access\n", devtoname(dev));
369		return(ENODEV);
370	}
371
372	return(0);
373}
374
375static int
376xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
377{
378
379	return(0);
380}
381
382/*
383 * Don't automatically grab the xpt softc lock here even though this is going
384 * through the xpt device.  The xpt device is really just a back door for
385 * accessing other devices and SIMs, so the right thing to do is to grab
386 * the appropriate SIM lock once the bus/SIM is located.
387 */
388static int
389xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
390{
391	int error;
392
393	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
394		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
395	}
396	return (error);
397}
398
399static int
400xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
401{
402	int error;
403
404	error = 0;
405
406	switch(cmd) {
407	/*
408	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
409	 * to accept CCB types that don't quite make sense to send through a
410	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
411	 * in the CAM spec.
412	 */
413	case CAMIOCOMMAND: {
414		union ccb *ccb;
415		union ccb *inccb;
416		struct cam_eb *bus;
417
418		inccb = (union ccb *)addr;
419
420		bus = xpt_find_bus(inccb->ccb_h.path_id);
421		if (bus == NULL)
422			return (EINVAL);
423
424		switch (inccb->ccb_h.func_code) {
425		case XPT_SCAN_BUS:
426		case XPT_RESET_BUS:
427			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
428			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
429				xpt_release_bus(bus);
430				return (EINVAL);
431			}
432			break;
433		case XPT_SCAN_TGT:
434			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
435			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
436				xpt_release_bus(bus);
437				return (EINVAL);
438			}
439			break;
440		default:
441			break;
442		}
443
444		switch(inccb->ccb_h.func_code) {
445		case XPT_SCAN_BUS:
446		case XPT_RESET_BUS:
447		case XPT_PATH_INQ:
448		case XPT_ENG_INQ:
449		case XPT_SCAN_LUN:
450		case XPT_SCAN_TGT:
451
452			ccb = xpt_alloc_ccb();
453
454			/*
455			 * Create a path using the bus, target, and lun the
456			 * user passed in.
457			 */
458			if (xpt_create_path(&ccb->ccb_h.path, NULL,
459					    inccb->ccb_h.path_id,
460					    inccb->ccb_h.target_id,
461					    inccb->ccb_h.target_lun) !=
462					    CAM_REQ_CMP){
463				error = EINVAL;
464				xpt_free_ccb(ccb);
465				break;
466			}
467			/* Ensure all of our fields are correct */
468			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
469				      inccb->ccb_h.pinfo.priority);
470			xpt_merge_ccb(ccb, inccb);
471			xpt_path_lock(ccb->ccb_h.path);
472			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
473			xpt_path_unlock(ccb->ccb_h.path);
474			bcopy(ccb, inccb, sizeof(union ccb));
475			xpt_free_path(ccb->ccb_h.path);
476			xpt_free_ccb(ccb);
477			break;
478
479		case XPT_DEBUG: {
480			union ccb ccb;
481
482			/*
483			 * This is an immediate CCB, so it's okay to
484			 * allocate it on the stack.
485			 */
486
487			/*
488			 * Create a path using the bus, target, and lun the
489			 * user passed in.
490			 */
491			if (xpt_create_path(&ccb.ccb_h.path, NULL,
492					    inccb->ccb_h.path_id,
493					    inccb->ccb_h.target_id,
494					    inccb->ccb_h.target_lun) !=
495					    CAM_REQ_CMP){
496				error = EINVAL;
497				break;
498			}
499			/* Ensure all of our fields are correct */
500			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
501				      inccb->ccb_h.pinfo.priority);
502			xpt_merge_ccb(&ccb, inccb);
503			xpt_action(&ccb);
504			bcopy(&ccb, inccb, sizeof(union ccb));
505			xpt_free_path(ccb.ccb_h.path);
506			break;
507
508		}
509		case XPT_DEV_MATCH: {
510			struct cam_periph_map_info mapinfo;
511			struct cam_path *old_path;
512
513			/*
514			 * We can't deal with physical addresses for this
515			 * type of transaction.
516			 */
517			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
518			    CAM_DATA_VADDR) {
519				error = EINVAL;
520				break;
521			}
522
523			/*
524			 * Save this in case the caller had it set to
525			 * something in particular.
526			 */
527			old_path = inccb->ccb_h.path;
528
529			/*
530			 * We really don't need a path for the matching
531			 * code.  The path is needed because of the
532			 * debugging statements in xpt_action().  They
533			 * assume that the CCB has a valid path.
534			 */
535			inccb->ccb_h.path = xpt_periph->path;
536
537			bzero(&mapinfo, sizeof(mapinfo));
538
539			/*
540			 * Map the pattern and match buffers into kernel
541			 * virtual address space.
542			 */
543			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
544
545			if (error) {
546				inccb->ccb_h.path = old_path;
547				break;
548			}
549
550			/*
551			 * This is an immediate CCB, we can send it on directly.
552			 */
553			xpt_action(inccb);
554
555			/*
556			 * Map the buffers back into user space.
557			 */
558			cam_periph_unmapmem(inccb, &mapinfo);
559
560			inccb->ccb_h.path = old_path;
561
562			error = 0;
563			break;
564		}
565		default:
566			error = ENOTSUP;
567			break;
568		}
569		xpt_release_bus(bus);
570		break;
571	}
572	/*
573	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
574	 * with the periphal driver name and unit name filled in.  The other
575	 * fields don't really matter as input.  The passthrough driver name
576	 * ("pass"), and unit number are passed back in the ccb.  The current
577	 * device generation number, and the index into the device peripheral
578	 * driver list, and the status are also passed back.  Note that
579	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
580	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
581	 * (or rather should be) impossible for the device peripheral driver
582	 * list to change since we look at the whole thing in one pass, and
583	 * we do it with lock protection.
584	 *
585	 */
586	case CAMGETPASSTHRU: {
587		union ccb *ccb;
588		struct cam_periph *periph;
589		struct periph_driver **p_drv;
590		char   *name;
591		u_int unit;
592		int base_periph_found;
593
594		ccb = (union ccb *)addr;
595		unit = ccb->cgdl.unit_number;
596		name = ccb->cgdl.periph_name;
597		base_periph_found = 0;
598
599		/*
600		 * Sanity check -- make sure we don't get a null peripheral
601		 * driver name.
602		 */
603		if (*ccb->cgdl.periph_name == '\0') {
604			error = EINVAL;
605			break;
606		}
607
608		/* Keep the list from changing while we traverse it */
609		xpt_lock_buses();
610
611		/* first find our driver in the list of drivers */
612		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
613			if (strcmp((*p_drv)->driver_name, name) == 0)
614				break;
615
616		if (*p_drv == NULL) {
617			xpt_unlock_buses();
618			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
619			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
620			*ccb->cgdl.periph_name = '\0';
621			ccb->cgdl.unit_number = 0;
622			error = ENOENT;
623			break;
624		}
625
626		/*
627		 * Run through every peripheral instance of this driver
628		 * and check to see whether it matches the unit passed
629		 * in by the user.  If it does, get out of the loops and
630		 * find the passthrough driver associated with that
631		 * peripheral driver.
632		 */
633		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
634		     periph = TAILQ_NEXT(periph, unit_links)) {
635
636			if (periph->unit_number == unit)
637				break;
638		}
639		/*
640		 * If we found the peripheral driver that the user passed
641		 * in, go through all of the peripheral drivers for that
642		 * particular device and look for a passthrough driver.
643		 */
644		if (periph != NULL) {
645			struct cam_ed *device;
646			int i;
647
648			base_periph_found = 1;
649			device = periph->path->device;
650			for (i = 0, periph = SLIST_FIRST(&device->periphs);
651			     periph != NULL;
652			     periph = SLIST_NEXT(periph, periph_links), i++) {
653				/*
654				 * Check to see whether we have a
655				 * passthrough device or not.
656				 */
657				if (strcmp(periph->periph_name, "pass") == 0) {
658					/*
659					 * Fill in the getdevlist fields.
660					 */
661					strcpy(ccb->cgdl.periph_name,
662					       periph->periph_name);
663					ccb->cgdl.unit_number =
664						periph->unit_number;
665					if (SLIST_NEXT(periph, periph_links))
666						ccb->cgdl.status =
667							CAM_GDEVLIST_MORE_DEVS;
668					else
669						ccb->cgdl.status =
670						       CAM_GDEVLIST_LAST_DEVICE;
671					ccb->cgdl.generation =
672						device->generation;
673					ccb->cgdl.index = i;
674					/*
675					 * Fill in some CCB header fields
676					 * that the user may want.
677					 */
678					ccb->ccb_h.path_id =
679						periph->path->bus->path_id;
680					ccb->ccb_h.target_id =
681						periph->path->target->target_id;
682					ccb->ccb_h.target_lun =
683						periph->path->device->lun_id;
684					ccb->ccb_h.status = CAM_REQ_CMP;
685					break;
686				}
687			}
688		}
689
690		/*
691		 * If the periph is null here, one of two things has
692		 * happened.  The first possibility is that we couldn't
693		 * find the unit number of the particular peripheral driver
694		 * that the user is asking about.  e.g. the user asks for
695		 * the passthrough driver for "da11".  We find the list of
696		 * "da" peripherals all right, but there is no unit 11.
697		 * The other possibility is that we went through the list
698		 * of peripheral drivers attached to the device structure,
699		 * but didn't find one with the name "pass".  Either way,
700		 * we return ENOENT, since we couldn't find something.
701		 */
702		if (periph == NULL) {
703			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
704			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
705			*ccb->cgdl.periph_name = '\0';
706			ccb->cgdl.unit_number = 0;
707			error = ENOENT;
708			/*
709			 * It is unfortunate that this is even necessary,
710			 * but there are many, many clueless users out there.
711			 * If this is true, the user is looking for the
712			 * passthrough driver, but doesn't have one in his
713			 * kernel.
714			 */
715			if (base_periph_found == 1) {
716				printf("xptioctl: pass driver is not in the "
717				       "kernel\n");
718				printf("xptioctl: put \"device pass\" in "
719				       "your kernel config file\n");
720			}
721		}
722		xpt_unlock_buses();
723		break;
724		}
725	default:
726		error = ENOTTY;
727		break;
728	}
729
730	return(error);
731}
732
733static int
734cam_module_event_handler(module_t mod, int what, void *arg)
735{
736	int error;
737
738	switch (what) {
739	case MOD_LOAD:
740		if ((error = xpt_init(NULL)) != 0)
741			return (error);
742		break;
743	case MOD_UNLOAD:
744		return EBUSY;
745	default:
746		return EOPNOTSUPP;
747	}
748
749	return 0;
750}
751
752static void
753xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
754{
755
756	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
757		xpt_free_path(done_ccb->ccb_h.path);
758		xpt_free_ccb(done_ccb);
759	} else {
760		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
761		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
762	}
763	xpt_release_boot();
764}
765
766/* thread to handle bus rescans */
767static void
768xpt_scanner_thread(void *dummy)
769{
770	union ccb	*ccb;
771	struct cam_path	 path;
772
773	xpt_lock_buses();
774	for (;;) {
775		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
776			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
777			       "-", 0);
778		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
779			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
780			xpt_unlock_buses();
781
782			/*
783			 * Since lock can be dropped inside and path freed
784			 * by completion callback even before return here,
785			 * take our own path copy for reference.
786			 */
787			xpt_copy_path(&path, ccb->ccb_h.path);
788			xpt_path_lock(&path);
789			xpt_action(ccb);
790			xpt_path_unlock(&path);
791			xpt_release_path(&path);
792
793			xpt_lock_buses();
794		}
795	}
796}
797
798void
799xpt_rescan(union ccb *ccb)
800{
801	struct ccb_hdr *hdr;
802
803	/* Prepare request */
804	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
805	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
806		ccb->ccb_h.func_code = XPT_SCAN_BUS;
807	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
808	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
809		ccb->ccb_h.func_code = XPT_SCAN_TGT;
810	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
811	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
812		ccb->ccb_h.func_code = XPT_SCAN_LUN;
813	else {
814		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
815		xpt_free_path(ccb->ccb_h.path);
816		xpt_free_ccb(ccb);
817		return;
818	}
819	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
820	ccb->ccb_h.cbfcnp = xpt_rescan_done;
821	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
822	/* Don't make duplicate entries for the same paths. */
823	xpt_lock_buses();
824	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
825		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
826			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
827				wakeup(&xsoftc.ccb_scanq);
828				xpt_unlock_buses();
829				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
830				xpt_free_path(ccb->ccb_h.path);
831				xpt_free_ccb(ccb);
832				return;
833			}
834		}
835	}
836	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
837	xsoftc.buses_to_config++;
838	wakeup(&xsoftc.ccb_scanq);
839	xpt_unlock_buses();
840}
841
842/* Functions accessed by the peripheral drivers */
843static int
844xpt_init(void *dummy)
845{
846	struct cam_sim *xpt_sim;
847	struct cam_path *path;
848	struct cam_devq *devq;
849	cam_status status;
850	int error, i;
851
852	TAILQ_INIT(&xsoftc.xpt_busses);
853	TAILQ_INIT(&xsoftc.ccb_scanq);
854	STAILQ_INIT(&xsoftc.highpowerq);
855	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
856
857	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
858	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
859	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
860	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
861
862#ifdef CAM_BOOT_DELAY
863	/*
864	 * Override this value at compile time to assist our users
865	 * who don't use loader to boot a kernel.
866	 */
867	xsoftc.boot_delay = CAM_BOOT_DELAY;
868#endif
869	/*
870	 * The xpt layer is, itself, the equivelent of a SIM.
871	 * Allow 16 ccbs in the ccb pool for it.  This should
872	 * give decent parallelism when we probe busses and
873	 * perform other XPT functions.
874	 */
875	devq = cam_simq_alloc(16);
876	xpt_sim = cam_sim_alloc(xptaction,
877				xptpoll,
878				"xpt",
879				/*softc*/NULL,
880				/*unit*/0,
881				/*mtx*/&xsoftc.xpt_lock,
882				/*max_dev_transactions*/0,
883				/*max_tagged_dev_transactions*/0,
884				devq);
885	if (xpt_sim == NULL)
886		return (ENOMEM);
887
888	mtx_lock(&xsoftc.xpt_lock);
889	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
890		mtx_unlock(&xsoftc.xpt_lock);
891		printf("xpt_init: xpt_bus_register failed with status %#x,"
892		       " failing attach\n", status);
893		return (EINVAL);
894	}
895	mtx_unlock(&xsoftc.xpt_lock);
896
897	/*
898	 * Looking at the XPT from the SIM layer, the XPT is
899	 * the equivelent of a peripheral driver.  Allocate
900	 * a peripheral driver entry for us.
901	 */
902	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
903				      CAM_TARGET_WILDCARD,
904				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
905		printf("xpt_init: xpt_create_path failed with status %#x,"
906		       " failing attach\n", status);
907		return (EINVAL);
908	}
909	xpt_path_lock(path);
910	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
911			 path, NULL, 0, xpt_sim);
912	xpt_path_unlock(path);
913	xpt_free_path(path);
914
915	if (cam_num_doneqs < 1)
916		cam_num_doneqs = 1 + mp_ncpus / 6;
917	else if (cam_num_doneqs > MAXCPU)
918		cam_num_doneqs = MAXCPU;
919	for (i = 0; i < cam_num_doneqs; i++) {
920		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
921		    MTX_DEF);
922		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
923		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
924		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
925		if (error != 0) {
926			cam_num_doneqs = i;
927			break;
928		}
929	}
930	if (cam_num_doneqs < 1) {
931		printf("xpt_init: Cannot init completion queues "
932		       "- failing attach\n");
933		return (ENOMEM);
934	}
935	/*
936	 * Register a callback for when interrupts are enabled.
937	 */
938	xsoftc.xpt_config_hook =
939	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
940					      M_CAMXPT, M_NOWAIT | M_ZERO);
941	if (xsoftc.xpt_config_hook == NULL) {
942		printf("xpt_init: Cannot malloc config hook "
943		       "- failing attach\n");
944		return (ENOMEM);
945	}
946	xsoftc.xpt_config_hook->ich_func = xpt_config;
947	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
948		free (xsoftc.xpt_config_hook, M_CAMXPT);
949		printf("xpt_init: config_intrhook_establish failed "
950		       "- failing attach\n");
951	}
952
953	return (0);
954}
955
956static cam_status
957xptregister(struct cam_periph *periph, void *arg)
958{
959	struct cam_sim *xpt_sim;
960
961	if (periph == NULL) {
962		printf("xptregister: periph was NULL!!\n");
963		return(CAM_REQ_CMP_ERR);
964	}
965
966	xpt_sim = (struct cam_sim *)arg;
967	xpt_sim->softc = periph;
968	xpt_periph = periph;
969	periph->softc = NULL;
970
971	return(CAM_REQ_CMP);
972}
973
974int32_t
975xpt_add_periph(struct cam_periph *periph)
976{
977	struct cam_ed *device;
978	int32_t	 status;
979
980	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
981	device = periph->path->device;
982	status = CAM_REQ_CMP;
983	if (device != NULL) {
984		mtx_lock(&device->target->bus->eb_mtx);
985		device->generation++;
986		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
987		mtx_unlock(&device->target->bus->eb_mtx);
988		atomic_add_32(&xsoftc.xpt_generation, 1);
989	}
990
991	return (status);
992}
993
994void
995xpt_remove_periph(struct cam_periph *periph)
996{
997	struct cam_ed *device;
998
999	device = periph->path->device;
1000	if (device != NULL) {
1001		mtx_lock(&device->target->bus->eb_mtx);
1002		device->generation++;
1003		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1004		mtx_unlock(&device->target->bus->eb_mtx);
1005		atomic_add_32(&xsoftc.xpt_generation, 1);
1006	}
1007}
1008
1009
1010void
1011xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1012{
1013	struct	cam_path *path = periph->path;
1014
1015	cam_periph_assert(periph, MA_OWNED);
1016	periph->flags |= CAM_PERIPH_ANNOUNCED;
1017
1018	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1019	       periph->periph_name, periph->unit_number,
1020	       path->bus->sim->sim_name,
1021	       path->bus->sim->unit_number,
1022	       path->bus->sim->bus_id,
1023	       path->bus->path_id,
1024	       path->target->target_id,
1025	       (uintmax_t)path->device->lun_id);
1026	printf("%s%d: ", periph->periph_name, periph->unit_number);
1027	if (path->device->protocol == PROTO_SCSI)
1028		scsi_print_inquiry(&path->device->inq_data);
1029	else if (path->device->protocol == PROTO_ATA ||
1030	    path->device->protocol == PROTO_SATAPM)
1031		ata_print_ident(&path->device->ident_data);
1032	else if (path->device->protocol == PROTO_SEMB)
1033		semb_print_ident(
1034		    (struct sep_identify_data *)&path->device->ident_data);
1035	else
1036		printf("Unknown protocol device\n");
1037	if (path->device->serial_num_len > 0) {
1038		/* Don't wrap the screen  - print only the first 60 chars */
1039		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1040		       periph->unit_number, path->device->serial_num);
1041	}
1042	/* Announce transport details. */
1043	(*(path->bus->xport->announce))(periph);
1044	/* Announce command queueing. */
1045	if (path->device->inq_flags & SID_CmdQue
1046	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1047		printf("%s%d: Command Queueing enabled\n",
1048		       periph->periph_name, periph->unit_number);
1049	}
1050	/* Announce caller's details if they've passed in. */
1051	if (announce_string != NULL)
1052		printf("%s%d: %s\n", periph->periph_name,
1053		       periph->unit_number, announce_string);
1054}
1055
1056void
1057xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1058{
1059	if (quirks != 0) {
1060		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1061		    periph->unit_number, quirks, bit_string);
1062	}
1063}
1064
1065void
1066xpt_denounce_periph(struct cam_periph *periph)
1067{
1068	struct	cam_path *path = periph->path;
1069
1070	cam_periph_assert(periph, MA_OWNED);
1071	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1072	       periph->periph_name, periph->unit_number,
1073	       path->bus->sim->sim_name,
1074	       path->bus->sim->unit_number,
1075	       path->bus->sim->bus_id,
1076	       path->bus->path_id,
1077	       path->target->target_id,
1078	       (uintmax_t)path->device->lun_id);
1079	printf("%s%d: ", periph->periph_name, periph->unit_number);
1080	if (path->device->protocol == PROTO_SCSI)
1081		scsi_print_inquiry_short(&path->device->inq_data);
1082	else if (path->device->protocol == PROTO_ATA ||
1083	    path->device->protocol == PROTO_SATAPM)
1084		ata_print_ident_short(&path->device->ident_data);
1085	else if (path->device->protocol == PROTO_SEMB)
1086		semb_print_ident_short(
1087		    (struct sep_identify_data *)&path->device->ident_data);
1088	else
1089		printf("Unknown protocol device");
1090	if (path->device->serial_num_len > 0)
1091		printf(" s/n %.60s", path->device->serial_num);
1092	printf(" detached\n");
1093}
1094
1095
1096int
1097xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1098{
1099	int ret = -1, l;
1100	struct ccb_dev_advinfo cdai;
1101	struct scsi_vpd_id_descriptor *idd;
1102
1103	xpt_path_assert(path, MA_OWNED);
1104
1105	memset(&cdai, 0, sizeof(cdai));
1106	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1107	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1108	cdai.bufsiz = len;
1109
1110	if (!strcmp(attr, "GEOM::ident"))
1111		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1112	else if (!strcmp(attr, "GEOM::physpath"))
1113		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1114	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1115		 strcmp(attr, "GEOM::lunname") == 0) {
1116		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1117		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1118	} else
1119		goto out;
1120
1121	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1122	if (cdai.buf == NULL) {
1123		ret = ENOMEM;
1124		goto out;
1125	}
1126	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1127	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1128		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1129	if (cdai.provsiz == 0)
1130		goto out;
1131	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1132		if (strcmp(attr, "GEOM::lunid") == 0) {
1133			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1134			    cdai.provsiz, scsi_devid_is_lun_naa);
1135			if (idd == NULL)
1136				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1137				    cdai.provsiz, scsi_devid_is_lun_eui64);
1138		} else
1139			idd = NULL;
1140		if (idd == NULL)
1141			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1142			    cdai.provsiz, scsi_devid_is_lun_t10);
1143		if (idd == NULL)
1144			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1145			    cdai.provsiz, scsi_devid_is_lun_name);
1146		if (idd == NULL)
1147			goto out;
1148		ret = 0;
1149		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1150			if (idd->length < len) {
1151				for (l = 0; l < idd->length; l++)
1152					buf[l] = idd->identifier[l] ?
1153					    idd->identifier[l] : ' ';
1154				buf[l] = 0;
1155			} else
1156				ret = EFAULT;
1157		} else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1158			l = strnlen(idd->identifier, idd->length);
1159			if (l < len) {
1160				bcopy(idd->identifier, buf, l);
1161				buf[l] = 0;
1162			} else
1163				ret = EFAULT;
1164		} else {
1165			if (idd->length * 2 < len) {
1166				for (l = 0; l < idd->length; l++)
1167					sprintf(buf + l * 2, "%02x",
1168					    idd->identifier[l]);
1169			} else
1170				ret = EFAULT;
1171		}
1172	} else {
1173		ret = 0;
1174		if (strlcpy(buf, cdai.buf, len) >= len)
1175			ret = EFAULT;
1176	}
1177
1178out:
1179	if (cdai.buf != NULL)
1180		free(cdai.buf, M_CAMXPT);
1181	return ret;
1182}
1183
1184static dev_match_ret
1185xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1186	    struct cam_eb *bus)
1187{
1188	dev_match_ret retval;
1189	u_int i;
1190
1191	retval = DM_RET_NONE;
1192
1193	/*
1194	 * If we aren't given something to match against, that's an error.
1195	 */
1196	if (bus == NULL)
1197		return(DM_RET_ERROR);
1198
1199	/*
1200	 * If there are no match entries, then this bus matches no
1201	 * matter what.
1202	 */
1203	if ((patterns == NULL) || (num_patterns == 0))
1204		return(DM_RET_DESCEND | DM_RET_COPY);
1205
1206	for (i = 0; i < num_patterns; i++) {
1207		struct bus_match_pattern *cur_pattern;
1208
1209		/*
1210		 * If the pattern in question isn't for a bus node, we
1211		 * aren't interested.  However, we do indicate to the
1212		 * calling routine that we should continue descending the
1213		 * tree, since the user wants to match against lower-level
1214		 * EDT elements.
1215		 */
1216		if (patterns[i].type != DEV_MATCH_BUS) {
1217			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1218				retval |= DM_RET_DESCEND;
1219			continue;
1220		}
1221
1222		cur_pattern = &patterns[i].pattern.bus_pattern;
1223
1224		/*
1225		 * If they want to match any bus node, we give them any
1226		 * device node.
1227		 */
1228		if (cur_pattern->flags == BUS_MATCH_ANY) {
1229			/* set the copy flag */
1230			retval |= DM_RET_COPY;
1231
1232			/*
1233			 * If we've already decided on an action, go ahead
1234			 * and return.
1235			 */
1236			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1237				return(retval);
1238		}
1239
1240		/*
1241		 * Not sure why someone would do this...
1242		 */
1243		if (cur_pattern->flags == BUS_MATCH_NONE)
1244			continue;
1245
1246		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1247		 && (cur_pattern->path_id != bus->path_id))
1248			continue;
1249
1250		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1251		 && (cur_pattern->bus_id != bus->sim->bus_id))
1252			continue;
1253
1254		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1255		 && (cur_pattern->unit_number != bus->sim->unit_number))
1256			continue;
1257
1258		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1259		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1260			     DEV_IDLEN) != 0))
1261			continue;
1262
1263		/*
1264		 * If we get to this point, the user definitely wants
1265		 * information on this bus.  So tell the caller to copy the
1266		 * data out.
1267		 */
1268		retval |= DM_RET_COPY;
1269
1270		/*
1271		 * If the return action has been set to descend, then we
1272		 * know that we've already seen a non-bus matching
1273		 * expression, therefore we need to further descend the tree.
1274		 * This won't change by continuing around the loop, so we
1275		 * go ahead and return.  If we haven't seen a non-bus
1276		 * matching expression, we keep going around the loop until
1277		 * we exhaust the matching expressions.  We'll set the stop
1278		 * flag once we fall out of the loop.
1279		 */
1280		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1281			return(retval);
1282	}
1283
1284	/*
1285	 * If the return action hasn't been set to descend yet, that means
1286	 * we haven't seen anything other than bus matching patterns.  So
1287	 * tell the caller to stop descending the tree -- the user doesn't
1288	 * want to match against lower level tree elements.
1289	 */
1290	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1291		retval |= DM_RET_STOP;
1292
1293	return(retval);
1294}
1295
1296static dev_match_ret
1297xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1298	       struct cam_ed *device)
1299{
1300	dev_match_ret retval;
1301	u_int i;
1302
1303	retval = DM_RET_NONE;
1304
1305	/*
1306	 * If we aren't given something to match against, that's an error.
1307	 */
1308	if (device == NULL)
1309		return(DM_RET_ERROR);
1310
1311	/*
1312	 * If there are no match entries, then this device matches no
1313	 * matter what.
1314	 */
1315	if ((patterns == NULL) || (num_patterns == 0))
1316		return(DM_RET_DESCEND | DM_RET_COPY);
1317
1318	for (i = 0; i < num_patterns; i++) {
1319		struct device_match_pattern *cur_pattern;
1320		struct scsi_vpd_device_id *device_id_page;
1321
1322		/*
1323		 * If the pattern in question isn't for a device node, we
1324		 * aren't interested.
1325		 */
1326		if (patterns[i].type != DEV_MATCH_DEVICE) {
1327			if ((patterns[i].type == DEV_MATCH_PERIPH)
1328			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1329				retval |= DM_RET_DESCEND;
1330			continue;
1331		}
1332
1333		cur_pattern = &patterns[i].pattern.device_pattern;
1334
1335		/* Error out if mutually exclusive options are specified. */
1336		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1337		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1338			return(DM_RET_ERROR);
1339
1340		/*
1341		 * If they want to match any device node, we give them any
1342		 * device node.
1343		 */
1344		if (cur_pattern->flags == DEV_MATCH_ANY)
1345			goto copy_dev_node;
1346
1347		/*
1348		 * Not sure why someone would do this...
1349		 */
1350		if (cur_pattern->flags == DEV_MATCH_NONE)
1351			continue;
1352
1353		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1354		 && (cur_pattern->path_id != device->target->bus->path_id))
1355			continue;
1356
1357		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1358		 && (cur_pattern->target_id != device->target->target_id))
1359			continue;
1360
1361		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1362		 && (cur_pattern->target_lun != device->lun_id))
1363			continue;
1364
1365		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1366		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1367				    (caddr_t)&cur_pattern->data.inq_pat,
1368				    1, sizeof(cur_pattern->data.inq_pat),
1369				    scsi_static_inquiry_match) == NULL))
1370			continue;
1371
1372		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1373		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1374		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1375		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1376				      device->device_id_len
1377				    - SVPD_DEVICE_ID_HDR_LEN,
1378				      cur_pattern->data.devid_pat.id,
1379				      cur_pattern->data.devid_pat.id_len) != 0))
1380			continue;
1381
1382copy_dev_node:
1383		/*
1384		 * If we get to this point, the user definitely wants
1385		 * information on this device.  So tell the caller to copy
1386		 * the data out.
1387		 */
1388		retval |= DM_RET_COPY;
1389
1390		/*
1391		 * If the return action has been set to descend, then we
1392		 * know that we've already seen a peripheral matching
1393		 * expression, therefore we need to further descend the tree.
1394		 * This won't change by continuing around the loop, so we
1395		 * go ahead and return.  If we haven't seen a peripheral
1396		 * matching expression, we keep going around the loop until
1397		 * we exhaust the matching expressions.  We'll set the stop
1398		 * flag once we fall out of the loop.
1399		 */
1400		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1401			return(retval);
1402	}
1403
1404	/*
1405	 * If the return action hasn't been set to descend yet, that means
1406	 * we haven't seen any peripheral matching patterns.  So tell the
1407	 * caller to stop descending the tree -- the user doesn't want to
1408	 * match against lower level tree elements.
1409	 */
1410	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1411		retval |= DM_RET_STOP;
1412
1413	return(retval);
1414}
1415
1416/*
1417 * Match a single peripheral against any number of match patterns.
1418 */
1419static dev_match_ret
1420xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1421	       struct cam_periph *periph)
1422{
1423	dev_match_ret retval;
1424	u_int i;
1425
1426	/*
1427	 * If we aren't given something to match against, that's an error.
1428	 */
1429	if (periph == NULL)
1430		return(DM_RET_ERROR);
1431
1432	/*
1433	 * If there are no match entries, then this peripheral matches no
1434	 * matter what.
1435	 */
1436	if ((patterns == NULL) || (num_patterns == 0))
1437		return(DM_RET_STOP | DM_RET_COPY);
1438
1439	/*
1440	 * There aren't any nodes below a peripheral node, so there's no
1441	 * reason to descend the tree any further.
1442	 */
1443	retval = DM_RET_STOP;
1444
1445	for (i = 0; i < num_patterns; i++) {
1446		struct periph_match_pattern *cur_pattern;
1447
1448		/*
1449		 * If the pattern in question isn't for a peripheral, we
1450		 * aren't interested.
1451		 */
1452		if (patterns[i].type != DEV_MATCH_PERIPH)
1453			continue;
1454
1455		cur_pattern = &patterns[i].pattern.periph_pattern;
1456
1457		/*
1458		 * If they want to match on anything, then we will do so.
1459		 */
1460		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1461			/* set the copy flag */
1462			retval |= DM_RET_COPY;
1463
1464			/*
1465			 * We've already set the return action to stop,
1466			 * since there are no nodes below peripherals in
1467			 * the tree.
1468			 */
1469			return(retval);
1470		}
1471
1472		/*
1473		 * Not sure why someone would do this...
1474		 */
1475		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1476			continue;
1477
1478		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1479		 && (cur_pattern->path_id != periph->path->bus->path_id))
1480			continue;
1481
1482		/*
1483		 * For the target and lun id's, we have to make sure the
1484		 * target and lun pointers aren't NULL.  The xpt peripheral
1485		 * has a wildcard target and device.
1486		 */
1487		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1488		 && ((periph->path->target == NULL)
1489		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1490			continue;
1491
1492		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1493		 && ((periph->path->device == NULL)
1494		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1495			continue;
1496
1497		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1498		 && (cur_pattern->unit_number != periph->unit_number))
1499			continue;
1500
1501		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1502		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1503			     DEV_IDLEN) != 0))
1504			continue;
1505
1506		/*
1507		 * If we get to this point, the user definitely wants
1508		 * information on this peripheral.  So tell the caller to
1509		 * copy the data out.
1510		 */
1511		retval |= DM_RET_COPY;
1512
1513		/*
1514		 * The return action has already been set to stop, since
1515		 * peripherals don't have any nodes below them in the EDT.
1516		 */
1517		return(retval);
1518	}
1519
1520	/*
1521	 * If we get to this point, the peripheral that was passed in
1522	 * doesn't match any of the patterns.
1523	 */
1524	return(retval);
1525}
1526
1527static int
1528xptedtbusfunc(struct cam_eb *bus, void *arg)
1529{
1530	struct ccb_dev_match *cdm;
1531	struct cam_et *target;
1532	dev_match_ret retval;
1533
1534	cdm = (struct ccb_dev_match *)arg;
1535
1536	/*
1537	 * If our position is for something deeper in the tree, that means
1538	 * that we've already seen this node.  So, we keep going down.
1539	 */
1540	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1541	 && (cdm->pos.cookie.bus == bus)
1542	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1543	 && (cdm->pos.cookie.target != NULL))
1544		retval = DM_RET_DESCEND;
1545	else
1546		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1547
1548	/*
1549	 * If we got an error, bail out of the search.
1550	 */
1551	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1552		cdm->status = CAM_DEV_MATCH_ERROR;
1553		return(0);
1554	}
1555
1556	/*
1557	 * If the copy flag is set, copy this bus out.
1558	 */
1559	if (retval & DM_RET_COPY) {
1560		int spaceleft, j;
1561
1562		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1563			sizeof(struct dev_match_result));
1564
1565		/*
1566		 * If we don't have enough space to put in another
1567		 * match result, save our position and tell the
1568		 * user there are more devices to check.
1569		 */
1570		if (spaceleft < sizeof(struct dev_match_result)) {
1571			bzero(&cdm->pos, sizeof(cdm->pos));
1572			cdm->pos.position_type =
1573				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1574
1575			cdm->pos.cookie.bus = bus;
1576			cdm->pos.generations[CAM_BUS_GENERATION]=
1577				xsoftc.bus_generation;
1578			cdm->status = CAM_DEV_MATCH_MORE;
1579			return(0);
1580		}
1581		j = cdm->num_matches;
1582		cdm->num_matches++;
1583		cdm->matches[j].type = DEV_MATCH_BUS;
1584		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1585		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1586		cdm->matches[j].result.bus_result.unit_number =
1587			bus->sim->unit_number;
1588		strncpy(cdm->matches[j].result.bus_result.dev_name,
1589			bus->sim->sim_name, DEV_IDLEN);
1590	}
1591
1592	/*
1593	 * If the user is only interested in busses, there's no
1594	 * reason to descend to the next level in the tree.
1595	 */
1596	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1597		return(1);
1598
1599	/*
1600	 * If there is a target generation recorded, check it to
1601	 * make sure the target list hasn't changed.
1602	 */
1603	mtx_lock(&bus->eb_mtx);
1604	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1605	 && (cdm->pos.cookie.bus == bus)
1606	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1607	 && (cdm->pos.cookie.target != NULL)) {
1608		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1609		    bus->generation)) {
1610			mtx_unlock(&bus->eb_mtx);
1611			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1612			return (0);
1613		}
1614		target = (struct cam_et *)cdm->pos.cookie.target;
1615		target->refcount++;
1616	} else
1617		target = NULL;
1618	mtx_unlock(&bus->eb_mtx);
1619
1620	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1621}
1622
1623static int
1624xptedttargetfunc(struct cam_et *target, void *arg)
1625{
1626	struct ccb_dev_match *cdm;
1627	struct cam_eb *bus;
1628	struct cam_ed *device;
1629
1630	cdm = (struct ccb_dev_match *)arg;
1631	bus = target->bus;
1632
1633	/*
1634	 * If there is a device list generation recorded, check it to
1635	 * make sure the device list hasn't changed.
1636	 */
1637	mtx_lock(&bus->eb_mtx);
1638	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1639	 && (cdm->pos.cookie.bus == bus)
1640	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1641	 && (cdm->pos.cookie.target == target)
1642	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1643	 && (cdm->pos.cookie.device != NULL)) {
1644		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1645		    target->generation) {
1646			mtx_unlock(&bus->eb_mtx);
1647			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1648			return(0);
1649		}
1650		device = (struct cam_ed *)cdm->pos.cookie.device;
1651		device->refcount++;
1652	} else
1653		device = NULL;
1654	mtx_unlock(&bus->eb_mtx);
1655
1656	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1657}
1658
1659static int
1660xptedtdevicefunc(struct cam_ed *device, void *arg)
1661{
1662	struct cam_eb *bus;
1663	struct cam_periph *periph;
1664	struct ccb_dev_match *cdm;
1665	dev_match_ret retval;
1666
1667	cdm = (struct ccb_dev_match *)arg;
1668	bus = device->target->bus;
1669
1670	/*
1671	 * If our position is for something deeper in the tree, that means
1672	 * that we've already seen this node.  So, we keep going down.
1673	 */
1674	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1675	 && (cdm->pos.cookie.device == device)
1676	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1677	 && (cdm->pos.cookie.periph != NULL))
1678		retval = DM_RET_DESCEND;
1679	else
1680		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1681					device);
1682
1683	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1684		cdm->status = CAM_DEV_MATCH_ERROR;
1685		return(0);
1686	}
1687
1688	/*
1689	 * If the copy flag is set, copy this device out.
1690	 */
1691	if (retval & DM_RET_COPY) {
1692		int spaceleft, j;
1693
1694		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1695			sizeof(struct dev_match_result));
1696
1697		/*
1698		 * If we don't have enough space to put in another
1699		 * match result, save our position and tell the
1700		 * user there are more devices to check.
1701		 */
1702		if (spaceleft < sizeof(struct dev_match_result)) {
1703			bzero(&cdm->pos, sizeof(cdm->pos));
1704			cdm->pos.position_type =
1705				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1706				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1707
1708			cdm->pos.cookie.bus = device->target->bus;
1709			cdm->pos.generations[CAM_BUS_GENERATION]=
1710				xsoftc.bus_generation;
1711			cdm->pos.cookie.target = device->target;
1712			cdm->pos.generations[CAM_TARGET_GENERATION] =
1713				device->target->bus->generation;
1714			cdm->pos.cookie.device = device;
1715			cdm->pos.generations[CAM_DEV_GENERATION] =
1716				device->target->generation;
1717			cdm->status = CAM_DEV_MATCH_MORE;
1718			return(0);
1719		}
1720		j = cdm->num_matches;
1721		cdm->num_matches++;
1722		cdm->matches[j].type = DEV_MATCH_DEVICE;
1723		cdm->matches[j].result.device_result.path_id =
1724			device->target->bus->path_id;
1725		cdm->matches[j].result.device_result.target_id =
1726			device->target->target_id;
1727		cdm->matches[j].result.device_result.target_lun =
1728			device->lun_id;
1729		cdm->matches[j].result.device_result.protocol =
1730			device->protocol;
1731		bcopy(&device->inq_data,
1732		      &cdm->matches[j].result.device_result.inq_data,
1733		      sizeof(struct scsi_inquiry_data));
1734		bcopy(&device->ident_data,
1735		      &cdm->matches[j].result.device_result.ident_data,
1736		      sizeof(struct ata_params));
1737
1738		/* Let the user know whether this device is unconfigured */
1739		if (device->flags & CAM_DEV_UNCONFIGURED)
1740			cdm->matches[j].result.device_result.flags =
1741				DEV_RESULT_UNCONFIGURED;
1742		else
1743			cdm->matches[j].result.device_result.flags =
1744				DEV_RESULT_NOFLAG;
1745	}
1746
1747	/*
1748	 * If the user isn't interested in peripherals, don't descend
1749	 * the tree any further.
1750	 */
1751	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1752		return(1);
1753
1754	/*
1755	 * If there is a peripheral list generation recorded, make sure
1756	 * it hasn't changed.
1757	 */
1758	xpt_lock_buses();
1759	mtx_lock(&bus->eb_mtx);
1760	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1761	 && (cdm->pos.cookie.bus == bus)
1762	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1763	 && (cdm->pos.cookie.target == device->target)
1764	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1765	 && (cdm->pos.cookie.device == device)
1766	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1767	 && (cdm->pos.cookie.periph != NULL)) {
1768		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1769		    device->generation) {
1770			mtx_unlock(&bus->eb_mtx);
1771			xpt_unlock_buses();
1772			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1773			return(0);
1774		}
1775		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1776		periph->refcount++;
1777	} else
1778		periph = NULL;
1779	mtx_unlock(&bus->eb_mtx);
1780	xpt_unlock_buses();
1781
1782	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1783}
1784
1785static int
1786xptedtperiphfunc(struct cam_periph *periph, void *arg)
1787{
1788	struct ccb_dev_match *cdm;
1789	dev_match_ret retval;
1790
1791	cdm = (struct ccb_dev_match *)arg;
1792
1793	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1794
1795	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1796		cdm->status = CAM_DEV_MATCH_ERROR;
1797		return(0);
1798	}
1799
1800	/*
1801	 * If the copy flag is set, copy this peripheral out.
1802	 */
1803	if (retval & DM_RET_COPY) {
1804		int spaceleft, j;
1805
1806		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1807			sizeof(struct dev_match_result));
1808
1809		/*
1810		 * If we don't have enough space to put in another
1811		 * match result, save our position and tell the
1812		 * user there are more devices to check.
1813		 */
1814		if (spaceleft < sizeof(struct dev_match_result)) {
1815			bzero(&cdm->pos, sizeof(cdm->pos));
1816			cdm->pos.position_type =
1817				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1818				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1819				CAM_DEV_POS_PERIPH;
1820
1821			cdm->pos.cookie.bus = periph->path->bus;
1822			cdm->pos.generations[CAM_BUS_GENERATION]=
1823				xsoftc.bus_generation;
1824			cdm->pos.cookie.target = periph->path->target;
1825			cdm->pos.generations[CAM_TARGET_GENERATION] =
1826				periph->path->bus->generation;
1827			cdm->pos.cookie.device = periph->path->device;
1828			cdm->pos.generations[CAM_DEV_GENERATION] =
1829				periph->path->target->generation;
1830			cdm->pos.cookie.periph = periph;
1831			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1832				periph->path->device->generation;
1833			cdm->status = CAM_DEV_MATCH_MORE;
1834			return(0);
1835		}
1836
1837		j = cdm->num_matches;
1838		cdm->num_matches++;
1839		cdm->matches[j].type = DEV_MATCH_PERIPH;
1840		cdm->matches[j].result.periph_result.path_id =
1841			periph->path->bus->path_id;
1842		cdm->matches[j].result.periph_result.target_id =
1843			periph->path->target->target_id;
1844		cdm->matches[j].result.periph_result.target_lun =
1845			periph->path->device->lun_id;
1846		cdm->matches[j].result.periph_result.unit_number =
1847			periph->unit_number;
1848		strncpy(cdm->matches[j].result.periph_result.periph_name,
1849			periph->periph_name, DEV_IDLEN);
1850	}
1851
1852	return(1);
1853}
1854
1855static int
1856xptedtmatch(struct ccb_dev_match *cdm)
1857{
1858	struct cam_eb *bus;
1859	int ret;
1860
1861	cdm->num_matches = 0;
1862
1863	/*
1864	 * Check the bus list generation.  If it has changed, the user
1865	 * needs to reset everything and start over.
1866	 */
1867	xpt_lock_buses();
1868	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1869	 && (cdm->pos.cookie.bus != NULL)) {
1870		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1871		    xsoftc.bus_generation) {
1872			xpt_unlock_buses();
1873			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1874			return(0);
1875		}
1876		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1877		bus->refcount++;
1878	} else
1879		bus = NULL;
1880	xpt_unlock_buses();
1881
1882	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1883
1884	/*
1885	 * If we get back 0, that means that we had to stop before fully
1886	 * traversing the EDT.  It also means that one of the subroutines
1887	 * has set the status field to the proper value.  If we get back 1,
1888	 * we've fully traversed the EDT and copied out any matching entries.
1889	 */
1890	if (ret == 1)
1891		cdm->status = CAM_DEV_MATCH_LAST;
1892
1893	return(ret);
1894}
1895
1896static int
1897xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1898{
1899	struct cam_periph *periph;
1900	struct ccb_dev_match *cdm;
1901
1902	cdm = (struct ccb_dev_match *)arg;
1903
1904	xpt_lock_buses();
1905	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1906	 && (cdm->pos.cookie.pdrv == pdrv)
1907	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1908	 && (cdm->pos.cookie.periph != NULL)) {
1909		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1910		    (*pdrv)->generation) {
1911			xpt_unlock_buses();
1912			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1913			return(0);
1914		}
1915		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1916		periph->refcount++;
1917	} else
1918		periph = NULL;
1919	xpt_unlock_buses();
1920
1921	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1922}
1923
1924static int
1925xptplistperiphfunc(struct cam_periph *periph, void *arg)
1926{
1927	struct ccb_dev_match *cdm;
1928	dev_match_ret retval;
1929
1930	cdm = (struct ccb_dev_match *)arg;
1931
1932	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1933
1934	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1935		cdm->status = CAM_DEV_MATCH_ERROR;
1936		return(0);
1937	}
1938
1939	/*
1940	 * If the copy flag is set, copy this peripheral out.
1941	 */
1942	if (retval & DM_RET_COPY) {
1943		int spaceleft, j;
1944
1945		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1946			sizeof(struct dev_match_result));
1947
1948		/*
1949		 * If we don't have enough space to put in another
1950		 * match result, save our position and tell the
1951		 * user there are more devices to check.
1952		 */
1953		if (spaceleft < sizeof(struct dev_match_result)) {
1954			struct periph_driver **pdrv;
1955
1956			pdrv = NULL;
1957			bzero(&cdm->pos, sizeof(cdm->pos));
1958			cdm->pos.position_type =
1959				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1960				CAM_DEV_POS_PERIPH;
1961
1962			/*
1963			 * This may look a bit non-sensical, but it is
1964			 * actually quite logical.  There are very few
1965			 * peripheral drivers, and bloating every peripheral
1966			 * structure with a pointer back to its parent
1967			 * peripheral driver linker set entry would cost
1968			 * more in the long run than doing this quick lookup.
1969			 */
1970			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1971				if (strcmp((*pdrv)->driver_name,
1972				    periph->periph_name) == 0)
1973					break;
1974			}
1975
1976			if (*pdrv == NULL) {
1977				cdm->status = CAM_DEV_MATCH_ERROR;
1978				return(0);
1979			}
1980
1981			cdm->pos.cookie.pdrv = pdrv;
1982			/*
1983			 * The periph generation slot does double duty, as
1984			 * does the periph pointer slot.  They are used for
1985			 * both edt and pdrv lookups and positioning.
1986			 */
1987			cdm->pos.cookie.periph = periph;
1988			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1989				(*pdrv)->generation;
1990			cdm->status = CAM_DEV_MATCH_MORE;
1991			return(0);
1992		}
1993
1994		j = cdm->num_matches;
1995		cdm->num_matches++;
1996		cdm->matches[j].type = DEV_MATCH_PERIPH;
1997		cdm->matches[j].result.periph_result.path_id =
1998			periph->path->bus->path_id;
1999
2000		/*
2001		 * The transport layer peripheral doesn't have a target or
2002		 * lun.
2003		 */
2004		if (periph->path->target)
2005			cdm->matches[j].result.periph_result.target_id =
2006				periph->path->target->target_id;
2007		else
2008			cdm->matches[j].result.periph_result.target_id =
2009				CAM_TARGET_WILDCARD;
2010
2011		if (periph->path->device)
2012			cdm->matches[j].result.periph_result.target_lun =
2013				periph->path->device->lun_id;
2014		else
2015			cdm->matches[j].result.periph_result.target_lun =
2016				CAM_LUN_WILDCARD;
2017
2018		cdm->matches[j].result.periph_result.unit_number =
2019			periph->unit_number;
2020		strncpy(cdm->matches[j].result.periph_result.periph_name,
2021			periph->periph_name, DEV_IDLEN);
2022	}
2023
2024	return(1);
2025}
2026
2027static int
2028xptperiphlistmatch(struct ccb_dev_match *cdm)
2029{
2030	int ret;
2031
2032	cdm->num_matches = 0;
2033
2034	/*
2035	 * At this point in the edt traversal function, we check the bus
2036	 * list generation to make sure that no busses have been added or
2037	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2038	 * For the peripheral driver list traversal function, however, we
2039	 * don't have to worry about new peripheral driver types coming or
2040	 * going; they're in a linker set, and therefore can't change
2041	 * without a recompile.
2042	 */
2043
2044	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2045	 && (cdm->pos.cookie.pdrv != NULL))
2046		ret = xptpdrvtraverse(
2047				(struct periph_driver **)cdm->pos.cookie.pdrv,
2048				xptplistpdrvfunc, cdm);
2049	else
2050		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2051
2052	/*
2053	 * If we get back 0, that means that we had to stop before fully
2054	 * traversing the peripheral driver tree.  It also means that one of
2055	 * the subroutines has set the status field to the proper value.  If
2056	 * we get back 1, we've fully traversed the EDT and copied out any
2057	 * matching entries.
2058	 */
2059	if (ret == 1)
2060		cdm->status = CAM_DEV_MATCH_LAST;
2061
2062	return(ret);
2063}
2064
2065static int
2066xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2067{
2068	struct cam_eb *bus, *next_bus;
2069	int retval;
2070
2071	retval = 1;
2072	if (start_bus)
2073		bus = start_bus;
2074	else {
2075		xpt_lock_buses();
2076		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2077		if (bus == NULL) {
2078			xpt_unlock_buses();
2079			return (retval);
2080		}
2081		bus->refcount++;
2082		xpt_unlock_buses();
2083	}
2084	for (; bus != NULL; bus = next_bus) {
2085		retval = tr_func(bus, arg);
2086		if (retval == 0) {
2087			xpt_release_bus(bus);
2088			break;
2089		}
2090		xpt_lock_buses();
2091		next_bus = TAILQ_NEXT(bus, links);
2092		if (next_bus)
2093			next_bus->refcount++;
2094		xpt_unlock_buses();
2095		xpt_release_bus(bus);
2096	}
2097	return(retval);
2098}
2099
2100static int
2101xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2102		  xpt_targetfunc_t *tr_func, void *arg)
2103{
2104	struct cam_et *target, *next_target;
2105	int retval;
2106
2107	retval = 1;
2108	if (start_target)
2109		target = start_target;
2110	else {
2111		mtx_lock(&bus->eb_mtx);
2112		target = TAILQ_FIRST(&bus->et_entries);
2113		if (target == NULL) {
2114			mtx_unlock(&bus->eb_mtx);
2115			return (retval);
2116		}
2117		target->refcount++;
2118		mtx_unlock(&bus->eb_mtx);
2119	}
2120	for (; target != NULL; target = next_target) {
2121		retval = tr_func(target, arg);
2122		if (retval == 0) {
2123			xpt_release_target(target);
2124			break;
2125		}
2126		mtx_lock(&bus->eb_mtx);
2127		next_target = TAILQ_NEXT(target, links);
2128		if (next_target)
2129			next_target->refcount++;
2130		mtx_unlock(&bus->eb_mtx);
2131		xpt_release_target(target);
2132	}
2133	return(retval);
2134}
2135
2136static int
2137xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2138		  xpt_devicefunc_t *tr_func, void *arg)
2139{
2140	struct cam_eb *bus;
2141	struct cam_ed *device, *next_device;
2142	int retval;
2143
2144	retval = 1;
2145	bus = target->bus;
2146	if (start_device)
2147		device = start_device;
2148	else {
2149		mtx_lock(&bus->eb_mtx);
2150		device = TAILQ_FIRST(&target->ed_entries);
2151		if (device == NULL) {
2152			mtx_unlock(&bus->eb_mtx);
2153			return (retval);
2154		}
2155		device->refcount++;
2156		mtx_unlock(&bus->eb_mtx);
2157	}
2158	for (; device != NULL; device = next_device) {
2159		mtx_lock(&device->device_mtx);
2160		retval = tr_func(device, arg);
2161		mtx_unlock(&device->device_mtx);
2162		if (retval == 0) {
2163			xpt_release_device(device);
2164			break;
2165		}
2166		mtx_lock(&bus->eb_mtx);
2167		next_device = TAILQ_NEXT(device, links);
2168		if (next_device)
2169			next_device->refcount++;
2170		mtx_unlock(&bus->eb_mtx);
2171		xpt_release_device(device);
2172	}
2173	return(retval);
2174}
2175
2176static int
2177xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2178		  xpt_periphfunc_t *tr_func, void *arg)
2179{
2180	struct cam_eb *bus;
2181	struct cam_periph *periph, *next_periph;
2182	int retval;
2183
2184	retval = 1;
2185
2186	bus = device->target->bus;
2187	if (start_periph)
2188		periph = start_periph;
2189	else {
2190		xpt_lock_buses();
2191		mtx_lock(&bus->eb_mtx);
2192		periph = SLIST_FIRST(&device->periphs);
2193		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2194			periph = SLIST_NEXT(periph, periph_links);
2195		if (periph == NULL) {
2196			mtx_unlock(&bus->eb_mtx);
2197			xpt_unlock_buses();
2198			return (retval);
2199		}
2200		periph->refcount++;
2201		mtx_unlock(&bus->eb_mtx);
2202		xpt_unlock_buses();
2203	}
2204	for (; periph != NULL; periph = next_periph) {
2205		retval = tr_func(periph, arg);
2206		if (retval == 0) {
2207			cam_periph_release_locked(periph);
2208			break;
2209		}
2210		xpt_lock_buses();
2211		mtx_lock(&bus->eb_mtx);
2212		next_periph = SLIST_NEXT(periph, periph_links);
2213		while (next_periph != NULL &&
2214		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2215			next_periph = SLIST_NEXT(next_periph, periph_links);
2216		if (next_periph)
2217			next_periph->refcount++;
2218		mtx_unlock(&bus->eb_mtx);
2219		xpt_unlock_buses();
2220		cam_periph_release_locked(periph);
2221	}
2222	return(retval);
2223}
2224
2225static int
2226xptpdrvtraverse(struct periph_driver **start_pdrv,
2227		xpt_pdrvfunc_t *tr_func, void *arg)
2228{
2229	struct periph_driver **pdrv;
2230	int retval;
2231
2232	retval = 1;
2233
2234	/*
2235	 * We don't traverse the peripheral driver list like we do the
2236	 * other lists, because it is a linker set, and therefore cannot be
2237	 * changed during runtime.  If the peripheral driver list is ever
2238	 * re-done to be something other than a linker set (i.e. it can
2239	 * change while the system is running), the list traversal should
2240	 * be modified to work like the other traversal functions.
2241	 */
2242	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2243	     *pdrv != NULL; pdrv++) {
2244		retval = tr_func(pdrv, arg);
2245
2246		if (retval == 0)
2247			return(retval);
2248	}
2249
2250	return(retval);
2251}
2252
2253static int
2254xptpdperiphtraverse(struct periph_driver **pdrv,
2255		    struct cam_periph *start_periph,
2256		    xpt_periphfunc_t *tr_func, void *arg)
2257{
2258	struct cam_periph *periph, *next_periph;
2259	int retval;
2260
2261	retval = 1;
2262
2263	if (start_periph)
2264		periph = start_periph;
2265	else {
2266		xpt_lock_buses();
2267		periph = TAILQ_FIRST(&(*pdrv)->units);
2268		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2269			periph = TAILQ_NEXT(periph, unit_links);
2270		if (periph == NULL) {
2271			xpt_unlock_buses();
2272			return (retval);
2273		}
2274		periph->refcount++;
2275		xpt_unlock_buses();
2276	}
2277	for (; periph != NULL; periph = next_periph) {
2278		cam_periph_lock(periph);
2279		retval = tr_func(periph, arg);
2280		cam_periph_unlock(periph);
2281		if (retval == 0) {
2282			cam_periph_release(periph);
2283			break;
2284		}
2285		xpt_lock_buses();
2286		next_periph = TAILQ_NEXT(periph, unit_links);
2287		while (next_periph != NULL &&
2288		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2289			next_periph = TAILQ_NEXT(next_periph, unit_links);
2290		if (next_periph)
2291			next_periph->refcount++;
2292		xpt_unlock_buses();
2293		cam_periph_release(periph);
2294	}
2295	return(retval);
2296}
2297
2298static int
2299xptdefbusfunc(struct cam_eb *bus, void *arg)
2300{
2301	struct xpt_traverse_config *tr_config;
2302
2303	tr_config = (struct xpt_traverse_config *)arg;
2304
2305	if (tr_config->depth == XPT_DEPTH_BUS) {
2306		xpt_busfunc_t *tr_func;
2307
2308		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2309
2310		return(tr_func(bus, tr_config->tr_arg));
2311	} else
2312		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2313}
2314
2315static int
2316xptdeftargetfunc(struct cam_et *target, void *arg)
2317{
2318	struct xpt_traverse_config *tr_config;
2319
2320	tr_config = (struct xpt_traverse_config *)arg;
2321
2322	if (tr_config->depth == XPT_DEPTH_TARGET) {
2323		xpt_targetfunc_t *tr_func;
2324
2325		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2326
2327		return(tr_func(target, tr_config->tr_arg));
2328	} else
2329		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2330}
2331
2332static int
2333xptdefdevicefunc(struct cam_ed *device, void *arg)
2334{
2335	struct xpt_traverse_config *tr_config;
2336
2337	tr_config = (struct xpt_traverse_config *)arg;
2338
2339	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2340		xpt_devicefunc_t *tr_func;
2341
2342		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2343
2344		return(tr_func(device, tr_config->tr_arg));
2345	} else
2346		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2347}
2348
2349static int
2350xptdefperiphfunc(struct cam_periph *periph, void *arg)
2351{
2352	struct xpt_traverse_config *tr_config;
2353	xpt_periphfunc_t *tr_func;
2354
2355	tr_config = (struct xpt_traverse_config *)arg;
2356
2357	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2358
2359	/*
2360	 * Unlike the other default functions, we don't check for depth
2361	 * here.  The peripheral driver level is the last level in the EDT,
2362	 * so if we're here, we should execute the function in question.
2363	 */
2364	return(tr_func(periph, tr_config->tr_arg));
2365}
2366
2367/*
2368 * Execute the given function for every bus in the EDT.
2369 */
2370static int
2371xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2372{
2373	struct xpt_traverse_config tr_config;
2374
2375	tr_config.depth = XPT_DEPTH_BUS;
2376	tr_config.tr_func = tr_func;
2377	tr_config.tr_arg = arg;
2378
2379	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2380}
2381
2382/*
2383 * Execute the given function for every device in the EDT.
2384 */
2385static int
2386xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2387{
2388	struct xpt_traverse_config tr_config;
2389
2390	tr_config.depth = XPT_DEPTH_DEVICE;
2391	tr_config.tr_func = tr_func;
2392	tr_config.tr_arg = arg;
2393
2394	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2395}
2396
2397static int
2398xptsetasyncfunc(struct cam_ed *device, void *arg)
2399{
2400	struct cam_path path;
2401	struct ccb_getdev cgd;
2402	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2403
2404	/*
2405	 * Don't report unconfigured devices (Wildcard devs,
2406	 * devices only for target mode, device instances
2407	 * that have been invalidated but are waiting for
2408	 * their last reference count to be released).
2409	 */
2410	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2411		return (1);
2412
2413	xpt_compile_path(&path,
2414			 NULL,
2415			 device->target->bus->path_id,
2416			 device->target->target_id,
2417			 device->lun_id);
2418	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2419	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2420	xpt_action((union ccb *)&cgd);
2421	csa->callback(csa->callback_arg,
2422			    AC_FOUND_DEVICE,
2423			    &path, &cgd);
2424	xpt_release_path(&path);
2425
2426	return(1);
2427}
2428
2429static int
2430xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2431{
2432	struct cam_path path;
2433	struct ccb_pathinq cpi;
2434	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2435
2436	xpt_compile_path(&path, /*periph*/NULL,
2437			 bus->path_id,
2438			 CAM_TARGET_WILDCARD,
2439			 CAM_LUN_WILDCARD);
2440	xpt_path_lock(&path);
2441	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2442	cpi.ccb_h.func_code = XPT_PATH_INQ;
2443	xpt_action((union ccb *)&cpi);
2444	csa->callback(csa->callback_arg,
2445			    AC_PATH_REGISTERED,
2446			    &path, &cpi);
2447	xpt_path_unlock(&path);
2448	xpt_release_path(&path);
2449
2450	return(1);
2451}
2452
2453void
2454xpt_action(union ccb *start_ccb)
2455{
2456
2457	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2458
2459	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2460	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2461}
2462
2463void
2464xpt_action_default(union ccb *start_ccb)
2465{
2466	struct cam_path *path;
2467	struct cam_sim *sim;
2468	int lock;
2469
2470	path = start_ccb->ccb_h.path;
2471	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2472
2473	switch (start_ccb->ccb_h.func_code) {
2474	case XPT_SCSI_IO:
2475	{
2476		struct cam_ed *device;
2477
2478		/*
2479		 * For the sake of compatibility with SCSI-1
2480		 * devices that may not understand the identify
2481		 * message, we include lun information in the
2482		 * second byte of all commands.  SCSI-1 specifies
2483		 * that luns are a 3 bit value and reserves only 3
2484		 * bits for lun information in the CDB.  Later
2485		 * revisions of the SCSI spec allow for more than 8
2486		 * luns, but have deprecated lun information in the
2487		 * CDB.  So, if the lun won't fit, we must omit.
2488		 *
2489		 * Also be aware that during initial probing for devices,
2490		 * the inquiry information is unknown but initialized to 0.
2491		 * This means that this code will be exercised while probing
2492		 * devices with an ANSI revision greater than 2.
2493		 */
2494		device = path->device;
2495		if (device->protocol_version <= SCSI_REV_2
2496		 && start_ccb->ccb_h.target_lun < 8
2497		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2498
2499			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2500			    start_ccb->ccb_h.target_lun << 5;
2501		}
2502		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2503	}
2504	/* FALLTHROUGH */
2505	case XPT_TARGET_IO:
2506	case XPT_CONT_TARGET_IO:
2507		start_ccb->csio.sense_resid = 0;
2508		start_ccb->csio.resid = 0;
2509		/* FALLTHROUGH */
2510	case XPT_ATA_IO:
2511		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2512			start_ccb->ataio.resid = 0;
2513		/* FALLTHROUGH */
2514	case XPT_RESET_DEV:
2515	case XPT_ENG_EXEC:
2516	case XPT_SMP_IO:
2517	{
2518		struct cam_devq *devq;
2519
2520		devq = path->bus->sim->devq;
2521		mtx_lock(&devq->send_mtx);
2522		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2523		if (xpt_schedule_devq(devq, path->device) != 0)
2524			xpt_run_devq(devq);
2525		mtx_unlock(&devq->send_mtx);
2526		break;
2527	}
2528	case XPT_CALC_GEOMETRY:
2529		/* Filter out garbage */
2530		if (start_ccb->ccg.block_size == 0
2531		 || start_ccb->ccg.volume_size == 0) {
2532			start_ccb->ccg.cylinders = 0;
2533			start_ccb->ccg.heads = 0;
2534			start_ccb->ccg.secs_per_track = 0;
2535			start_ccb->ccb_h.status = CAM_REQ_CMP;
2536			break;
2537		}
2538#if defined(PC98) || defined(__sparc64__)
2539		/*
2540		 * In a PC-98 system, geometry translation depens on
2541		 * the "real" device geometry obtained from mode page 4.
2542		 * SCSI geometry translation is performed in the
2543		 * initialization routine of the SCSI BIOS and the result
2544		 * stored in host memory.  If the translation is available
2545		 * in host memory, use it.  If not, rely on the default
2546		 * translation the device driver performs.
2547		 * For sparc64, we may need adjust the geometry of large
2548		 * disks in order to fit the limitations of the 16-bit
2549		 * fields of the VTOC8 disk label.
2550		 */
2551		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2552			start_ccb->ccb_h.status = CAM_REQ_CMP;
2553			break;
2554		}
2555#endif
2556		goto call_sim;
2557	case XPT_ABORT:
2558	{
2559		union ccb* abort_ccb;
2560
2561		abort_ccb = start_ccb->cab.abort_ccb;
2562		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2563
2564			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2565				struct cam_ccbq *ccbq;
2566				struct cam_ed *device;
2567
2568				device = abort_ccb->ccb_h.path->device;
2569				ccbq = &device->ccbq;
2570				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2571				abort_ccb->ccb_h.status =
2572				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2573				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2574				xpt_done(abort_ccb);
2575				start_ccb->ccb_h.status = CAM_REQ_CMP;
2576				break;
2577			}
2578			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2579			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2580				/*
2581				 * We've caught this ccb en route to
2582				 * the SIM.  Flag it for abort and the
2583				 * SIM will do so just before starting
2584				 * real work on the CCB.
2585				 */
2586				abort_ccb->ccb_h.status =
2587				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2588				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2589				start_ccb->ccb_h.status = CAM_REQ_CMP;
2590				break;
2591			}
2592		}
2593		if (XPT_FC_IS_QUEUED(abort_ccb)
2594		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2595			/*
2596			 * It's already completed but waiting
2597			 * for our SWI to get to it.
2598			 */
2599			start_ccb->ccb_h.status = CAM_UA_ABORT;
2600			break;
2601		}
2602		/*
2603		 * If we weren't able to take care of the abort request
2604		 * in the XPT, pass the request down to the SIM for processing.
2605		 */
2606	}
2607	/* FALLTHROUGH */
2608	case XPT_ACCEPT_TARGET_IO:
2609	case XPT_EN_LUN:
2610	case XPT_IMMED_NOTIFY:
2611	case XPT_NOTIFY_ACK:
2612	case XPT_RESET_BUS:
2613	case XPT_IMMEDIATE_NOTIFY:
2614	case XPT_NOTIFY_ACKNOWLEDGE:
2615	case XPT_GET_SIM_KNOB:
2616	case XPT_SET_SIM_KNOB:
2617	case XPT_GET_TRAN_SETTINGS:
2618	case XPT_SET_TRAN_SETTINGS:
2619	case XPT_PATH_INQ:
2620call_sim:
2621		sim = path->bus->sim;
2622		lock = (mtx_owned(sim->mtx) == 0);
2623		if (lock)
2624			CAM_SIM_LOCK(sim);
2625		(*(sim->sim_action))(sim, start_ccb);
2626		if (lock)
2627			CAM_SIM_UNLOCK(sim);
2628		break;
2629	case XPT_PATH_STATS:
2630		start_ccb->cpis.last_reset = path->bus->last_reset;
2631		start_ccb->ccb_h.status = CAM_REQ_CMP;
2632		break;
2633	case XPT_GDEV_TYPE:
2634	{
2635		struct cam_ed *dev;
2636
2637		dev = path->device;
2638		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2639			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2640		} else {
2641			struct ccb_getdev *cgd;
2642
2643			cgd = &start_ccb->cgd;
2644			cgd->protocol = dev->protocol;
2645			cgd->inq_data = dev->inq_data;
2646			cgd->ident_data = dev->ident_data;
2647			cgd->inq_flags = dev->inq_flags;
2648			cgd->ccb_h.status = CAM_REQ_CMP;
2649			cgd->serial_num_len = dev->serial_num_len;
2650			if ((dev->serial_num_len > 0)
2651			 && (dev->serial_num != NULL))
2652				bcopy(dev->serial_num, cgd->serial_num,
2653				      dev->serial_num_len);
2654		}
2655		break;
2656	}
2657	case XPT_GDEV_STATS:
2658	{
2659		struct cam_ed *dev;
2660
2661		dev = path->device;
2662		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2663			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2664		} else {
2665			struct ccb_getdevstats *cgds;
2666			struct cam_eb *bus;
2667			struct cam_et *tar;
2668			struct cam_devq *devq;
2669
2670			cgds = &start_ccb->cgds;
2671			bus = path->bus;
2672			tar = path->target;
2673			devq = bus->sim->devq;
2674			mtx_lock(&devq->send_mtx);
2675			cgds->dev_openings = dev->ccbq.dev_openings;
2676			cgds->dev_active = dev->ccbq.dev_active;
2677			cgds->allocated = dev->ccbq.allocated;
2678			cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2679			cgds->held = cgds->allocated - cgds->dev_active -
2680			    cgds->queued;
2681			cgds->last_reset = tar->last_reset;
2682			cgds->maxtags = dev->maxtags;
2683			cgds->mintags = dev->mintags;
2684			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2685				cgds->last_reset = bus->last_reset;
2686			mtx_unlock(&devq->send_mtx);
2687			cgds->ccb_h.status = CAM_REQ_CMP;
2688		}
2689		break;
2690	}
2691	case XPT_GDEVLIST:
2692	{
2693		struct cam_periph	*nperiph;
2694		struct periph_list	*periph_head;
2695		struct ccb_getdevlist	*cgdl;
2696		u_int			i;
2697		struct cam_ed		*device;
2698		int			found;
2699
2700
2701		found = 0;
2702
2703		/*
2704		 * Don't want anyone mucking with our data.
2705		 */
2706		device = path->device;
2707		periph_head = &device->periphs;
2708		cgdl = &start_ccb->cgdl;
2709
2710		/*
2711		 * Check and see if the list has changed since the user
2712		 * last requested a list member.  If so, tell them that the
2713		 * list has changed, and therefore they need to start over
2714		 * from the beginning.
2715		 */
2716		if ((cgdl->index != 0) &&
2717		    (cgdl->generation != device->generation)) {
2718			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2719			break;
2720		}
2721
2722		/*
2723		 * Traverse the list of peripherals and attempt to find
2724		 * the requested peripheral.
2725		 */
2726		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2727		     (nperiph != NULL) && (i <= cgdl->index);
2728		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2729			if (i == cgdl->index) {
2730				strncpy(cgdl->periph_name,
2731					nperiph->periph_name,
2732					DEV_IDLEN);
2733				cgdl->unit_number = nperiph->unit_number;
2734				found = 1;
2735			}
2736		}
2737		if (found == 0) {
2738			cgdl->status = CAM_GDEVLIST_ERROR;
2739			break;
2740		}
2741
2742		if (nperiph == NULL)
2743			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2744		else
2745			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2746
2747		cgdl->index++;
2748		cgdl->generation = device->generation;
2749
2750		cgdl->ccb_h.status = CAM_REQ_CMP;
2751		break;
2752	}
2753	case XPT_DEV_MATCH:
2754	{
2755		dev_pos_type position_type;
2756		struct ccb_dev_match *cdm;
2757
2758		cdm = &start_ccb->cdm;
2759
2760		/*
2761		 * There are two ways of getting at information in the EDT.
2762		 * The first way is via the primary EDT tree.  It starts
2763		 * with a list of busses, then a list of targets on a bus,
2764		 * then devices/luns on a target, and then peripherals on a
2765		 * device/lun.  The "other" way is by the peripheral driver
2766		 * lists.  The peripheral driver lists are organized by
2767		 * peripheral driver.  (obviously)  So it makes sense to
2768		 * use the peripheral driver list if the user is looking
2769		 * for something like "da1", or all "da" devices.  If the
2770		 * user is looking for something on a particular bus/target
2771		 * or lun, it's generally better to go through the EDT tree.
2772		 */
2773
2774		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2775			position_type = cdm->pos.position_type;
2776		else {
2777			u_int i;
2778
2779			position_type = CAM_DEV_POS_NONE;
2780
2781			for (i = 0; i < cdm->num_patterns; i++) {
2782				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2783				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2784					position_type = CAM_DEV_POS_EDT;
2785					break;
2786				}
2787			}
2788
2789			if (cdm->num_patterns == 0)
2790				position_type = CAM_DEV_POS_EDT;
2791			else if (position_type == CAM_DEV_POS_NONE)
2792				position_type = CAM_DEV_POS_PDRV;
2793		}
2794
2795		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2796		case CAM_DEV_POS_EDT:
2797			xptedtmatch(cdm);
2798			break;
2799		case CAM_DEV_POS_PDRV:
2800			xptperiphlistmatch(cdm);
2801			break;
2802		default:
2803			cdm->status = CAM_DEV_MATCH_ERROR;
2804			break;
2805		}
2806
2807		if (cdm->status == CAM_DEV_MATCH_ERROR)
2808			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2809		else
2810			start_ccb->ccb_h.status = CAM_REQ_CMP;
2811
2812		break;
2813	}
2814	case XPT_SASYNC_CB:
2815	{
2816		struct ccb_setasync *csa;
2817		struct async_node *cur_entry;
2818		struct async_list *async_head;
2819		u_int32_t added;
2820
2821		csa = &start_ccb->csa;
2822		added = csa->event_enable;
2823		async_head = &path->device->asyncs;
2824
2825		/*
2826		 * If there is already an entry for us, simply
2827		 * update it.
2828		 */
2829		cur_entry = SLIST_FIRST(async_head);
2830		while (cur_entry != NULL) {
2831			if ((cur_entry->callback_arg == csa->callback_arg)
2832			 && (cur_entry->callback == csa->callback))
2833				break;
2834			cur_entry = SLIST_NEXT(cur_entry, links);
2835		}
2836
2837		if (cur_entry != NULL) {
2838		 	/*
2839			 * If the request has no flags set,
2840			 * remove the entry.
2841			 */
2842			added &= ~cur_entry->event_enable;
2843			if (csa->event_enable == 0) {
2844				SLIST_REMOVE(async_head, cur_entry,
2845					     async_node, links);
2846				xpt_release_device(path->device);
2847				free(cur_entry, M_CAMXPT);
2848			} else {
2849				cur_entry->event_enable = csa->event_enable;
2850			}
2851			csa->event_enable = added;
2852		} else {
2853			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2854					   M_NOWAIT);
2855			if (cur_entry == NULL) {
2856				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2857				break;
2858			}
2859			cur_entry->event_enable = csa->event_enable;
2860			cur_entry->event_lock =
2861			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2862			cur_entry->callback_arg = csa->callback_arg;
2863			cur_entry->callback = csa->callback;
2864			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2865			xpt_acquire_device(path->device);
2866		}
2867		start_ccb->ccb_h.status = CAM_REQ_CMP;
2868		break;
2869	}
2870	case XPT_REL_SIMQ:
2871	{
2872		struct ccb_relsim *crs;
2873		struct cam_ed *dev;
2874
2875		crs = &start_ccb->crs;
2876		dev = path->device;
2877		if (dev == NULL) {
2878
2879			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2880			break;
2881		}
2882
2883		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2884
2885			/* Don't ever go below one opening */
2886			if (crs->openings > 0) {
2887				xpt_dev_ccbq_resize(path, crs->openings);
2888				if (bootverbose) {
2889					xpt_print(path,
2890					    "number of openings is now %d\n",
2891					    crs->openings);
2892				}
2893			}
2894		}
2895
2896		mtx_lock(&dev->sim->devq->send_mtx);
2897		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2898
2899			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2900
2901				/*
2902				 * Just extend the old timeout and decrement
2903				 * the freeze count so that a single timeout
2904				 * is sufficient for releasing the queue.
2905				 */
2906				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2907				callout_stop(&dev->callout);
2908			} else {
2909
2910				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2911			}
2912
2913			callout_reset_sbt(&dev->callout,
2914			    SBT_1MS * crs->release_timeout, 0,
2915			    xpt_release_devq_timeout, dev, 0);
2916
2917			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2918
2919		}
2920
2921		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2922
2923			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2924				/*
2925				 * Decrement the freeze count so that a single
2926				 * completion is still sufficient to unfreeze
2927				 * the queue.
2928				 */
2929				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2930			} else {
2931
2932				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2933				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2934			}
2935		}
2936
2937		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2938
2939			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2940			 || (dev->ccbq.dev_active == 0)) {
2941
2942				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2943			} else {
2944
2945				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2946				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2947			}
2948		}
2949		mtx_unlock(&dev->sim->devq->send_mtx);
2950
2951		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2952			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2953		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2954		start_ccb->ccb_h.status = CAM_REQ_CMP;
2955		break;
2956	}
2957	case XPT_DEBUG: {
2958		struct cam_path *oldpath;
2959
2960		/* Check that all request bits are supported. */
2961		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2962			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2963			break;
2964		}
2965
2966		cam_dflags = CAM_DEBUG_NONE;
2967		if (cam_dpath != NULL) {
2968			oldpath = cam_dpath;
2969			cam_dpath = NULL;
2970			xpt_free_path(oldpath);
2971		}
2972		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2973			if (xpt_create_path(&cam_dpath, NULL,
2974					    start_ccb->ccb_h.path_id,
2975					    start_ccb->ccb_h.target_id,
2976					    start_ccb->ccb_h.target_lun) !=
2977					    CAM_REQ_CMP) {
2978				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2979			} else {
2980				cam_dflags = start_ccb->cdbg.flags;
2981				start_ccb->ccb_h.status = CAM_REQ_CMP;
2982				xpt_print(cam_dpath, "debugging flags now %x\n",
2983				    cam_dflags);
2984			}
2985		} else
2986			start_ccb->ccb_h.status = CAM_REQ_CMP;
2987		break;
2988	}
2989	case XPT_NOOP:
2990		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2991			xpt_freeze_devq(path, 1);
2992		start_ccb->ccb_h.status = CAM_REQ_CMP;
2993		break;
2994	case XPT_REPROBE_LUN:
2995		xpt_async(AC_INQ_CHANGED, path, NULL);
2996		start_ccb->ccb_h.status = CAM_REQ_CMP;
2997		xpt_done(start_ccb);
2998		break;
2999	default:
3000	case XPT_SDEV_TYPE:
3001	case XPT_TERM_IO:
3002	case XPT_ENG_INQ:
3003		/* XXX Implement */
3004		printf("%s: CCB type %#x not supported\n", __func__,
3005		       start_ccb->ccb_h.func_code);
3006		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3007		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3008			xpt_done(start_ccb);
3009		}
3010		break;
3011	}
3012}
3013
3014void
3015xpt_polled_action(union ccb *start_ccb)
3016{
3017	u_int32_t timeout;
3018	struct	  cam_sim *sim;
3019	struct	  cam_devq *devq;
3020	struct	  cam_ed *dev;
3021
3022	timeout = start_ccb->ccb_h.timeout * 10;
3023	sim = start_ccb->ccb_h.path->bus->sim;
3024	devq = sim->devq;
3025	dev = start_ccb->ccb_h.path->device;
3026
3027	mtx_unlock(&dev->device_mtx);
3028
3029	/*
3030	 * Steal an opening so that no other queued requests
3031	 * can get it before us while we simulate interrupts.
3032	 */
3033	mtx_lock(&devq->send_mtx);
3034	dev->ccbq.dev_openings--;
3035	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3036	    (--timeout > 0)) {
3037		mtx_unlock(&devq->send_mtx);
3038		DELAY(100);
3039		CAM_SIM_LOCK(sim);
3040		(*(sim->sim_poll))(sim);
3041		CAM_SIM_UNLOCK(sim);
3042		camisr_runqueue();
3043		mtx_lock(&devq->send_mtx);
3044	}
3045	dev->ccbq.dev_openings++;
3046	mtx_unlock(&devq->send_mtx);
3047
3048	if (timeout != 0) {
3049		xpt_action(start_ccb);
3050		while(--timeout > 0) {
3051			CAM_SIM_LOCK(sim);
3052			(*(sim->sim_poll))(sim);
3053			CAM_SIM_UNLOCK(sim);
3054			camisr_runqueue();
3055			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3056			    != CAM_REQ_INPROG)
3057				break;
3058			DELAY(100);
3059		}
3060		if (timeout == 0) {
3061			/*
3062			 * XXX Is it worth adding a sim_timeout entry
3063			 * point so we can attempt recovery?  If
3064			 * this is only used for dumps, I don't think
3065			 * it is.
3066			 */
3067			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3068		}
3069	} else {
3070		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3071	}
3072
3073	mtx_lock(&dev->device_mtx);
3074}
3075
3076/*
3077 * Schedule a peripheral driver to receive a ccb when its
3078 * target device has space for more transactions.
3079 */
3080void
3081xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3082{
3083
3084	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3085	cam_periph_assert(periph, MA_OWNED);
3086	if (new_priority < periph->scheduled_priority) {
3087		periph->scheduled_priority = new_priority;
3088		xpt_run_allocq(periph, 0);
3089	}
3090}
3091
3092
3093/*
3094 * Schedule a device to run on a given queue.
3095 * If the device was inserted as a new entry on the queue,
3096 * return 1 meaning the device queue should be run. If we
3097 * were already queued, implying someone else has already
3098 * started the queue, return 0 so the caller doesn't attempt
3099 * to run the queue.
3100 */
3101static int
3102xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3103		 u_int32_t new_priority)
3104{
3105	int retval;
3106	u_int32_t old_priority;
3107
3108	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3109
3110	old_priority = pinfo->priority;
3111
3112	/*
3113	 * Are we already queued?
3114	 */
3115	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3116		/* Simply reorder based on new priority */
3117		if (new_priority < old_priority) {
3118			camq_change_priority(queue, pinfo->index,
3119					     new_priority);
3120			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3121					("changed priority to %d\n",
3122					 new_priority));
3123			retval = 1;
3124		} else
3125			retval = 0;
3126	} else {
3127		/* New entry on the queue */
3128		if (new_priority < old_priority)
3129			pinfo->priority = new_priority;
3130
3131		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3132				("Inserting onto queue\n"));
3133		pinfo->generation = ++queue->generation;
3134		camq_insert(queue, pinfo);
3135		retval = 1;
3136	}
3137	return (retval);
3138}
3139
3140static void
3141xpt_run_allocq_task(void *context, int pending)
3142{
3143	struct cam_periph *periph = context;
3144
3145	cam_periph_lock(periph);
3146	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3147	xpt_run_allocq(periph, 1);
3148	cam_periph_unlock(periph);
3149	cam_periph_release(periph);
3150}
3151
3152static void
3153xpt_run_allocq(struct cam_periph *periph, int sleep)
3154{
3155	struct cam_ed	*device;
3156	union ccb	*ccb;
3157	uint32_t	 prio;
3158
3159	cam_periph_assert(periph, MA_OWNED);
3160	if (periph->periph_allocating)
3161		return;
3162	periph->periph_allocating = 1;
3163	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3164	device = periph->path->device;
3165	ccb = NULL;
3166restart:
3167	while ((prio = min(periph->scheduled_priority,
3168	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3169	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3170	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3171
3172		if (ccb == NULL &&
3173		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3174			if (sleep) {
3175				ccb = xpt_get_ccb(periph);
3176				goto restart;
3177			}
3178			if (periph->flags & CAM_PERIPH_RUN_TASK)
3179				break;
3180			cam_periph_doacquire(periph);
3181			periph->flags |= CAM_PERIPH_RUN_TASK;
3182			taskqueue_enqueue(xsoftc.xpt_taskq,
3183			    &periph->periph_run_task);
3184			break;
3185		}
3186		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3187		if (prio == periph->immediate_priority) {
3188			periph->immediate_priority = CAM_PRIORITY_NONE;
3189			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3190					("waking cam_periph_getccb()\n"));
3191			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3192					  periph_links.sle);
3193			wakeup(&periph->ccb_list);
3194		} else {
3195			periph->scheduled_priority = CAM_PRIORITY_NONE;
3196			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3197					("calling periph_start()\n"));
3198			periph->periph_start(periph, ccb);
3199		}
3200		ccb = NULL;
3201	}
3202	if (ccb != NULL)
3203		xpt_release_ccb(ccb);
3204	periph->periph_allocating = 0;
3205}
3206
3207static void
3208xpt_run_devq(struct cam_devq *devq)
3209{
3210	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3211	int lock;
3212
3213	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3214
3215	devq->send_queue.qfrozen_cnt++;
3216	while ((devq->send_queue.entries > 0)
3217	    && (devq->send_openings > 0)
3218	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3219		struct	cam_ed *device;
3220		union ccb *work_ccb;
3221		struct	cam_sim *sim;
3222
3223		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3224							   CAMQ_HEAD);
3225		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3226				("running device %p\n", device));
3227
3228		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3229		if (work_ccb == NULL) {
3230			printf("device on run queue with no ccbs???\n");
3231			continue;
3232		}
3233
3234		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3235
3236			mtx_lock(&xsoftc.xpt_highpower_lock);
3237		 	if (xsoftc.num_highpower <= 0) {
3238				/*
3239				 * We got a high power command, but we
3240				 * don't have any available slots.  Freeze
3241				 * the device queue until we have a slot
3242				 * available.
3243				 */
3244				xpt_freeze_devq_device(device, 1);
3245				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3246						   highpowerq_entry);
3247
3248				mtx_unlock(&xsoftc.xpt_highpower_lock);
3249				continue;
3250			} else {
3251				/*
3252				 * Consume a high power slot while
3253				 * this ccb runs.
3254				 */
3255				xsoftc.num_highpower--;
3256			}
3257			mtx_unlock(&xsoftc.xpt_highpower_lock);
3258		}
3259		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3260		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3261		devq->send_openings--;
3262		devq->send_active++;
3263		xpt_schedule_devq(devq, device);
3264		mtx_unlock(&devq->send_mtx);
3265
3266		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3267			/*
3268			 * The client wants to freeze the queue
3269			 * after this CCB is sent.
3270			 */
3271			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3272		}
3273
3274		/* In Target mode, the peripheral driver knows best... */
3275		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3276			if ((device->inq_flags & SID_CmdQue) != 0
3277			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3278				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3279			else
3280				/*
3281				 * Clear this in case of a retried CCB that
3282				 * failed due to a rejected tag.
3283				 */
3284				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3285		}
3286
3287		switch (work_ccb->ccb_h.func_code) {
3288		case XPT_SCSI_IO:
3289			CAM_DEBUG(work_ccb->ccb_h.path,
3290			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3291			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3292					  &device->inq_data),
3293			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3294					     cdb_str, sizeof(cdb_str))));
3295			break;
3296		case XPT_ATA_IO:
3297			CAM_DEBUG(work_ccb->ccb_h.path,
3298			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3299			     ata_op_string(&work_ccb->ataio.cmd),
3300			     ata_cmd_string(&work_ccb->ataio.cmd,
3301					    cdb_str, sizeof(cdb_str))));
3302			break;
3303		default:
3304			break;
3305		}
3306
3307		/*
3308		 * Device queues can be shared among multiple SIM instances
3309		 * that reside on different busses.  Use the SIM from the
3310		 * queued device, rather than the one from the calling bus.
3311		 */
3312		sim = device->sim;
3313		lock = (mtx_owned(sim->mtx) == 0);
3314		if (lock)
3315			CAM_SIM_LOCK(sim);
3316		(*(sim->sim_action))(sim, work_ccb);
3317		if (lock)
3318			CAM_SIM_UNLOCK(sim);
3319		mtx_lock(&devq->send_mtx);
3320	}
3321	devq->send_queue.qfrozen_cnt--;
3322}
3323
3324/*
3325 * This function merges stuff from the slave ccb into the master ccb, while
3326 * keeping important fields in the master ccb constant.
3327 */
3328void
3329xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3330{
3331
3332	/*
3333	 * Pull fields that are valid for peripheral drivers to set
3334	 * into the master CCB along with the CCB "payload".
3335	 */
3336	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3337	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3338	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3339	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3340	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3341	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3342}
3343
3344void
3345xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3346		    u_int32_t priority, u_int32_t flags)
3347{
3348
3349	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3350	ccb_h->pinfo.priority = priority;
3351	ccb_h->path = path;
3352	ccb_h->path_id = path->bus->path_id;
3353	if (path->target)
3354		ccb_h->target_id = path->target->target_id;
3355	else
3356		ccb_h->target_id = CAM_TARGET_WILDCARD;
3357	if (path->device) {
3358		ccb_h->target_lun = path->device->lun_id;
3359		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3360	} else {
3361		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3362	}
3363	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3364	ccb_h->flags = flags;
3365	ccb_h->xflags = 0;
3366}
3367
3368void
3369xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3370{
3371	xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3372}
3373
3374/* Path manipulation functions */
3375cam_status
3376xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3377		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3378{
3379	struct	   cam_path *path;
3380	cam_status status;
3381
3382	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3383
3384	if (path == NULL) {
3385		status = CAM_RESRC_UNAVAIL;
3386		return(status);
3387	}
3388	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3389	if (status != CAM_REQ_CMP) {
3390		free(path, M_CAMPATH);
3391		path = NULL;
3392	}
3393	*new_path_ptr = path;
3394	return (status);
3395}
3396
3397cam_status
3398xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3399			 struct cam_periph *periph, path_id_t path_id,
3400			 target_id_t target_id, lun_id_t lun_id)
3401{
3402
3403	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3404	    lun_id));
3405}
3406
3407cam_status
3408xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3409		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3410{
3411	struct	     cam_eb *bus;
3412	struct	     cam_et *target;
3413	struct	     cam_ed *device;
3414	cam_status   status;
3415
3416	status = CAM_REQ_CMP;	/* Completed without error */
3417	target = NULL;		/* Wildcarded */
3418	device = NULL;		/* Wildcarded */
3419
3420	/*
3421	 * We will potentially modify the EDT, so block interrupts
3422	 * that may attempt to create cam paths.
3423	 */
3424	bus = xpt_find_bus(path_id);
3425	if (bus == NULL) {
3426		status = CAM_PATH_INVALID;
3427	} else {
3428		xpt_lock_buses();
3429		mtx_lock(&bus->eb_mtx);
3430		target = xpt_find_target(bus, target_id);
3431		if (target == NULL) {
3432			/* Create one */
3433			struct cam_et *new_target;
3434
3435			new_target = xpt_alloc_target(bus, target_id);
3436			if (new_target == NULL) {
3437				status = CAM_RESRC_UNAVAIL;
3438			} else {
3439				target = new_target;
3440			}
3441		}
3442		xpt_unlock_buses();
3443		if (target != NULL) {
3444			device = xpt_find_device(target, lun_id);
3445			if (device == NULL) {
3446				/* Create one */
3447				struct cam_ed *new_device;
3448
3449				new_device =
3450				    (*(bus->xport->alloc_device))(bus,
3451								      target,
3452								      lun_id);
3453				if (new_device == NULL) {
3454					status = CAM_RESRC_UNAVAIL;
3455				} else {
3456					device = new_device;
3457				}
3458			}
3459		}
3460		mtx_unlock(&bus->eb_mtx);
3461	}
3462
3463	/*
3464	 * Only touch the user's data if we are successful.
3465	 */
3466	if (status == CAM_REQ_CMP) {
3467		new_path->periph = perph;
3468		new_path->bus = bus;
3469		new_path->target = target;
3470		new_path->device = device;
3471		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3472	} else {
3473		if (device != NULL)
3474			xpt_release_device(device);
3475		if (target != NULL)
3476			xpt_release_target(target);
3477		if (bus != NULL)
3478			xpt_release_bus(bus);
3479	}
3480	return (status);
3481}
3482
3483cam_status
3484xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3485{
3486	struct	   cam_path *new_path;
3487
3488	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3489	if (new_path == NULL)
3490		return(CAM_RESRC_UNAVAIL);
3491	xpt_copy_path(new_path, path);
3492	*new_path_ptr = new_path;
3493	return (CAM_REQ_CMP);
3494}
3495
3496void
3497xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3498{
3499
3500	*new_path = *path;
3501	if (path->bus != NULL)
3502		xpt_acquire_bus(path->bus);
3503	if (path->target != NULL)
3504		xpt_acquire_target(path->target);
3505	if (path->device != NULL)
3506		xpt_acquire_device(path->device);
3507}
3508
3509void
3510xpt_release_path(struct cam_path *path)
3511{
3512	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3513	if (path->device != NULL) {
3514		xpt_release_device(path->device);
3515		path->device = NULL;
3516	}
3517	if (path->target != NULL) {
3518		xpt_release_target(path->target);
3519		path->target = NULL;
3520	}
3521	if (path->bus != NULL) {
3522		xpt_release_bus(path->bus);
3523		path->bus = NULL;
3524	}
3525}
3526
3527void
3528xpt_free_path(struct cam_path *path)
3529{
3530
3531	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3532	xpt_release_path(path);
3533	free(path, M_CAMPATH);
3534}
3535
3536void
3537xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3538    uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3539{
3540
3541	xpt_lock_buses();
3542	if (bus_ref) {
3543		if (path->bus)
3544			*bus_ref = path->bus->refcount;
3545		else
3546			*bus_ref = 0;
3547	}
3548	if (periph_ref) {
3549		if (path->periph)
3550			*periph_ref = path->periph->refcount;
3551		else
3552			*periph_ref = 0;
3553	}
3554	xpt_unlock_buses();
3555	if (target_ref) {
3556		if (path->target)
3557			*target_ref = path->target->refcount;
3558		else
3559			*target_ref = 0;
3560	}
3561	if (device_ref) {
3562		if (path->device)
3563			*device_ref = path->device->refcount;
3564		else
3565			*device_ref = 0;
3566	}
3567}
3568
3569/*
3570 * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3571 * in path1, 2 for match with wildcards in path2.
3572 */
3573int
3574xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3575{
3576	int retval = 0;
3577
3578	if (path1->bus != path2->bus) {
3579		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3580			retval = 1;
3581		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3582			retval = 2;
3583		else
3584			return (-1);
3585	}
3586	if (path1->target != path2->target) {
3587		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3588			if (retval == 0)
3589				retval = 1;
3590		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3591			retval = 2;
3592		else
3593			return (-1);
3594	}
3595	if (path1->device != path2->device) {
3596		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3597			if (retval == 0)
3598				retval = 1;
3599		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3600			retval = 2;
3601		else
3602			return (-1);
3603	}
3604	return (retval);
3605}
3606
3607int
3608xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3609{
3610	int retval = 0;
3611
3612	if (path->bus != dev->target->bus) {
3613		if (path->bus->path_id == CAM_BUS_WILDCARD)
3614			retval = 1;
3615		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3616			retval = 2;
3617		else
3618			return (-1);
3619	}
3620	if (path->target != dev->target) {
3621		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3622			if (retval == 0)
3623				retval = 1;
3624		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3625			retval = 2;
3626		else
3627			return (-1);
3628	}
3629	if (path->device != dev) {
3630		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3631			if (retval == 0)
3632				retval = 1;
3633		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3634			retval = 2;
3635		else
3636			return (-1);
3637	}
3638	return (retval);
3639}
3640
3641void
3642xpt_print_path(struct cam_path *path)
3643{
3644
3645	if (path == NULL)
3646		printf("(nopath): ");
3647	else {
3648		if (path->periph != NULL)
3649			printf("(%s%d:", path->periph->periph_name,
3650			       path->periph->unit_number);
3651		else
3652			printf("(noperiph:");
3653
3654		if (path->bus != NULL)
3655			printf("%s%d:%d:", path->bus->sim->sim_name,
3656			       path->bus->sim->unit_number,
3657			       path->bus->sim->bus_id);
3658		else
3659			printf("nobus:");
3660
3661		if (path->target != NULL)
3662			printf("%d:", path->target->target_id);
3663		else
3664			printf("X:");
3665
3666		if (path->device != NULL)
3667			printf("%jx): ", (uintmax_t)path->device->lun_id);
3668		else
3669			printf("X): ");
3670	}
3671}
3672
3673void
3674xpt_print_device(struct cam_ed *device)
3675{
3676
3677	if (device == NULL)
3678		printf("(nopath): ");
3679	else {
3680		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3681		       device->sim->unit_number,
3682		       device->sim->bus_id,
3683		       device->target->target_id,
3684		       (uintmax_t)device->lun_id);
3685	}
3686}
3687
3688void
3689xpt_print(struct cam_path *path, const char *fmt, ...)
3690{
3691	va_list ap;
3692	xpt_print_path(path);
3693	va_start(ap, fmt);
3694	vprintf(fmt, ap);
3695	va_end(ap);
3696}
3697
3698int
3699xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3700{
3701	struct sbuf sb;
3702
3703	sbuf_new(&sb, str, str_len, 0);
3704
3705	if (path == NULL)
3706		sbuf_printf(&sb, "(nopath): ");
3707	else {
3708		if (path->periph != NULL)
3709			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3710				    path->periph->unit_number);
3711		else
3712			sbuf_printf(&sb, "(noperiph:");
3713
3714		if (path->bus != NULL)
3715			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3716				    path->bus->sim->unit_number,
3717				    path->bus->sim->bus_id);
3718		else
3719			sbuf_printf(&sb, "nobus:");
3720
3721		if (path->target != NULL)
3722			sbuf_printf(&sb, "%d:", path->target->target_id);
3723		else
3724			sbuf_printf(&sb, "X:");
3725
3726		if (path->device != NULL)
3727			sbuf_printf(&sb, "%jx): ",
3728			    (uintmax_t)path->device->lun_id);
3729		else
3730			sbuf_printf(&sb, "X): ");
3731	}
3732	sbuf_finish(&sb);
3733
3734	return(sbuf_len(&sb));
3735}
3736
3737path_id_t
3738xpt_path_path_id(struct cam_path *path)
3739{
3740	return(path->bus->path_id);
3741}
3742
3743target_id_t
3744xpt_path_target_id(struct cam_path *path)
3745{
3746	if (path->target != NULL)
3747		return (path->target->target_id);
3748	else
3749		return (CAM_TARGET_WILDCARD);
3750}
3751
3752lun_id_t
3753xpt_path_lun_id(struct cam_path *path)
3754{
3755	if (path->device != NULL)
3756		return (path->device->lun_id);
3757	else
3758		return (CAM_LUN_WILDCARD);
3759}
3760
3761struct cam_sim *
3762xpt_path_sim(struct cam_path *path)
3763{
3764
3765	return (path->bus->sim);
3766}
3767
3768struct cam_periph*
3769xpt_path_periph(struct cam_path *path)
3770{
3771
3772	return (path->periph);
3773}
3774
3775int
3776xpt_path_legacy_ata_id(struct cam_path *path)
3777{
3778	struct cam_eb *bus;
3779	int bus_id;
3780
3781	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3782	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3783	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3784	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3785		return (-1);
3786
3787	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3788	    path->bus->sim->unit_number < 2) {
3789		bus_id = path->bus->sim->unit_number;
3790	} else {
3791		bus_id = 2;
3792		xpt_lock_buses();
3793		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3794			if (bus == path->bus)
3795				break;
3796			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3797			     bus->sim->unit_number >= 2) ||
3798			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3799			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3800			    strcmp(bus->sim->sim_name, "siisch") == 0)
3801				bus_id++;
3802		}
3803		xpt_unlock_buses();
3804	}
3805	if (path->target != NULL) {
3806		if (path->target->target_id < 2)
3807			return (bus_id * 2 + path->target->target_id);
3808		else
3809			return (-1);
3810	} else
3811		return (bus_id * 2);
3812}
3813
3814/*
3815 * Release a CAM control block for the caller.  Remit the cost of the structure
3816 * to the device referenced by the path.  If the this device had no 'credits'
3817 * and peripheral drivers have registered async callbacks for this notification
3818 * call them now.
3819 */
3820void
3821xpt_release_ccb(union ccb *free_ccb)
3822{
3823	struct	 cam_ed *device;
3824	struct	 cam_periph *periph;
3825
3826	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3827	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3828	device = free_ccb->ccb_h.path->device;
3829	periph = free_ccb->ccb_h.path->periph;
3830
3831	xpt_free_ccb(free_ccb);
3832	periph->periph_allocated--;
3833	cam_ccbq_release_opening(&device->ccbq);
3834	xpt_run_allocq(periph, 0);
3835}
3836
3837/* Functions accessed by SIM drivers */
3838
3839static struct xpt_xport xport_default = {
3840	.alloc_device = xpt_alloc_device_default,
3841	.action = xpt_action_default,
3842	.async = xpt_dev_async_default,
3843};
3844
3845/*
3846 * A sim structure, listing the SIM entry points and instance
3847 * identification info is passed to xpt_bus_register to hook the SIM
3848 * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3849 * for this new bus and places it in the array of busses and assigns
3850 * it a path_id.  The path_id may be influenced by "hard wiring"
3851 * information specified by the user.  Once interrupt services are
3852 * available, the bus will be probed.
3853 */
3854int32_t
3855xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3856{
3857	struct cam_eb *new_bus;
3858	struct cam_eb *old_bus;
3859	struct ccb_pathinq cpi;
3860	struct cam_path *path;
3861	cam_status status;
3862
3863	mtx_assert(sim->mtx, MA_OWNED);
3864
3865	sim->bus_id = bus;
3866	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3867					  M_CAMXPT, M_NOWAIT|M_ZERO);
3868	if (new_bus == NULL) {
3869		/* Couldn't satisfy request */
3870		return (CAM_RESRC_UNAVAIL);
3871	}
3872
3873	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3874	TAILQ_INIT(&new_bus->et_entries);
3875	cam_sim_hold(sim);
3876	new_bus->sim = sim;
3877	timevalclear(&new_bus->last_reset);
3878	new_bus->flags = 0;
3879	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3880	new_bus->generation = 0;
3881
3882	xpt_lock_buses();
3883	sim->path_id = new_bus->path_id =
3884	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3885	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3886	while (old_bus != NULL
3887	    && old_bus->path_id < new_bus->path_id)
3888		old_bus = TAILQ_NEXT(old_bus, links);
3889	if (old_bus != NULL)
3890		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3891	else
3892		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3893	xsoftc.bus_generation++;
3894	xpt_unlock_buses();
3895
3896	/*
3897	 * Set a default transport so that a PATH_INQ can be issued to
3898	 * the SIM.  This will then allow for probing and attaching of
3899	 * a more appropriate transport.
3900	 */
3901	new_bus->xport = &xport_default;
3902
3903	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3904				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3905	if (status != CAM_REQ_CMP) {
3906		xpt_release_bus(new_bus);
3907		free(path, M_CAMXPT);
3908		return (CAM_RESRC_UNAVAIL);
3909	}
3910
3911	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3912	cpi.ccb_h.func_code = XPT_PATH_INQ;
3913	xpt_action((union ccb *)&cpi);
3914
3915	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3916		switch (cpi.transport) {
3917		case XPORT_SPI:
3918		case XPORT_SAS:
3919		case XPORT_FC:
3920		case XPORT_USB:
3921		case XPORT_ISCSI:
3922		case XPORT_SRP:
3923		case XPORT_PPB:
3924			new_bus->xport = scsi_get_xport();
3925			break;
3926		case XPORT_ATA:
3927		case XPORT_SATA:
3928			new_bus->xport = ata_get_xport();
3929			break;
3930		default:
3931			new_bus->xport = &xport_default;
3932			break;
3933		}
3934	}
3935
3936	/* Notify interested parties */
3937	if (sim->path_id != CAM_XPT_PATH_ID) {
3938
3939		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3940		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3941			union	ccb *scan_ccb;
3942
3943			/* Initiate bus rescan. */
3944			scan_ccb = xpt_alloc_ccb_nowait();
3945			if (scan_ccb != NULL) {
3946				scan_ccb->ccb_h.path = path;
3947				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3948				scan_ccb->crcn.flags = 0;
3949				xpt_rescan(scan_ccb);
3950			} else {
3951				xpt_print(path,
3952					  "Can't allocate CCB to scan bus\n");
3953				xpt_free_path(path);
3954			}
3955		} else
3956			xpt_free_path(path);
3957	} else
3958		xpt_free_path(path);
3959	return (CAM_SUCCESS);
3960}
3961
3962int32_t
3963xpt_bus_deregister(path_id_t pathid)
3964{
3965	struct cam_path bus_path;
3966	cam_status status;
3967
3968	status = xpt_compile_path(&bus_path, NULL, pathid,
3969				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3970	if (status != CAM_REQ_CMP)
3971		return (status);
3972
3973	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3974	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3975
3976	/* Release the reference count held while registered. */
3977	xpt_release_bus(bus_path.bus);
3978	xpt_release_path(&bus_path);
3979
3980	return (CAM_REQ_CMP);
3981}
3982
3983static path_id_t
3984xptnextfreepathid(void)
3985{
3986	struct cam_eb *bus;
3987	path_id_t pathid;
3988	const char *strval;
3989
3990	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3991	pathid = 0;
3992	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3993retry:
3994	/* Find an unoccupied pathid */
3995	while (bus != NULL && bus->path_id <= pathid) {
3996		if (bus->path_id == pathid)
3997			pathid++;
3998		bus = TAILQ_NEXT(bus, links);
3999	}
4000
4001	/*
4002	 * Ensure that this pathid is not reserved for
4003	 * a bus that may be registered in the future.
4004	 */
4005	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4006		++pathid;
4007		/* Start the search over */
4008		goto retry;
4009	}
4010	return (pathid);
4011}
4012
4013static path_id_t
4014xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4015{
4016	path_id_t pathid;
4017	int i, dunit, val;
4018	char buf[32];
4019	const char *dname;
4020
4021	pathid = CAM_XPT_PATH_ID;
4022	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4023	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4024		return (pathid);
4025	i = 0;
4026	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4027		if (strcmp(dname, "scbus")) {
4028			/* Avoid a bit of foot shooting. */
4029			continue;
4030		}
4031		if (dunit < 0)		/* unwired?! */
4032			continue;
4033		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4034			if (sim_bus == val) {
4035				pathid = dunit;
4036				break;
4037			}
4038		} else if (sim_bus == 0) {
4039			/* Unspecified matches bus 0 */
4040			pathid = dunit;
4041			break;
4042		} else {
4043			printf("Ambiguous scbus configuration for %s%d "
4044			       "bus %d, cannot wire down.  The kernel "
4045			       "config entry for scbus%d should "
4046			       "specify a controller bus.\n"
4047			       "Scbus will be assigned dynamically.\n",
4048			       sim_name, sim_unit, sim_bus, dunit);
4049			break;
4050		}
4051	}
4052
4053	if (pathid == CAM_XPT_PATH_ID)
4054		pathid = xptnextfreepathid();
4055	return (pathid);
4056}
4057
4058static const char *
4059xpt_async_string(u_int32_t async_code)
4060{
4061
4062	switch (async_code) {
4063	case AC_BUS_RESET: return ("AC_BUS_RESET");
4064	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4065	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4066	case AC_SENT_BDR: return ("AC_SENT_BDR");
4067	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4068	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4069	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4070	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4071	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4072	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4073	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4074	case AC_CONTRACT: return ("AC_CONTRACT");
4075	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4076	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4077	}
4078	return ("AC_UNKNOWN");
4079}
4080
4081static int
4082xpt_async_size(u_int32_t async_code)
4083{
4084
4085	switch (async_code) {
4086	case AC_BUS_RESET: return (0);
4087	case AC_UNSOL_RESEL: return (0);
4088	case AC_SCSI_AEN: return (0);
4089	case AC_SENT_BDR: return (0);
4090	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4091	case AC_PATH_DEREGISTERED: return (0);
4092	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4093	case AC_LOST_DEVICE: return (0);
4094	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4095	case AC_INQ_CHANGED: return (0);
4096	case AC_GETDEV_CHANGED: return (0);
4097	case AC_CONTRACT: return (sizeof(struct ac_contract));
4098	case AC_ADVINFO_CHANGED: return (-1);
4099	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4100	}
4101	return (0);
4102}
4103
4104static int
4105xpt_async_process_dev(struct cam_ed *device, void *arg)
4106{
4107	union ccb *ccb = arg;
4108	struct cam_path *path = ccb->ccb_h.path;
4109	void *async_arg = ccb->casync.async_arg_ptr;
4110	u_int32_t async_code = ccb->casync.async_code;
4111	int relock;
4112
4113	if (path->device != device
4114	 && path->device->lun_id != CAM_LUN_WILDCARD
4115	 && device->lun_id != CAM_LUN_WILDCARD)
4116		return (1);
4117
4118	/*
4119	 * The async callback could free the device.
4120	 * If it is a broadcast async, it doesn't hold
4121	 * device reference, so take our own reference.
4122	 */
4123	xpt_acquire_device(device);
4124
4125	/*
4126	 * If async for specific device is to be delivered to
4127	 * the wildcard client, take the specific device lock.
4128	 * XXX: We may need a way for client to specify it.
4129	 */
4130	if ((device->lun_id == CAM_LUN_WILDCARD &&
4131	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4132	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4133	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4134	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4135	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4136		mtx_unlock(&device->device_mtx);
4137		xpt_path_lock(path);
4138		relock = 1;
4139	} else
4140		relock = 0;
4141
4142	(*(device->target->bus->xport->async))(async_code,
4143	    device->target->bus, device->target, device, async_arg);
4144	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4145
4146	if (relock) {
4147		xpt_path_unlock(path);
4148		mtx_lock(&device->device_mtx);
4149	}
4150	xpt_release_device(device);
4151	return (1);
4152}
4153
4154static int
4155xpt_async_process_tgt(struct cam_et *target, void *arg)
4156{
4157	union ccb *ccb = arg;
4158	struct cam_path *path = ccb->ccb_h.path;
4159
4160	if (path->target != target
4161	 && path->target->target_id != CAM_TARGET_WILDCARD
4162	 && target->target_id != CAM_TARGET_WILDCARD)
4163		return (1);
4164
4165	if (ccb->casync.async_code == AC_SENT_BDR) {
4166		/* Update our notion of when the last reset occurred */
4167		microtime(&target->last_reset);
4168	}
4169
4170	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4171}
4172
4173static void
4174xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4175{
4176	struct cam_eb *bus;
4177	struct cam_path *path;
4178	void *async_arg;
4179	u_int32_t async_code;
4180
4181	path = ccb->ccb_h.path;
4182	async_code = ccb->casync.async_code;
4183	async_arg = ccb->casync.async_arg_ptr;
4184	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4185	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4186	bus = path->bus;
4187
4188	if (async_code == AC_BUS_RESET) {
4189		/* Update our notion of when the last reset occurred */
4190		microtime(&bus->last_reset);
4191	}
4192
4193	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4194
4195	/*
4196	 * If this wasn't a fully wildcarded async, tell all
4197	 * clients that want all async events.
4198	 */
4199	if (bus != xpt_periph->path->bus) {
4200		xpt_path_lock(xpt_periph->path);
4201		xpt_async_process_dev(xpt_periph->path->device, ccb);
4202		xpt_path_unlock(xpt_periph->path);
4203	}
4204
4205	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4206		xpt_release_devq(path, 1, TRUE);
4207	else
4208		xpt_release_simq(path->bus->sim, TRUE);
4209	if (ccb->casync.async_arg_size > 0)
4210		free(async_arg, M_CAMXPT);
4211	xpt_free_path(path);
4212	xpt_free_ccb(ccb);
4213}
4214
4215static void
4216xpt_async_bcast(struct async_list *async_head,
4217		u_int32_t async_code,
4218		struct cam_path *path, void *async_arg)
4219{
4220	struct async_node *cur_entry;
4221	int lock;
4222
4223	cur_entry = SLIST_FIRST(async_head);
4224	while (cur_entry != NULL) {
4225		struct async_node *next_entry;
4226		/*
4227		 * Grab the next list entry before we call the current
4228		 * entry's callback.  This is because the callback function
4229		 * can delete its async callback entry.
4230		 */
4231		next_entry = SLIST_NEXT(cur_entry, links);
4232		if ((cur_entry->event_enable & async_code) != 0) {
4233			lock = cur_entry->event_lock;
4234			if (lock)
4235				CAM_SIM_LOCK(path->device->sim);
4236			cur_entry->callback(cur_entry->callback_arg,
4237					    async_code, path,
4238					    async_arg);
4239			if (lock)
4240				CAM_SIM_UNLOCK(path->device->sim);
4241		}
4242		cur_entry = next_entry;
4243	}
4244}
4245
4246void
4247xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4248{
4249	union ccb *ccb;
4250	int size;
4251
4252	ccb = xpt_alloc_ccb_nowait();
4253	if (ccb == NULL) {
4254		xpt_print(path, "Can't allocate CCB to send %s\n",
4255		    xpt_async_string(async_code));
4256		return;
4257	}
4258
4259	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4260		xpt_print(path, "Can't allocate path to send %s\n",
4261		    xpt_async_string(async_code));
4262		xpt_free_ccb(ccb);
4263		return;
4264	}
4265	ccb->ccb_h.path->periph = NULL;
4266	ccb->ccb_h.func_code = XPT_ASYNC;
4267	ccb->ccb_h.cbfcnp = xpt_async_process;
4268	ccb->ccb_h.flags |= CAM_UNLOCKED;
4269	ccb->casync.async_code = async_code;
4270	ccb->casync.async_arg_size = 0;
4271	size = xpt_async_size(async_code);
4272	if (size > 0 && async_arg != NULL) {
4273		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4274		if (ccb->casync.async_arg_ptr == NULL) {
4275			xpt_print(path, "Can't allocate argument to send %s\n",
4276			    xpt_async_string(async_code));
4277			xpt_free_path(ccb->ccb_h.path);
4278			xpt_free_ccb(ccb);
4279			return;
4280		}
4281		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4282		ccb->casync.async_arg_size = size;
4283	} else if (size < 0) {
4284		ccb->casync.async_arg_ptr = async_arg;
4285		ccb->casync.async_arg_size = size;
4286	}
4287	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4288		xpt_freeze_devq(path, 1);
4289	else
4290		xpt_freeze_simq(path->bus->sim, 1);
4291	xpt_done(ccb);
4292}
4293
4294static void
4295xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4296		      struct cam_et *target, struct cam_ed *device,
4297		      void *async_arg)
4298{
4299
4300	/*
4301	 * We only need to handle events for real devices.
4302	 */
4303	if (target->target_id == CAM_TARGET_WILDCARD
4304	 || device->lun_id == CAM_LUN_WILDCARD)
4305		return;
4306
4307	printf("%s called\n", __func__);
4308}
4309
4310static uint32_t
4311xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4312{
4313	struct cam_devq	*devq;
4314	uint32_t freeze;
4315
4316	devq = dev->sim->devq;
4317	mtx_assert(&devq->send_mtx, MA_OWNED);
4318	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4319	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4320	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4321	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4322	/* Remove frozen device from sendq. */
4323	if (device_is_queued(dev))
4324		camq_remove(&devq->send_queue, dev->devq_entry.index);
4325	return (freeze);
4326}
4327
4328u_int32_t
4329xpt_freeze_devq(struct cam_path *path, u_int count)
4330{
4331	struct cam_ed	*dev = path->device;
4332	struct cam_devq	*devq;
4333	uint32_t	 freeze;
4334
4335	devq = dev->sim->devq;
4336	mtx_lock(&devq->send_mtx);
4337	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4338	freeze = xpt_freeze_devq_device(dev, count);
4339	mtx_unlock(&devq->send_mtx);
4340	return (freeze);
4341}
4342
4343u_int32_t
4344xpt_freeze_simq(struct cam_sim *sim, u_int count)
4345{
4346	struct cam_devq	*devq;
4347	uint32_t	 freeze;
4348
4349	devq = sim->devq;
4350	mtx_lock(&devq->send_mtx);
4351	freeze = (devq->send_queue.qfrozen_cnt += count);
4352	mtx_unlock(&devq->send_mtx);
4353	return (freeze);
4354}
4355
4356static void
4357xpt_release_devq_timeout(void *arg)
4358{
4359	struct cam_ed *dev;
4360	struct cam_devq *devq;
4361
4362	dev = (struct cam_ed *)arg;
4363	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4364	devq = dev->sim->devq;
4365	mtx_assert(&devq->send_mtx, MA_OWNED);
4366	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4367		xpt_run_devq(devq);
4368}
4369
4370void
4371xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4372{
4373	struct cam_ed *dev;
4374	struct cam_devq *devq;
4375
4376	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4377	    count, run_queue));
4378	dev = path->device;
4379	devq = dev->sim->devq;
4380	mtx_lock(&devq->send_mtx);
4381	if (xpt_release_devq_device(dev, count, run_queue))
4382		xpt_run_devq(dev->sim->devq);
4383	mtx_unlock(&devq->send_mtx);
4384}
4385
4386static int
4387xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4388{
4389
4390	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4391	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4392	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4393	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4394	if (count > dev->ccbq.queue.qfrozen_cnt) {
4395#ifdef INVARIANTS
4396		printf("xpt_release_devq(): requested %u > present %u\n",
4397		    count, dev->ccbq.queue.qfrozen_cnt);
4398#endif
4399		count = dev->ccbq.queue.qfrozen_cnt;
4400	}
4401	dev->ccbq.queue.qfrozen_cnt -= count;
4402	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4403		/*
4404		 * No longer need to wait for a successful
4405		 * command completion.
4406		 */
4407		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4408		/*
4409		 * Remove any timeouts that might be scheduled
4410		 * to release this queue.
4411		 */
4412		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4413			callout_stop(&dev->callout);
4414			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4415		}
4416		/*
4417		 * Now that we are unfrozen schedule the
4418		 * device so any pending transactions are
4419		 * run.
4420		 */
4421		xpt_schedule_devq(dev->sim->devq, dev);
4422	} else
4423		run_queue = 0;
4424	return (run_queue);
4425}
4426
4427void
4428xpt_release_simq(struct cam_sim *sim, int run_queue)
4429{
4430	struct cam_devq	*devq;
4431
4432	devq = sim->devq;
4433	mtx_lock(&devq->send_mtx);
4434	if (devq->send_queue.qfrozen_cnt <= 0) {
4435#ifdef INVARIANTS
4436		printf("xpt_release_simq: requested 1 > present %u\n",
4437		    devq->send_queue.qfrozen_cnt);
4438#endif
4439	} else
4440		devq->send_queue.qfrozen_cnt--;
4441	if (devq->send_queue.qfrozen_cnt == 0) {
4442		/*
4443		 * If there is a timeout scheduled to release this
4444		 * sim queue, remove it.  The queue frozen count is
4445		 * already at 0.
4446		 */
4447		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4448			callout_stop(&sim->callout);
4449			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4450		}
4451		if (run_queue) {
4452			/*
4453			 * Now that we are unfrozen run the send queue.
4454			 */
4455			xpt_run_devq(sim->devq);
4456		}
4457	}
4458	mtx_unlock(&devq->send_mtx);
4459}
4460
4461/*
4462 * XXX Appears to be unused.
4463 */
4464static void
4465xpt_release_simq_timeout(void *arg)
4466{
4467	struct cam_sim *sim;
4468
4469	sim = (struct cam_sim *)arg;
4470	xpt_release_simq(sim, /* run_queue */ TRUE);
4471}
4472
4473void
4474xpt_done(union ccb *done_ccb)
4475{
4476	struct cam_doneq *queue;
4477	int	run, hash;
4478
4479	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4480	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4481		return;
4482
4483	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4484	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4485	queue = &cam_doneqs[hash];
4486	mtx_lock(&queue->cam_doneq_mtx);
4487	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4488	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4489	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4490	mtx_unlock(&queue->cam_doneq_mtx);
4491	if (run)
4492		wakeup(&queue->cam_doneq);
4493}
4494
4495void
4496xpt_done_direct(union ccb *done_ccb)
4497{
4498
4499	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4500	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4501		return;
4502
4503	xpt_done_process(&done_ccb->ccb_h);
4504}
4505
4506union ccb *
4507xpt_alloc_ccb()
4508{
4509	union ccb *new_ccb;
4510
4511	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4512	return (new_ccb);
4513}
4514
4515union ccb *
4516xpt_alloc_ccb_nowait()
4517{
4518	union ccb *new_ccb;
4519
4520	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4521	return (new_ccb);
4522}
4523
4524void
4525xpt_free_ccb(union ccb *free_ccb)
4526{
4527	free(free_ccb, M_CAMCCB);
4528}
4529
4530
4531
4532/* Private XPT functions */
4533
4534/*
4535 * Get a CAM control block for the caller. Charge the structure to the device
4536 * referenced by the path.  If we don't have sufficient resources to allocate
4537 * more ccbs, we return NULL.
4538 */
4539static union ccb *
4540xpt_get_ccb_nowait(struct cam_periph *periph)
4541{
4542	union ccb *new_ccb;
4543
4544	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4545	if (new_ccb == NULL)
4546		return (NULL);
4547	periph->periph_allocated++;
4548	cam_ccbq_take_opening(&periph->path->device->ccbq);
4549	return (new_ccb);
4550}
4551
4552static union ccb *
4553xpt_get_ccb(struct cam_periph *periph)
4554{
4555	union ccb *new_ccb;
4556
4557	cam_periph_unlock(periph);
4558	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4559	cam_periph_lock(periph);
4560	periph->periph_allocated++;
4561	cam_ccbq_take_opening(&periph->path->device->ccbq);
4562	return (new_ccb);
4563}
4564
4565union ccb *
4566cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4567{
4568	struct ccb_hdr *ccb_h;
4569
4570	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4571	cam_periph_assert(periph, MA_OWNED);
4572	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4573	    ccb_h->pinfo.priority != priority) {
4574		if (priority < periph->immediate_priority) {
4575			periph->immediate_priority = priority;
4576			xpt_run_allocq(periph, 0);
4577		} else
4578			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4579			    "cgticb", 0);
4580	}
4581	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4582	return ((union ccb *)ccb_h);
4583}
4584
4585static void
4586xpt_acquire_bus(struct cam_eb *bus)
4587{
4588
4589	xpt_lock_buses();
4590	bus->refcount++;
4591	xpt_unlock_buses();
4592}
4593
4594static void
4595xpt_release_bus(struct cam_eb *bus)
4596{
4597
4598	xpt_lock_buses();
4599	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4600	if (--bus->refcount > 0) {
4601		xpt_unlock_buses();
4602		return;
4603	}
4604	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4605	xsoftc.bus_generation++;
4606	xpt_unlock_buses();
4607	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4608	    ("destroying bus, but target list is not empty"));
4609	cam_sim_release(bus->sim);
4610	mtx_destroy(&bus->eb_mtx);
4611	free(bus, M_CAMXPT);
4612}
4613
4614static struct cam_et *
4615xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4616{
4617	struct cam_et *cur_target, *target;
4618
4619	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4620	mtx_assert(&bus->eb_mtx, MA_OWNED);
4621	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4622					 M_NOWAIT|M_ZERO);
4623	if (target == NULL)
4624		return (NULL);
4625
4626	TAILQ_INIT(&target->ed_entries);
4627	target->bus = bus;
4628	target->target_id = target_id;
4629	target->refcount = 1;
4630	target->generation = 0;
4631	target->luns = NULL;
4632	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4633	timevalclear(&target->last_reset);
4634	/*
4635	 * Hold a reference to our parent bus so it
4636	 * will not go away before we do.
4637	 */
4638	bus->refcount++;
4639
4640	/* Insertion sort into our bus's target list */
4641	cur_target = TAILQ_FIRST(&bus->et_entries);
4642	while (cur_target != NULL && cur_target->target_id < target_id)
4643		cur_target = TAILQ_NEXT(cur_target, links);
4644	if (cur_target != NULL) {
4645		TAILQ_INSERT_BEFORE(cur_target, target, links);
4646	} else {
4647		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4648	}
4649	bus->generation++;
4650	return (target);
4651}
4652
4653static void
4654xpt_acquire_target(struct cam_et *target)
4655{
4656	struct cam_eb *bus = target->bus;
4657
4658	mtx_lock(&bus->eb_mtx);
4659	target->refcount++;
4660	mtx_unlock(&bus->eb_mtx);
4661}
4662
4663static void
4664xpt_release_target(struct cam_et *target)
4665{
4666	struct cam_eb *bus = target->bus;
4667
4668	mtx_lock(&bus->eb_mtx);
4669	if (--target->refcount > 0) {
4670		mtx_unlock(&bus->eb_mtx);
4671		return;
4672	}
4673	TAILQ_REMOVE(&bus->et_entries, target, links);
4674	bus->generation++;
4675	mtx_unlock(&bus->eb_mtx);
4676	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4677	    ("destroying target, but device list is not empty"));
4678	xpt_release_bus(bus);
4679	mtx_destroy(&target->luns_mtx);
4680	if (target->luns)
4681		free(target->luns, M_CAMXPT);
4682	free(target, M_CAMXPT);
4683}
4684
4685static struct cam_ed *
4686xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4687			 lun_id_t lun_id)
4688{
4689	struct cam_ed *device;
4690
4691	device = xpt_alloc_device(bus, target, lun_id);
4692	if (device == NULL)
4693		return (NULL);
4694
4695	device->mintags = 1;
4696	device->maxtags = 1;
4697	return (device);
4698}
4699
4700static void
4701xpt_destroy_device(void *context, int pending)
4702{
4703	struct cam_ed	*device = context;
4704
4705	mtx_lock(&device->device_mtx);
4706	mtx_destroy(&device->device_mtx);
4707	free(device, M_CAMDEV);
4708}
4709
4710struct cam_ed *
4711xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4712{
4713	struct cam_ed	*cur_device, *device;
4714	struct cam_devq	*devq;
4715	cam_status status;
4716
4717	mtx_assert(&bus->eb_mtx, MA_OWNED);
4718	/* Make space for us in the device queue on our bus */
4719	devq = bus->sim->devq;
4720	mtx_lock(&devq->send_mtx);
4721	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4722	mtx_unlock(&devq->send_mtx);
4723	if (status != CAM_REQ_CMP)
4724		return (NULL);
4725
4726	device = (struct cam_ed *)malloc(sizeof(*device),
4727					 M_CAMDEV, M_NOWAIT|M_ZERO);
4728	if (device == NULL)
4729		return (NULL);
4730
4731	cam_init_pinfo(&device->devq_entry);
4732	device->target = target;
4733	device->lun_id = lun_id;
4734	device->sim = bus->sim;
4735	if (cam_ccbq_init(&device->ccbq,
4736			  bus->sim->max_dev_openings) != 0) {
4737		free(device, M_CAMDEV);
4738		return (NULL);
4739	}
4740	SLIST_INIT(&device->asyncs);
4741	SLIST_INIT(&device->periphs);
4742	device->generation = 0;
4743	device->flags = CAM_DEV_UNCONFIGURED;
4744	device->tag_delay_count = 0;
4745	device->tag_saved_openings = 0;
4746	device->refcount = 1;
4747	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4748	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4749	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4750	/*
4751	 * Hold a reference to our parent bus so it
4752	 * will not go away before we do.
4753	 */
4754	target->refcount++;
4755
4756	cur_device = TAILQ_FIRST(&target->ed_entries);
4757	while (cur_device != NULL && cur_device->lun_id < lun_id)
4758		cur_device = TAILQ_NEXT(cur_device, links);
4759	if (cur_device != NULL)
4760		TAILQ_INSERT_BEFORE(cur_device, device, links);
4761	else
4762		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4763	target->generation++;
4764	return (device);
4765}
4766
4767void
4768xpt_acquire_device(struct cam_ed *device)
4769{
4770	struct cam_eb *bus = device->target->bus;
4771
4772	mtx_lock(&bus->eb_mtx);
4773	device->refcount++;
4774	mtx_unlock(&bus->eb_mtx);
4775}
4776
4777void
4778xpt_release_device(struct cam_ed *device)
4779{
4780	struct cam_eb *bus = device->target->bus;
4781	struct cam_devq *devq;
4782
4783	mtx_lock(&bus->eb_mtx);
4784	if (--device->refcount > 0) {
4785		mtx_unlock(&bus->eb_mtx);
4786		return;
4787	}
4788
4789	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4790	device->target->generation++;
4791	mtx_unlock(&bus->eb_mtx);
4792
4793	/* Release our slot in the devq */
4794	devq = bus->sim->devq;
4795	mtx_lock(&devq->send_mtx);
4796	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4797	mtx_unlock(&devq->send_mtx);
4798
4799	KASSERT(SLIST_EMPTY(&device->periphs),
4800	    ("destroying device, but periphs list is not empty"));
4801	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4802	    ("destroying device while still queued for ccbs"));
4803
4804	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4805		callout_stop(&device->callout);
4806
4807	xpt_release_target(device->target);
4808
4809	cam_ccbq_fini(&device->ccbq);
4810	/*
4811	 * Free allocated memory.  free(9) does nothing if the
4812	 * supplied pointer is NULL, so it is safe to call without
4813	 * checking.
4814	 */
4815	free(device->supported_vpds, M_CAMXPT);
4816	free(device->device_id, M_CAMXPT);
4817	free(device->ext_inq, M_CAMXPT);
4818	free(device->physpath, M_CAMXPT);
4819	free(device->rcap_buf, M_CAMXPT);
4820	free(device->serial_num, M_CAMXPT);
4821	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4822}
4823
4824u_int32_t
4825xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4826{
4827	int	result;
4828	struct	cam_ed *dev;
4829
4830	dev = path->device;
4831	mtx_lock(&dev->sim->devq->send_mtx);
4832	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4833	mtx_unlock(&dev->sim->devq->send_mtx);
4834	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4835	 || (dev->inq_flags & SID_CmdQue) != 0)
4836		dev->tag_saved_openings = newopenings;
4837	return (result);
4838}
4839
4840static struct cam_eb *
4841xpt_find_bus(path_id_t path_id)
4842{
4843	struct cam_eb *bus;
4844
4845	xpt_lock_buses();
4846	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4847	     bus != NULL;
4848	     bus = TAILQ_NEXT(bus, links)) {
4849		if (bus->path_id == path_id) {
4850			bus->refcount++;
4851			break;
4852		}
4853	}
4854	xpt_unlock_buses();
4855	return (bus);
4856}
4857
4858static struct cam_et *
4859xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4860{
4861	struct cam_et *target;
4862
4863	mtx_assert(&bus->eb_mtx, MA_OWNED);
4864	for (target = TAILQ_FIRST(&bus->et_entries);
4865	     target != NULL;
4866	     target = TAILQ_NEXT(target, links)) {
4867		if (target->target_id == target_id) {
4868			target->refcount++;
4869			break;
4870		}
4871	}
4872	return (target);
4873}
4874
4875static struct cam_ed *
4876xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4877{
4878	struct cam_ed *device;
4879
4880	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4881	for (device = TAILQ_FIRST(&target->ed_entries);
4882	     device != NULL;
4883	     device = TAILQ_NEXT(device, links)) {
4884		if (device->lun_id == lun_id) {
4885			device->refcount++;
4886			break;
4887		}
4888	}
4889	return (device);
4890}
4891
4892void
4893xpt_start_tags(struct cam_path *path)
4894{
4895	struct ccb_relsim crs;
4896	struct cam_ed *device;
4897	struct cam_sim *sim;
4898	int    newopenings;
4899
4900	device = path->device;
4901	sim = path->bus->sim;
4902	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4903	xpt_freeze_devq(path, /*count*/1);
4904	device->inq_flags |= SID_CmdQue;
4905	if (device->tag_saved_openings != 0)
4906		newopenings = device->tag_saved_openings;
4907	else
4908		newopenings = min(device->maxtags,
4909				  sim->max_tagged_dev_openings);
4910	xpt_dev_ccbq_resize(path, newopenings);
4911	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4912	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4913	crs.ccb_h.func_code = XPT_REL_SIMQ;
4914	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4915	crs.openings
4916	    = crs.release_timeout
4917	    = crs.qfrozen_cnt
4918	    = 0;
4919	xpt_action((union ccb *)&crs);
4920}
4921
4922void
4923xpt_stop_tags(struct cam_path *path)
4924{
4925	struct ccb_relsim crs;
4926	struct cam_ed *device;
4927	struct cam_sim *sim;
4928
4929	device = path->device;
4930	sim = path->bus->sim;
4931	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4932	device->tag_delay_count = 0;
4933	xpt_freeze_devq(path, /*count*/1);
4934	device->inq_flags &= ~SID_CmdQue;
4935	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4936	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4937	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4938	crs.ccb_h.func_code = XPT_REL_SIMQ;
4939	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4940	crs.openings
4941	    = crs.release_timeout
4942	    = crs.qfrozen_cnt
4943	    = 0;
4944	xpt_action((union ccb *)&crs);
4945}
4946
4947static void
4948xpt_boot_delay(void *arg)
4949{
4950
4951	xpt_release_boot();
4952}
4953
4954static void
4955xpt_config(void *arg)
4956{
4957	/*
4958	 * Now that interrupts are enabled, go find our devices
4959	 */
4960	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4961		printf("xpt_config: failed to create taskqueue thread.\n");
4962
4963	/* Setup debugging path */
4964	if (cam_dflags != CAM_DEBUG_NONE) {
4965		if (xpt_create_path(&cam_dpath, NULL,
4966				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4967				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4968			printf("xpt_config: xpt_create_path() failed for debug"
4969			       " target %d:%d:%d, debugging disabled\n",
4970			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4971			cam_dflags = CAM_DEBUG_NONE;
4972		}
4973	} else
4974		cam_dpath = NULL;
4975
4976	periphdriver_init(1);
4977	xpt_hold_boot();
4978	callout_init(&xsoftc.boot_callout, 1);
4979	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4980	    xpt_boot_delay, NULL, 0);
4981	/* Fire up rescan thread. */
4982	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4983	    "cam", "scanner")) {
4984		printf("xpt_config: failed to create rescan thread.\n");
4985	}
4986}
4987
4988void
4989xpt_hold_boot(void)
4990{
4991	xpt_lock_buses();
4992	xsoftc.buses_to_config++;
4993	xpt_unlock_buses();
4994}
4995
4996void
4997xpt_release_boot(void)
4998{
4999	xpt_lock_buses();
5000	xsoftc.buses_to_config--;
5001	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
5002		struct	xpt_task *task;
5003
5004		xsoftc.buses_config_done = 1;
5005		xpt_unlock_buses();
5006		/* Call manually because we don't have any busses */
5007		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5008		if (task != NULL) {
5009			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5010			taskqueue_enqueue(taskqueue_thread, &task->task);
5011		}
5012	} else
5013		xpt_unlock_buses();
5014}
5015
5016/*
5017 * If the given device only has one peripheral attached to it, and if that
5018 * peripheral is the passthrough driver, announce it.  This insures that the
5019 * user sees some sort of announcement for every peripheral in their system.
5020 */
5021static int
5022xptpassannouncefunc(struct cam_ed *device, void *arg)
5023{
5024	struct cam_periph *periph;
5025	int i;
5026
5027	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5028	     periph = SLIST_NEXT(periph, periph_links), i++);
5029
5030	periph = SLIST_FIRST(&device->periphs);
5031	if ((i == 1)
5032	 && (strncmp(periph->periph_name, "pass", 4) == 0))
5033		xpt_announce_periph(periph, NULL);
5034
5035	return(1);
5036}
5037
5038static void
5039xpt_finishconfig_task(void *context, int pending)
5040{
5041
5042	periphdriver_init(2);
5043	/*
5044	 * Check for devices with no "standard" peripheral driver
5045	 * attached.  For any devices like that, announce the
5046	 * passthrough driver so the user will see something.
5047	 */
5048	if (!bootverbose)
5049		xpt_for_all_devices(xptpassannouncefunc, NULL);
5050
5051	/* Release our hook so that the boot can continue. */
5052	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5053	free(xsoftc.xpt_config_hook, M_CAMXPT);
5054	xsoftc.xpt_config_hook = NULL;
5055
5056	free(context, M_CAMXPT);
5057}
5058
5059cam_status
5060xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5061		   struct cam_path *path)
5062{
5063	struct ccb_setasync csa;
5064	cam_status status;
5065	int xptpath = 0;
5066
5067	if (path == NULL) {
5068		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5069					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5070		if (status != CAM_REQ_CMP)
5071			return (status);
5072		xpt_path_lock(path);
5073		xptpath = 1;
5074	}
5075
5076	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5077	csa.ccb_h.func_code = XPT_SASYNC_CB;
5078	csa.event_enable = event;
5079	csa.callback = cbfunc;
5080	csa.callback_arg = cbarg;
5081	xpt_action((union ccb *)&csa);
5082	status = csa.ccb_h.status;
5083
5084	if (xptpath) {
5085		xpt_path_unlock(path);
5086		xpt_free_path(path);
5087	}
5088
5089	if ((status == CAM_REQ_CMP) &&
5090	    (csa.event_enable & AC_FOUND_DEVICE)) {
5091		/*
5092		 * Get this peripheral up to date with all
5093		 * the currently existing devices.
5094		 */
5095		xpt_for_all_devices(xptsetasyncfunc, &csa);
5096	}
5097	if ((status == CAM_REQ_CMP) &&
5098	    (csa.event_enable & AC_PATH_REGISTERED)) {
5099		/*
5100		 * Get this peripheral up to date with all
5101		 * the currently existing busses.
5102		 */
5103		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5104	}
5105
5106	return (status);
5107}
5108
5109static void
5110xptaction(struct cam_sim *sim, union ccb *work_ccb)
5111{
5112	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5113
5114	switch (work_ccb->ccb_h.func_code) {
5115	/* Common cases first */
5116	case XPT_PATH_INQ:		/* Path routing inquiry */
5117	{
5118		struct ccb_pathinq *cpi;
5119
5120		cpi = &work_ccb->cpi;
5121		cpi->version_num = 1; /* XXX??? */
5122		cpi->hba_inquiry = 0;
5123		cpi->target_sprt = 0;
5124		cpi->hba_misc = 0;
5125		cpi->hba_eng_cnt = 0;
5126		cpi->max_target = 0;
5127		cpi->max_lun = 0;
5128		cpi->initiator_id = 0;
5129		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5130		strncpy(cpi->hba_vid, "", HBA_IDLEN);
5131		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5132		cpi->unit_number = sim->unit_number;
5133		cpi->bus_id = sim->bus_id;
5134		cpi->base_transfer_speed = 0;
5135		cpi->protocol = PROTO_UNSPECIFIED;
5136		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5137		cpi->transport = XPORT_UNSPECIFIED;
5138		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5139		cpi->ccb_h.status = CAM_REQ_CMP;
5140		xpt_done(work_ccb);
5141		break;
5142	}
5143	default:
5144		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5145		xpt_done(work_ccb);
5146		break;
5147	}
5148}
5149
5150/*
5151 * The xpt as a "controller" has no interrupt sources, so polling
5152 * is a no-op.
5153 */
5154static void
5155xptpoll(struct cam_sim *sim)
5156{
5157}
5158
5159void
5160xpt_lock_buses(void)
5161{
5162	mtx_lock(&xsoftc.xpt_topo_lock);
5163}
5164
5165void
5166xpt_unlock_buses(void)
5167{
5168	mtx_unlock(&xsoftc.xpt_topo_lock);
5169}
5170
5171struct mtx *
5172xpt_path_mtx(struct cam_path *path)
5173{
5174
5175	return (&path->device->device_mtx);
5176}
5177
5178static void
5179xpt_done_process(struct ccb_hdr *ccb_h)
5180{
5181	struct cam_sim *sim;
5182	struct cam_devq *devq;
5183	struct mtx *mtx = NULL;
5184
5185	if (ccb_h->flags & CAM_HIGH_POWER) {
5186		struct highpowerlist	*hphead;
5187		struct cam_ed		*device;
5188
5189		mtx_lock(&xsoftc.xpt_highpower_lock);
5190		hphead = &xsoftc.highpowerq;
5191
5192		device = STAILQ_FIRST(hphead);
5193
5194		/*
5195		 * Increment the count since this command is done.
5196		 */
5197		xsoftc.num_highpower++;
5198
5199		/*
5200		 * Any high powered commands queued up?
5201		 */
5202		if (device != NULL) {
5203
5204			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5205			mtx_unlock(&xsoftc.xpt_highpower_lock);
5206
5207			mtx_lock(&device->sim->devq->send_mtx);
5208			xpt_release_devq_device(device,
5209					 /*count*/1, /*runqueue*/TRUE);
5210			mtx_unlock(&device->sim->devq->send_mtx);
5211		} else
5212			mtx_unlock(&xsoftc.xpt_highpower_lock);
5213	}
5214
5215	sim = ccb_h->path->bus->sim;
5216
5217	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5218		xpt_release_simq(sim, /*run_queue*/FALSE);
5219		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5220	}
5221
5222	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5223	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5224		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5225		ccb_h->status &= ~CAM_DEV_QFRZN;
5226	}
5227
5228	devq = sim->devq;
5229	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5230		struct cam_ed *dev = ccb_h->path->device;
5231
5232		mtx_lock(&devq->send_mtx);
5233		devq->send_active--;
5234		devq->send_openings++;
5235		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5236
5237		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5238		  && (dev->ccbq.dev_active == 0))) {
5239			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5240			xpt_release_devq_device(dev, /*count*/1,
5241					 /*run_queue*/FALSE);
5242		}
5243
5244		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5245		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5246			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5247			xpt_release_devq_device(dev, /*count*/1,
5248					 /*run_queue*/FALSE);
5249		}
5250
5251		if (!device_is_queued(dev))
5252			(void)xpt_schedule_devq(devq, dev);
5253		xpt_run_devq(devq);
5254		mtx_unlock(&devq->send_mtx);
5255
5256		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5257			mtx = xpt_path_mtx(ccb_h->path);
5258			mtx_lock(mtx);
5259
5260			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5261			 && (--dev->tag_delay_count == 0))
5262				xpt_start_tags(ccb_h->path);
5263		}
5264	}
5265
5266	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5267		if (mtx == NULL) {
5268			mtx = xpt_path_mtx(ccb_h->path);
5269			mtx_lock(mtx);
5270		}
5271	} else {
5272		if (mtx != NULL) {
5273			mtx_unlock(mtx);
5274			mtx = NULL;
5275		}
5276	}
5277
5278	/* Call the peripheral driver's callback */
5279	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5280	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5281	if (mtx != NULL)
5282		mtx_unlock(mtx);
5283}
5284
5285void
5286xpt_done_td(void *arg)
5287{
5288	struct cam_doneq *queue = arg;
5289	struct ccb_hdr *ccb_h;
5290	STAILQ_HEAD(, ccb_hdr)	doneq;
5291
5292	STAILQ_INIT(&doneq);
5293	mtx_lock(&queue->cam_doneq_mtx);
5294	while (1) {
5295		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5296			queue->cam_doneq_sleep = 1;
5297			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5298			    PRIBIO, "-", 0);
5299			queue->cam_doneq_sleep = 0;
5300		}
5301		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5302		mtx_unlock(&queue->cam_doneq_mtx);
5303
5304		THREAD_NO_SLEEPING();
5305		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5306			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5307			xpt_done_process(ccb_h);
5308		}
5309		THREAD_SLEEPING_OK();
5310
5311		mtx_lock(&queue->cam_doneq_mtx);
5312	}
5313}
5314
5315static void
5316camisr_runqueue(void)
5317{
5318	struct	ccb_hdr *ccb_h;
5319	struct cam_doneq *queue;
5320	int i;
5321
5322	/* Process global queues. */
5323	for (i = 0; i < cam_num_doneqs; i++) {
5324		queue = &cam_doneqs[i];
5325		mtx_lock(&queue->cam_doneq_mtx);
5326		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5327			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5328			mtx_unlock(&queue->cam_doneq_mtx);
5329			xpt_done_process(ccb_h);
5330			mtx_lock(&queue->cam_doneq_mtx);
5331		}
5332		mtx_unlock(&queue->cam_doneq_mtx);
5333	}
5334}
5335