cam_xpt.c revision 315937 
1/*-
2 * Implementation of the Common Access Method Transport (XPT) layer.
3 *
4 * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5 * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions, and the following disclaimer,
13 *    without modification, immediately at the beginning of the file.
14 * 2. The name of the author may not be used to endorse or promote products
15 *    derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/10/sys/cam/cam_xpt.c 315937 2017-03-25 11:36:06Z mav $");
32
33#include <sys/param.h>
34#include <sys/bus.h>
35#include <sys/systm.h>
36#include <sys/types.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/time.h>
40#include <sys/conf.h>
41#include <sys/fcntl.h>
42#include <sys/interrupt.h>
43#include <sys/proc.h>
44#include <sys/sbuf.h>
45#include <sys/smp.h>
46#include <sys/taskqueue.h>
47
48#include <sys/lock.h>
49#include <sys/mutex.h>
50#include <sys/sysctl.h>
51#include <sys/kthread.h>
52
53#include <cam/cam.h>
54#include <cam/cam_ccb.h>
55#include <cam/cam_periph.h>
56#include <cam/cam_queue.h>
57#include <cam/cam_sim.h>
58#include <cam/cam_xpt.h>
59#include <cam/cam_xpt_sim.h>
60#include <cam/cam_xpt_periph.h>
61#include <cam/cam_xpt_internal.h>
62#include <cam/cam_debug.h>
63#include <cam/cam_compat.h>
64
65#include <cam/scsi/scsi_all.h>
66#include <cam/scsi/scsi_message.h>
67#include <cam/scsi/scsi_pass.h>
68
69#include <machine/md_var.h>	/* geometry translation */
70#include <machine/stdarg.h>	/* for xpt_print below */
71
72#include "opt_cam.h"
73
74/*
75 * This is the maximum number of high powered commands (e.g. start unit)
76 * that can be outstanding at a particular time.
77 */
78#ifndef CAM_MAX_HIGHPOWER
79#define CAM_MAX_HIGHPOWER  4
80#endif
81
82/* Datastructures internal to the xpt layer */
83MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87
88/* Object for defering XPT actions to a taskqueue */
89struct xpt_task {
90	struct task	task;
91	void		*data1;
92	uintptr_t	data2;
93};
94
95struct xpt_softc {
96	uint32_t		xpt_generation;
97
98	/* number of high powered commands that can go through right now */
99	struct mtx		xpt_highpower_lock;
100	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
101	int			num_highpower;
102
103	/* queue for handling async rescan requests. */
104	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
105	int buses_to_config;
106	int buses_config_done;
107
108	/* Registered busses */
109	TAILQ_HEAD(,cam_eb)	xpt_busses;
110	u_int			bus_generation;
111
112	struct intr_config_hook	*xpt_config_hook;
113
114	int			boot_delay;
115	struct callout 		boot_callout;
116
117	struct mtx		xpt_topo_lock;
118	struct mtx		xpt_lock;
119	struct taskqueue	*xpt_taskq;
120};
121
122typedef enum {
123	DM_RET_COPY		= 0x01,
124	DM_RET_FLAG_MASK	= 0x0f,
125	DM_RET_NONE		= 0x00,
126	DM_RET_STOP		= 0x10,
127	DM_RET_DESCEND		= 0x20,
128	DM_RET_ERROR		= 0x30,
129	DM_RET_ACTION_MASK	= 0xf0
130} dev_match_ret;
131
132typedef enum {
133	XPT_DEPTH_BUS,
134	XPT_DEPTH_TARGET,
135	XPT_DEPTH_DEVICE,
136	XPT_DEPTH_PERIPH
137} xpt_traverse_depth;
138
139struct xpt_traverse_config {
140	xpt_traverse_depth	depth;
141	void			*tr_func;
142	void			*tr_arg;
143};
144
145typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
146typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
147typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
148typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
149typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
150
151/* Transport layer configuration information */
152static struct xpt_softc xsoftc;
153
154MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
155
156TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
157SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
158           &xsoftc.boot_delay, 0, "Bus registration wait time");
159SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
160	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
161
162struct cam_doneq {
163	struct mtx_padalign	cam_doneq_mtx;
164	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
165	int			cam_doneq_sleep;
166};
167
168static struct cam_doneq cam_doneqs[MAXCPU];
169static int cam_num_doneqs;
170static struct proc *cam_proc;
171
172TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
173SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
174           &cam_num_doneqs, 0, "Number of completion queues/threads");
175
176struct cam_periph *xpt_periph;
177
178static periph_init_t xpt_periph_init;
179
180static struct periph_driver xpt_driver =
181{
182	xpt_periph_init, "xpt",
183	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
184	CAM_PERIPH_DRV_EARLY
185};
186
187PERIPHDRIVER_DECLARE(xpt, xpt_driver);
188
189static d_open_t xptopen;
190static d_close_t xptclose;
191static d_ioctl_t xptioctl;
192static d_ioctl_t xptdoioctl;
193
194static struct cdevsw xpt_cdevsw = {
195	.d_version =	D_VERSION,
196	.d_flags =	0,
197	.d_open =	xptopen,
198	.d_close =	xptclose,
199	.d_ioctl =	xptioctl,
200	.d_name =	"xpt",
201};
202
203/* Storage for debugging datastructures */
204struct cam_path *cam_dpath;
205u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
206TUNABLE_INT("kern.cam.dflags", &cam_dflags);
207SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
208	&cam_dflags, 0, "Enabled debug flags");
209u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
210TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
211SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
212	&cam_debug_delay, 0, "Delay in us after each debug message");
213
214/* Our boot-time initialization hook */
215static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
216
217static moduledata_t cam_moduledata = {
218	"cam",
219	cam_module_event_handler,
220	NULL
221};
222
223static int	xpt_init(void *);
224
225DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
226MODULE_VERSION(cam, 1);
227
228
229static void		xpt_async_bcast(struct async_list *async_head,
230					u_int32_t async_code,
231					struct cam_path *path,
232					void *async_arg);
233static path_id_t xptnextfreepathid(void);
234static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
235static union ccb *xpt_get_ccb(struct cam_periph *periph);
236static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
237static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
238static void	 xpt_run_allocq_task(void *context, int pending);
239static void	 xpt_run_devq(struct cam_devq *devq);
240static timeout_t xpt_release_devq_timeout;
241static void	 xpt_release_simq_timeout(void *arg) __unused;
242static void	 xpt_acquire_bus(struct cam_eb *bus);
243static void	 xpt_release_bus(struct cam_eb *bus);
244static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
245static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
246		    int run_queue);
247static struct cam_et*
248		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
249static void	 xpt_acquire_target(struct cam_et *target);
250static void	 xpt_release_target(struct cam_et *target);
251static struct cam_eb*
252		 xpt_find_bus(path_id_t path_id);
253static struct cam_et*
254		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
255static struct cam_ed*
256		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
257static void	 xpt_config(void *arg);
258static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
259				 u_int32_t new_priority);
260static xpt_devicefunc_t xptpassannouncefunc;
261static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
262static void	 xptpoll(struct cam_sim *sim);
263static void	 camisr_runqueue(void);
264static void	 xpt_done_process(struct ccb_hdr *ccb_h);
265static void	 xpt_done_td(void *);
266static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
267				    u_int num_patterns, struct cam_eb *bus);
268static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
269				       u_int num_patterns,
270				       struct cam_ed *device);
271static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
272				       u_int num_patterns,
273				       struct cam_periph *periph);
274static xpt_busfunc_t	xptedtbusfunc;
275static xpt_targetfunc_t	xptedttargetfunc;
276static xpt_devicefunc_t	xptedtdevicefunc;
277static xpt_periphfunc_t	xptedtperiphfunc;
278static xpt_pdrvfunc_t	xptplistpdrvfunc;
279static xpt_periphfunc_t	xptplistperiphfunc;
280static int		xptedtmatch(struct ccb_dev_match *cdm);
281static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
282static int		xptbustraverse(struct cam_eb *start_bus,
283				       xpt_busfunc_t *tr_func, void *arg);
284static int		xpttargettraverse(struct cam_eb *bus,
285					  struct cam_et *start_target,
286					  xpt_targetfunc_t *tr_func, void *arg);
287static int		xptdevicetraverse(struct cam_et *target,
288					  struct cam_ed *start_device,
289					  xpt_devicefunc_t *tr_func, void *arg);
290static int		xptperiphtraverse(struct cam_ed *device,
291					  struct cam_periph *start_periph,
292					  xpt_periphfunc_t *tr_func, void *arg);
293static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
294					xpt_pdrvfunc_t *tr_func, void *arg);
295static int		xptpdperiphtraverse(struct periph_driver **pdrv,
296					    struct cam_periph *start_periph,
297					    xpt_periphfunc_t *tr_func,
298					    void *arg);
299static xpt_busfunc_t	xptdefbusfunc;
300static xpt_targetfunc_t	xptdeftargetfunc;
301static xpt_devicefunc_t	xptdefdevicefunc;
302static xpt_periphfunc_t	xptdefperiphfunc;
303static void		xpt_finishconfig_task(void *context, int pending);
304static void		xpt_dev_async_default(u_int32_t async_code,
305					      struct cam_eb *bus,
306					      struct cam_et *target,
307					      struct cam_ed *device,
308					      void *async_arg);
309static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
310						 struct cam_et *target,
311						 lun_id_t lun_id);
312static xpt_devicefunc_t	xptsetasyncfunc;
313static xpt_busfunc_t	xptsetasyncbusfunc;
314static cam_status	xptregister(struct cam_periph *periph,
315				    void *arg);
316static __inline int device_is_queued(struct cam_ed *device);
317
318static __inline int
319xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
320{
321	int	retval;
322
323	mtx_assert(&devq->send_mtx, MA_OWNED);
324	if ((dev->ccbq.queue.entries > 0) &&
325	    (dev->ccbq.dev_openings > 0) &&
326	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
327		/*
328		 * The priority of a device waiting for controller
329		 * resources is that of the highest priority CCB
330		 * enqueued.
331		 */
332		retval =
333		    xpt_schedule_dev(&devq->send_queue,
334				     &dev->devq_entry,
335				     CAMQ_GET_PRIO(&dev->ccbq.queue));
336	} else {
337		retval = 0;
338	}
339	return (retval);
340}
341
342static __inline int
343device_is_queued(struct cam_ed *device)
344{
345	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
346}
347
348static void
349xpt_periph_init()
350{
351	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
352}
353
354static int
355xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
356{
357
358	/*
359	 * Only allow read-write access.
360	 */
361	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
362		return(EPERM);
363
364	/*
365	 * We don't allow nonblocking access.
366	 */
367	if ((flags & O_NONBLOCK) != 0) {
368		printf("%s: can't do nonblocking access\n", devtoname(dev));
369		return(ENODEV);
370	}
371
372	return(0);
373}
374
375static int
376xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
377{
378
379	return(0);
380}
381
382/*
383 * Don't automatically grab the xpt softc lock here even though this is going
384 * through the xpt device.  The xpt device is really just a back door for
385 * accessing other devices and SIMs, so the right thing to do is to grab
386 * the appropriate SIM lock once the bus/SIM is located.
387 */
388static int
389xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
390{
391	int error;
392
393	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
394		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
395	}
396	return (error);
397}
398
399static int
400xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
401{
402	int error;
403
404	error = 0;
405
406	switch(cmd) {
407	/*
408	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
409	 * to accept CCB types that don't quite make sense to send through a
410	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
411	 * in the CAM spec.
412	 */
413	case CAMIOCOMMAND: {
414		union ccb *ccb;
415		union ccb *inccb;
416		struct cam_eb *bus;
417
418		inccb = (union ccb *)addr;
419
420		bus = xpt_find_bus(inccb->ccb_h.path_id);
421		if (bus == NULL)
422			return (EINVAL);
423
424		switch (inccb->ccb_h.func_code) {
425		case XPT_SCAN_BUS:
426		case XPT_RESET_BUS:
427			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
428			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
429				xpt_release_bus(bus);
430				return (EINVAL);
431			}
432			break;
433		case XPT_SCAN_TGT:
434			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
435			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
436				xpt_release_bus(bus);
437				return (EINVAL);
438			}
439			break;
440		default:
441			break;
442		}
443
444		switch(inccb->ccb_h.func_code) {
445		case XPT_SCAN_BUS:
446		case XPT_RESET_BUS:
447		case XPT_PATH_INQ:
448		case XPT_ENG_INQ:
449		case XPT_SCAN_LUN:
450		case XPT_SCAN_TGT:
451
452			ccb = xpt_alloc_ccb();
453
454			/*
455			 * Create a path using the bus, target, and lun the
456			 * user passed in.
457			 */
458			if (xpt_create_path(&ccb->ccb_h.path, NULL,
459					    inccb->ccb_h.path_id,
460					    inccb->ccb_h.target_id,
461					    inccb->ccb_h.target_lun) !=
462					    CAM_REQ_CMP){
463				error = EINVAL;
464				xpt_free_ccb(ccb);
465				break;
466			}
467			/* Ensure all of our fields are correct */
468			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
469				      inccb->ccb_h.pinfo.priority);
470			xpt_merge_ccb(ccb, inccb);
471			xpt_path_lock(ccb->ccb_h.path);
472			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
473			xpt_path_unlock(ccb->ccb_h.path);
474			bcopy(ccb, inccb, sizeof(union ccb));
475			xpt_free_path(ccb->ccb_h.path);
476			xpt_free_ccb(ccb);
477			break;
478
479		case XPT_DEBUG: {
480			union ccb ccb;
481
482			/*
483			 * This is an immediate CCB, so it's okay to
484			 * allocate it on the stack.
485			 */
486
487			/*
488			 * Create a path using the bus, target, and lun the
489			 * user passed in.
490			 */
491			if (xpt_create_path(&ccb.ccb_h.path, NULL,
492					    inccb->ccb_h.path_id,
493					    inccb->ccb_h.target_id,
494					    inccb->ccb_h.target_lun) !=
495					    CAM_REQ_CMP){
496				error = EINVAL;
497				break;
498			}
499			/* Ensure all of our fields are correct */
500			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
501				      inccb->ccb_h.pinfo.priority);
502			xpt_merge_ccb(&ccb, inccb);
503			xpt_action(&ccb);
504			bcopy(&ccb, inccb, sizeof(union ccb));
505			xpt_free_path(ccb.ccb_h.path);
506			break;
507
508		}
509		case XPT_DEV_MATCH: {
510			struct cam_periph_map_info mapinfo;
511			struct cam_path *old_path;
512
513			/*
514			 * We can't deal with physical addresses for this
515			 * type of transaction.
516			 */
517			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
518			    CAM_DATA_VADDR) {
519				error = EINVAL;
520				break;
521			}
522
523			/*
524			 * Save this in case the caller had it set to
525			 * something in particular.
526			 */
527			old_path = inccb->ccb_h.path;
528
529			/*
530			 * We really don't need a path for the matching
531			 * code.  The path is needed because of the
532			 * debugging statements in xpt_action().  They
533			 * assume that the CCB has a valid path.
534			 */
535			inccb->ccb_h.path = xpt_periph->path;
536
537			bzero(&mapinfo, sizeof(mapinfo));
538
539			/*
540			 * Map the pattern and match buffers into kernel
541			 * virtual address space.
542			 */
543			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
544
545			if (error) {
546				inccb->ccb_h.path = old_path;
547				break;
548			}
549
550			/*
551			 * This is an immediate CCB, we can send it on directly.
552			 */
553			xpt_action(inccb);
554
555			/*
556			 * Map the buffers back into user space.
557			 */
558			cam_periph_unmapmem(inccb, &mapinfo);
559
560			inccb->ccb_h.path = old_path;
561
562			error = 0;
563			break;
564		}
565		default:
566			error = ENOTSUP;
567			break;
568		}
569		xpt_release_bus(bus);
570		break;
571	}
572	/*
573	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
574	 * with the periphal driver name and unit name filled in.  The other
575	 * fields don't really matter as input.  The passthrough driver name
576	 * ("pass"), and unit number are passed back in the ccb.  The current
577	 * device generation number, and the index into the device peripheral
578	 * driver list, and the status are also passed back.  Note that
579	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
580	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
581	 * (or rather should be) impossible for the device peripheral driver
582	 * list to change since we look at the whole thing in one pass, and
583	 * we do it with lock protection.
584	 *
585	 */
586	case CAMGETPASSTHRU: {
587		union ccb *ccb;
588		struct cam_periph *periph;
589		struct periph_driver **p_drv;
590		char   *name;
591		u_int unit;
592		int base_periph_found;
593
594		ccb = (union ccb *)addr;
595		unit = ccb->cgdl.unit_number;
596		name = ccb->cgdl.periph_name;
597		base_periph_found = 0;
598
599		/*
600		 * Sanity check -- make sure we don't get a null peripheral
601		 * driver name.
602		 */
603		if (*ccb->cgdl.periph_name == '\0') {
604			error = EINVAL;
605			break;
606		}
607
608		/* Keep the list from changing while we traverse it */
609		xpt_lock_buses();
610
611		/* first find our driver in the list of drivers */
612		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
613			if (strcmp((*p_drv)->driver_name, name) == 0)
614				break;
615
616		if (*p_drv == NULL) {
617			xpt_unlock_buses();
618			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
619			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
620			*ccb->cgdl.periph_name = '\0';
621			ccb->cgdl.unit_number = 0;
622			error = ENOENT;
623			break;
624		}
625
626		/*
627		 * Run through every peripheral instance of this driver
628		 * and check to see whether it matches the unit passed
629		 * in by the user.  If it does, get out of the loops and
630		 * find the passthrough driver associated with that
631		 * peripheral driver.
632		 */
633		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
634		     periph = TAILQ_NEXT(periph, unit_links)) {
635
636			if (periph->unit_number == unit)
637				break;
638		}
639		/*
640		 * If we found the peripheral driver that the user passed
641		 * in, go through all of the peripheral drivers for that
642		 * particular device and look for a passthrough driver.
643		 */
644		if (periph != NULL) {
645			struct cam_ed *device;
646			int i;
647
648			base_periph_found = 1;
649			device = periph->path->device;
650			for (i = 0, periph = SLIST_FIRST(&device->periphs);
651			     periph != NULL;
652			     periph = SLIST_NEXT(periph, periph_links), i++) {
653				/*
654				 * Check to see whether we have a
655				 * passthrough device or not.
656				 */
657				if (strcmp(periph->periph_name, "pass") == 0) {
658					/*
659					 * Fill in the getdevlist fields.
660					 */
661					strcpy(ccb->cgdl.periph_name,
662					       periph->periph_name);
663					ccb->cgdl.unit_number =
664						periph->unit_number;
665					if (SLIST_NEXT(periph, periph_links))
666						ccb->cgdl.status =
667							CAM_GDEVLIST_MORE_DEVS;
668					else
669						ccb->cgdl.status =
670						       CAM_GDEVLIST_LAST_DEVICE;
671					ccb->cgdl.generation =
672						device->generation;
673					ccb->cgdl.index = i;
674					/*
675					 * Fill in some CCB header fields
676					 * that the user may want.
677					 */
678					ccb->ccb_h.path_id =
679						periph->path->bus->path_id;
680					ccb->ccb_h.target_id =
681						periph->path->target->target_id;
682					ccb->ccb_h.target_lun =
683						periph->path->device->lun_id;
684					ccb->ccb_h.status = CAM_REQ_CMP;
685					break;
686				}
687			}
688		}
689
690		/*
691		 * If the periph is null here, one of two things has
692		 * happened.  The first possibility is that we couldn't
693		 * find the unit number of the particular peripheral driver
694		 * that the user is asking about.  e.g. the user asks for
695		 * the passthrough driver for "da11".  We find the list of
696		 * "da" peripherals all right, but there is no unit 11.
697		 * The other possibility is that we went through the list
698		 * of peripheral drivers attached to the device structure,
699		 * but didn't find one with the name "pass".  Either way,
700		 * we return ENOENT, since we couldn't find something.
701		 */
702		if (periph == NULL) {
703			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
704			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
705			*ccb->cgdl.periph_name = '\0';
706			ccb->cgdl.unit_number = 0;
707			error = ENOENT;
708			/*
709			 * It is unfortunate that this is even necessary,
710			 * but there are many, many clueless users out there.
711			 * If this is true, the user is looking for the
712			 * passthrough driver, but doesn't have one in his
713			 * kernel.
714			 */
715			if (base_periph_found == 1) {
716				printf("xptioctl: pass driver is not in the "
717				       "kernel\n");
718				printf("xptioctl: put \"device pass\" in "
719				       "your kernel config file\n");
720			}
721		}
722		xpt_unlock_buses();
723		break;
724		}
725	default:
726		error = ENOTTY;
727		break;
728	}
729
730	return(error);
731}
732
733static int
734cam_module_event_handler(module_t mod, int what, void *arg)
735{
736	int error;
737
738	switch (what) {
739	case MOD_LOAD:
740		if ((error = xpt_init(NULL)) != 0)
741			return (error);
742		break;
743	case MOD_UNLOAD:
744		return EBUSY;
745	default:
746		return EOPNOTSUPP;
747	}
748
749	return 0;
750}
751
752static void
753xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
754{
755
756	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
757		xpt_free_path(done_ccb->ccb_h.path);
758		xpt_free_ccb(done_ccb);
759	} else {
760		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
761		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
762	}
763	xpt_release_boot();
764}
765
766/* thread to handle bus rescans */
767static void
768xpt_scanner_thread(void *dummy)
769{
770	union ccb	*ccb;
771	struct cam_path	 path;
772
773	xpt_lock_buses();
774	for (;;) {
775		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
776			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
777			       "-", 0);
778		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
779			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
780			xpt_unlock_buses();
781
782			/*
783			 * Since lock can be dropped inside and path freed
784			 * by completion callback even before return here,
785			 * take our own path copy for reference.
786			 */
787			xpt_copy_path(&path, ccb->ccb_h.path);
788			xpt_path_lock(&path);
789			xpt_action(ccb);
790			xpt_path_unlock(&path);
791			xpt_release_path(&path);
792
793			xpt_lock_buses();
794		}
795	}
796}
797
798void
799xpt_rescan(union ccb *ccb)
800{
801	struct ccb_hdr *hdr;
802
803	/* Prepare request */
804	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
805	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
806		ccb->ccb_h.func_code = XPT_SCAN_BUS;
807	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
808	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
809		ccb->ccb_h.func_code = XPT_SCAN_TGT;
810	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
811	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
812		ccb->ccb_h.func_code = XPT_SCAN_LUN;
813	else {
814		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
815		xpt_free_path(ccb->ccb_h.path);
816		xpt_free_ccb(ccb);
817		return;
818	}
819	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
820	ccb->ccb_h.cbfcnp = xpt_rescan_done;
821	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
822	/* Don't make duplicate entries for the same paths. */
823	xpt_lock_buses();
824	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
825		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
826			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
827				wakeup(&xsoftc.ccb_scanq);
828				xpt_unlock_buses();
829				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
830				xpt_free_path(ccb->ccb_h.path);
831				xpt_free_ccb(ccb);
832				return;
833			}
834		}
835	}
836	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
837	xsoftc.buses_to_config++;
838	wakeup(&xsoftc.ccb_scanq);
839	xpt_unlock_buses();
840}
841
842/* Functions accessed by the peripheral drivers */
843static int
844xpt_init(void *dummy)
845{
846	struct cam_sim *xpt_sim;
847	struct cam_path *path;
848	struct cam_devq *devq;
849	cam_status status;
850	int error, i;
851
852	TAILQ_INIT(&xsoftc.xpt_busses);
853	TAILQ_INIT(&xsoftc.ccb_scanq);
854	STAILQ_INIT(&xsoftc.highpowerq);
855	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
856
857	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
858	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
859	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
860	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
861
862#ifdef CAM_BOOT_DELAY
863	/*
864	 * Override this value at compile time to assist our users
865	 * who don't use loader to boot a kernel.
866	 */
867	xsoftc.boot_delay = CAM_BOOT_DELAY;
868#endif
869	/*
870	 * The xpt layer is, itself, the equivalent of a SIM.
871	 * Allow 16 ccbs in the ccb pool for it.  This should
872	 * give decent parallelism when we probe busses and
873	 * perform other XPT functions.
874	 */
875	devq = cam_simq_alloc(16);
876	xpt_sim = cam_sim_alloc(xptaction,
877				xptpoll,
878				"xpt",
879				/*softc*/NULL,
880				/*unit*/0,
881				/*mtx*/&xsoftc.xpt_lock,
882				/*max_dev_transactions*/0,
883				/*max_tagged_dev_transactions*/0,
884				devq);
885	if (xpt_sim == NULL)
886		return (ENOMEM);
887
888	mtx_lock(&xsoftc.xpt_lock);
889	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
890		mtx_unlock(&xsoftc.xpt_lock);
891		printf("xpt_init: xpt_bus_register failed with status %#x,"
892		       " failing attach\n", status);
893		return (EINVAL);
894	}
895	mtx_unlock(&xsoftc.xpt_lock);
896
897	/*
898	 * Looking at the XPT from the SIM layer, the XPT is
899	 * the equivalent of a peripheral driver.  Allocate
900	 * a peripheral driver entry for us.
901	 */
902	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
903				      CAM_TARGET_WILDCARD,
904				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
905		printf("xpt_init: xpt_create_path failed with status %#x,"
906		       " failing attach\n", status);
907		return (EINVAL);
908	}
909	xpt_path_lock(path);
910	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
911			 path, NULL, 0, xpt_sim);
912	xpt_path_unlock(path);
913	xpt_free_path(path);
914
915	if (cam_num_doneqs < 1)
916		cam_num_doneqs = 1 + mp_ncpus / 6;
917	else if (cam_num_doneqs > MAXCPU)
918		cam_num_doneqs = MAXCPU;
919	for (i = 0; i < cam_num_doneqs; i++) {
920		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
921		    MTX_DEF);
922		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
923		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
924		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
925		if (error != 0) {
926			cam_num_doneqs = i;
927			break;
928		}
929	}
930	if (cam_num_doneqs < 1) {
931		printf("xpt_init: Cannot init completion queues "
932		       "- failing attach\n");
933		return (ENOMEM);
934	}
935	/*
936	 * Register a callback for when interrupts are enabled.
937	 */
938	xsoftc.xpt_config_hook =
939	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
940					      M_CAMXPT, M_NOWAIT | M_ZERO);
941	if (xsoftc.xpt_config_hook == NULL) {
942		printf("xpt_init: Cannot malloc config hook "
943		       "- failing attach\n");
944		return (ENOMEM);
945	}
946	xsoftc.xpt_config_hook->ich_func = xpt_config;
947	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
948		free (xsoftc.xpt_config_hook, M_CAMXPT);
949		printf("xpt_init: config_intrhook_establish failed "
950		       "- failing attach\n");
951	}
952
953	return (0);
954}
955
956static cam_status
957xptregister(struct cam_periph *periph, void *arg)
958{
959	struct cam_sim *xpt_sim;
960
961	if (periph == NULL) {
962		printf("xptregister: periph was NULL!!\n");
963		return(CAM_REQ_CMP_ERR);
964	}
965
966	xpt_sim = (struct cam_sim *)arg;
967	xpt_sim->softc = periph;
968	xpt_periph = periph;
969	periph->softc = NULL;
970
971	return(CAM_REQ_CMP);
972}
973
974int32_t
975xpt_add_periph(struct cam_periph *periph)
976{
977	struct cam_ed *device;
978	int32_t	 status;
979
980	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
981	device = periph->path->device;
982	status = CAM_REQ_CMP;
983	if (device != NULL) {
984		mtx_lock(&device->target->bus->eb_mtx);
985		device->generation++;
986		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
987		mtx_unlock(&device->target->bus->eb_mtx);
988		atomic_add_32(&xsoftc.xpt_generation, 1);
989	}
990
991	return (status);
992}
993
994void
995xpt_remove_periph(struct cam_periph *periph)
996{
997	struct cam_ed *device;
998
999	device = periph->path->device;
1000	if (device != NULL) {
1001		mtx_lock(&device->target->bus->eb_mtx);
1002		device->generation++;
1003		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1004		mtx_unlock(&device->target->bus->eb_mtx);
1005		atomic_add_32(&xsoftc.xpt_generation, 1);
1006	}
1007}
1008
1009
1010void
1011xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1012{
1013	struct	cam_path *path = periph->path;
1014
1015	cam_periph_assert(periph, MA_OWNED);
1016	periph->flags |= CAM_PERIPH_ANNOUNCED;
1017
1018	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1019	       periph->periph_name, periph->unit_number,
1020	       path->bus->sim->sim_name,
1021	       path->bus->sim->unit_number,
1022	       path->bus->sim->bus_id,
1023	       path->bus->path_id,
1024	       path->target->target_id,
1025	       (uintmax_t)path->device->lun_id);
1026	printf("%s%d: ", periph->periph_name, periph->unit_number);
1027	if (path->device->protocol == PROTO_SCSI)
1028		scsi_print_inquiry(&path->device->inq_data);
1029	else if (path->device->protocol == PROTO_ATA ||
1030	    path->device->protocol == PROTO_SATAPM)
1031		ata_print_ident(&path->device->ident_data);
1032	else if (path->device->protocol == PROTO_SEMB)
1033		semb_print_ident(
1034		    (struct sep_identify_data *)&path->device->ident_data);
1035	else
1036		printf("Unknown protocol device\n");
1037	if (path->device->serial_num_len > 0) {
1038		/* Don't wrap the screen  - print only the first 60 chars */
1039		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1040		       periph->unit_number, path->device->serial_num);
1041	}
1042	/* Announce transport details. */
1043	(*(path->bus->xport->announce))(periph);
1044	/* Announce command queueing. */
1045	if (path->device->inq_flags & SID_CmdQue
1046	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1047		printf("%s%d: Command Queueing enabled\n",
1048		       periph->periph_name, periph->unit_number);
1049	}
1050	/* Announce caller's details if they've passed in. */
1051	if (announce_string != NULL)
1052		printf("%s%d: %s\n", periph->periph_name,
1053		       periph->unit_number, announce_string);
1054}
1055
1056void
1057xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1058{
1059	if (quirks != 0) {
1060		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1061		    periph->unit_number, quirks, bit_string);
1062	}
1063}
1064
1065void
1066xpt_denounce_periph(struct cam_periph *periph)
1067{
1068	struct	cam_path *path = periph->path;
1069
1070	cam_periph_assert(periph, MA_OWNED);
1071	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1072	       periph->periph_name, periph->unit_number,
1073	       path->bus->sim->sim_name,
1074	       path->bus->sim->unit_number,
1075	       path->bus->sim->bus_id,
1076	       path->bus->path_id,
1077	       path->target->target_id,
1078	       (uintmax_t)path->device->lun_id);
1079	printf("%s%d: ", periph->periph_name, periph->unit_number);
1080	if (path->device->protocol == PROTO_SCSI)
1081		scsi_print_inquiry_short(&path->device->inq_data);
1082	else if (path->device->protocol == PROTO_ATA ||
1083	    path->device->protocol == PROTO_SATAPM)
1084		ata_print_ident_short(&path->device->ident_data);
1085	else if (path->device->protocol == PROTO_SEMB)
1086		semb_print_ident_short(
1087		    (struct sep_identify_data *)&path->device->ident_data);
1088	else
1089		printf("Unknown protocol device");
1090	if (path->device->serial_num_len > 0)
1091		printf(" s/n %.60s", path->device->serial_num);
1092	printf(" detached\n");
1093}
1094
1095
1096int
1097xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1098{
1099	int ret = -1, l, o;
1100	struct ccb_dev_advinfo cdai;
1101	struct scsi_vpd_id_descriptor *idd;
1102
1103	xpt_path_assert(path, MA_OWNED);
1104
1105	memset(&cdai, 0, sizeof(cdai));
1106	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1107	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1108	cdai.flags = CDAI_FLAG_NONE;
1109	cdai.bufsiz = len;
1110
1111	if (!strcmp(attr, "GEOM::ident"))
1112		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1113	else if (!strcmp(attr, "GEOM::physpath"))
1114		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1115	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1116		 strcmp(attr, "GEOM::lunname") == 0) {
1117		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1118		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1119	} else
1120		goto out;
1121
1122	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1123	if (cdai.buf == NULL) {
1124		ret = ENOMEM;
1125		goto out;
1126	}
1127	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1128	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1129		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1130	if (cdai.provsiz == 0)
1131		goto out;
1132	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1133		if (strcmp(attr, "GEOM::lunid") == 0) {
1134			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1135			    cdai.provsiz, scsi_devid_is_lun_naa);
1136			if (idd == NULL)
1137				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1138				    cdai.provsiz, scsi_devid_is_lun_eui64);
1139			if (idd == NULL)
1140				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1141				    cdai.provsiz, scsi_devid_is_lun_uuid);
1142			if (idd == NULL)
1143				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1144				    cdai.provsiz, scsi_devid_is_lun_md5);
1145		} else
1146			idd = NULL;
1147		if (idd == NULL)
1148			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1149			    cdai.provsiz, scsi_devid_is_lun_t10);
1150		if (idd == NULL)
1151			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1152			    cdai.provsiz, scsi_devid_is_lun_name);
1153		if (idd == NULL)
1154			goto out;
1155		ret = 0;
1156		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1157			if (idd->length < len) {
1158				for (l = 0; l < idd->length; l++)
1159					buf[l] = idd->identifier[l] ?
1160					    idd->identifier[l] : ' ';
1161				buf[l] = 0;
1162			} else
1163				ret = EFAULT;
1164		} else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1165			l = strnlen(idd->identifier, idd->length);
1166			if (l < len) {
1167				bcopy(idd->identifier, buf, l);
1168				buf[l] = 0;
1169			} else
1170				ret = EFAULT;
1171		} else if ((idd->id_type & SVPD_ID_TYPE_MASK) == SVPD_ID_TYPE_UUID
1172		    && idd->identifier[0] == 0x10) {
1173			if ((idd->length - 2) * 2 + 4 < len) {
1174				for (l = 2, o = 0; l < idd->length; l++) {
1175					if (l == 6 || l == 8 || l == 10 || l == 12)
1176					    o += sprintf(buf + o, "-");
1177					o += sprintf(buf + o, "%02x",
1178					    idd->identifier[l]);
1179				}
1180			} else
1181				ret = EFAULT;
1182		} else {
1183			if (idd->length * 2 < len) {
1184				for (l = 0; l < idd->length; l++)
1185					sprintf(buf + l * 2, "%02x",
1186					    idd->identifier[l]);
1187			} else
1188				ret = EFAULT;
1189		}
1190	} else {
1191		ret = 0;
1192		if (strlcpy(buf, cdai.buf, len) >= len)
1193			ret = EFAULT;
1194	}
1195
1196out:
1197	if (cdai.buf != NULL)
1198		free(cdai.buf, M_CAMXPT);
1199	return ret;
1200}
1201
1202static dev_match_ret
1203xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1204	    struct cam_eb *bus)
1205{
1206	dev_match_ret retval;
1207	u_int i;
1208
1209	retval = DM_RET_NONE;
1210
1211	/*
1212	 * If we aren't given something to match against, that's an error.
1213	 */
1214	if (bus == NULL)
1215		return(DM_RET_ERROR);
1216
1217	/*
1218	 * If there are no match entries, then this bus matches no
1219	 * matter what.
1220	 */
1221	if ((patterns == NULL) || (num_patterns == 0))
1222		return(DM_RET_DESCEND | DM_RET_COPY);
1223
1224	for (i = 0; i < num_patterns; i++) {
1225		struct bus_match_pattern *cur_pattern;
1226
1227		/*
1228		 * If the pattern in question isn't for a bus node, we
1229		 * aren't interested.  However, we do indicate to the
1230		 * calling routine that we should continue descending the
1231		 * tree, since the user wants to match against lower-level
1232		 * EDT elements.
1233		 */
1234		if (patterns[i].type != DEV_MATCH_BUS) {
1235			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1236				retval |= DM_RET_DESCEND;
1237			continue;
1238		}
1239
1240		cur_pattern = &patterns[i].pattern.bus_pattern;
1241
1242		/*
1243		 * If they want to match any bus node, we give them any
1244		 * device node.
1245		 */
1246		if (cur_pattern->flags == BUS_MATCH_ANY) {
1247			/* set the copy flag */
1248			retval |= DM_RET_COPY;
1249
1250			/*
1251			 * If we've already decided on an action, go ahead
1252			 * and return.
1253			 */
1254			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1255				return(retval);
1256		}
1257
1258		/*
1259		 * Not sure why someone would do this...
1260		 */
1261		if (cur_pattern->flags == BUS_MATCH_NONE)
1262			continue;
1263
1264		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1265		 && (cur_pattern->path_id != bus->path_id))
1266			continue;
1267
1268		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1269		 && (cur_pattern->bus_id != bus->sim->bus_id))
1270			continue;
1271
1272		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1273		 && (cur_pattern->unit_number != bus->sim->unit_number))
1274			continue;
1275
1276		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1277		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1278			     DEV_IDLEN) != 0))
1279			continue;
1280
1281		/*
1282		 * If we get to this point, the user definitely wants
1283		 * information on this bus.  So tell the caller to copy the
1284		 * data out.
1285		 */
1286		retval |= DM_RET_COPY;
1287
1288		/*
1289		 * If the return action has been set to descend, then we
1290		 * know that we've already seen a non-bus matching
1291		 * expression, therefore we need to further descend the tree.
1292		 * This won't change by continuing around the loop, so we
1293		 * go ahead and return.  If we haven't seen a non-bus
1294		 * matching expression, we keep going around the loop until
1295		 * we exhaust the matching expressions.  We'll set the stop
1296		 * flag once we fall out of the loop.
1297		 */
1298		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1299			return(retval);
1300	}
1301
1302	/*
1303	 * If the return action hasn't been set to descend yet, that means
1304	 * we haven't seen anything other than bus matching patterns.  So
1305	 * tell the caller to stop descending the tree -- the user doesn't
1306	 * want to match against lower level tree elements.
1307	 */
1308	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1309		retval |= DM_RET_STOP;
1310
1311	return(retval);
1312}
1313
1314static dev_match_ret
1315xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1316	       struct cam_ed *device)
1317{
1318	dev_match_ret retval;
1319	u_int i;
1320
1321	retval = DM_RET_NONE;
1322
1323	/*
1324	 * If we aren't given something to match against, that's an error.
1325	 */
1326	if (device == NULL)
1327		return(DM_RET_ERROR);
1328
1329	/*
1330	 * If there are no match entries, then this device matches no
1331	 * matter what.
1332	 */
1333	if ((patterns == NULL) || (num_patterns == 0))
1334		return(DM_RET_DESCEND | DM_RET_COPY);
1335
1336	for (i = 0; i < num_patterns; i++) {
1337		struct device_match_pattern *cur_pattern;
1338		struct scsi_vpd_device_id *device_id_page;
1339
1340		/*
1341		 * If the pattern in question isn't for a device node, we
1342		 * aren't interested.
1343		 */
1344		if (patterns[i].type != DEV_MATCH_DEVICE) {
1345			if ((patterns[i].type == DEV_MATCH_PERIPH)
1346			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1347				retval |= DM_RET_DESCEND;
1348			continue;
1349		}
1350
1351		cur_pattern = &patterns[i].pattern.device_pattern;
1352
1353		/* Error out if mutually exclusive options are specified. */
1354		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1355		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1356			return(DM_RET_ERROR);
1357
1358		/*
1359		 * If they want to match any device node, we give them any
1360		 * device node.
1361		 */
1362		if (cur_pattern->flags == DEV_MATCH_ANY)
1363			goto copy_dev_node;
1364
1365		/*
1366		 * Not sure why someone would do this...
1367		 */
1368		if (cur_pattern->flags == DEV_MATCH_NONE)
1369			continue;
1370
1371		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1372		 && (cur_pattern->path_id != device->target->bus->path_id))
1373			continue;
1374
1375		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1376		 && (cur_pattern->target_id != device->target->target_id))
1377			continue;
1378
1379		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1380		 && (cur_pattern->target_lun != device->lun_id))
1381			continue;
1382
1383		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1384		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1385				    (caddr_t)&cur_pattern->data.inq_pat,
1386				    1, sizeof(cur_pattern->data.inq_pat),
1387				    scsi_static_inquiry_match) == NULL))
1388			continue;
1389
1390		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1391		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1392		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1393		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1394				      device->device_id_len
1395				    - SVPD_DEVICE_ID_HDR_LEN,
1396				      cur_pattern->data.devid_pat.id,
1397				      cur_pattern->data.devid_pat.id_len) != 0))
1398			continue;
1399
1400copy_dev_node:
1401		/*
1402		 * If we get to this point, the user definitely wants
1403		 * information on this device.  So tell the caller to copy
1404		 * the data out.
1405		 */
1406		retval |= DM_RET_COPY;
1407
1408		/*
1409		 * If the return action has been set to descend, then we
1410		 * know that we've already seen a peripheral matching
1411		 * expression, therefore we need to further descend the tree.
1412		 * This won't change by continuing around the loop, so we
1413		 * go ahead and return.  If we haven't seen a peripheral
1414		 * matching expression, we keep going around the loop until
1415		 * we exhaust the matching expressions.  We'll set the stop
1416		 * flag once we fall out of the loop.
1417		 */
1418		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1419			return(retval);
1420	}
1421
1422	/*
1423	 * If the return action hasn't been set to descend yet, that means
1424	 * we haven't seen any peripheral matching patterns.  So tell the
1425	 * caller to stop descending the tree -- the user doesn't want to
1426	 * match against lower level tree elements.
1427	 */
1428	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1429		retval |= DM_RET_STOP;
1430
1431	return(retval);
1432}
1433
1434/*
1435 * Match a single peripheral against any number of match patterns.
1436 */
1437static dev_match_ret
1438xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1439	       struct cam_periph *periph)
1440{
1441	dev_match_ret retval;
1442	u_int i;
1443
1444	/*
1445	 * If we aren't given something to match against, that's an error.
1446	 */
1447	if (periph == NULL)
1448		return(DM_RET_ERROR);
1449
1450	/*
1451	 * If there are no match entries, then this peripheral matches no
1452	 * matter what.
1453	 */
1454	if ((patterns == NULL) || (num_patterns == 0))
1455		return(DM_RET_STOP | DM_RET_COPY);
1456
1457	/*
1458	 * There aren't any nodes below a peripheral node, so there's no
1459	 * reason to descend the tree any further.
1460	 */
1461	retval = DM_RET_STOP;
1462
1463	for (i = 0; i < num_patterns; i++) {
1464		struct periph_match_pattern *cur_pattern;
1465
1466		/*
1467		 * If the pattern in question isn't for a peripheral, we
1468		 * aren't interested.
1469		 */
1470		if (patterns[i].type != DEV_MATCH_PERIPH)
1471			continue;
1472
1473		cur_pattern = &patterns[i].pattern.periph_pattern;
1474
1475		/*
1476		 * If they want to match on anything, then we will do so.
1477		 */
1478		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1479			/* set the copy flag */
1480			retval |= DM_RET_COPY;
1481
1482			/*
1483			 * We've already set the return action to stop,
1484			 * since there are no nodes below peripherals in
1485			 * the tree.
1486			 */
1487			return(retval);
1488		}
1489
1490		/*
1491		 * Not sure why someone would do this...
1492		 */
1493		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1494			continue;
1495
1496		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1497		 && (cur_pattern->path_id != periph->path->bus->path_id))
1498			continue;
1499
1500		/*
1501		 * For the target and lun id's, we have to make sure the
1502		 * target and lun pointers aren't NULL.  The xpt peripheral
1503		 * has a wildcard target and device.
1504		 */
1505		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1506		 && ((periph->path->target == NULL)
1507		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1508			continue;
1509
1510		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1511		 && ((periph->path->device == NULL)
1512		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1513			continue;
1514
1515		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1516		 && (cur_pattern->unit_number != periph->unit_number))
1517			continue;
1518
1519		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1520		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1521			     DEV_IDLEN) != 0))
1522			continue;
1523
1524		/*
1525		 * If we get to this point, the user definitely wants
1526		 * information on this peripheral.  So tell the caller to
1527		 * copy the data out.
1528		 */
1529		retval |= DM_RET_COPY;
1530
1531		/*
1532		 * The return action has already been set to stop, since
1533		 * peripherals don't have any nodes below them in the EDT.
1534		 */
1535		return(retval);
1536	}
1537
1538	/*
1539	 * If we get to this point, the peripheral that was passed in
1540	 * doesn't match any of the patterns.
1541	 */
1542	return(retval);
1543}
1544
1545static int
1546xptedtbusfunc(struct cam_eb *bus, void *arg)
1547{
1548	struct ccb_dev_match *cdm;
1549	struct cam_et *target;
1550	dev_match_ret retval;
1551
1552	cdm = (struct ccb_dev_match *)arg;
1553
1554	/*
1555	 * If our position is for something deeper in the tree, that means
1556	 * that we've already seen this node.  So, we keep going down.
1557	 */
1558	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1559	 && (cdm->pos.cookie.bus == bus)
1560	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1561	 && (cdm->pos.cookie.target != NULL))
1562		retval = DM_RET_DESCEND;
1563	else
1564		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1565
1566	/*
1567	 * If we got an error, bail out of the search.
1568	 */
1569	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1570		cdm->status = CAM_DEV_MATCH_ERROR;
1571		return(0);
1572	}
1573
1574	/*
1575	 * If the copy flag is set, copy this bus out.
1576	 */
1577	if (retval & DM_RET_COPY) {
1578		int spaceleft, j;
1579
1580		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1581			sizeof(struct dev_match_result));
1582
1583		/*
1584		 * If we don't have enough space to put in another
1585		 * match result, save our position and tell the
1586		 * user there are more devices to check.
1587		 */
1588		if (spaceleft < sizeof(struct dev_match_result)) {
1589			bzero(&cdm->pos, sizeof(cdm->pos));
1590			cdm->pos.position_type =
1591				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1592
1593			cdm->pos.cookie.bus = bus;
1594			cdm->pos.generations[CAM_BUS_GENERATION]=
1595				xsoftc.bus_generation;
1596			cdm->status = CAM_DEV_MATCH_MORE;
1597			return(0);
1598		}
1599		j = cdm->num_matches;
1600		cdm->num_matches++;
1601		cdm->matches[j].type = DEV_MATCH_BUS;
1602		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1603		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1604		cdm->matches[j].result.bus_result.unit_number =
1605			bus->sim->unit_number;
1606		strncpy(cdm->matches[j].result.bus_result.dev_name,
1607			bus->sim->sim_name, DEV_IDLEN);
1608	}
1609
1610	/*
1611	 * If the user is only interested in busses, there's no
1612	 * reason to descend to the next level in the tree.
1613	 */
1614	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1615		return(1);
1616
1617	/*
1618	 * If there is a target generation recorded, check it to
1619	 * make sure the target list hasn't changed.
1620	 */
1621	mtx_lock(&bus->eb_mtx);
1622	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1623	 && (cdm->pos.cookie.bus == bus)
1624	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1625	 && (cdm->pos.cookie.target != NULL)) {
1626		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1627		    bus->generation)) {
1628			mtx_unlock(&bus->eb_mtx);
1629			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1630			return (0);
1631		}
1632		target = (struct cam_et *)cdm->pos.cookie.target;
1633		target->refcount++;
1634	} else
1635		target = NULL;
1636	mtx_unlock(&bus->eb_mtx);
1637
1638	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1639}
1640
1641static int
1642xptedttargetfunc(struct cam_et *target, void *arg)
1643{
1644	struct ccb_dev_match *cdm;
1645	struct cam_eb *bus;
1646	struct cam_ed *device;
1647
1648	cdm = (struct ccb_dev_match *)arg;
1649	bus = target->bus;
1650
1651	/*
1652	 * If there is a device list generation recorded, check it to
1653	 * make sure the device list hasn't changed.
1654	 */
1655	mtx_lock(&bus->eb_mtx);
1656	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1657	 && (cdm->pos.cookie.bus == bus)
1658	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1659	 && (cdm->pos.cookie.target == target)
1660	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1661	 && (cdm->pos.cookie.device != NULL)) {
1662		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1663		    target->generation) {
1664			mtx_unlock(&bus->eb_mtx);
1665			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1666			return(0);
1667		}
1668		device = (struct cam_ed *)cdm->pos.cookie.device;
1669		device->refcount++;
1670	} else
1671		device = NULL;
1672	mtx_unlock(&bus->eb_mtx);
1673
1674	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1675}
1676
1677static int
1678xptedtdevicefunc(struct cam_ed *device, void *arg)
1679{
1680	struct cam_eb *bus;
1681	struct cam_periph *periph;
1682	struct ccb_dev_match *cdm;
1683	dev_match_ret retval;
1684
1685	cdm = (struct ccb_dev_match *)arg;
1686	bus = device->target->bus;
1687
1688	/*
1689	 * If our position is for something deeper in the tree, that means
1690	 * that we've already seen this node.  So, we keep going down.
1691	 */
1692	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1693	 && (cdm->pos.cookie.device == device)
1694	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1695	 && (cdm->pos.cookie.periph != NULL))
1696		retval = DM_RET_DESCEND;
1697	else
1698		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1699					device);
1700
1701	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1702		cdm->status = CAM_DEV_MATCH_ERROR;
1703		return(0);
1704	}
1705
1706	/*
1707	 * If the copy flag is set, copy this device out.
1708	 */
1709	if (retval & DM_RET_COPY) {
1710		int spaceleft, j;
1711
1712		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1713			sizeof(struct dev_match_result));
1714
1715		/*
1716		 * If we don't have enough space to put in another
1717		 * match result, save our position and tell the
1718		 * user there are more devices to check.
1719		 */
1720		if (spaceleft < sizeof(struct dev_match_result)) {
1721			bzero(&cdm->pos, sizeof(cdm->pos));
1722			cdm->pos.position_type =
1723				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1724				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1725
1726			cdm->pos.cookie.bus = device->target->bus;
1727			cdm->pos.generations[CAM_BUS_GENERATION]=
1728				xsoftc.bus_generation;
1729			cdm->pos.cookie.target = device->target;
1730			cdm->pos.generations[CAM_TARGET_GENERATION] =
1731				device->target->bus->generation;
1732			cdm->pos.cookie.device = device;
1733			cdm->pos.generations[CAM_DEV_GENERATION] =
1734				device->target->generation;
1735			cdm->status = CAM_DEV_MATCH_MORE;
1736			return(0);
1737		}
1738		j = cdm->num_matches;
1739		cdm->num_matches++;
1740		cdm->matches[j].type = DEV_MATCH_DEVICE;
1741		cdm->matches[j].result.device_result.path_id =
1742			device->target->bus->path_id;
1743		cdm->matches[j].result.device_result.target_id =
1744			device->target->target_id;
1745		cdm->matches[j].result.device_result.target_lun =
1746			device->lun_id;
1747		cdm->matches[j].result.device_result.protocol =
1748			device->protocol;
1749		bcopy(&device->inq_data,
1750		      &cdm->matches[j].result.device_result.inq_data,
1751		      sizeof(struct scsi_inquiry_data));
1752		bcopy(&device->ident_data,
1753		      &cdm->matches[j].result.device_result.ident_data,
1754		      sizeof(struct ata_params));
1755
1756		/* Let the user know whether this device is unconfigured */
1757		if (device->flags & CAM_DEV_UNCONFIGURED)
1758			cdm->matches[j].result.device_result.flags =
1759				DEV_RESULT_UNCONFIGURED;
1760		else
1761			cdm->matches[j].result.device_result.flags =
1762				DEV_RESULT_NOFLAG;
1763	}
1764
1765	/*
1766	 * If the user isn't interested in peripherals, don't descend
1767	 * the tree any further.
1768	 */
1769	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1770		return(1);
1771
1772	/*
1773	 * If there is a peripheral list generation recorded, make sure
1774	 * it hasn't changed.
1775	 */
1776	xpt_lock_buses();
1777	mtx_lock(&bus->eb_mtx);
1778	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1779	 && (cdm->pos.cookie.bus == bus)
1780	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1781	 && (cdm->pos.cookie.target == device->target)
1782	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1783	 && (cdm->pos.cookie.device == device)
1784	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1785	 && (cdm->pos.cookie.periph != NULL)) {
1786		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1787		    device->generation) {
1788			mtx_unlock(&bus->eb_mtx);
1789			xpt_unlock_buses();
1790			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1791			return(0);
1792		}
1793		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1794		periph->refcount++;
1795	} else
1796		periph = NULL;
1797	mtx_unlock(&bus->eb_mtx);
1798	xpt_unlock_buses();
1799
1800	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1801}
1802
1803static int
1804xptedtperiphfunc(struct cam_periph *periph, void *arg)
1805{
1806	struct ccb_dev_match *cdm;
1807	dev_match_ret retval;
1808
1809	cdm = (struct ccb_dev_match *)arg;
1810
1811	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1812
1813	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1814		cdm->status = CAM_DEV_MATCH_ERROR;
1815		return(0);
1816	}
1817
1818	/*
1819	 * If the copy flag is set, copy this peripheral out.
1820	 */
1821	if (retval & DM_RET_COPY) {
1822		int spaceleft, j;
1823
1824		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1825			sizeof(struct dev_match_result));
1826
1827		/*
1828		 * If we don't have enough space to put in another
1829		 * match result, save our position and tell the
1830		 * user there are more devices to check.
1831		 */
1832		if (spaceleft < sizeof(struct dev_match_result)) {
1833			bzero(&cdm->pos, sizeof(cdm->pos));
1834			cdm->pos.position_type =
1835				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1836				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1837				CAM_DEV_POS_PERIPH;
1838
1839			cdm->pos.cookie.bus = periph->path->bus;
1840			cdm->pos.generations[CAM_BUS_GENERATION]=
1841				xsoftc.bus_generation;
1842			cdm->pos.cookie.target = periph->path->target;
1843			cdm->pos.generations[CAM_TARGET_GENERATION] =
1844				periph->path->bus->generation;
1845			cdm->pos.cookie.device = periph->path->device;
1846			cdm->pos.generations[CAM_DEV_GENERATION] =
1847				periph->path->target->generation;
1848			cdm->pos.cookie.periph = periph;
1849			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1850				periph->path->device->generation;
1851			cdm->status = CAM_DEV_MATCH_MORE;
1852			return(0);
1853		}
1854
1855		j = cdm->num_matches;
1856		cdm->num_matches++;
1857		cdm->matches[j].type = DEV_MATCH_PERIPH;
1858		cdm->matches[j].result.periph_result.path_id =
1859			periph->path->bus->path_id;
1860		cdm->matches[j].result.periph_result.target_id =
1861			periph->path->target->target_id;
1862		cdm->matches[j].result.periph_result.target_lun =
1863			periph->path->device->lun_id;
1864		cdm->matches[j].result.periph_result.unit_number =
1865			periph->unit_number;
1866		strncpy(cdm->matches[j].result.periph_result.periph_name,
1867			periph->periph_name, DEV_IDLEN);
1868	}
1869
1870	return(1);
1871}
1872
1873static int
1874xptedtmatch(struct ccb_dev_match *cdm)
1875{
1876	struct cam_eb *bus;
1877	int ret;
1878
1879	cdm->num_matches = 0;
1880
1881	/*
1882	 * Check the bus list generation.  If it has changed, the user
1883	 * needs to reset everything and start over.
1884	 */
1885	xpt_lock_buses();
1886	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1887	 && (cdm->pos.cookie.bus != NULL)) {
1888		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1889		    xsoftc.bus_generation) {
1890			xpt_unlock_buses();
1891			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1892			return(0);
1893		}
1894		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1895		bus->refcount++;
1896	} else
1897		bus = NULL;
1898	xpt_unlock_buses();
1899
1900	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1901
1902	/*
1903	 * If we get back 0, that means that we had to stop before fully
1904	 * traversing the EDT.  It also means that one of the subroutines
1905	 * has set the status field to the proper value.  If we get back 1,
1906	 * we've fully traversed the EDT and copied out any matching entries.
1907	 */
1908	if (ret == 1)
1909		cdm->status = CAM_DEV_MATCH_LAST;
1910
1911	return(ret);
1912}
1913
1914static int
1915xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1916{
1917	struct cam_periph *periph;
1918	struct ccb_dev_match *cdm;
1919
1920	cdm = (struct ccb_dev_match *)arg;
1921
1922	xpt_lock_buses();
1923	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1924	 && (cdm->pos.cookie.pdrv == pdrv)
1925	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1926	 && (cdm->pos.cookie.periph != NULL)) {
1927		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1928		    (*pdrv)->generation) {
1929			xpt_unlock_buses();
1930			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1931			return(0);
1932		}
1933		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1934		periph->refcount++;
1935	} else
1936		periph = NULL;
1937	xpt_unlock_buses();
1938
1939	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1940}
1941
1942static int
1943xptplistperiphfunc(struct cam_periph *periph, void *arg)
1944{
1945	struct ccb_dev_match *cdm;
1946	dev_match_ret retval;
1947
1948	cdm = (struct ccb_dev_match *)arg;
1949
1950	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1951
1952	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1953		cdm->status = CAM_DEV_MATCH_ERROR;
1954		return(0);
1955	}
1956
1957	/*
1958	 * If the copy flag is set, copy this peripheral out.
1959	 */
1960	if (retval & DM_RET_COPY) {
1961		int spaceleft, j;
1962
1963		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1964			sizeof(struct dev_match_result));
1965
1966		/*
1967		 * If we don't have enough space to put in another
1968		 * match result, save our position and tell the
1969		 * user there are more devices to check.
1970		 */
1971		if (spaceleft < sizeof(struct dev_match_result)) {
1972			struct periph_driver **pdrv;
1973
1974			pdrv = NULL;
1975			bzero(&cdm->pos, sizeof(cdm->pos));
1976			cdm->pos.position_type =
1977				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1978				CAM_DEV_POS_PERIPH;
1979
1980			/*
1981			 * This may look a bit non-sensical, but it is
1982			 * actually quite logical.  There are very few
1983			 * peripheral drivers, and bloating every peripheral
1984			 * structure with a pointer back to its parent
1985			 * peripheral driver linker set entry would cost
1986			 * more in the long run than doing this quick lookup.
1987			 */
1988			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1989				if (strcmp((*pdrv)->driver_name,
1990				    periph->periph_name) == 0)
1991					break;
1992			}
1993
1994			if (*pdrv == NULL) {
1995				cdm->status = CAM_DEV_MATCH_ERROR;
1996				return(0);
1997			}
1998
1999			cdm->pos.cookie.pdrv = pdrv;
2000			/*
2001			 * The periph generation slot does double duty, as
2002			 * does the periph pointer slot.  They are used for
2003			 * both edt and pdrv lookups and positioning.
2004			 */
2005			cdm->pos.cookie.periph = periph;
2006			cdm->pos.generations[CAM_PERIPH_GENERATION] =
2007				(*pdrv)->generation;
2008			cdm->status = CAM_DEV_MATCH_MORE;
2009			return(0);
2010		}
2011
2012		j = cdm->num_matches;
2013		cdm->num_matches++;
2014		cdm->matches[j].type = DEV_MATCH_PERIPH;
2015		cdm->matches[j].result.periph_result.path_id =
2016			periph->path->bus->path_id;
2017
2018		/*
2019		 * The transport layer peripheral doesn't have a target or
2020		 * lun.
2021		 */
2022		if (periph->path->target)
2023			cdm->matches[j].result.periph_result.target_id =
2024				periph->path->target->target_id;
2025		else
2026			cdm->matches[j].result.periph_result.target_id =
2027				CAM_TARGET_WILDCARD;
2028
2029		if (periph->path->device)
2030			cdm->matches[j].result.periph_result.target_lun =
2031				periph->path->device->lun_id;
2032		else
2033			cdm->matches[j].result.periph_result.target_lun =
2034				CAM_LUN_WILDCARD;
2035
2036		cdm->matches[j].result.periph_result.unit_number =
2037			periph->unit_number;
2038		strncpy(cdm->matches[j].result.periph_result.periph_name,
2039			periph->periph_name, DEV_IDLEN);
2040	}
2041
2042	return(1);
2043}
2044
2045static int
2046xptperiphlistmatch(struct ccb_dev_match *cdm)
2047{
2048	int ret;
2049
2050	cdm->num_matches = 0;
2051
2052	/*
2053	 * At this point in the edt traversal function, we check the bus
2054	 * list generation to make sure that no busses have been added or
2055	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2056	 * For the peripheral driver list traversal function, however, we
2057	 * don't have to worry about new peripheral driver types coming or
2058	 * going; they're in a linker set, and therefore can't change
2059	 * without a recompile.
2060	 */
2061
2062	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2063	 && (cdm->pos.cookie.pdrv != NULL))
2064		ret = xptpdrvtraverse(
2065				(struct periph_driver **)cdm->pos.cookie.pdrv,
2066				xptplistpdrvfunc, cdm);
2067	else
2068		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2069
2070	/*
2071	 * If we get back 0, that means that we had to stop before fully
2072	 * traversing the peripheral driver tree.  It also means that one of
2073	 * the subroutines has set the status field to the proper value.  If
2074	 * we get back 1, we've fully traversed the EDT and copied out any
2075	 * matching entries.
2076	 */
2077	if (ret == 1)
2078		cdm->status = CAM_DEV_MATCH_LAST;
2079
2080	return(ret);
2081}
2082
2083static int
2084xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2085{
2086	struct cam_eb *bus, *next_bus;
2087	int retval;
2088
2089	retval = 1;
2090	if (start_bus)
2091		bus = start_bus;
2092	else {
2093		xpt_lock_buses();
2094		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2095		if (bus == NULL) {
2096			xpt_unlock_buses();
2097			return (retval);
2098		}
2099		bus->refcount++;
2100		xpt_unlock_buses();
2101	}
2102	for (; bus != NULL; bus = next_bus) {
2103		retval = tr_func(bus, arg);
2104		if (retval == 0) {
2105			xpt_release_bus(bus);
2106			break;
2107		}
2108		xpt_lock_buses();
2109		next_bus = TAILQ_NEXT(bus, links);
2110		if (next_bus)
2111			next_bus->refcount++;
2112		xpt_unlock_buses();
2113		xpt_release_bus(bus);
2114	}
2115	return(retval);
2116}
2117
2118static int
2119xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2120		  xpt_targetfunc_t *tr_func, void *arg)
2121{
2122	struct cam_et *target, *next_target;
2123	int retval;
2124
2125	retval = 1;
2126	if (start_target)
2127		target = start_target;
2128	else {
2129		mtx_lock(&bus->eb_mtx);
2130		target = TAILQ_FIRST(&bus->et_entries);
2131		if (target == NULL) {
2132			mtx_unlock(&bus->eb_mtx);
2133			return (retval);
2134		}
2135		target->refcount++;
2136		mtx_unlock(&bus->eb_mtx);
2137	}
2138	for (; target != NULL; target = next_target) {
2139		retval = tr_func(target, arg);
2140		if (retval == 0) {
2141			xpt_release_target(target);
2142			break;
2143		}
2144		mtx_lock(&bus->eb_mtx);
2145		next_target = TAILQ_NEXT(target, links);
2146		if (next_target)
2147			next_target->refcount++;
2148		mtx_unlock(&bus->eb_mtx);
2149		xpt_release_target(target);
2150	}
2151	return(retval);
2152}
2153
2154static int
2155xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2156		  xpt_devicefunc_t *tr_func, void *arg)
2157{
2158	struct cam_eb *bus;
2159	struct cam_ed *device, *next_device;
2160	int retval;
2161
2162	retval = 1;
2163	bus = target->bus;
2164	if (start_device)
2165		device = start_device;
2166	else {
2167		mtx_lock(&bus->eb_mtx);
2168		device = TAILQ_FIRST(&target->ed_entries);
2169		if (device == NULL) {
2170			mtx_unlock(&bus->eb_mtx);
2171			return (retval);
2172		}
2173		device->refcount++;
2174		mtx_unlock(&bus->eb_mtx);
2175	}
2176	for (; device != NULL; device = next_device) {
2177		mtx_lock(&device->device_mtx);
2178		retval = tr_func(device, arg);
2179		mtx_unlock(&device->device_mtx);
2180		if (retval == 0) {
2181			xpt_release_device(device);
2182			break;
2183		}
2184		mtx_lock(&bus->eb_mtx);
2185		next_device = TAILQ_NEXT(device, links);
2186		if (next_device)
2187			next_device->refcount++;
2188		mtx_unlock(&bus->eb_mtx);
2189		xpt_release_device(device);
2190	}
2191	return(retval);
2192}
2193
2194static int
2195xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2196		  xpt_periphfunc_t *tr_func, void *arg)
2197{
2198	struct cam_eb *bus;
2199	struct cam_periph *periph, *next_periph;
2200	int retval;
2201
2202	retval = 1;
2203
2204	bus = device->target->bus;
2205	if (start_periph)
2206		periph = start_periph;
2207	else {
2208		xpt_lock_buses();
2209		mtx_lock(&bus->eb_mtx);
2210		periph = SLIST_FIRST(&device->periphs);
2211		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2212			periph = SLIST_NEXT(periph, periph_links);
2213		if (periph == NULL) {
2214			mtx_unlock(&bus->eb_mtx);
2215			xpt_unlock_buses();
2216			return (retval);
2217		}
2218		periph->refcount++;
2219		mtx_unlock(&bus->eb_mtx);
2220		xpt_unlock_buses();
2221	}
2222	for (; periph != NULL; periph = next_periph) {
2223		retval = tr_func(periph, arg);
2224		if (retval == 0) {
2225			cam_periph_release_locked(periph);
2226			break;
2227		}
2228		xpt_lock_buses();
2229		mtx_lock(&bus->eb_mtx);
2230		next_periph = SLIST_NEXT(periph, periph_links);
2231		while (next_periph != NULL &&
2232		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2233			next_periph = SLIST_NEXT(next_periph, periph_links);
2234		if (next_periph)
2235			next_periph->refcount++;
2236		mtx_unlock(&bus->eb_mtx);
2237		xpt_unlock_buses();
2238		cam_periph_release_locked(periph);
2239	}
2240	return(retval);
2241}
2242
2243static int
2244xptpdrvtraverse(struct periph_driver **start_pdrv,
2245		xpt_pdrvfunc_t *tr_func, void *arg)
2246{
2247	struct periph_driver **pdrv;
2248	int retval;
2249
2250	retval = 1;
2251
2252	/*
2253	 * We don't traverse the peripheral driver list like we do the
2254	 * other lists, because it is a linker set, and therefore cannot be
2255	 * changed during runtime.  If the peripheral driver list is ever
2256	 * re-done to be something other than a linker set (i.e. it can
2257	 * change while the system is running), the list traversal should
2258	 * be modified to work like the other traversal functions.
2259	 */
2260	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2261	     *pdrv != NULL; pdrv++) {
2262		retval = tr_func(pdrv, arg);
2263
2264		if (retval == 0)
2265			return(retval);
2266	}
2267
2268	return(retval);
2269}
2270
2271static int
2272xptpdperiphtraverse(struct periph_driver **pdrv,
2273		    struct cam_periph *start_periph,
2274		    xpt_periphfunc_t *tr_func, void *arg)
2275{
2276	struct cam_periph *periph, *next_periph;
2277	int retval;
2278
2279	retval = 1;
2280
2281	if (start_periph)
2282		periph = start_periph;
2283	else {
2284		xpt_lock_buses();
2285		periph = TAILQ_FIRST(&(*pdrv)->units);
2286		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2287			periph = TAILQ_NEXT(periph, unit_links);
2288		if (periph == NULL) {
2289			xpt_unlock_buses();
2290			return (retval);
2291		}
2292		periph->refcount++;
2293		xpt_unlock_buses();
2294	}
2295	for (; periph != NULL; periph = next_periph) {
2296		cam_periph_lock(periph);
2297		retval = tr_func(periph, arg);
2298		cam_periph_unlock(periph);
2299		if (retval == 0) {
2300			cam_periph_release(periph);
2301			break;
2302		}
2303		xpt_lock_buses();
2304		next_periph = TAILQ_NEXT(periph, unit_links);
2305		while (next_periph != NULL &&
2306		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2307			next_periph = TAILQ_NEXT(next_periph, unit_links);
2308		if (next_periph)
2309			next_periph->refcount++;
2310		xpt_unlock_buses();
2311		cam_periph_release(periph);
2312	}
2313	return(retval);
2314}
2315
2316static int
2317xptdefbusfunc(struct cam_eb *bus, void *arg)
2318{
2319	struct xpt_traverse_config *tr_config;
2320
2321	tr_config = (struct xpt_traverse_config *)arg;
2322
2323	if (tr_config->depth == XPT_DEPTH_BUS) {
2324		xpt_busfunc_t *tr_func;
2325
2326		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2327
2328		return(tr_func(bus, tr_config->tr_arg));
2329	} else
2330		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2331}
2332
2333static int
2334xptdeftargetfunc(struct cam_et *target, void *arg)
2335{
2336	struct xpt_traverse_config *tr_config;
2337
2338	tr_config = (struct xpt_traverse_config *)arg;
2339
2340	if (tr_config->depth == XPT_DEPTH_TARGET) {
2341		xpt_targetfunc_t *tr_func;
2342
2343		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2344
2345		return(tr_func(target, tr_config->tr_arg));
2346	} else
2347		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2348}
2349
2350static int
2351xptdefdevicefunc(struct cam_ed *device, void *arg)
2352{
2353	struct xpt_traverse_config *tr_config;
2354
2355	tr_config = (struct xpt_traverse_config *)arg;
2356
2357	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2358		xpt_devicefunc_t *tr_func;
2359
2360		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2361
2362		return(tr_func(device, tr_config->tr_arg));
2363	} else
2364		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2365}
2366
2367static int
2368xptdefperiphfunc(struct cam_periph *periph, void *arg)
2369{
2370	struct xpt_traverse_config *tr_config;
2371	xpt_periphfunc_t *tr_func;
2372
2373	tr_config = (struct xpt_traverse_config *)arg;
2374
2375	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2376
2377	/*
2378	 * Unlike the other default functions, we don't check for depth
2379	 * here.  The peripheral driver level is the last level in the EDT,
2380	 * so if we're here, we should execute the function in question.
2381	 */
2382	return(tr_func(periph, tr_config->tr_arg));
2383}
2384
2385/*
2386 * Execute the given function for every bus in the EDT.
2387 */
2388static int
2389xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2390{
2391	struct xpt_traverse_config tr_config;
2392
2393	tr_config.depth = XPT_DEPTH_BUS;
2394	tr_config.tr_func = tr_func;
2395	tr_config.tr_arg = arg;
2396
2397	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2398}
2399
2400/*
2401 * Execute the given function for every device in the EDT.
2402 */
2403static int
2404xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2405{
2406	struct xpt_traverse_config tr_config;
2407
2408	tr_config.depth = XPT_DEPTH_DEVICE;
2409	tr_config.tr_func = tr_func;
2410	tr_config.tr_arg = arg;
2411
2412	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2413}
2414
2415static int
2416xptsetasyncfunc(struct cam_ed *device, void *arg)
2417{
2418	struct cam_path path;
2419	struct ccb_getdev cgd;
2420	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2421
2422	/*
2423	 * Don't report unconfigured devices (Wildcard devs,
2424	 * devices only for target mode, device instances
2425	 * that have been invalidated but are waiting for
2426	 * their last reference count to be released).
2427	 */
2428	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2429		return (1);
2430
2431	xpt_compile_path(&path,
2432			 NULL,
2433			 device->target->bus->path_id,
2434			 device->target->target_id,
2435			 device->lun_id);
2436	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2437	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2438	xpt_action((union ccb *)&cgd);
2439	csa->callback(csa->callback_arg,
2440			    AC_FOUND_DEVICE,
2441			    &path, &cgd);
2442	xpt_release_path(&path);
2443
2444	return(1);
2445}
2446
2447static int
2448xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2449{
2450	struct cam_path path;
2451	struct ccb_pathinq cpi;
2452	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2453
2454	xpt_compile_path(&path, /*periph*/NULL,
2455			 bus->path_id,
2456			 CAM_TARGET_WILDCARD,
2457			 CAM_LUN_WILDCARD);
2458	xpt_path_lock(&path);
2459	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2460	cpi.ccb_h.func_code = XPT_PATH_INQ;
2461	xpt_action((union ccb *)&cpi);
2462	csa->callback(csa->callback_arg,
2463			    AC_PATH_REGISTERED,
2464			    &path, &cpi);
2465	xpt_path_unlock(&path);
2466	xpt_release_path(&path);
2467
2468	return(1);
2469}
2470
2471void
2472xpt_action(union ccb *start_ccb)
2473{
2474
2475	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2476
2477	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2478	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2479}
2480
2481void
2482xpt_action_default(union ccb *start_ccb)
2483{
2484	struct cam_path *path;
2485	struct cam_sim *sim;
2486	int lock;
2487
2488	path = start_ccb->ccb_h.path;
2489	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2490
2491	switch (start_ccb->ccb_h.func_code) {
2492	case XPT_SCSI_IO:
2493	{
2494		struct cam_ed *device;
2495
2496		/*
2497		 * For the sake of compatibility with SCSI-1
2498		 * devices that may not understand the identify
2499		 * message, we include lun information in the
2500		 * second byte of all commands.  SCSI-1 specifies
2501		 * that luns are a 3 bit value and reserves only 3
2502		 * bits for lun information in the CDB.  Later
2503		 * revisions of the SCSI spec allow for more than 8
2504		 * luns, but have deprecated lun information in the
2505		 * CDB.  So, if the lun won't fit, we must omit.
2506		 *
2507		 * Also be aware that during initial probing for devices,
2508		 * the inquiry information is unknown but initialized to 0.
2509		 * This means that this code will be exercised while probing
2510		 * devices with an ANSI revision greater than 2.
2511		 */
2512		device = path->device;
2513		if (device->protocol_version <= SCSI_REV_2
2514		 && start_ccb->ccb_h.target_lun < 8
2515		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2516
2517			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2518			    start_ccb->ccb_h.target_lun << 5;
2519		}
2520		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2521	}
2522	/* FALLTHROUGH */
2523	case XPT_TARGET_IO:
2524	case XPT_CONT_TARGET_IO:
2525		start_ccb->csio.sense_resid = 0;
2526		start_ccb->csio.resid = 0;
2527		/* FALLTHROUGH */
2528	case XPT_ATA_IO:
2529		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2530			start_ccb->ataio.resid = 0;
2531		/* FALLTHROUGH */
2532	case XPT_RESET_DEV:
2533	case XPT_ENG_EXEC:
2534	case XPT_SMP_IO:
2535	{
2536		struct cam_devq *devq;
2537
2538		devq = path->bus->sim->devq;
2539		mtx_lock(&devq->send_mtx);
2540		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2541		if (xpt_schedule_devq(devq, path->device) != 0)
2542			xpt_run_devq(devq);
2543		mtx_unlock(&devq->send_mtx);
2544		break;
2545	}
2546	case XPT_CALC_GEOMETRY:
2547		/* Filter out garbage */
2548		if (start_ccb->ccg.block_size == 0
2549		 || start_ccb->ccg.volume_size == 0) {
2550			start_ccb->ccg.cylinders = 0;
2551			start_ccb->ccg.heads = 0;
2552			start_ccb->ccg.secs_per_track = 0;
2553			start_ccb->ccb_h.status = CAM_REQ_CMP;
2554			break;
2555		}
2556#if defined(PC98) || defined(__sparc64__)
2557		/*
2558		 * In a PC-98 system, geometry translation depens on
2559		 * the "real" device geometry obtained from mode page 4.
2560		 * SCSI geometry translation is performed in the
2561		 * initialization routine of the SCSI BIOS and the result
2562		 * stored in host memory.  If the translation is available
2563		 * in host memory, use it.  If not, rely on the default
2564		 * translation the device driver performs.
2565		 * For sparc64, we may need adjust the geometry of large
2566		 * disks in order to fit the limitations of the 16-bit
2567		 * fields of the VTOC8 disk label.
2568		 */
2569		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2570			start_ccb->ccb_h.status = CAM_REQ_CMP;
2571			break;
2572		}
2573#endif
2574		goto call_sim;
2575	case XPT_ABORT:
2576	{
2577		union ccb* abort_ccb;
2578
2579		abort_ccb = start_ccb->cab.abort_ccb;
2580		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2581
2582			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2583				struct cam_ccbq *ccbq;
2584				struct cam_ed *device;
2585
2586				device = abort_ccb->ccb_h.path->device;
2587				ccbq = &device->ccbq;
2588				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2589				abort_ccb->ccb_h.status =
2590				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2591				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2592				xpt_done(abort_ccb);
2593				start_ccb->ccb_h.status = CAM_REQ_CMP;
2594				break;
2595			}
2596			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2597			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2598				/*
2599				 * We've caught this ccb en route to
2600				 * the SIM.  Flag it for abort and the
2601				 * SIM will do so just before starting
2602				 * real work on the CCB.
2603				 */
2604				abort_ccb->ccb_h.status =
2605				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2606				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2607				start_ccb->ccb_h.status = CAM_REQ_CMP;
2608				break;
2609			}
2610		}
2611		if (XPT_FC_IS_QUEUED(abort_ccb)
2612		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2613			/*
2614			 * It's already completed but waiting
2615			 * for our SWI to get to it.
2616			 */
2617			start_ccb->ccb_h.status = CAM_UA_ABORT;
2618			break;
2619		}
2620		/*
2621		 * If we weren't able to take care of the abort request
2622		 * in the XPT, pass the request down to the SIM for processing.
2623		 */
2624	}
2625	/* FALLTHROUGH */
2626	case XPT_ACCEPT_TARGET_IO:
2627	case XPT_EN_LUN:
2628	case XPT_IMMED_NOTIFY:
2629	case XPT_NOTIFY_ACK:
2630	case XPT_RESET_BUS:
2631	case XPT_IMMEDIATE_NOTIFY:
2632	case XPT_NOTIFY_ACKNOWLEDGE:
2633	case XPT_GET_SIM_KNOB:
2634	case XPT_SET_SIM_KNOB:
2635	case XPT_GET_TRAN_SETTINGS:
2636	case XPT_SET_TRAN_SETTINGS:
2637	case XPT_PATH_INQ:
2638call_sim:
2639		sim = path->bus->sim;
2640		lock = (mtx_owned(sim->mtx) == 0);
2641		if (lock)
2642			CAM_SIM_LOCK(sim);
2643		(*(sim->sim_action))(sim, start_ccb);
2644		if (lock)
2645			CAM_SIM_UNLOCK(sim);
2646		break;
2647	case XPT_PATH_STATS:
2648		start_ccb->cpis.last_reset = path->bus->last_reset;
2649		start_ccb->ccb_h.status = CAM_REQ_CMP;
2650		break;
2651	case XPT_GDEV_TYPE:
2652	{
2653		struct cam_ed *dev;
2654
2655		dev = path->device;
2656		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2657			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2658		} else {
2659			struct ccb_getdev *cgd;
2660
2661			cgd = &start_ccb->cgd;
2662			cgd->protocol = dev->protocol;
2663			cgd->inq_data = dev->inq_data;
2664			cgd->ident_data = dev->ident_data;
2665			cgd->inq_flags = dev->inq_flags;
2666			cgd->ccb_h.status = CAM_REQ_CMP;
2667			cgd->serial_num_len = dev->serial_num_len;
2668			if ((dev->serial_num_len > 0)
2669			 && (dev->serial_num != NULL))
2670				bcopy(dev->serial_num, cgd->serial_num,
2671				      dev->serial_num_len);
2672		}
2673		break;
2674	}
2675	case XPT_GDEV_STATS:
2676	{
2677		struct ccb_getdevstats *cgds = &start_ccb->cgds;
2678		struct cam_ed *dev = path->device;
2679		struct cam_eb *bus = path->bus;
2680		struct cam_et *tar = path->target;
2681		struct cam_devq *devq = bus->sim->devq;
2682
2683		mtx_lock(&devq->send_mtx);
2684		cgds->dev_openings = dev->ccbq.dev_openings;
2685		cgds->dev_active = dev->ccbq.dev_active;
2686		cgds->allocated = dev->ccbq.allocated;
2687		cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2688		cgds->held = cgds->allocated - cgds->dev_active - cgds->queued;
2689		cgds->last_reset = tar->last_reset;
2690		cgds->maxtags = dev->maxtags;
2691		cgds->mintags = dev->mintags;
2692		if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2693			cgds->last_reset = bus->last_reset;
2694		mtx_unlock(&devq->send_mtx);
2695		cgds->ccb_h.status = CAM_REQ_CMP;
2696		break;
2697	}
2698	case XPT_GDEVLIST:
2699	{
2700		struct cam_periph	*nperiph;
2701		struct periph_list	*periph_head;
2702		struct ccb_getdevlist	*cgdl;
2703		u_int			i;
2704		struct cam_ed		*device;
2705		int			found;
2706
2707
2708		found = 0;
2709
2710		/*
2711		 * Don't want anyone mucking with our data.
2712		 */
2713		device = path->device;
2714		periph_head = &device->periphs;
2715		cgdl = &start_ccb->cgdl;
2716
2717		/*
2718		 * Check and see if the list has changed since the user
2719		 * last requested a list member.  If so, tell them that the
2720		 * list has changed, and therefore they need to start over
2721		 * from the beginning.
2722		 */
2723		if ((cgdl->index != 0) &&
2724		    (cgdl->generation != device->generation)) {
2725			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2726			break;
2727		}
2728
2729		/*
2730		 * Traverse the list of peripherals and attempt to find
2731		 * the requested peripheral.
2732		 */
2733		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2734		     (nperiph != NULL) && (i <= cgdl->index);
2735		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2736			if (i == cgdl->index) {
2737				strncpy(cgdl->periph_name,
2738					nperiph->periph_name,
2739					DEV_IDLEN);
2740				cgdl->unit_number = nperiph->unit_number;
2741				found = 1;
2742			}
2743		}
2744		if (found == 0) {
2745			cgdl->status = CAM_GDEVLIST_ERROR;
2746			break;
2747		}
2748
2749		if (nperiph == NULL)
2750			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2751		else
2752			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2753
2754		cgdl->index++;
2755		cgdl->generation = device->generation;
2756
2757		cgdl->ccb_h.status = CAM_REQ_CMP;
2758		break;
2759	}
2760	case XPT_DEV_MATCH:
2761	{
2762		dev_pos_type position_type;
2763		struct ccb_dev_match *cdm;
2764
2765		cdm = &start_ccb->cdm;
2766
2767		/*
2768		 * There are two ways of getting at information in the EDT.
2769		 * The first way is via the primary EDT tree.  It starts
2770		 * with a list of busses, then a list of targets on a bus,
2771		 * then devices/luns on a target, and then peripherals on a
2772		 * device/lun.  The "other" way is by the peripheral driver
2773		 * lists.  The peripheral driver lists are organized by
2774		 * peripheral driver.  (obviously)  So it makes sense to
2775		 * use the peripheral driver list if the user is looking
2776		 * for something like "da1", or all "da" devices.  If the
2777		 * user is looking for something on a particular bus/target
2778		 * or lun, it's generally better to go through the EDT tree.
2779		 */
2780
2781		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2782			position_type = cdm->pos.position_type;
2783		else {
2784			u_int i;
2785
2786			position_type = CAM_DEV_POS_NONE;
2787
2788			for (i = 0; i < cdm->num_patterns; i++) {
2789				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2790				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2791					position_type = CAM_DEV_POS_EDT;
2792					break;
2793				}
2794			}
2795
2796			if (cdm->num_patterns == 0)
2797				position_type = CAM_DEV_POS_EDT;
2798			else if (position_type == CAM_DEV_POS_NONE)
2799				position_type = CAM_DEV_POS_PDRV;
2800		}
2801
2802		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2803		case CAM_DEV_POS_EDT:
2804			xptedtmatch(cdm);
2805			break;
2806		case CAM_DEV_POS_PDRV:
2807			xptperiphlistmatch(cdm);
2808			break;
2809		default:
2810			cdm->status = CAM_DEV_MATCH_ERROR;
2811			break;
2812		}
2813
2814		if (cdm->status == CAM_DEV_MATCH_ERROR)
2815			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2816		else
2817			start_ccb->ccb_h.status = CAM_REQ_CMP;
2818
2819		break;
2820	}
2821	case XPT_SASYNC_CB:
2822	{
2823		struct ccb_setasync *csa;
2824		struct async_node *cur_entry;
2825		struct async_list *async_head;
2826		u_int32_t added;
2827
2828		csa = &start_ccb->csa;
2829		added = csa->event_enable;
2830		async_head = &path->device->asyncs;
2831
2832		/*
2833		 * If there is already an entry for us, simply
2834		 * update it.
2835		 */
2836		cur_entry = SLIST_FIRST(async_head);
2837		while (cur_entry != NULL) {
2838			if ((cur_entry->callback_arg == csa->callback_arg)
2839			 && (cur_entry->callback == csa->callback))
2840				break;
2841			cur_entry = SLIST_NEXT(cur_entry, links);
2842		}
2843
2844		if (cur_entry != NULL) {
2845		 	/*
2846			 * If the request has no flags set,
2847			 * remove the entry.
2848			 */
2849			added &= ~cur_entry->event_enable;
2850			if (csa->event_enable == 0) {
2851				SLIST_REMOVE(async_head, cur_entry,
2852					     async_node, links);
2853				xpt_release_device(path->device);
2854				free(cur_entry, M_CAMXPT);
2855			} else {
2856				cur_entry->event_enable = csa->event_enable;
2857			}
2858			csa->event_enable = added;
2859		} else {
2860			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2861					   M_NOWAIT);
2862			if (cur_entry == NULL) {
2863				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2864				break;
2865			}
2866			cur_entry->event_enable = csa->event_enable;
2867			cur_entry->event_lock =
2868			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2869			cur_entry->callback_arg = csa->callback_arg;
2870			cur_entry->callback = csa->callback;
2871			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2872			xpt_acquire_device(path->device);
2873		}
2874		start_ccb->ccb_h.status = CAM_REQ_CMP;
2875		break;
2876	}
2877	case XPT_REL_SIMQ:
2878	{
2879		struct ccb_relsim *crs;
2880		struct cam_ed *dev;
2881
2882		crs = &start_ccb->crs;
2883		dev = path->device;
2884		if (dev == NULL) {
2885
2886			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2887			break;
2888		}
2889
2890		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2891
2892			/* Don't ever go below one opening */
2893			if (crs->openings > 0) {
2894				xpt_dev_ccbq_resize(path, crs->openings);
2895				if (bootverbose) {
2896					xpt_print(path,
2897					    "number of openings is now %d\n",
2898					    crs->openings);
2899				}
2900			}
2901		}
2902
2903		mtx_lock(&dev->sim->devq->send_mtx);
2904		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2905
2906			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2907
2908				/*
2909				 * Just extend the old timeout and decrement
2910				 * the freeze count so that a single timeout
2911				 * is sufficient for releasing the queue.
2912				 */
2913				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2914				callout_stop(&dev->callout);
2915			} else {
2916
2917				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2918			}
2919
2920			callout_reset_sbt(&dev->callout,
2921			    SBT_1MS * crs->release_timeout, 0,
2922			    xpt_release_devq_timeout, dev, 0);
2923
2924			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2925
2926		}
2927
2928		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2929
2930			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2931				/*
2932				 * Decrement the freeze count so that a single
2933				 * completion is still sufficient to unfreeze
2934				 * the queue.
2935				 */
2936				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2937			} else {
2938
2939				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2940				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2941			}
2942		}
2943
2944		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2945
2946			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2947			 || (dev->ccbq.dev_active == 0)) {
2948
2949				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2950			} else {
2951
2952				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2953				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2954			}
2955		}
2956		mtx_unlock(&dev->sim->devq->send_mtx);
2957
2958		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2959			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2960		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2961		start_ccb->ccb_h.status = CAM_REQ_CMP;
2962		break;
2963	}
2964	case XPT_DEBUG: {
2965		struct cam_path *oldpath;
2966
2967		/* Check that all request bits are supported. */
2968		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2969			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2970			break;
2971		}
2972
2973		cam_dflags = CAM_DEBUG_NONE;
2974		if (cam_dpath != NULL) {
2975			oldpath = cam_dpath;
2976			cam_dpath = NULL;
2977			xpt_free_path(oldpath);
2978		}
2979		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2980			if (xpt_create_path(&cam_dpath, NULL,
2981					    start_ccb->ccb_h.path_id,
2982					    start_ccb->ccb_h.target_id,
2983					    start_ccb->ccb_h.target_lun) !=
2984					    CAM_REQ_CMP) {
2985				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2986			} else {
2987				cam_dflags = start_ccb->cdbg.flags;
2988				start_ccb->ccb_h.status = CAM_REQ_CMP;
2989				xpt_print(cam_dpath, "debugging flags now %x\n",
2990				    cam_dflags);
2991			}
2992		} else
2993			start_ccb->ccb_h.status = CAM_REQ_CMP;
2994		break;
2995	}
2996	case XPT_NOOP:
2997		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2998			xpt_freeze_devq(path, 1);
2999		start_ccb->ccb_h.status = CAM_REQ_CMP;
3000		break;
3001	case XPT_REPROBE_LUN:
3002		xpt_async(AC_INQ_CHANGED, path, NULL);
3003		start_ccb->ccb_h.status = CAM_REQ_CMP;
3004		xpt_done(start_ccb);
3005		break;
3006	default:
3007	case XPT_SDEV_TYPE:
3008	case XPT_TERM_IO:
3009	case XPT_ENG_INQ:
3010		/* XXX Implement */
3011		printf("%s: CCB type %#x not supported\n", __func__,
3012		       start_ccb->ccb_h.func_code);
3013		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3014		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3015			xpt_done(start_ccb);
3016		}
3017		break;
3018	}
3019}
3020
3021void
3022xpt_polled_action(union ccb *start_ccb)
3023{
3024	u_int32_t timeout;
3025	struct	  cam_sim *sim;
3026	struct	  cam_devq *devq;
3027	struct	  cam_ed *dev;
3028
3029	timeout = start_ccb->ccb_h.timeout * 10;
3030	sim = start_ccb->ccb_h.path->bus->sim;
3031	devq = sim->devq;
3032	dev = start_ccb->ccb_h.path->device;
3033
3034	mtx_unlock(&dev->device_mtx);
3035
3036	/*
3037	 * Steal an opening so that no other queued requests
3038	 * can get it before us while we simulate interrupts.
3039	 */
3040	mtx_lock(&devq->send_mtx);
3041	dev->ccbq.dev_openings--;
3042	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3043	    (--timeout > 0)) {
3044		mtx_unlock(&devq->send_mtx);
3045		DELAY(100);
3046		CAM_SIM_LOCK(sim);
3047		(*(sim->sim_poll))(sim);
3048		CAM_SIM_UNLOCK(sim);
3049		camisr_runqueue();
3050		mtx_lock(&devq->send_mtx);
3051	}
3052	dev->ccbq.dev_openings++;
3053	mtx_unlock(&devq->send_mtx);
3054
3055	if (timeout != 0) {
3056		xpt_action(start_ccb);
3057		while(--timeout > 0) {
3058			CAM_SIM_LOCK(sim);
3059			(*(sim->sim_poll))(sim);
3060			CAM_SIM_UNLOCK(sim);
3061			camisr_runqueue();
3062			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3063			    != CAM_REQ_INPROG)
3064				break;
3065			DELAY(100);
3066		}
3067		if (timeout == 0) {
3068			/*
3069			 * XXX Is it worth adding a sim_timeout entry
3070			 * point so we can attempt recovery?  If
3071			 * this is only used for dumps, I don't think
3072			 * it is.
3073			 */
3074			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3075		}
3076	} else {
3077		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3078	}
3079
3080	mtx_lock(&dev->device_mtx);
3081}
3082
3083/*
3084 * Schedule a peripheral driver to receive a ccb when its
3085 * target device has space for more transactions.
3086 */
3087void
3088xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3089{
3090
3091	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3092	cam_periph_assert(periph, MA_OWNED);
3093	if (new_priority < periph->scheduled_priority) {
3094		periph->scheduled_priority = new_priority;
3095		xpt_run_allocq(periph, 0);
3096	}
3097}
3098
3099
3100/*
3101 * Schedule a device to run on a given queue.
3102 * If the device was inserted as a new entry on the queue,
3103 * return 1 meaning the device queue should be run. If we
3104 * were already queued, implying someone else has already
3105 * started the queue, return 0 so the caller doesn't attempt
3106 * to run the queue.
3107 */
3108static int
3109xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3110		 u_int32_t new_priority)
3111{
3112	int retval;
3113	u_int32_t old_priority;
3114
3115	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3116
3117	old_priority = pinfo->priority;
3118
3119	/*
3120	 * Are we already queued?
3121	 */
3122	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3123		/* Simply reorder based on new priority */
3124		if (new_priority < old_priority) {
3125			camq_change_priority(queue, pinfo->index,
3126					     new_priority);
3127			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3128					("changed priority to %d\n",
3129					 new_priority));
3130			retval = 1;
3131		} else
3132			retval = 0;
3133	} else {
3134		/* New entry on the queue */
3135		if (new_priority < old_priority)
3136			pinfo->priority = new_priority;
3137
3138		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3139				("Inserting onto queue\n"));
3140		pinfo->generation = ++queue->generation;
3141		camq_insert(queue, pinfo);
3142		retval = 1;
3143	}
3144	return (retval);
3145}
3146
3147static void
3148xpt_run_allocq_task(void *context, int pending)
3149{
3150	struct cam_periph *periph = context;
3151
3152	cam_periph_lock(periph);
3153	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3154	xpt_run_allocq(periph, 1);
3155	cam_periph_unlock(periph);
3156	cam_periph_release(periph);
3157}
3158
3159static void
3160xpt_run_allocq(struct cam_periph *periph, int sleep)
3161{
3162	struct cam_ed	*device;
3163	union ccb	*ccb;
3164	uint32_t	 prio;
3165
3166	cam_periph_assert(periph, MA_OWNED);
3167	if (periph->periph_allocating)
3168		return;
3169	periph->periph_allocating = 1;
3170	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3171	device = periph->path->device;
3172	ccb = NULL;
3173restart:
3174	while ((prio = min(periph->scheduled_priority,
3175	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3176	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3177	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3178
3179		if (ccb == NULL &&
3180		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3181			if (sleep) {
3182				ccb = xpt_get_ccb(periph);
3183				goto restart;
3184			}
3185			if (periph->flags & CAM_PERIPH_RUN_TASK)
3186				break;
3187			cam_periph_doacquire(periph);
3188			periph->flags |= CAM_PERIPH_RUN_TASK;
3189			taskqueue_enqueue(xsoftc.xpt_taskq,
3190			    &periph->periph_run_task);
3191			break;
3192		}
3193		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3194		if (prio == periph->immediate_priority) {
3195			periph->immediate_priority = CAM_PRIORITY_NONE;
3196			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3197					("waking cam_periph_getccb()\n"));
3198			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3199					  periph_links.sle);
3200			wakeup(&periph->ccb_list);
3201		} else {
3202			periph->scheduled_priority = CAM_PRIORITY_NONE;
3203			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3204					("calling periph_start()\n"));
3205			periph->periph_start(periph, ccb);
3206		}
3207		ccb = NULL;
3208	}
3209	if (ccb != NULL)
3210		xpt_release_ccb(ccb);
3211	periph->periph_allocating = 0;
3212}
3213
3214static void
3215xpt_run_devq(struct cam_devq *devq)
3216{
3217	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3218	int lock;
3219
3220	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3221
3222	devq->send_queue.qfrozen_cnt++;
3223	while ((devq->send_queue.entries > 0)
3224	    && (devq->send_openings > 0)
3225	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3226		struct	cam_ed *device;
3227		union ccb *work_ccb;
3228		struct	cam_sim *sim;
3229
3230		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3231							   CAMQ_HEAD);
3232		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3233				("running device %p\n", device));
3234
3235		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3236		if (work_ccb == NULL) {
3237			printf("device on run queue with no ccbs???\n");
3238			continue;
3239		}
3240
3241		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3242
3243			mtx_lock(&xsoftc.xpt_highpower_lock);
3244		 	if (xsoftc.num_highpower <= 0) {
3245				/*
3246				 * We got a high power command, but we
3247				 * don't have any available slots.  Freeze
3248				 * the device queue until we have a slot
3249				 * available.
3250				 */
3251				xpt_freeze_devq_device(device, 1);
3252				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3253						   highpowerq_entry);
3254
3255				mtx_unlock(&xsoftc.xpt_highpower_lock);
3256				continue;
3257			} else {
3258				/*
3259				 * Consume a high power slot while
3260				 * this ccb runs.
3261				 */
3262				xsoftc.num_highpower--;
3263			}
3264			mtx_unlock(&xsoftc.xpt_highpower_lock);
3265		}
3266		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3267		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3268		devq->send_openings--;
3269		devq->send_active++;
3270		xpt_schedule_devq(devq, device);
3271		mtx_unlock(&devq->send_mtx);
3272
3273		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3274			/*
3275			 * The client wants to freeze the queue
3276			 * after this CCB is sent.
3277			 */
3278			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3279		}
3280
3281		/* In Target mode, the peripheral driver knows best... */
3282		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3283			if ((device->inq_flags & SID_CmdQue) != 0
3284			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3285				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3286			else
3287				/*
3288				 * Clear this in case of a retried CCB that
3289				 * failed due to a rejected tag.
3290				 */
3291				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3292		}
3293
3294		switch (work_ccb->ccb_h.func_code) {
3295		case XPT_SCSI_IO:
3296			CAM_DEBUG(work_ccb->ccb_h.path,
3297			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3298			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3299					  &device->inq_data),
3300			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3301					     cdb_str, sizeof(cdb_str))));
3302			break;
3303		case XPT_ATA_IO:
3304			CAM_DEBUG(work_ccb->ccb_h.path,
3305			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3306			     ata_op_string(&work_ccb->ataio.cmd),
3307			     ata_cmd_string(&work_ccb->ataio.cmd,
3308					    cdb_str, sizeof(cdb_str))));
3309			break;
3310		default:
3311			break;
3312		}
3313
3314		/*
3315		 * Device queues can be shared among multiple SIM instances
3316		 * that reside on different busses.  Use the SIM from the
3317		 * queued device, rather than the one from the calling bus.
3318		 */
3319		sim = device->sim;
3320		lock = (mtx_owned(sim->mtx) == 0);
3321		if (lock)
3322			CAM_SIM_LOCK(sim);
3323		(*(sim->sim_action))(sim, work_ccb);
3324		if (lock)
3325			CAM_SIM_UNLOCK(sim);
3326		mtx_lock(&devq->send_mtx);
3327	}
3328	devq->send_queue.qfrozen_cnt--;
3329}
3330
3331/*
3332 * This function merges stuff from the slave ccb into the master ccb, while
3333 * keeping important fields in the master ccb constant.
3334 */
3335void
3336xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3337{
3338
3339	/*
3340	 * Pull fields that are valid for peripheral drivers to set
3341	 * into the master CCB along with the CCB "payload".
3342	 */
3343	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3344	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3345	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3346	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3347	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3348	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3349}
3350
3351void
3352xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3353		    u_int32_t priority, u_int32_t flags)
3354{
3355
3356	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3357	ccb_h->pinfo.priority = priority;
3358	ccb_h->path = path;
3359	ccb_h->path_id = path->bus->path_id;
3360	if (path->target)
3361		ccb_h->target_id = path->target->target_id;
3362	else
3363		ccb_h->target_id = CAM_TARGET_WILDCARD;
3364	if (path->device) {
3365		ccb_h->target_lun = path->device->lun_id;
3366		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3367	} else {
3368		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3369	}
3370	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3371	ccb_h->flags = flags;
3372	ccb_h->xflags = 0;
3373}
3374
3375void
3376xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3377{
3378	xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3379}
3380
3381/* Path manipulation functions */
3382cam_status
3383xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3384		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3385{
3386	struct	   cam_path *path;
3387	cam_status status;
3388
3389	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3390
3391	if (path == NULL) {
3392		status = CAM_RESRC_UNAVAIL;
3393		return(status);
3394	}
3395	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3396	if (status != CAM_REQ_CMP) {
3397		free(path, M_CAMPATH);
3398		path = NULL;
3399	}
3400	*new_path_ptr = path;
3401	return (status);
3402}
3403
3404cam_status
3405xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3406			 struct cam_periph *periph, path_id_t path_id,
3407			 target_id_t target_id, lun_id_t lun_id)
3408{
3409
3410	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3411	    lun_id));
3412}
3413
3414cam_status
3415xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3416		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3417{
3418	struct	     cam_eb *bus;
3419	struct	     cam_et *target;
3420	struct	     cam_ed *device;
3421	cam_status   status;
3422
3423	status = CAM_REQ_CMP;	/* Completed without error */
3424	target = NULL;		/* Wildcarded */
3425	device = NULL;		/* Wildcarded */
3426
3427	/*
3428	 * We will potentially modify the EDT, so block interrupts
3429	 * that may attempt to create cam paths.
3430	 */
3431	bus = xpt_find_bus(path_id);
3432	if (bus == NULL) {
3433		status = CAM_PATH_INVALID;
3434	} else {
3435		xpt_lock_buses();
3436		mtx_lock(&bus->eb_mtx);
3437		target = xpt_find_target(bus, target_id);
3438		if (target == NULL) {
3439			/* Create one */
3440			struct cam_et *new_target;
3441
3442			new_target = xpt_alloc_target(bus, target_id);
3443			if (new_target == NULL) {
3444				status = CAM_RESRC_UNAVAIL;
3445			} else {
3446				target = new_target;
3447			}
3448		}
3449		xpt_unlock_buses();
3450		if (target != NULL) {
3451			device = xpt_find_device(target, lun_id);
3452			if (device == NULL) {
3453				/* Create one */
3454				struct cam_ed *new_device;
3455
3456				new_device =
3457				    (*(bus->xport->alloc_device))(bus,
3458								      target,
3459								      lun_id);
3460				if (new_device == NULL) {
3461					status = CAM_RESRC_UNAVAIL;
3462				} else {
3463					device = new_device;
3464				}
3465			}
3466		}
3467		mtx_unlock(&bus->eb_mtx);
3468	}
3469
3470	/*
3471	 * Only touch the user's data if we are successful.
3472	 */
3473	if (status == CAM_REQ_CMP) {
3474		new_path->periph = perph;
3475		new_path->bus = bus;
3476		new_path->target = target;
3477		new_path->device = device;
3478		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3479	} else {
3480		if (device != NULL)
3481			xpt_release_device(device);
3482		if (target != NULL)
3483			xpt_release_target(target);
3484		if (bus != NULL)
3485			xpt_release_bus(bus);
3486	}
3487	return (status);
3488}
3489
3490cam_status
3491xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3492{
3493	struct	   cam_path *new_path;
3494
3495	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3496	if (new_path == NULL)
3497		return(CAM_RESRC_UNAVAIL);
3498	xpt_copy_path(new_path, path);
3499	*new_path_ptr = new_path;
3500	return (CAM_REQ_CMP);
3501}
3502
3503void
3504xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3505{
3506
3507	*new_path = *path;
3508	if (path->bus != NULL)
3509		xpt_acquire_bus(path->bus);
3510	if (path->target != NULL)
3511		xpt_acquire_target(path->target);
3512	if (path->device != NULL)
3513		xpt_acquire_device(path->device);
3514}
3515
3516void
3517xpt_release_path(struct cam_path *path)
3518{
3519	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3520	if (path->device != NULL) {
3521		xpt_release_device(path->device);
3522		path->device = NULL;
3523	}
3524	if (path->target != NULL) {
3525		xpt_release_target(path->target);
3526		path->target = NULL;
3527	}
3528	if (path->bus != NULL) {
3529		xpt_release_bus(path->bus);
3530		path->bus = NULL;
3531	}
3532}
3533
3534void
3535xpt_free_path(struct cam_path *path)
3536{
3537
3538	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3539	xpt_release_path(path);
3540	free(path, M_CAMPATH);
3541}
3542
3543void
3544xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3545    uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3546{
3547
3548	xpt_lock_buses();
3549	if (bus_ref) {
3550		if (path->bus)
3551			*bus_ref = path->bus->refcount;
3552		else
3553			*bus_ref = 0;
3554	}
3555	if (periph_ref) {
3556		if (path->periph)
3557			*periph_ref = path->periph->refcount;
3558		else
3559			*periph_ref = 0;
3560	}
3561	xpt_unlock_buses();
3562	if (target_ref) {
3563		if (path->target)
3564			*target_ref = path->target->refcount;
3565		else
3566			*target_ref = 0;
3567	}
3568	if (device_ref) {
3569		if (path->device)
3570			*device_ref = path->device->refcount;
3571		else
3572			*device_ref = 0;
3573	}
3574}
3575
3576/*
3577 * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3578 * in path1, 2 for match with wildcards in path2.
3579 */
3580int
3581xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3582{
3583	int retval = 0;
3584
3585	if (path1->bus != path2->bus) {
3586		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3587			retval = 1;
3588		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3589			retval = 2;
3590		else
3591			return (-1);
3592	}
3593	if (path1->target != path2->target) {
3594		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3595			if (retval == 0)
3596				retval = 1;
3597		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3598			retval = 2;
3599		else
3600			return (-1);
3601	}
3602	if (path1->device != path2->device) {
3603		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3604			if (retval == 0)
3605				retval = 1;
3606		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3607			retval = 2;
3608		else
3609			return (-1);
3610	}
3611	return (retval);
3612}
3613
3614int
3615xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3616{
3617	int retval = 0;
3618
3619	if (path->bus != dev->target->bus) {
3620		if (path->bus->path_id == CAM_BUS_WILDCARD)
3621			retval = 1;
3622		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3623			retval = 2;
3624		else
3625			return (-1);
3626	}
3627	if (path->target != dev->target) {
3628		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3629			if (retval == 0)
3630				retval = 1;
3631		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3632			retval = 2;
3633		else
3634			return (-1);
3635	}
3636	if (path->device != dev) {
3637		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3638			if (retval == 0)
3639				retval = 1;
3640		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3641			retval = 2;
3642		else
3643			return (-1);
3644	}
3645	return (retval);
3646}
3647
3648void
3649xpt_print_path(struct cam_path *path)
3650{
3651
3652	if (path == NULL)
3653		printf("(nopath): ");
3654	else {
3655		if (path->periph != NULL)
3656			printf("(%s%d:", path->periph->periph_name,
3657			       path->periph->unit_number);
3658		else
3659			printf("(noperiph:");
3660
3661		if (path->bus != NULL)
3662			printf("%s%d:%d:", path->bus->sim->sim_name,
3663			       path->bus->sim->unit_number,
3664			       path->bus->sim->bus_id);
3665		else
3666			printf("nobus:");
3667
3668		if (path->target != NULL)
3669			printf("%d:", path->target->target_id);
3670		else
3671			printf("X:");
3672
3673		if (path->device != NULL)
3674			printf("%jx): ", (uintmax_t)path->device->lun_id);
3675		else
3676			printf("X): ");
3677	}
3678}
3679
3680void
3681xpt_print_device(struct cam_ed *device)
3682{
3683
3684	if (device == NULL)
3685		printf("(nopath): ");
3686	else {
3687		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3688		       device->sim->unit_number,
3689		       device->sim->bus_id,
3690		       device->target->target_id,
3691		       (uintmax_t)device->lun_id);
3692	}
3693}
3694
3695void
3696xpt_print(struct cam_path *path, const char *fmt, ...)
3697{
3698	va_list ap;
3699	xpt_print_path(path);
3700	va_start(ap, fmt);
3701	vprintf(fmt, ap);
3702	va_end(ap);
3703}
3704
3705int
3706xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3707{
3708	struct sbuf sb;
3709
3710	sbuf_new(&sb, str, str_len, 0);
3711
3712	if (path == NULL)
3713		sbuf_printf(&sb, "(nopath): ");
3714	else {
3715		if (path->periph != NULL)
3716			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3717				    path->periph->unit_number);
3718		else
3719			sbuf_printf(&sb, "(noperiph:");
3720
3721		if (path->bus != NULL)
3722			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3723				    path->bus->sim->unit_number,
3724				    path->bus->sim->bus_id);
3725		else
3726			sbuf_printf(&sb, "nobus:");
3727
3728		if (path->target != NULL)
3729			sbuf_printf(&sb, "%d:", path->target->target_id);
3730		else
3731			sbuf_printf(&sb, "X:");
3732
3733		if (path->device != NULL)
3734			sbuf_printf(&sb, "%jx): ",
3735			    (uintmax_t)path->device->lun_id);
3736		else
3737			sbuf_printf(&sb, "X): ");
3738	}
3739	sbuf_finish(&sb);
3740
3741	return(sbuf_len(&sb));
3742}
3743
3744path_id_t
3745xpt_path_path_id(struct cam_path *path)
3746{
3747	return(path->bus->path_id);
3748}
3749
3750target_id_t
3751xpt_path_target_id(struct cam_path *path)
3752{
3753	if (path->target != NULL)
3754		return (path->target->target_id);
3755	else
3756		return (CAM_TARGET_WILDCARD);
3757}
3758
3759lun_id_t
3760xpt_path_lun_id(struct cam_path *path)
3761{
3762	if (path->device != NULL)
3763		return (path->device->lun_id);
3764	else
3765		return (CAM_LUN_WILDCARD);
3766}
3767
3768struct cam_sim *
3769xpt_path_sim(struct cam_path *path)
3770{
3771
3772	return (path->bus->sim);
3773}
3774
3775struct cam_periph*
3776xpt_path_periph(struct cam_path *path)
3777{
3778
3779	return (path->periph);
3780}
3781
3782int
3783xpt_path_legacy_ata_id(struct cam_path *path)
3784{
3785	struct cam_eb *bus;
3786	int bus_id;
3787
3788	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3789	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3790	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3791	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3792		return (-1);
3793
3794	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3795	    path->bus->sim->unit_number < 2) {
3796		bus_id = path->bus->sim->unit_number;
3797	} else {
3798		bus_id = 2;
3799		xpt_lock_buses();
3800		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3801			if (bus == path->bus)
3802				break;
3803			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3804			     bus->sim->unit_number >= 2) ||
3805			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3806			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3807			    strcmp(bus->sim->sim_name, "siisch") == 0)
3808				bus_id++;
3809		}
3810		xpt_unlock_buses();
3811	}
3812	if (path->target != NULL) {
3813		if (path->target->target_id < 2)
3814			return (bus_id * 2 + path->target->target_id);
3815		else
3816			return (-1);
3817	} else
3818		return (bus_id * 2);
3819}
3820
3821/*
3822 * Release a CAM control block for the caller.  Remit the cost of the structure
3823 * to the device referenced by the path.  If the this device had no 'credits'
3824 * and peripheral drivers have registered async callbacks for this notification
3825 * call them now.
3826 */
3827void
3828xpt_release_ccb(union ccb *free_ccb)
3829{
3830	struct	 cam_ed *device;
3831	struct	 cam_periph *periph;
3832
3833	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3834	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3835	device = free_ccb->ccb_h.path->device;
3836	periph = free_ccb->ccb_h.path->periph;
3837
3838	xpt_free_ccb(free_ccb);
3839	periph->periph_allocated--;
3840	cam_ccbq_release_opening(&device->ccbq);
3841	xpt_run_allocq(periph, 0);
3842}
3843
3844/* Functions accessed by SIM drivers */
3845
3846static struct xpt_xport xport_default = {
3847	.alloc_device = xpt_alloc_device_default,
3848	.action = xpt_action_default,
3849	.async = xpt_dev_async_default,
3850};
3851
3852/*
3853 * A sim structure, listing the SIM entry points and instance
3854 * identification info is passed to xpt_bus_register to hook the SIM
3855 * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3856 * for this new bus and places it in the array of busses and assigns
3857 * it a path_id.  The path_id may be influenced by "hard wiring"
3858 * information specified by the user.  Once interrupt services are
3859 * available, the bus will be probed.
3860 */
3861int32_t
3862xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3863{
3864	struct cam_eb *new_bus;
3865	struct cam_eb *old_bus;
3866	struct ccb_pathinq cpi;
3867	struct cam_path *path;
3868	cam_status status;
3869
3870	mtx_assert(sim->mtx, MA_OWNED);
3871
3872	sim->bus_id = bus;
3873	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3874					  M_CAMXPT, M_NOWAIT|M_ZERO);
3875	if (new_bus == NULL) {
3876		/* Couldn't satisfy request */
3877		return (CAM_RESRC_UNAVAIL);
3878	}
3879
3880	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3881	TAILQ_INIT(&new_bus->et_entries);
3882	cam_sim_hold(sim);
3883	new_bus->sim = sim;
3884	timevalclear(&new_bus->last_reset);
3885	new_bus->flags = 0;
3886	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3887	new_bus->generation = 0;
3888
3889	xpt_lock_buses();
3890	sim->path_id = new_bus->path_id =
3891	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3892	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3893	while (old_bus != NULL
3894	    && old_bus->path_id < new_bus->path_id)
3895		old_bus = TAILQ_NEXT(old_bus, links);
3896	if (old_bus != NULL)
3897		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3898	else
3899		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3900	xsoftc.bus_generation++;
3901	xpt_unlock_buses();
3902
3903	/*
3904	 * Set a default transport so that a PATH_INQ can be issued to
3905	 * the SIM.  This will then allow for probing and attaching of
3906	 * a more appropriate transport.
3907	 */
3908	new_bus->xport = &xport_default;
3909
3910	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3911				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3912	if (status != CAM_REQ_CMP) {
3913		xpt_release_bus(new_bus);
3914		free(path, M_CAMXPT);
3915		return (CAM_RESRC_UNAVAIL);
3916	}
3917
3918	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3919	cpi.ccb_h.func_code = XPT_PATH_INQ;
3920	xpt_action((union ccb *)&cpi);
3921
3922	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3923		switch (cpi.transport) {
3924		case XPORT_SPI:
3925		case XPORT_SAS:
3926		case XPORT_FC:
3927		case XPORT_USB:
3928		case XPORT_ISCSI:
3929		case XPORT_SRP:
3930		case XPORT_PPB:
3931			new_bus->xport = scsi_get_xport();
3932			break;
3933		case XPORT_ATA:
3934		case XPORT_SATA:
3935			new_bus->xport = ata_get_xport();
3936			break;
3937		default:
3938			new_bus->xport = &xport_default;
3939			break;
3940		}
3941	}
3942
3943	/* Notify interested parties */
3944	if (sim->path_id != CAM_XPT_PATH_ID) {
3945
3946		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3947		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3948			union	ccb *scan_ccb;
3949
3950			/* Initiate bus rescan. */
3951			scan_ccb = xpt_alloc_ccb_nowait();
3952			if (scan_ccb != NULL) {
3953				scan_ccb->ccb_h.path = path;
3954				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3955				scan_ccb->crcn.flags = 0;
3956				xpt_rescan(scan_ccb);
3957			} else {
3958				xpt_print(path,
3959					  "Can't allocate CCB to scan bus\n");
3960				xpt_free_path(path);
3961			}
3962		} else
3963			xpt_free_path(path);
3964	} else
3965		xpt_free_path(path);
3966	return (CAM_SUCCESS);
3967}
3968
3969int32_t
3970xpt_bus_deregister(path_id_t pathid)
3971{
3972	struct cam_path bus_path;
3973	cam_status status;
3974
3975	status = xpt_compile_path(&bus_path, NULL, pathid,
3976				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3977	if (status != CAM_REQ_CMP)
3978		return (status);
3979
3980	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3981	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3982
3983	/* Release the reference count held while registered. */
3984	xpt_release_bus(bus_path.bus);
3985	xpt_release_path(&bus_path);
3986
3987	return (CAM_REQ_CMP);
3988}
3989
3990static path_id_t
3991xptnextfreepathid(void)
3992{
3993	struct cam_eb *bus;
3994	path_id_t pathid;
3995	const char *strval;
3996
3997	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3998	pathid = 0;
3999	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4000retry:
4001	/* Find an unoccupied pathid */
4002	while (bus != NULL && bus->path_id <= pathid) {
4003		if (bus->path_id == pathid)
4004			pathid++;
4005		bus = TAILQ_NEXT(bus, links);
4006	}
4007
4008	/*
4009	 * Ensure that this pathid is not reserved for
4010	 * a bus that may be registered in the future.
4011	 */
4012	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4013		++pathid;
4014		/* Start the search over */
4015		goto retry;
4016	}
4017	return (pathid);
4018}
4019
4020static path_id_t
4021xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4022{
4023	path_id_t pathid;
4024	int i, dunit, val;
4025	char buf[32];
4026	const char *dname;
4027
4028	pathid = CAM_XPT_PATH_ID;
4029	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4030	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4031		return (pathid);
4032	i = 0;
4033	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4034		if (strcmp(dname, "scbus")) {
4035			/* Avoid a bit of foot shooting. */
4036			continue;
4037		}
4038		if (dunit < 0)		/* unwired?! */
4039			continue;
4040		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4041			if (sim_bus == val) {
4042				pathid = dunit;
4043				break;
4044			}
4045		} else if (sim_bus == 0) {
4046			/* Unspecified matches bus 0 */
4047			pathid = dunit;
4048			break;
4049		} else {
4050			printf("Ambiguous scbus configuration for %s%d "
4051			       "bus %d, cannot wire down.  The kernel "
4052			       "config entry for scbus%d should "
4053			       "specify a controller bus.\n"
4054			       "Scbus will be assigned dynamically.\n",
4055			       sim_name, sim_unit, sim_bus, dunit);
4056			break;
4057		}
4058	}
4059
4060	if (pathid == CAM_XPT_PATH_ID)
4061		pathid = xptnextfreepathid();
4062	return (pathid);
4063}
4064
4065static const char *
4066xpt_async_string(u_int32_t async_code)
4067{
4068
4069	switch (async_code) {
4070	case AC_BUS_RESET: return ("AC_BUS_RESET");
4071	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4072	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4073	case AC_SENT_BDR: return ("AC_SENT_BDR");
4074	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4075	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4076	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4077	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4078	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4079	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4080	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4081	case AC_CONTRACT: return ("AC_CONTRACT");
4082	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4083	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4084	}
4085	return ("AC_UNKNOWN");
4086}
4087
4088static int
4089xpt_async_size(u_int32_t async_code)
4090{
4091
4092	switch (async_code) {
4093	case AC_BUS_RESET: return (0);
4094	case AC_UNSOL_RESEL: return (0);
4095	case AC_SCSI_AEN: return (0);
4096	case AC_SENT_BDR: return (0);
4097	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4098	case AC_PATH_DEREGISTERED: return (0);
4099	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4100	case AC_LOST_DEVICE: return (0);
4101	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4102	case AC_INQ_CHANGED: return (0);
4103	case AC_GETDEV_CHANGED: return (0);
4104	case AC_CONTRACT: return (sizeof(struct ac_contract));
4105	case AC_ADVINFO_CHANGED: return (-1);
4106	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4107	}
4108	return (0);
4109}
4110
4111static int
4112xpt_async_process_dev(struct cam_ed *device, void *arg)
4113{
4114	union ccb *ccb = arg;
4115	struct cam_path *path = ccb->ccb_h.path;
4116	void *async_arg = ccb->casync.async_arg_ptr;
4117	u_int32_t async_code = ccb->casync.async_code;
4118	int relock;
4119
4120	if (path->device != device
4121	 && path->device->lun_id != CAM_LUN_WILDCARD
4122	 && device->lun_id != CAM_LUN_WILDCARD)
4123		return (1);
4124
4125	/*
4126	 * The async callback could free the device.
4127	 * If it is a broadcast async, it doesn't hold
4128	 * device reference, so take our own reference.
4129	 */
4130	xpt_acquire_device(device);
4131
4132	/*
4133	 * If async for specific device is to be delivered to
4134	 * the wildcard client, take the specific device lock.
4135	 * XXX: We may need a way for client to specify it.
4136	 */
4137	if ((device->lun_id == CAM_LUN_WILDCARD &&
4138	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4139	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4140	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4141	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4142	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4143		mtx_unlock(&device->device_mtx);
4144		xpt_path_lock(path);
4145		relock = 1;
4146	} else
4147		relock = 0;
4148
4149	(*(device->target->bus->xport->async))(async_code,
4150	    device->target->bus, device->target, device, async_arg);
4151	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4152
4153	if (relock) {
4154		xpt_path_unlock(path);
4155		mtx_lock(&device->device_mtx);
4156	}
4157	xpt_release_device(device);
4158	return (1);
4159}
4160
4161static int
4162xpt_async_process_tgt(struct cam_et *target, void *arg)
4163{
4164	union ccb *ccb = arg;
4165	struct cam_path *path = ccb->ccb_h.path;
4166
4167	if (path->target != target
4168	 && path->target->target_id != CAM_TARGET_WILDCARD
4169	 && target->target_id != CAM_TARGET_WILDCARD)
4170		return (1);
4171
4172	if (ccb->casync.async_code == AC_SENT_BDR) {
4173		/* Update our notion of when the last reset occurred */
4174		microtime(&target->last_reset);
4175	}
4176
4177	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4178}
4179
4180static void
4181xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4182{
4183	struct cam_eb *bus;
4184	struct cam_path *path;
4185	void *async_arg;
4186	u_int32_t async_code;
4187
4188	path = ccb->ccb_h.path;
4189	async_code = ccb->casync.async_code;
4190	async_arg = ccb->casync.async_arg_ptr;
4191	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4192	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4193	bus = path->bus;
4194
4195	if (async_code == AC_BUS_RESET) {
4196		/* Update our notion of when the last reset occurred */
4197		microtime(&bus->last_reset);
4198	}
4199
4200	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4201
4202	/*
4203	 * If this wasn't a fully wildcarded async, tell all
4204	 * clients that want all async events.
4205	 */
4206	if (bus != xpt_periph->path->bus) {
4207		xpt_path_lock(xpt_periph->path);
4208		xpt_async_process_dev(xpt_periph->path->device, ccb);
4209		xpt_path_unlock(xpt_periph->path);
4210	}
4211
4212	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4213		xpt_release_devq(path, 1, TRUE);
4214	else
4215		xpt_release_simq(path->bus->sim, TRUE);
4216	if (ccb->casync.async_arg_size > 0)
4217		free(async_arg, M_CAMXPT);
4218	xpt_free_path(path);
4219	xpt_free_ccb(ccb);
4220}
4221
4222static void
4223xpt_async_bcast(struct async_list *async_head,
4224		u_int32_t async_code,
4225		struct cam_path *path, void *async_arg)
4226{
4227	struct async_node *cur_entry;
4228	int lock;
4229
4230	cur_entry = SLIST_FIRST(async_head);
4231	while (cur_entry != NULL) {
4232		struct async_node *next_entry;
4233		/*
4234		 * Grab the next list entry before we call the current
4235		 * entry's callback.  This is because the callback function
4236		 * can delete its async callback entry.
4237		 */
4238		next_entry = SLIST_NEXT(cur_entry, links);
4239		if ((cur_entry->event_enable & async_code) != 0) {
4240			lock = cur_entry->event_lock;
4241			if (lock)
4242				CAM_SIM_LOCK(path->device->sim);
4243			cur_entry->callback(cur_entry->callback_arg,
4244					    async_code, path,
4245					    async_arg);
4246			if (lock)
4247				CAM_SIM_UNLOCK(path->device->sim);
4248		}
4249		cur_entry = next_entry;
4250	}
4251}
4252
4253void
4254xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4255{
4256	union ccb *ccb;
4257	int size;
4258
4259	ccb = xpt_alloc_ccb_nowait();
4260	if (ccb == NULL) {
4261		xpt_print(path, "Can't allocate CCB to send %s\n",
4262		    xpt_async_string(async_code));
4263		return;
4264	}
4265
4266	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4267		xpt_print(path, "Can't allocate path to send %s\n",
4268		    xpt_async_string(async_code));
4269		xpt_free_ccb(ccb);
4270		return;
4271	}
4272	ccb->ccb_h.path->periph = NULL;
4273	ccb->ccb_h.func_code = XPT_ASYNC;
4274	ccb->ccb_h.cbfcnp = xpt_async_process;
4275	ccb->ccb_h.flags |= CAM_UNLOCKED;
4276	ccb->casync.async_code = async_code;
4277	ccb->casync.async_arg_size = 0;
4278	size = xpt_async_size(async_code);
4279	if (size > 0 && async_arg != NULL) {
4280		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4281		if (ccb->casync.async_arg_ptr == NULL) {
4282			xpt_print(path, "Can't allocate argument to send %s\n",
4283			    xpt_async_string(async_code));
4284			xpt_free_path(ccb->ccb_h.path);
4285			xpt_free_ccb(ccb);
4286			return;
4287		}
4288		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4289		ccb->casync.async_arg_size = size;
4290	} else if (size < 0) {
4291		ccb->casync.async_arg_ptr = async_arg;
4292		ccb->casync.async_arg_size = size;
4293	}
4294	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4295		xpt_freeze_devq(path, 1);
4296	else
4297		xpt_freeze_simq(path->bus->sim, 1);
4298	xpt_done(ccb);
4299}
4300
4301static void
4302xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4303		      struct cam_et *target, struct cam_ed *device,
4304		      void *async_arg)
4305{
4306
4307	/*
4308	 * We only need to handle events for real devices.
4309	 */
4310	if (target->target_id == CAM_TARGET_WILDCARD
4311	 || device->lun_id == CAM_LUN_WILDCARD)
4312		return;
4313
4314	printf("%s called\n", __func__);
4315}
4316
4317static uint32_t
4318xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4319{
4320	struct cam_devq	*devq;
4321	uint32_t freeze;
4322
4323	devq = dev->sim->devq;
4324	mtx_assert(&devq->send_mtx, MA_OWNED);
4325	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4326	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4327	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4328	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4329	/* Remove frozen device from sendq. */
4330	if (device_is_queued(dev))
4331		camq_remove(&devq->send_queue, dev->devq_entry.index);
4332	return (freeze);
4333}
4334
4335u_int32_t
4336xpt_freeze_devq(struct cam_path *path, u_int count)
4337{
4338	struct cam_ed	*dev = path->device;
4339	struct cam_devq	*devq;
4340	uint32_t	 freeze;
4341
4342	devq = dev->sim->devq;
4343	mtx_lock(&devq->send_mtx);
4344	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4345	freeze = xpt_freeze_devq_device(dev, count);
4346	mtx_unlock(&devq->send_mtx);
4347	return (freeze);
4348}
4349
4350u_int32_t
4351xpt_freeze_simq(struct cam_sim *sim, u_int count)
4352{
4353	struct cam_devq	*devq;
4354	uint32_t	 freeze;
4355
4356	devq = sim->devq;
4357	mtx_lock(&devq->send_mtx);
4358	freeze = (devq->send_queue.qfrozen_cnt += count);
4359	mtx_unlock(&devq->send_mtx);
4360	return (freeze);
4361}
4362
4363static void
4364xpt_release_devq_timeout(void *arg)
4365{
4366	struct cam_ed *dev;
4367	struct cam_devq *devq;
4368
4369	dev = (struct cam_ed *)arg;
4370	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4371	devq = dev->sim->devq;
4372	mtx_assert(&devq->send_mtx, MA_OWNED);
4373	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4374		xpt_run_devq(devq);
4375}
4376
4377void
4378xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4379{
4380	struct cam_ed *dev;
4381	struct cam_devq *devq;
4382
4383	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4384	    count, run_queue));
4385	dev = path->device;
4386	devq = dev->sim->devq;
4387	mtx_lock(&devq->send_mtx);
4388	if (xpt_release_devq_device(dev, count, run_queue))
4389		xpt_run_devq(dev->sim->devq);
4390	mtx_unlock(&devq->send_mtx);
4391}
4392
4393static int
4394xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4395{
4396
4397	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4398	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4399	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4400	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4401	if (count > dev->ccbq.queue.qfrozen_cnt) {
4402#ifdef INVARIANTS
4403		printf("xpt_release_devq(): requested %u > present %u\n",
4404		    count, dev->ccbq.queue.qfrozen_cnt);
4405#endif
4406		count = dev->ccbq.queue.qfrozen_cnt;
4407	}
4408	dev->ccbq.queue.qfrozen_cnt -= count;
4409	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4410		/*
4411		 * No longer need to wait for a successful
4412		 * command completion.
4413		 */
4414		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4415		/*
4416		 * Remove any timeouts that might be scheduled
4417		 * to release this queue.
4418		 */
4419		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4420			callout_stop(&dev->callout);
4421			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4422		}
4423		/*
4424		 * Now that we are unfrozen schedule the
4425		 * device so any pending transactions are
4426		 * run.
4427		 */
4428		xpt_schedule_devq(dev->sim->devq, dev);
4429	} else
4430		run_queue = 0;
4431	return (run_queue);
4432}
4433
4434void
4435xpt_release_simq(struct cam_sim *sim, int run_queue)
4436{
4437	struct cam_devq	*devq;
4438
4439	devq = sim->devq;
4440	mtx_lock(&devq->send_mtx);
4441	if (devq->send_queue.qfrozen_cnt <= 0) {
4442#ifdef INVARIANTS
4443		printf("xpt_release_simq: requested 1 > present %u\n",
4444		    devq->send_queue.qfrozen_cnt);
4445#endif
4446	} else
4447		devq->send_queue.qfrozen_cnt--;
4448	if (devq->send_queue.qfrozen_cnt == 0) {
4449		/*
4450		 * If there is a timeout scheduled to release this
4451		 * sim queue, remove it.  The queue frozen count is
4452		 * already at 0.
4453		 */
4454		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4455			callout_stop(&sim->callout);
4456			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4457		}
4458		if (run_queue) {
4459			/*
4460			 * Now that we are unfrozen run the send queue.
4461			 */
4462			xpt_run_devq(sim->devq);
4463		}
4464	}
4465	mtx_unlock(&devq->send_mtx);
4466}
4467
4468/*
4469 * XXX Appears to be unused.
4470 */
4471static void
4472xpt_release_simq_timeout(void *arg)
4473{
4474	struct cam_sim *sim;
4475
4476	sim = (struct cam_sim *)arg;
4477	xpt_release_simq(sim, /* run_queue */ TRUE);
4478}
4479
4480void
4481xpt_done(union ccb *done_ccb)
4482{
4483	struct cam_doneq *queue;
4484	int	run, hash;
4485
4486	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4487	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4488		return;
4489
4490	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4491	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4492	queue = &cam_doneqs[hash];
4493	mtx_lock(&queue->cam_doneq_mtx);
4494	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4495	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4496	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4497	mtx_unlock(&queue->cam_doneq_mtx);
4498	if (run)
4499		wakeup(&queue->cam_doneq);
4500}
4501
4502void
4503xpt_done_direct(union ccb *done_ccb)
4504{
4505
4506	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4507	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4508		return;
4509
4510	xpt_done_process(&done_ccb->ccb_h);
4511}
4512
4513union ccb *
4514xpt_alloc_ccb()
4515{
4516	union ccb *new_ccb;
4517
4518	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4519	return (new_ccb);
4520}
4521
4522union ccb *
4523xpt_alloc_ccb_nowait()
4524{
4525	union ccb *new_ccb;
4526
4527	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4528	return (new_ccb);
4529}
4530
4531void
4532xpt_free_ccb(union ccb *free_ccb)
4533{
4534	free(free_ccb, M_CAMCCB);
4535}
4536
4537
4538
4539/* Private XPT functions */
4540
4541/*
4542 * Get a CAM control block for the caller. Charge the structure to the device
4543 * referenced by the path.  If we don't have sufficient resources to allocate
4544 * more ccbs, we return NULL.
4545 */
4546static union ccb *
4547xpt_get_ccb_nowait(struct cam_periph *periph)
4548{
4549	union ccb *new_ccb;
4550
4551	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4552	if (new_ccb == NULL)
4553		return (NULL);
4554	periph->periph_allocated++;
4555	cam_ccbq_take_opening(&periph->path->device->ccbq);
4556	return (new_ccb);
4557}
4558
4559static union ccb *
4560xpt_get_ccb(struct cam_periph *periph)
4561{
4562	union ccb *new_ccb;
4563
4564	cam_periph_unlock(periph);
4565	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4566	cam_periph_lock(periph);
4567	periph->periph_allocated++;
4568	cam_ccbq_take_opening(&periph->path->device->ccbq);
4569	return (new_ccb);
4570}
4571
4572union ccb *
4573cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4574{
4575	struct ccb_hdr *ccb_h;
4576
4577	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4578	cam_periph_assert(periph, MA_OWNED);
4579	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4580	    ccb_h->pinfo.priority != priority) {
4581		if (priority < periph->immediate_priority) {
4582			periph->immediate_priority = priority;
4583			xpt_run_allocq(periph, 0);
4584		} else
4585			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4586			    "cgticb", 0);
4587	}
4588	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4589	return ((union ccb *)ccb_h);
4590}
4591
4592static void
4593xpt_acquire_bus(struct cam_eb *bus)
4594{
4595
4596	xpt_lock_buses();
4597	bus->refcount++;
4598	xpt_unlock_buses();
4599}
4600
4601static void
4602xpt_release_bus(struct cam_eb *bus)
4603{
4604
4605	xpt_lock_buses();
4606	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4607	if (--bus->refcount > 0) {
4608		xpt_unlock_buses();
4609		return;
4610	}
4611	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4612	xsoftc.bus_generation++;
4613	xpt_unlock_buses();
4614	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4615	    ("destroying bus, but target list is not empty"));
4616	cam_sim_release(bus->sim);
4617	mtx_destroy(&bus->eb_mtx);
4618	free(bus, M_CAMXPT);
4619}
4620
4621static struct cam_et *
4622xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4623{
4624	struct cam_et *cur_target, *target;
4625
4626	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4627	mtx_assert(&bus->eb_mtx, MA_OWNED);
4628	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4629					 M_NOWAIT|M_ZERO);
4630	if (target == NULL)
4631		return (NULL);
4632
4633	TAILQ_INIT(&target->ed_entries);
4634	target->bus = bus;
4635	target->target_id = target_id;
4636	target->refcount = 1;
4637	target->generation = 0;
4638	target->luns = NULL;
4639	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4640	timevalclear(&target->last_reset);
4641	/*
4642	 * Hold a reference to our parent bus so it
4643	 * will not go away before we do.
4644	 */
4645	bus->refcount++;
4646
4647	/* Insertion sort into our bus's target list */
4648	cur_target = TAILQ_FIRST(&bus->et_entries);
4649	while (cur_target != NULL && cur_target->target_id < target_id)
4650		cur_target = TAILQ_NEXT(cur_target, links);
4651	if (cur_target != NULL) {
4652		TAILQ_INSERT_BEFORE(cur_target, target, links);
4653	} else {
4654		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4655	}
4656	bus->generation++;
4657	return (target);
4658}
4659
4660static void
4661xpt_acquire_target(struct cam_et *target)
4662{
4663	struct cam_eb *bus = target->bus;
4664
4665	mtx_lock(&bus->eb_mtx);
4666	target->refcount++;
4667	mtx_unlock(&bus->eb_mtx);
4668}
4669
4670static void
4671xpt_release_target(struct cam_et *target)
4672{
4673	struct cam_eb *bus = target->bus;
4674
4675	mtx_lock(&bus->eb_mtx);
4676	if (--target->refcount > 0) {
4677		mtx_unlock(&bus->eb_mtx);
4678		return;
4679	}
4680	TAILQ_REMOVE(&bus->et_entries, target, links);
4681	bus->generation++;
4682	mtx_unlock(&bus->eb_mtx);
4683	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4684	    ("destroying target, but device list is not empty"));
4685	xpt_release_bus(bus);
4686	mtx_destroy(&target->luns_mtx);
4687	if (target->luns)
4688		free(target->luns, M_CAMXPT);
4689	free(target, M_CAMXPT);
4690}
4691
4692static struct cam_ed *
4693xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4694			 lun_id_t lun_id)
4695{
4696	struct cam_ed *device;
4697
4698	device = xpt_alloc_device(bus, target, lun_id);
4699	if (device == NULL)
4700		return (NULL);
4701
4702	device->mintags = 1;
4703	device->maxtags = 1;
4704	return (device);
4705}
4706
4707static void
4708xpt_destroy_device(void *context, int pending)
4709{
4710	struct cam_ed	*device = context;
4711
4712	mtx_lock(&device->device_mtx);
4713	mtx_destroy(&device->device_mtx);
4714	free(device, M_CAMDEV);
4715}
4716
4717struct cam_ed *
4718xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4719{
4720	struct cam_ed	*cur_device, *device;
4721	struct cam_devq	*devq;
4722	cam_status status;
4723
4724	mtx_assert(&bus->eb_mtx, MA_OWNED);
4725	/* Make space for us in the device queue on our bus */
4726	devq = bus->sim->devq;
4727	mtx_lock(&devq->send_mtx);
4728	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4729	mtx_unlock(&devq->send_mtx);
4730	if (status != CAM_REQ_CMP)
4731		return (NULL);
4732
4733	device = (struct cam_ed *)malloc(sizeof(*device),
4734					 M_CAMDEV, M_NOWAIT|M_ZERO);
4735	if (device == NULL)
4736		return (NULL);
4737
4738	cam_init_pinfo(&device->devq_entry);
4739	device->target = target;
4740	device->lun_id = lun_id;
4741	device->sim = bus->sim;
4742	if (cam_ccbq_init(&device->ccbq,
4743			  bus->sim->max_dev_openings) != 0) {
4744		free(device, M_CAMDEV);
4745		return (NULL);
4746	}
4747	SLIST_INIT(&device->asyncs);
4748	SLIST_INIT(&device->periphs);
4749	device->generation = 0;
4750	device->flags = CAM_DEV_UNCONFIGURED;
4751	device->tag_delay_count = 0;
4752	device->tag_saved_openings = 0;
4753	device->refcount = 1;
4754	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4755	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4756	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4757	/*
4758	 * Hold a reference to our parent bus so it
4759	 * will not go away before we do.
4760	 */
4761	target->refcount++;
4762
4763	cur_device = TAILQ_FIRST(&target->ed_entries);
4764	while (cur_device != NULL && cur_device->lun_id < lun_id)
4765		cur_device = TAILQ_NEXT(cur_device, links);
4766	if (cur_device != NULL)
4767		TAILQ_INSERT_BEFORE(cur_device, device, links);
4768	else
4769		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4770	target->generation++;
4771	return (device);
4772}
4773
4774void
4775xpt_acquire_device(struct cam_ed *device)
4776{
4777	struct cam_eb *bus = device->target->bus;
4778
4779	mtx_lock(&bus->eb_mtx);
4780	device->refcount++;
4781	mtx_unlock(&bus->eb_mtx);
4782}
4783
4784void
4785xpt_release_device(struct cam_ed *device)
4786{
4787	struct cam_eb *bus = device->target->bus;
4788	struct cam_devq *devq;
4789
4790	mtx_lock(&bus->eb_mtx);
4791	if (--device->refcount > 0) {
4792		mtx_unlock(&bus->eb_mtx);
4793		return;
4794	}
4795
4796	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4797	device->target->generation++;
4798	mtx_unlock(&bus->eb_mtx);
4799
4800	/* Release our slot in the devq */
4801	devq = bus->sim->devq;
4802	mtx_lock(&devq->send_mtx);
4803	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4804	mtx_unlock(&devq->send_mtx);
4805
4806	KASSERT(SLIST_EMPTY(&device->periphs),
4807	    ("destroying device, but periphs list is not empty"));
4808	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4809	    ("destroying device while still queued for ccbs"));
4810
4811	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4812		callout_stop(&device->callout);
4813
4814	xpt_release_target(device->target);
4815
4816	cam_ccbq_fini(&device->ccbq);
4817	/*
4818	 * Free allocated memory.  free(9) does nothing if the
4819	 * supplied pointer is NULL, so it is safe to call without
4820	 * checking.
4821	 */
4822	free(device->supported_vpds, M_CAMXPT);
4823	free(device->device_id, M_CAMXPT);
4824	free(device->ext_inq, M_CAMXPT);
4825	free(device->physpath, M_CAMXPT);
4826	free(device->rcap_buf, M_CAMXPT);
4827	free(device->serial_num, M_CAMXPT);
4828	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4829}
4830
4831u_int32_t
4832xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4833{
4834	int	result;
4835	struct	cam_ed *dev;
4836
4837	dev = path->device;
4838	mtx_lock(&dev->sim->devq->send_mtx);
4839	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4840	mtx_unlock(&dev->sim->devq->send_mtx);
4841	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4842	 || (dev->inq_flags & SID_CmdQue) != 0)
4843		dev->tag_saved_openings = newopenings;
4844	return (result);
4845}
4846
4847static struct cam_eb *
4848xpt_find_bus(path_id_t path_id)
4849{
4850	struct cam_eb *bus;
4851
4852	xpt_lock_buses();
4853	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4854	     bus != NULL;
4855	     bus = TAILQ_NEXT(bus, links)) {
4856		if (bus->path_id == path_id) {
4857			bus->refcount++;
4858			break;
4859		}
4860	}
4861	xpt_unlock_buses();
4862	return (bus);
4863}
4864
4865static struct cam_et *
4866xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4867{
4868	struct cam_et *target;
4869
4870	mtx_assert(&bus->eb_mtx, MA_OWNED);
4871	for (target = TAILQ_FIRST(&bus->et_entries);
4872	     target != NULL;
4873	     target = TAILQ_NEXT(target, links)) {
4874		if (target->target_id == target_id) {
4875			target->refcount++;
4876			break;
4877		}
4878	}
4879	return (target);
4880}
4881
4882static struct cam_ed *
4883xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4884{
4885	struct cam_ed *device;
4886
4887	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4888	for (device = TAILQ_FIRST(&target->ed_entries);
4889	     device != NULL;
4890	     device = TAILQ_NEXT(device, links)) {
4891		if (device->lun_id == lun_id) {
4892			device->refcount++;
4893			break;
4894		}
4895	}
4896	return (device);
4897}
4898
4899void
4900xpt_start_tags(struct cam_path *path)
4901{
4902	struct ccb_relsim crs;
4903	struct cam_ed *device;
4904	struct cam_sim *sim;
4905	int    newopenings;
4906
4907	device = path->device;
4908	sim = path->bus->sim;
4909	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4910	xpt_freeze_devq(path, /*count*/1);
4911	device->inq_flags |= SID_CmdQue;
4912	if (device->tag_saved_openings != 0)
4913		newopenings = device->tag_saved_openings;
4914	else
4915		newopenings = min(device->maxtags,
4916				  sim->max_tagged_dev_openings);
4917	xpt_dev_ccbq_resize(path, newopenings);
4918	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4919	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4920	crs.ccb_h.func_code = XPT_REL_SIMQ;
4921	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4922	crs.openings
4923	    = crs.release_timeout
4924	    = crs.qfrozen_cnt
4925	    = 0;
4926	xpt_action((union ccb *)&crs);
4927}
4928
4929void
4930xpt_stop_tags(struct cam_path *path)
4931{
4932	struct ccb_relsim crs;
4933	struct cam_ed *device;
4934	struct cam_sim *sim;
4935
4936	device = path->device;
4937	sim = path->bus->sim;
4938	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4939	device->tag_delay_count = 0;
4940	xpt_freeze_devq(path, /*count*/1);
4941	device->inq_flags &= ~SID_CmdQue;
4942	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4943	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4944	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4945	crs.ccb_h.func_code = XPT_REL_SIMQ;
4946	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4947	crs.openings
4948	    = crs.release_timeout
4949	    = crs.qfrozen_cnt
4950	    = 0;
4951	xpt_action((union ccb *)&crs);
4952}
4953
4954static void
4955xpt_boot_delay(void *arg)
4956{
4957
4958	xpt_release_boot();
4959}
4960
4961static void
4962xpt_config(void *arg)
4963{
4964	/*
4965	 * Now that interrupts are enabled, go find our devices
4966	 */
4967	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4968		printf("xpt_config: failed to create taskqueue thread.\n");
4969
4970	/* Setup debugging path */
4971	if (cam_dflags != CAM_DEBUG_NONE) {
4972		if (xpt_create_path(&cam_dpath, NULL,
4973				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4974				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4975			printf("xpt_config: xpt_create_path() failed for debug"
4976			       " target %d:%d:%d, debugging disabled\n",
4977			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4978			cam_dflags = CAM_DEBUG_NONE;
4979		}
4980	} else
4981		cam_dpath = NULL;
4982
4983	periphdriver_init(1);
4984	xpt_hold_boot();
4985	callout_init(&xsoftc.boot_callout, 1);
4986	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4987	    xpt_boot_delay, NULL, 0);
4988	/* Fire up rescan thread. */
4989	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4990	    "cam", "scanner")) {
4991		printf("xpt_config: failed to create rescan thread.\n");
4992	}
4993}
4994
4995void
4996xpt_hold_boot(void)
4997{
4998	xpt_lock_buses();
4999	xsoftc.buses_to_config++;
5000	xpt_unlock_buses();
5001}
5002
5003void
5004xpt_release_boot(void)
5005{
5006	xpt_lock_buses();
5007	xsoftc.buses_to_config--;
5008	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
5009		struct	xpt_task *task;
5010
5011		xsoftc.buses_config_done = 1;
5012		xpt_unlock_buses();
5013		/* Call manually because we don't have any busses */
5014		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5015		if (task != NULL) {
5016			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5017			taskqueue_enqueue(taskqueue_thread, &task->task);
5018		}
5019	} else
5020		xpt_unlock_buses();
5021}
5022
5023/*
5024 * If the given device only has one peripheral attached to it, and if that
5025 * peripheral is the passthrough driver, announce it.  This insures that the
5026 * user sees some sort of announcement for every peripheral in their system.
5027 */
5028static int
5029xptpassannouncefunc(struct cam_ed *device, void *arg)
5030{
5031	struct cam_periph *periph;
5032	int i;
5033
5034	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5035	     periph = SLIST_NEXT(periph, periph_links), i++);
5036
5037	periph = SLIST_FIRST(&device->periphs);
5038	if ((i == 1)
5039	 && (strncmp(periph->periph_name, "pass", 4) == 0))
5040		xpt_announce_periph(periph, NULL);
5041
5042	return(1);
5043}
5044
5045static void
5046xpt_finishconfig_task(void *context, int pending)
5047{
5048
5049	periphdriver_init(2);
5050	/*
5051	 * Check for devices with no "standard" peripheral driver
5052	 * attached.  For any devices like that, announce the
5053	 * passthrough driver so the user will see something.
5054	 */
5055	if (!bootverbose)
5056		xpt_for_all_devices(xptpassannouncefunc, NULL);
5057
5058	/* Release our hook so that the boot can continue. */
5059	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5060	free(xsoftc.xpt_config_hook, M_CAMXPT);
5061	xsoftc.xpt_config_hook = NULL;
5062
5063	free(context, M_CAMXPT);
5064}
5065
5066cam_status
5067xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5068		   struct cam_path *path)
5069{
5070	struct ccb_setasync csa;
5071	cam_status status;
5072	int xptpath = 0;
5073
5074	if (path == NULL) {
5075		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5076					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5077		if (status != CAM_REQ_CMP)
5078			return (status);
5079		xpt_path_lock(path);
5080		xptpath = 1;
5081	}
5082
5083	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5084	csa.ccb_h.func_code = XPT_SASYNC_CB;
5085	csa.event_enable = event;
5086	csa.callback = cbfunc;
5087	csa.callback_arg = cbarg;
5088	xpt_action((union ccb *)&csa);
5089	status = csa.ccb_h.status;
5090
5091	if (xptpath) {
5092		xpt_path_unlock(path);
5093		xpt_free_path(path);
5094	}
5095
5096	if ((status == CAM_REQ_CMP) &&
5097	    (csa.event_enable & AC_FOUND_DEVICE)) {
5098		/*
5099		 * Get this peripheral up to date with all
5100		 * the currently existing devices.
5101		 */
5102		xpt_for_all_devices(xptsetasyncfunc, &csa);
5103	}
5104	if ((status == CAM_REQ_CMP) &&
5105	    (csa.event_enable & AC_PATH_REGISTERED)) {
5106		/*
5107		 * Get this peripheral up to date with all
5108		 * the currently existing busses.
5109		 */
5110		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5111	}
5112
5113	return (status);
5114}
5115
5116static void
5117xptaction(struct cam_sim *sim, union ccb *work_ccb)
5118{
5119	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5120
5121	switch (work_ccb->ccb_h.func_code) {
5122	/* Common cases first */
5123	case XPT_PATH_INQ:		/* Path routing inquiry */
5124	{
5125		struct ccb_pathinq *cpi;
5126
5127		cpi = &work_ccb->cpi;
5128		cpi->version_num = 1; /* XXX??? */
5129		cpi->hba_inquiry = 0;
5130		cpi->target_sprt = 0;
5131		cpi->hba_misc = 0;
5132		cpi->hba_eng_cnt = 0;
5133		cpi->max_target = 0;
5134		cpi->max_lun = 0;
5135		cpi->initiator_id = 0;
5136		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5137		strlcpy(cpi->hba_vid, "", HBA_IDLEN);
5138		strlcpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5139		cpi->unit_number = sim->unit_number;
5140		cpi->bus_id = sim->bus_id;
5141		cpi->base_transfer_speed = 0;
5142		cpi->protocol = PROTO_UNSPECIFIED;
5143		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5144		cpi->transport = XPORT_UNSPECIFIED;
5145		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5146		cpi->ccb_h.status = CAM_REQ_CMP;
5147		xpt_done(work_ccb);
5148		break;
5149	}
5150	default:
5151		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5152		xpt_done(work_ccb);
5153		break;
5154	}
5155}
5156
5157/*
5158 * The xpt as a "controller" has no interrupt sources, so polling
5159 * is a no-op.
5160 */
5161static void
5162xptpoll(struct cam_sim *sim)
5163{
5164}
5165
5166void
5167xpt_lock_buses(void)
5168{
5169	mtx_lock(&xsoftc.xpt_topo_lock);
5170}
5171
5172void
5173xpt_unlock_buses(void)
5174{
5175	mtx_unlock(&xsoftc.xpt_topo_lock);
5176}
5177
5178struct mtx *
5179xpt_path_mtx(struct cam_path *path)
5180{
5181
5182	return (&path->device->device_mtx);
5183}
5184
5185static void
5186xpt_done_process(struct ccb_hdr *ccb_h)
5187{
5188	struct cam_sim *sim;
5189	struct cam_devq *devq;
5190	struct mtx *mtx = NULL;
5191
5192	if (ccb_h->flags & CAM_HIGH_POWER) {
5193		struct highpowerlist	*hphead;
5194		struct cam_ed		*device;
5195
5196		mtx_lock(&xsoftc.xpt_highpower_lock);
5197		hphead = &xsoftc.highpowerq;
5198
5199		device = STAILQ_FIRST(hphead);
5200
5201		/*
5202		 * Increment the count since this command is done.
5203		 */
5204		xsoftc.num_highpower++;
5205
5206		/*
5207		 * Any high powered commands queued up?
5208		 */
5209		if (device != NULL) {
5210
5211			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5212			mtx_unlock(&xsoftc.xpt_highpower_lock);
5213
5214			mtx_lock(&device->sim->devq->send_mtx);
5215			xpt_release_devq_device(device,
5216					 /*count*/1, /*runqueue*/TRUE);
5217			mtx_unlock(&device->sim->devq->send_mtx);
5218		} else
5219			mtx_unlock(&xsoftc.xpt_highpower_lock);
5220	}
5221
5222	sim = ccb_h->path->bus->sim;
5223
5224	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5225		xpt_release_simq(sim, /*run_queue*/FALSE);
5226		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5227	}
5228
5229	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5230	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5231		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5232		ccb_h->status &= ~CAM_DEV_QFRZN;
5233	}
5234
5235	devq = sim->devq;
5236	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5237		struct cam_ed *dev = ccb_h->path->device;
5238
5239		mtx_lock(&devq->send_mtx);
5240		devq->send_active--;
5241		devq->send_openings++;
5242		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5243
5244		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5245		  && (dev->ccbq.dev_active == 0))) {
5246			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5247			xpt_release_devq_device(dev, /*count*/1,
5248					 /*run_queue*/FALSE);
5249		}
5250
5251		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5252		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5253			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5254			xpt_release_devq_device(dev, /*count*/1,
5255					 /*run_queue*/FALSE);
5256		}
5257
5258		if (!device_is_queued(dev))
5259			(void)xpt_schedule_devq(devq, dev);
5260		xpt_run_devq(devq);
5261		mtx_unlock(&devq->send_mtx);
5262
5263		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5264			mtx = xpt_path_mtx(ccb_h->path);
5265			mtx_lock(mtx);
5266
5267			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5268			 && (--dev->tag_delay_count == 0))
5269				xpt_start_tags(ccb_h->path);
5270		}
5271	}
5272
5273	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5274		if (mtx == NULL) {
5275			mtx = xpt_path_mtx(ccb_h->path);
5276			mtx_lock(mtx);
5277		}
5278	} else {
5279		if (mtx != NULL) {
5280			mtx_unlock(mtx);
5281			mtx = NULL;
5282		}
5283	}
5284
5285	/* Call the peripheral driver's callback */
5286	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5287	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5288	if (mtx != NULL)
5289		mtx_unlock(mtx);
5290}
5291
5292void
5293xpt_done_td(void *arg)
5294{
5295	struct cam_doneq *queue = arg;
5296	struct ccb_hdr *ccb_h;
5297	STAILQ_HEAD(, ccb_hdr)	doneq;
5298
5299	STAILQ_INIT(&doneq);
5300	mtx_lock(&queue->cam_doneq_mtx);
5301	while (1) {
5302		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5303			queue->cam_doneq_sleep = 1;
5304			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5305			    PRIBIO, "-", 0);
5306			queue->cam_doneq_sleep = 0;
5307		}
5308		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5309		mtx_unlock(&queue->cam_doneq_mtx);
5310
5311		THREAD_NO_SLEEPING();
5312		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5313			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5314			xpt_done_process(ccb_h);
5315		}
5316		THREAD_SLEEPING_OK();
5317
5318		mtx_lock(&queue->cam_doneq_mtx);
5319	}
5320}
5321
5322static void
5323camisr_runqueue(void)
5324{
5325	struct	ccb_hdr *ccb_h;
5326	struct cam_doneq *queue;
5327	int i;
5328
5329	/* Process global queues. */
5330	for (i = 0; i < cam_num_doneqs; i++) {
5331		queue = &cam_doneqs[i];
5332		mtx_lock(&queue->cam_doneq_mtx);
5333		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5334			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5335			mtx_unlock(&queue->cam_doneq_mtx);
5336			xpt_done_process(ccb_h);
5337			mtx_lock(&queue->cam_doneq_mtx);
5338		}
5339		mtx_unlock(&queue->cam_doneq_mtx);
5340	}
5341}
5342