1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2024 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8#include <sys/types.h>
9#include <sys/malloc.h>
10#include <sys/memdesc.h>
11#include <sys/refcount.h>
12
13#include <cam/cam.h>
14#include <cam/cam_ccb.h>
15#include <cam/cam_sim.h>
16#include <cam/cam_xpt_sim.h>
17#include <cam/cam_debug.h>
18
19#include <dev/nvmf/host/nvmf_var.h>
20
21/*
22 * The I/O completion may trigger after the received CQE if the I/O
23 * used a zero-copy mbuf that isn't harvested until after the NIC
24 * driver processes TX completions.  Use spriv_field0 to as a refcount.
25 *
26 * Store any I/O error returned in spriv_field1.
27 */
28static __inline u_int *
29ccb_refs(union ccb *ccb)
30{
31	return ((u_int *)&ccb->ccb_h.spriv_field0);
32}
33
34#define	spriv_ioerror	spriv_field1
35
36static void
37nvmf_ccb_done(union ccb *ccb)
38{
39	if (!refcount_release(ccb_refs(ccb)))
40		return;
41
42	if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) {
43		ccb->ccb_h.status = CAM_REQUEUE_REQ;
44		xpt_done(ccb);
45	} else if (ccb->nvmeio.cpl.status != 0) {
46		ccb->ccb_h.status = CAM_NVME_STATUS_ERROR;
47		xpt_done(ccb);
48	} else if (ccb->ccb_h.spriv_ioerror != 0) {
49		KASSERT(ccb->ccb_h.spriv_ioerror != EJUSTRETURN,
50		    ("%s: zero sized transfer without CQE error", __func__));
51		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
52		xpt_done(ccb);
53	} else {
54		ccb->ccb_h.status = CAM_REQ_CMP;
55		xpt_done_direct(ccb);
56	}
57}
58
59static void
60nvmf_ccb_io_complete(void *arg, size_t xfered, int error)
61{
62	union ccb *ccb = arg;
63
64	/*
65	 * TODO: Reporting partial completions requires extending
66	 * nvmeio to support resid and updating nda to handle partial
67	 * reads, either by returning partial success (or an error) to
68	 * the caller, or retrying all or part of the request.
69	 */
70	ccb->ccb_h.spriv_ioerror = error;
71	if (error == 0) {
72		if (xfered == 0) {
73#ifdef INVARIANTS
74			/*
75			 * If the request fails with an error in the CQE
76			 * there will be no data transferred but also no
77			 * I/O error.
78			 */
79			ccb->ccb_h.spriv_ioerror = EJUSTRETURN;
80#endif
81		} else
82			KASSERT(xfered == ccb->nvmeio.dxfer_len,
83			    ("%s: partial CCB completion", __func__));
84	}
85
86	nvmf_ccb_done(ccb);
87}
88
89static void
90nvmf_ccb_complete(void *arg, const struct nvme_completion *cqe)
91{
92	union ccb *ccb = arg;
93
94	ccb->nvmeio.cpl = *cqe;
95	nvmf_ccb_done(ccb);
96}
97
98static void
99nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb)
100{
101	struct ccb_nvmeio *nvmeio = &ccb->nvmeio;
102	struct memdesc mem;
103	struct nvmf_request *req;
104	struct nvmf_host_qpair *qp;
105
106	mtx_lock(&sc->sim_mtx);
107	if (sc->sim_disconnected) {
108		mtx_unlock(&sc->sim_mtx);
109		nvmeio->ccb_h.status = CAM_REQUEUE_REQ;
110		xpt_done(ccb);
111		return;
112	}
113	if (nvmeio->ccb_h.func_code == XPT_NVME_IO)
114		qp = nvmf_select_io_queue(sc);
115	else
116		qp = sc->admin;
117	req = nvmf_allocate_request(qp, &nvmeio->cmd, nvmf_ccb_complete,
118	    ccb, M_NOWAIT);
119	if (req == NULL) {
120		mtx_unlock(&sc->sim_mtx);
121		nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL;
122		xpt_done(ccb);
123		return;
124	}
125
126	if (nvmeio->dxfer_len != 0) {
127		refcount_init(ccb_refs(ccb), 2);
128		mem = memdesc_ccb(ccb);
129		nvmf_capsule_append_data(req->nc, &mem, nvmeio->dxfer_len,
130		    (ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT,
131		    nvmf_ccb_io_complete, ccb);
132	} else
133		refcount_init(ccb_refs(ccb), 1);
134
135	/*
136	 * Clear spriv_ioerror as it can hold an earlier error if this
137	 * CCB was aborted and has been retried.
138	 */
139	ccb->ccb_h.spriv_ioerror = 0;
140	KASSERT(ccb->ccb_h.status == CAM_REQ_INPROG,
141	    ("%s: incoming CCB is not in-progress", __func__));
142	ccb->ccb_h.status |= CAM_SIM_QUEUED;
143	nvmf_submit_request(req);
144	mtx_unlock(&sc->sim_mtx);
145}
146
147static void
148nvmf_sim_action(struct cam_sim *sim, union ccb *ccb)
149{
150	struct nvmf_softc *sc = cam_sim_softc(sim);
151
152	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
153	    ("nvmf_sim_action: func= %#x\n",
154		ccb->ccb_h.func_code));
155
156	switch (ccb->ccb_h.func_code) {
157	case XPT_PATH_INQ:	/* Path routing inquiry */
158	{
159		struct ccb_pathinq *cpi = &ccb->cpi;
160
161		cpi->version_num = 1;
162		cpi->hba_inquiry = 0;
163		cpi->target_sprt = 0;
164		cpi->hba_misc =  PIM_UNMAPPED | PIM_NOSCAN;
165		cpi->hba_eng_cnt = 0;
166		cpi->max_target = 0;
167		cpi->max_lun = sc->cdata->nn;
168		cpi->async_flags = 0;
169		cpi->hpath_id = 0;
170		cpi->initiator_id = 0;
171		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
172		strlcpy(cpi->hba_vid, "NVMeoF", HBA_IDLEN);
173		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
174		cpi->unit_number = cam_sim_unit(sim);
175		cpi->bus_id = 0;
176
177		/* XXX: Same as iSCSI. */
178		cpi->base_transfer_speed = 150000;
179		cpi->protocol = PROTO_NVME;
180		cpi->protocol_version = sc->vs;
181		cpi->transport = XPORT_NVMF;
182		cpi->transport_version = sc->vs;
183		cpi->xport_specific.nvmf.nsid =
184		    xpt_path_lun_id(ccb->ccb_h.path);
185		cpi->xport_specific.nvmf.trtype = sc->trtype;
186		strlcpy(cpi->xport_specific.nvmf.dev_name,
187		    device_get_nameunit(sc->dev),
188		    sizeof(cpi->xport_specific.nvmf.dev_name));
189		cpi->maxio = sc->max_xfer_size;
190		cpi->hba_vendor = 0;
191		cpi->hba_device = 0;
192		cpi->hba_subvendor = 0;
193		cpi->hba_subdevice = 0;
194		cpi->ccb_h.status = CAM_REQ_CMP;
195		break;
196	}
197	case XPT_GET_TRAN_SETTINGS:	/* Get transport settings */
198	{
199		struct ccb_trans_settings *cts = &ccb->cts;
200		struct ccb_trans_settings_nvme *nvme;
201		struct ccb_trans_settings_nvmf *nvmf;
202
203		cts->protocol = PROTO_NVME;
204		cts->protocol_version = sc->vs;
205		cts->transport = XPORT_NVMF;
206		cts->transport_version = sc->vs;
207
208		nvme = &cts->proto_specific.nvme;
209		nvme->valid = CTS_NVME_VALID_SPEC;
210		nvme->spec = sc->vs;
211
212		nvmf = &cts->xport_specific.nvmf;
213		nvmf->valid = CTS_NVMF_VALID_TRTYPE;
214		nvmf->trtype = sc->trtype;
215		cts->ccb_h.status = CAM_REQ_CMP;
216		break;
217	}
218	case XPT_SET_TRAN_SETTINGS:	/* Set transport settings */
219		/*
220		 * No transfer settings can be set, but nvme_xpt sends
221		 * this anyway.
222		 */
223		ccb->ccb_h.status = CAM_REQ_CMP;
224		break;
225	case XPT_NVME_IO:		/* Execute the requested I/O */
226	case XPT_NVME_ADMIN:		/* or Admin operation */
227		nvmf_sim_io(sc, ccb);
228		return;
229	default:
230		/* XXX */
231		device_printf(sc->dev, "unhandled sim function %#x\n",
232		    ccb->ccb_h.func_code);
233		ccb->ccb_h.status = CAM_REQ_INVALID;
234		break;
235	}
236	xpt_done(ccb);
237}
238
239int
240nvmf_init_sim(struct nvmf_softc *sc)
241{
242	struct cam_devq *devq;
243	int max_trans;
244
245	max_trans = sc->max_pending_io * 3 / 4;
246	devq = cam_simq_alloc(max_trans);
247	if (devq == NULL) {
248		device_printf(sc->dev, "Failed to allocate CAM simq\n");
249		return (ENOMEM);
250	}
251
252	mtx_init(&sc->sim_mtx, "nvmf sim", NULL, MTX_DEF);
253	sc->sim = cam_sim_alloc(nvmf_sim_action, NULL, "nvme", sc,
254	    device_get_unit(sc->dev), NULL, max_trans, max_trans, devq);
255	if (sc->sim == NULL) {
256		device_printf(sc->dev, "Failed to allocate CAM sim\n");
257		cam_simq_free(devq);
258		mtx_destroy(&sc->sim_mtx);
259		return (ENXIO);
260	}
261	if (xpt_bus_register(sc->sim, sc->dev, 0) != CAM_SUCCESS) {
262		device_printf(sc->dev, "Failed to create CAM bus\n");
263		cam_sim_free(sc->sim, TRUE);
264		mtx_destroy(&sc->sim_mtx);
265		return (ENXIO);
266	}
267	if (xpt_create_path(&sc->path, NULL, cam_sim_path(sc->sim),
268	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
269		device_printf(sc->dev, "Failed to create CAM path\n");
270		xpt_bus_deregister(cam_sim_path(sc->sim));
271		cam_sim_free(sc->sim, TRUE);
272		mtx_destroy(&sc->sim_mtx);
273		return (ENXIO);
274	}
275	return (0);
276}
277
278void
279nvmf_sim_rescan_ns(struct nvmf_softc *sc, uint32_t id)
280{
281	union ccb *ccb;
282
283	ccb = xpt_alloc_ccb_nowait();
284	if (ccb == NULL) {
285		device_printf(sc->dev,
286		    "unable to alloc CCB for rescan of namespace %u\n", id);
287		return;
288	}
289
290	/*
291	 * As with nvme_sim, map NVMe namespace IDs onto CAM unit
292	 * LUNs.
293	 */
294	if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->sim), 0,
295	    id) != CAM_REQ_CMP) {
296		device_printf(sc->dev,
297		    "Unable to create path for rescan of namespace %u\n", id);
298		xpt_free_ccb(ccb);
299		return;
300	}
301	xpt_rescan(ccb);
302}
303
304void
305nvmf_disconnect_sim(struct nvmf_softc *sc)
306{
307	mtx_lock(&sc->sim_mtx);
308	sc->sim_disconnected = true;
309	xpt_freeze_simq(sc->sim, 1);
310	mtx_unlock(&sc->sim_mtx);
311}
312
313void
314nvmf_reconnect_sim(struct nvmf_softc *sc)
315{
316	mtx_lock(&sc->sim_mtx);
317	sc->sim_disconnected = false;
318	mtx_unlock(&sc->sim_mtx);
319	xpt_release_simq(sc->sim, 1);
320}
321
322void
323nvmf_destroy_sim(struct nvmf_softc *sc)
324{
325	xpt_async(AC_LOST_DEVICE, sc->path, NULL);
326	if (sc->sim_disconnected)
327		xpt_release_simq(sc->sim, 1);
328	xpt_free_path(sc->path);
329	xpt_bus_deregister(cam_sim_path(sc->sim));
330	cam_sim_free(sc->sim, TRUE);
331	mtx_destroy(&sc->sim_mtx);
332}
333