nvme/libnvme/nvme_qpair.c

311 static void nvme_qpair_construct_tracker(struct nvme_tracker *tr,
314 	tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
315 	tr->cid = cid;
316 	tr->active = false;
343 				      struct nvme_tracker *tr)
345 	struct nvme_request *req = tr->req;
354 		   (int)tr->cid,
355 		   (int)tr->req->cmd.cid);
357 	qpair->tr[tr->cid].active = true;
368 					struct nvme_tracker *tr,
372 	struct nvme_request *req = tr->req;
377 		qpair->tr[cpl->cid].active = false;
389 	qpair->tr[cpl->cid].active = false;
397 		nvme_qpair_submit_tracker(qpair, tr);
407 	tr->req = NULL;
409 	LIST_REMOVE(tr, list);
410 	LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
443 					       struct nvme_tracker *tr,
453 	cpl.cid = tr->cid;
458 	nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
489 	struct nvme_tracker *tr;
491 	tr = LIST_FIRST(&qpair->outstanding_tr);
492 	while (tr != NULL) {
493 		nvme_assert(tr->req != NULL,
494 			    "tr->req == NULL in abort_aers\n");
495 		if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
496 			nvme_qpair_manual_complete_tracker(qpair, tr,
500 			tr = LIST_FIRST(&qpair->outstanding_tr);
503 		tr = LIST_NEXT(tr, list);
513 					    struct nvme_tracker *tr)
519 	nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
529 					    struct nvme_tracker *tr)
539 		_nvme_qpair_req_bad_phys(qpair, tr);
550 		tr->req->cmd.mptr = nvme_mem_vtophys(md_payload);
551 		if (tr->req->cmd.mptr == NVME_VTOPHYS_ERROR) {
552 			_nvme_qpair_req_bad_phys(qpair, tr);
557 	tr->req->cmd.psdt = NVME_PSDT_PRP;
558 	tr->req->cmd.dptr.prp.prp1 = phys_addr;
561 		tr->req->cmd.dptr.prp.prp2 = nvme_mem_vtophys(seg_addr);
564 		tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
569 				_nvme_qpair_req_bad_phys(qpair, tr);
572 			tr->u.prp[cur_nseg - 1] = phys_addr;
585 					    struct nvme_tracker *tr)
606 	sgl = tr->u.sgl;
615 			_nvme_qpair_req_bad_phys(qpair, tr);
622 			_nvme_qpair_req_bad_phys(qpair, tr);
648 		req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
649 		req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
653 		req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
666 					      struct nvme_tracker *tr)
694 			_nvme_qpair_req_bad_phys(qpair, tr);
723 				tr->req->cmd.dptr.prp.prp2 = phys_addr +
726 				tr->req->cmd.dptr.prp.prp2 = phys_addr;
728 			prp2 = tr->req->cmd.dptr.prp.prp2;
735 			tr->req->cmd.dptr.prp.prp2 =
736 				(uint64_t)tr->prp_sgl_bus_addr;
740 					tr->u.prp[0] = prp2;
741 					tr->u.prp[last_nseg + 1] = phys_addr +
744 					tr->u.prp[last_nseg] = phys_addr +
758 	struct nvme_tracker *tr, *tr_temp;
766 	LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) {
768 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
778 	struct nvme_tracker *tr, *temp;
796 	LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) {
798 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
848 	struct nvme_tracker *tr;
939 	 * trackers in tr[] via normal array indexing.
941 	qpair->tr = nvme_mem_alloc_node(sizeof(struct nvme_tracker) * trackers,
944 	if (!qpair->tr) {
948 	memset(qpair->tr, 0, sizeof(struct nvme_tracker) * trackers);
951 		   qpair->id, qpair->tr, phys_addr);
954 		tr = &qpair->tr[i];
955 		nvme_qpair_construct_tracker(tr, i, phys_addr);
956 		LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
986 	if (qpair->tr) {
987 		nvme_free(qpair->tr);
988 		qpair->tr = NULL;
1008 	struct nvme_tracker *tr;
1046 	tr = LIST_FIRST(&qpair->free_tr);
1047 	if (tr == NULL || !qpair->enabled || !STAILQ_EMPTY(&qpair->queued_req)) {
1060 		if (tr)
1065 	/* remove tr from free_tr */
1066 	LIST_REMOVE(tr, list);
1067 	LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
1068 	tr->req = req;
1069 	req->cmd.cid = tr->cid;
1075 		ret = _nvme_qpair_build_contig_request(qpair, req, tr);
1078 			ret = _nvme_qpair_build_hw_sgl_request(qpair, req, tr);
1080 			ret = _nvme_qpair_build_prps_sgl_request(qpair, req, tr);
1082 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
1089 		nvme_qpair_submit_tracker(qpair, tr);
1103 	struct nvme_tracker *tr;
1133 		tr = &qpair->tr[cpl->cid];
1134 		if (tr->active) {
1135 			nvme_qpair_complete_tracker(qpair, tr, cpl, true);
1208 	struct nvme_tracker *tr;
1232 		tr = LIST_FIRST(&qpair->outstanding_tr);
1233 		nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,