1/*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2021 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/bus.h> 38#include <sys/kernel.h> 39#include <sys/module.h> 40#include <sys/sysctl.h> 41#include <machine/bus.h> 42#include <linux/device.h> 43#include <sys/rman.h> 44 45#include "ice_rdma.h" 46#include "irdma_main.h" 47#include "icrdma_hw.h" 48 49#include "irdma_if.h" 50#include "irdma_di_if.h" 51 52/** 53 * Driver version 54 */ 55char irdma_driver_version[] = "1.2.36-k"; 56 57/** 58 * irdma_init_tunable - prepare tunables 59 * @rf: RDMA PCI function 60 * @pf_id: id of the pf 61 */ 62static void 63irdma_init_tunable(struct irdma_pci_f *rf, uint8_t pf_id) 64{ 65 struct sysctl_oid_list *irdma_oid_list; 66 struct irdma_tunable_info *t_info = &rf->tun_info; 67 char pf_name[16]; 68 69 snprintf(pf_name, 15, "irdma%d", pf_id); 70 sysctl_ctx_init(&t_info->irdma_sysctl_ctx); 71 72 t_info->irdma_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx, 73 SYSCTL_STATIC_CHILDREN(_dev), 74 OID_AUTO, pf_name, 75 CTLFLAG_RD, NULL, ""); 76 77 irdma_oid_list = SYSCTL_CHILDREN(t_info->irdma_sysctl_tree); 78 79 t_info->sws_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx, 80 irdma_oid_list, OID_AUTO, 81 "sw_stats", CTLFLAG_RD, 82 NULL, ""); 83 /* 84 * debug mask setting 85 */ 86 SYSCTL_ADD_S32(&t_info->irdma_sysctl_ctx, irdma_oid_list, 87 OID_AUTO, "debug", CTLFLAG_RWTUN, &rf->sc_dev.debug_mask, 88 0, "irdma debug"); 89 90 /* 91 * RoCEv2/iWARP setting RoCEv2 the default mode 92 */ 93 t_info->roce_ena = 1; 94 SYSCTL_ADD_U8(&t_info->irdma_sysctl_ctx, irdma_oid_list, OID_AUTO, 95 "roce_enable", CTLFLAG_RDTUN, &t_info->roce_ena, 0, 96 "RoCEv2 mode enable"); 97 98 rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY; 99 if (t_info->roce_ena == 1) 100 rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY; 101 else if (t_info->roce_ena != 0) 102 printf("%s:%d wrong roce_enable value (%d), using iWARP\n", 103 __func__, __LINE__, t_info->roce_ena); 104 printf("%s:%d protocol: %s, roce_enable value: %d\n", __func__, __LINE__, 105 (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2", 106 t_info->roce_ena); 107 108 snprintf(t_info->drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version); 109 SYSCTL_ADD_STRING(&t_info->irdma_sysctl_ctx, irdma_oid_list, 110 OID_AUTO, "drv_ver", CTLFLAG_RDTUN, t_info->drv_ver, 111 IRDMA_VER_LEN, "driver version"); 112 113 irdma_dcqcn_tunables_init(rf); 114 irdma_sysctl_settings(rf); 115} 116 117/** 118 * irdma_find_handler - obtain hdl object to identify pf 119 * @p_dev: the peer interface structure 120 */ 121static struct irdma_handler * 122irdma_find_handler(struct ice_rdma_peer *p_dev) 123{ 124 struct irdma_handler *hdl; 125 unsigned long flags; 126 127 spin_lock_irqsave(&irdma_handler_lock, flags); 128 list_for_each_entry(hdl, &irdma_handlers, list) { 129 if (!hdl->iwdev->rf->peer_info) 130 continue; 131 if (hdl->iwdev->rf->peer_info->dev == p_dev->dev) { 132 spin_unlock_irqrestore(&irdma_handler_lock, flags); 133 return hdl; 134 } 135 } 136 spin_unlock_irqrestore(&irdma_handler_lock, flags); 137 138 return NULL; 139} 140 141/** 142 * peer_to_iwdev - return iwdev based on peer 143 * @peer: the peer interface structure 144 */ 145static struct irdma_device * 146peer_to_iwdev(struct ice_rdma_peer *peer) 147{ 148 struct irdma_handler *hdl; 149 150 hdl = irdma_find_handler(peer); 151 if (!hdl) { 152 printf("%s:%d rdma handler not found\n", __func__, __LINE__); 153 return NULL; 154 } 155 156 return hdl->iwdev; 157} 158 159/** 160 * irdma_get_qos_info - save qos info from parameters to internal struct 161 * @l2params: destination, qos, tc, mtu info structure 162 * @qos_info: source, DCB settings structure 163 */ 164static void 165irdma_get_qos_info(struct irdma_pci_f *rf, struct irdma_l2params *l2params, 166 struct ice_qos_params *qos_info) 167{ 168 int i; 169 char txt[7][128] = {"", "", "", "", "", "", ""}; 170 u8 len; 171 172 l2params->num_tc = qos_info->num_tc; 173 l2params->num_apps = qos_info->num_apps; 174 l2params->vsi_prio_type = qos_info->vsi_priority_type; 175 l2params->vsi_rel_bw = qos_info->vsi_relative_bw; 176 for (i = 0; i < l2params->num_tc; i++) { 177 l2params->tc_info[i].egress_virt_up = 178 qos_info->tc_info[i].egress_virt_up; 179 l2params->tc_info[i].ingress_virt_up = 180 qos_info->tc_info[i].ingress_virt_up; 181 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type; 182 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw; 183 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx; 184 } 185 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 186 l2params->up2tc[i] = qos_info->up2tc[i]; 187 188 if (qos_info->pfc_mode == IRDMA_QOS_MODE_DSCP) { 189 l2params->dscp_mode = true; 190 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); 191 } 192 if (!(rf->sc_dev.debug_mask & IRDMA_DEBUG_DCB)) 193 return; 194 for (i = 0; i < l2params->num_tc; i++) { 195 len = strlen(txt[0]); 196 snprintf(txt[0] + len, sizeof(txt[0]) - 5, " %d", 197 l2params->tc_info[i].egress_virt_up); 198 len = strlen(txt[1]); 199 snprintf(txt[1] + len, sizeof(txt[1]) - 5, " %d", 200 l2params->tc_info[i].ingress_virt_up); 201 len = strlen(txt[2]); 202 snprintf(txt[2] + len, sizeof(txt[2]) - 5, " %d", 203 l2params->tc_info[i].prio_type); 204 len = strlen(txt[3]); 205 snprintf(txt[3] + len, sizeof(txt[3]) - 5, " %d", 206 l2params->tc_info[i].rel_bw); 207 len = strlen(txt[4]); 208 snprintf(txt[4] + len, sizeof(txt[4]) - 5, " %lu", 209 l2params->tc_info[i].tc_ctx); 210 } 211 len = strlen(txt[5]); 212 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 213 len += snprintf(txt[5] + len, sizeof(txt[5]) - 5, " %d", 214 l2params->up2tc[i]); 215 len = strlen(txt[6]); 216 for (i = 0; i < IRDMA_DSCP_NUM_VAL; i++) 217 len += snprintf(txt[6] + len, sizeof(txt[6]) - 5, " %d", 218 l2params->dscp_map[i]); 219 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_tc: %d\n", l2params->num_tc); 220 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_apps: %d\n", l2params->num_apps); 221 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_prio_type: %d\n", l2params->vsi_prio_type); 222 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_rel_bw: %d\n", l2params->vsi_rel_bw); 223 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "egress_virt_up: %s\n", txt[0]); 224 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "ingress_virt_up:%s\n", txt[1]); 225 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "prio_type: %s\n", txt[2]); 226 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "rel_bw: %s\n", txt[3]); 227 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "tc_ctx: %s\n", txt[4]); 228 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "up2tc: %s\n", txt[5]); 229 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "dscp_mode: %s\n", txt[6]); 230 231 irdma_debug_buf(&rf->sc_dev, IRDMA_DEBUG_DCB, "l2params", l2params, sizeof(*l2params)); 232} 233 234/** 235 * irdma_log_invalid_mtu - check mtu setting validity 236 * @mtu: mtu value 237 * @dev: hardware control device structure 238 */ 239static void 240irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev) 241{ 242 if (mtu < IRDMA_MIN_MTU_IPV4) 243 irdma_dev_warn(to_ibdev(dev), 244 "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", 245 mtu); 246 else if (mtu < IRDMA_MIN_MTU_IPV6) 247 irdma_dev_warn(to_ibdev(dev), 248 "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\n", 249 mtu); 250} 251 252/** 253 * irdma_get_event_name - convert type enum to string 254 * @type: event type enum 255 */ 256static const char * 257irdma_get_event_name(enum ice_rdma_event_type type) 258{ 259 switch (type) { 260 case ICE_RDMA_EVENT_LINK_CHANGE: 261 return "LINK CHANGE"; 262 case ICE_RDMA_EVENT_MTU_CHANGE: 263 return "MTU CHANGE"; 264 case ICE_RDMA_EVENT_TC_CHANGE: 265 return "TC CHANGE"; 266 case ICE_RDMA_EVENT_API_CHANGE: 267 return "API CHANGE"; 268 case ICE_RDMA_EVENT_CRIT_ERR: 269 return "CRITICAL ERROR"; 270 case ICE_RDMA_EVENT_RESET: 271 return "RESET"; 272 case ICE_RDMA_EVENT_QSET_REGISTER: 273 return "QSET REGISTER"; 274 case ICE_RDMA_EVENT_VSI_FILTER_UPDATE: 275 return "VSI FILTER UPDATE"; 276 default: 277 return "UNKNOWN"; 278 } 279} 280 281/** 282 * irdma_event_handler - handling events from lan driver 283 * @peer: the peer interface structure 284 * @event: event info structure 285 */ 286static void 287irdma_event_handler(struct ice_rdma_peer *peer, struct ice_rdma_event *event) 288{ 289 struct irdma_device *iwdev; 290 struct irdma_l2params l2params = {}; 291 292 printf("%s:%d event_handler %s (%x) on pf %d (%d)\n", __func__, __LINE__, 293 irdma_get_event_name(event->type), 294 event->type, peer->pf_id, if_getdunit(peer->ifp)); 295 iwdev = peer_to_iwdev(peer); 296 if (!iwdev) { 297 printf("%s:%d rdma device not found\n", __func__, __LINE__); 298 return; 299 } 300 301 switch (event->type) { 302 case ICE_RDMA_EVENT_LINK_CHANGE: 303 printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__, 304 peer->pf_id, if_getdunit(peer->ifp), event->linkstate, 305 event->baudrate); 306 break; 307 case ICE_RDMA_EVENT_MTU_CHANGE: 308 if (iwdev->vsi.mtu != event->mtu) { 309 l2params.mtu = event->mtu; 310 l2params.mtu_changed = true; 311 irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev); 312 irdma_change_l2params(&iwdev->vsi, &l2params); 313 } 314 break; 315 case ICE_RDMA_EVENT_TC_CHANGE: 316 /* 317 * 1. check if it is pre or post 2. check if it is currently being done 318 */ 319 if (event->prep == iwdev->vsi.tc_change_pending) { 320 printf("%s:%d can't process %s TC change if TC change is %spending\n", 321 __func__, __LINE__, 322 event->prep ? "pre" : "post", 323 event->prep ? " " : "not "); 324 goto done; 325 } 326 if (!atomic_inc_not_zero(&iwdev->rf->dev_ctx.event_rfcnt)) { 327 printf("%s:%d (%d) EVENT_TC_CHANGE received, but not processed %d\n", 328 __func__, __LINE__, if_getdunit(peer->ifp), 329 atomic_read(&iwdev->rf->dev_ctx.event_rfcnt)); 330 break; 331 } 332 if (event->prep) { 333 iwdev->vsi.tc_change_pending = true; 334 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND); 335 wait_event_timeout(iwdev->suspend_wq, 336 !atomic_read(&iwdev->vsi.qp_suspend_reqs), 337 IRDMA_EVENT_TIMEOUT_MS * 10); 338 irdma_ws_reset(&iwdev->vsi); 339 printf("%s:%d TC change preparation done\n", __func__, __LINE__); 340 } else { 341 l2params.tc_changed = true; 342 irdma_get_qos_info(iwdev->rf, &l2params, &event->port_qos); 343 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 344 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; 345 346 irdma_check_fc_for_tc_update(&iwdev->vsi, &l2params); 347 irdma_change_l2params(&iwdev->vsi, &l2params); 348 printf("%s:%d TC change done\n", __func__, __LINE__); 349 } 350 atomic_dec(&iwdev->rf->dev_ctx.event_rfcnt); 351 break; 352 case ICE_RDMA_EVENT_CRIT_ERR: 353 if (event->oicr_reg & IRDMAPFINT_OICR_PE_CRITERR_M) { 354 u32 pe_criterr; 355 356#define IRDMA_Q1_RESOURCE_ERR 0x0001024d 357 pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); 358 if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { 359 irdma_pr_err("critical PE Error, GLPE_CRITERR=0x%08x\n", 360 pe_criterr); 361 iwdev->rf->reset = true; 362 } else { 363 irdma_dev_warn(to_ibdev(&iwdev->rf->sc_dev), 364 "Q1 Resource Check\n"); 365 } 366 } 367 if (event->oicr_reg & IRDMAPFINT_OICR_HMC_ERR_M) { 368 irdma_pr_err("HMC Error\n"); 369 iwdev->rf->reset = true; 370 } 371 if (iwdev->rf->reset) 372 iwdev->rf->gen_ops.request_reset(iwdev->rf); 373 break; 374 case ICE_RDMA_EVENT_RESET: 375 iwdev->rf->reset = true; 376 break; 377 default: 378 printf("%s:%d event type unsupported: %d\n", __func__, __LINE__, event->type); 379 } 380done: 381 return; 382} 383 384/** 385 * irdma_link_change - Callback for link state change 386 * @peer: the peer interface structure 387 * @linkstate: state of the link 388 * @baudrate: speed of the link 389 */ 390static void 391irdma_link_change(struct ice_rdma_peer *peer, int linkstate, uint64_t baudrate) 392{ 393 printf("%s:%d PF: %x (%x), state: %d, speed: %lu\n", __func__, __LINE__, 394 peer->pf_id, if_getdunit(peer->ifp), linkstate, baudrate); 395} 396 397/** 398 * irdma_finalize_task - Finish open or close phase in a separate thread 399 * @context: instance holding peer and iwdev information 400 * 401 * Triggered from irdma_open or irdma_close to perform rt_init_hw or 402 * rt_deinit_hw respectively. Does registration and unregistration of 403 * the device. 404 */ 405static void 406irdma_finalize_task(void *context, int pending) 407{ 408 struct irdma_task_arg *task_arg = (struct irdma_task_arg *)context; 409 struct irdma_device *iwdev = task_arg->iwdev; 410 struct irdma_pci_f *rf = iwdev->rf; 411 struct ice_rdma_peer *peer = task_arg->peer; 412 struct irdma_l2params l2params = {{{0}}}; 413 struct ice_rdma_request req = {0}; 414 int status = 0; 415 416 if (iwdev->iw_status) { 417 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, 418 "Starting deferred closing %d (%d)\n", 419 rf->peer_info->pf_id, if_getdunit(peer->ifp)); 420 atomic_dec(&rf->dev_ctx.event_rfcnt); 421 wait_event_timeout(iwdev->suspend_wq, 422 !atomic_read(&rf->dev_ctx.event_rfcnt), 423 IRDMA_MAX_TIMEOUT); 424 if (atomic_read(&rf->dev_ctx.event_rfcnt) != 0) { 425 printf("%s:%d (%d) waiting for event_rfcnt (%d) timeout, proceed with unload\n", 426 __func__, __LINE__, if_getdunit(peer->ifp), 427 atomic_read(&rf->dev_ctx.event_rfcnt)); 428 } 429 irdma_dereg_ipaddr_event_cb(rf); 430 irdma_ib_unregister_device(iwdev); 431 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE; 432 req.enable_filter = false; 433 IRDMA_DI_REQ_HANDLER(peer, &req); 434 irdma_cleanup_dead_qps(&iwdev->vsi); 435 irdma_rt_deinit_hw(iwdev); 436 } else { 437 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, 438 "Starting deferred opening %d (%d)\n", 439 rf->peer_info->pf_id, if_getdunit(peer->ifp)); 440 irdma_get_qos_info(iwdev->rf, &l2params, &peer->initial_qos_info); 441 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 442 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; 443 444 l2params.mtu = peer->mtu; 445 status = irdma_rt_init_hw(iwdev, &l2params); 446 if (status) { 447 irdma_pr_err("RT init failed %d\n", status); 448 ib_dealloc_device(&iwdev->ibdev); 449 return; 450 } 451 status = irdma_ib_register_device(iwdev); 452 if (status) { 453 irdma_pr_err("Registration failed %d\n", status); 454 irdma_rt_deinit_hw(iwdev); 455 ib_dealloc_device(&iwdev->ibdev); 456 } 457 irdma_sw_stats_tunables_init(rf); 458 req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE; 459 req.enable_filter = true; 460 IRDMA_DI_REQ_HANDLER(peer, &req); 461 irdma_reg_ipaddr_event_cb(rf); 462 atomic_inc(&rf->dev_ctx.event_rfcnt); 463 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, 464 "Deferred opening finished %d (%d)\n", 465 rf->peer_info->pf_id, if_getdunit(peer->ifp)); 466 } 467} 468 469/** 470 * irdma_alloc_pcidev - allocate memory for pcidev and populate data 471 * @peer: the new peer interface structure 472 * @rf: RDMA PCI function 473 */ 474static int 475irdma_alloc_pcidev(struct ice_rdma_peer *peer, struct irdma_pci_f *rf) 476{ 477 rf->pcidev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL); 478 if (!rf->pcidev) { 479 return -ENOMEM; 480 } 481 if (linux_pci_attach_device(rf->dev_ctx.dev, NULL, NULL, rf->pcidev)) 482 return -ENOMEM; 483 484 return 0; 485} 486 487/** 488 * irdma_dealloc_pcidev - deallocate memory for pcidev 489 * @rf: RDMA PCI function 490 */ 491static void 492irdma_dealloc_pcidev(struct irdma_pci_f *rf) 493{ 494 linux_pci_detach_device(rf->pcidev); 495 kfree(rf->pcidev); 496} 497 498/** 499 * irdma_fill_device_info - assign initial values to rf variables 500 * @iwdev: irdma device 501 * @peer: the peer interface structure 502 */ 503static void 504irdma_fill_device_info(struct irdma_device *iwdev, 505 struct ice_rdma_peer *peer) 506{ 507 struct irdma_pci_f *rf = iwdev->rf; 508 509 rf->peer_info = peer; 510 rf->gen_ops.register_qset = irdma_register_qset; 511 rf->gen_ops.unregister_qset = irdma_unregister_qset; 512 513 rf->rdma_ver = IRDMA_GEN_2; 514 rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2; 515 rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; 516 rf->rst_to = IRDMA_RST_TIMEOUT_HZ; 517 rf->check_fc = irdma_check_fc_for_qp; 518 rf->gen_ops.request_reset = irdma_request_reset; 519 irdma_set_rf_user_cfg_params(rf); 520 521 rf->default_vsi.vsi_idx = peer->pf_vsi_num; 522 rf->dev_ctx.dev = peer->dev; 523 rf->dev_ctx.mem_bus_space_tag = rman_get_bustag(peer->pci_mem); 524 rf->dev_ctx.mem_bus_space_handle = rman_get_bushandle(peer->pci_mem); 525 rf->dev_ctx.mem_bus_space_size = rman_get_size(peer->pci_mem); 526 527 rf->hw.dev_context = &rf->dev_ctx; 528 rf->hw.hw_addr = (u8 *)rman_get_virtual(peer->pci_mem); 529 rf->msix_count = peer->msix.count; 530 rf->msix_info.entry = peer->msix.base; 531 rf->msix_info.vector = peer->msix.count; 532 printf("%s:%d msix_info: %d %d %d\n", __func__, __LINE__, 533 rf->msix_count, rf->msix_info.entry, rf->msix_info.vector); 534 535 rf->iwdev = iwdev; 536 iwdev->netdev = peer->ifp; 537 iwdev->init_state = INITIAL_STATE; 538 iwdev->vsi_num = peer->pf_vsi_num; 539 iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED; 540 iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 541 iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; 542 iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; 543 iwdev->roce_rtomin = 5; 544 545 if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) { 546 iwdev->roce_mode = true; 547 } 548} 549 550/** 551 * irdma_probe - Callback to probe a new RDMA peer device 552 * @peer: the new peer interface structure 553 * 554 * Callback implementing the RDMA_PROBE function. Called by the ice driver to 555 * notify the RDMA client driver that a new device has been created 556 */ 557static int 558irdma_probe(struct ice_rdma_peer *peer) 559{ 560 struct irdma_device *iwdev; 561 struct irdma_pci_f *rf; 562 struct irdma_handler *hdl; 563 int err = 0; 564 565 irdma_pr_info("probe: irdma-%s peer=%p, peer->pf_id=%d, peer->ifp=%p, peer->ifp->if_dunit=%d, peer->pci_mem->r_bustag=%p\n", 566 irdma_driver_version, peer, peer->pf_id, peer->ifp, 567 if_getdunit(peer->ifp), (void *)(uintptr_t)peer->pci_mem->r_bustag); 568 569 hdl = irdma_find_handler(peer); 570 if (hdl) 571 return -EBUSY; 572 573 hdl = kzalloc(sizeof(*hdl), GFP_KERNEL); 574 if (!hdl) 575 return -ENOMEM; 576 577 iwdev = (struct irdma_device *)ib_alloc_device(sizeof(*iwdev)); 578 if (!iwdev) { 579 kfree(hdl); 580 return -ENOMEM; 581 } 582 583 iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL); 584 if (!iwdev->rf) { 585 ib_dealloc_device(&iwdev->ibdev); 586 kfree(hdl); 587 return -ENOMEM; 588 } 589 hdl->iwdev = iwdev; 590 iwdev->hdl = hdl; 591 592 irdma_init_tunable(iwdev->rf, if_getdunit(peer->ifp)); 593 irdma_fill_device_info(iwdev, peer); 594 rf = iwdev->rf; 595 596 if (irdma_alloc_pcidev(peer, rf)) 597 goto err_pcidev; 598 599 irdma_add_handler(hdl); 600 601 if (irdma_ctrl_init_hw(rf)) { 602 err = -EIO; 603 goto err_ctrl_init; 604 } 605 606 rf->dev_ctx.task_arg.peer = peer; 607 rf->dev_ctx.task_arg.iwdev = iwdev; 608 rf->dev_ctx.task_arg.peer = peer; 609 610 TASK_INIT(&hdl->deferred_task, 0, irdma_finalize_task, &rf->dev_ctx.task_arg); 611 hdl->deferred_tq = taskqueue_create_fast("irdma_defer", 612 M_NOWAIT, taskqueue_thread_enqueue, 613 &hdl->deferred_tq); 614 taskqueue_start_threads(&hdl->deferred_tq, 1, PI_NET, "irdma_defer_t"); 615 616 taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task); 617 618 return 0; 619 620err_ctrl_init: 621 irdma_del_handler(hdl); 622 irdma_dealloc_pcidev(rf); 623err_pcidev: 624 kfree(iwdev->rf); 625 ib_dealloc_device(&iwdev->ibdev); 626 kfree(hdl); 627 628 return err; 629} 630 631/** 632 * irdma_remove - Callback to remove an RDMA peer device 633 * @peer: the new peer interface structure 634 * 635 * Callback implementing the RDMA_REMOVE function. Called by the ice driver to 636 * notify the RDMA client driver that the device wille be delated 637 */ 638static int 639irdma_remove(struct ice_rdma_peer *peer) 640{ 641 struct irdma_handler *hdl; 642 struct irdma_device *iwdev; 643 644 irdma_debug((struct irdma_sc_dev *)NULL, IRDMA_DEBUG_INIT, 645 "removing %s irdma%d\n", __func__, if_getdunit(peer->ifp)); 646 647 hdl = irdma_find_handler(peer); 648 if (!hdl) 649 return 0; 650 651 iwdev = hdl->iwdev; 652 653 if (iwdev->vsi.tc_change_pending) { 654 iwdev->vsi.tc_change_pending = false; 655 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_RESUME); 656 } 657 658 taskqueue_enqueue(hdl->deferred_tq, &hdl->deferred_task); 659 660 taskqueue_drain(hdl->deferred_tq, &hdl->deferred_task); 661 taskqueue_free(hdl->deferred_tq); 662 hdl->iwdev->rf->dev_ctx.task_arg.iwdev = NULL; 663 hdl->iwdev->rf->dev_ctx.task_arg.peer = NULL; 664 665 sysctl_ctx_free(&iwdev->rf->tun_info.irdma_sysctl_ctx); 666 hdl->iwdev->rf->tun_info.irdma_sysctl_tree = NULL; 667 hdl->iwdev->rf->tun_info.sws_sysctl_tree = NULL; 668 669 irdma_ctrl_deinit_hw(iwdev->rf); 670 671 irdma_dealloc_pcidev(iwdev->rf); 672 673 irdma_del_handler(iwdev->hdl); 674 kfree(iwdev->hdl); 675 kfree(iwdev->rf); 676 ib_dealloc_device(&iwdev->ibdev); 677 irdma_pr_info("IRDMA hardware deinitialization complete irdma%d\n", 678 if_getdunit(peer->ifp)); 679 680 return 0; 681} 682 683/** 684 * irdma_open - Callback for operation open for RDMA device 685 * @peer: the new peer interface structure 686 * 687 * Callback implementing the RDMA_OPEN function. Called by the ice driver to 688 * notify the RDMA client driver that a new device has been initialized. 689 */ 690static int 691irdma_open(struct ice_rdma_peer *peer) 692{ 693 struct irdma_device *iwdev; 694 struct ice_rdma_event event = {0}; 695 696 iwdev = peer_to_iwdev(peer); 697 if (iwdev) { 698 event.type = ICE_RDMA_EVENT_MTU_CHANGE; 699 event.mtu = peer->mtu; 700 701 irdma_event_handler(peer, &event); 702 } else { 703 irdma_probe(peer); 704 } 705 706 return 0; 707} 708 709/** 710 * irdma_close - Callback to notify that a peer device is down 711 * @peer: the RDMA peer device being stopped 712 * 713 * Callback implementing the RDMA_CLOSE function. Called by the ice driver to 714 * notify the RDMA client driver that a peer device is being stopped. 715 */ 716static int 717irdma_close(struct ice_rdma_peer *peer) 718{ 719 /* 720 * This is called when ifconfig down or pf-reset is about to happen. 721 */ 722 struct irdma_device *iwdev; 723 724 iwdev = peer_to_iwdev(peer); 725 if (iwdev && iwdev->rf->reset) 726 irdma_remove(peer); 727 728 return 0; 729} 730 731/** 732 * irdma_prep_for_unregister - ensure the driver is ready to unregister 733 */ 734static void 735irdma_prep_for_unregister(void) 736{ 737 struct irdma_handler *hdl; 738 unsigned long flags; 739 bool hdl_valid; 740 741 do { 742 hdl_valid = false; 743 spin_lock_irqsave(&irdma_handler_lock, flags); 744 list_for_each_entry(hdl, &irdma_handlers, list) { 745 if (!hdl->iwdev->rf->peer_info) 746 continue; 747 hdl_valid = true; 748 break; 749 } 750 spin_unlock_irqrestore(&irdma_handler_lock, flags); 751 if (!hdl || !hdl_valid) 752 break; 753 IRDMA_CLOSE(hdl->iwdev->rf->peer_info); 754 IRDMA_REMOVE(hdl->iwdev->rf->peer_info); 755 } while (1); 756} 757 758static kobj_method_t irdma_methods[] = { 759 KOBJMETHOD(irdma_probe, irdma_probe), 760 KOBJMETHOD(irdma_open, irdma_open), 761 KOBJMETHOD(irdma_close, irdma_close), 762 KOBJMETHOD(irdma_remove, irdma_remove), 763 KOBJMETHOD(irdma_link_change, irdma_link_change), 764 KOBJMETHOD(irdma_event_handler, irdma_event_handler), 765 KOBJMETHOD_END 766}; 767 768/* declare irdma_class which extends the ice_rdma_di class */ 769DEFINE_CLASS_1(irdma, irdma_class, irdma_methods, sizeof(struct ice_rdma_peer), ice_rdma_di_class); 770 771static struct ice_rdma_info irdma_info = { 772 .major_version = ICE_RDMA_MAJOR_VERSION, 773 .minor_version = ICE_RDMA_MINOR_VERSION, 774 .patch_version = ICE_RDMA_PATCH_VERSION, 775 .rdma_class = &irdma_class, 776}; 777 778/** 779 * irdma_module_event_handler - Module event handler callback 780 * @mod: unused mod argument 781 * @what: the module event to handle 782 * @arg: unused module event argument 783 * 784 * Callback used by the FreeBSD module stack to notify the driver of module 785 * events. Used to implement custom handling for certain module events such as 786 * load and unload. 787 */ 788static int 789irdma_module_event_handler(module_t __unused mod, int what, void __unused * arg) 790{ 791 switch (what) { 792 case MOD_LOAD: 793 printf("Loading irdma module\n"); 794 return ice_rdma_register(&irdma_info); 795 case MOD_UNLOAD: 796 printf("Unloading irdma module\n"); 797 irdma_prep_for_unregister(); 798 ice_rdma_unregister(); 799 return (0); 800 default: 801 return (EOPNOTSUPP); 802 } 803 804 return (0); 805} 806 807static moduledata_t irdma_moduledata = { 808 "irdma", 809 irdma_module_event_handler, 810 NULL 811}; 812 813DECLARE_MODULE(irdma, irdma_moduledata, SI_SUB_LAST, SI_ORDER_ANY); 814MODULE_VERSION(irdma, 1); 815MODULE_DEPEND(irdma, ice, 1, 1, 1); 816MODULE_DEPEND(irdma, ibcore, 1, 1, 1); 817