cma.c revision 325617
1/* 2 * Copyright (c) 2005 Voltaire Inc. All rights reserved. 3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved. 5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 6 * Copyright (c) 2016 Chelsio Communications. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 */ 36 37#define LINUXKPI_PARAM_PREFIX ibcore_ 38 39#include <linux/completion.h> 40#include <linux/in.h> 41#include <linux/in6.h> 42#include <linux/mutex.h> 43#include <linux/random.h> 44#include <linux/idr.h> 45#include <linux/inetdevice.h> 46 47#include <net/tcp.h> 48#include <net/ipv6.h> 49 50#include <rdma/rdma_cm.h> 51#include <rdma/rdma_cm_ib.h> 52#include <rdma/ib_cache.h> 53#include <rdma/ib_cm.h> 54#include <rdma/ib_sa.h> 55#include <rdma/iw_cm.h> 56 57MODULE_AUTHOR("Sean Hefty"); 58MODULE_DESCRIPTION("Generic RDMA CM Agent"); 59MODULE_LICENSE("Dual BSD/GPL"); 60 61static int tavor_quirk = 0; 62module_param_named(tavor_quirk, tavor_quirk, int, 0644); 63MODULE_PARM_DESC(tavor_quirk, "Tavor performance quirk: limit MTU to 1K if > 0"); 64 65#define CMA_CM_RESPONSE_TIMEOUT 20 66#define CMA_MAX_CM_RETRIES 15 67#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 68#define IBOE_PACKET_LIFETIME 18 69 70static int cma_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 71module_param_named(cma_response_timeout, cma_response_timeout, int, 0644); 72MODULE_PARM_DESC(cma_response_timeout, "CMA_CM_RESPONSE_TIMEOUT default=20"); 73 74static int def_prec2sl = 3; 75module_param_named(def_prec2sl, def_prec2sl, int, 0644); 76MODULE_PARM_DESC(def_prec2sl, "Default value for SL priority with RoCE. Valid values 0 - 7"); 77 78static void cma_add_one(struct ib_device *device); 79static void cma_remove_one(struct ib_device *device); 80 81static struct ib_client cma_client = { 82 .name = "cma", 83 .add = cma_add_one, 84 .remove = cma_remove_one 85}; 86 87static struct ib_sa_client sa_client; 88static struct rdma_addr_client addr_client; 89static LIST_HEAD(dev_list); 90static LIST_HEAD(listen_any_list); 91static DEFINE_MUTEX(lock); 92static struct workqueue_struct *cma_wq; 93static DEFINE_IDR(sdp_ps); 94static DEFINE_IDR(tcp_ps); 95static DEFINE_IDR(udp_ps); 96static DEFINE_IDR(ipoib_ps); 97#if defined(INET) 98static int next_port; 99#endif 100 101struct cma_device { 102 struct list_head list; 103 struct ib_device *device; 104 struct completion comp; 105 atomic_t refcount; 106 struct list_head id_list; 107}; 108 109enum cma_state { 110 CMA_IDLE, 111 CMA_ADDR_QUERY, 112 CMA_ADDR_RESOLVED, 113 CMA_ROUTE_QUERY, 114 CMA_ROUTE_RESOLVED, 115 CMA_CONNECT, 116 CMA_DISCONNECT, 117 CMA_ADDR_BOUND, 118 CMA_LISTEN, 119 CMA_DEVICE_REMOVAL, 120 CMA_DESTROYING 121}; 122 123struct rdma_bind_list { 124 struct idr *ps; 125 struct hlist_head owners; 126 unsigned short port; 127}; 128 129/* 130 * Device removal can occur at anytime, so we need extra handling to 131 * serialize notifying the user of device removal with other callbacks. 132 * We do this by disabling removal notification while a callback is in process, 133 * and reporting it after the callback completes. 134 */ 135struct rdma_id_private { 136 struct rdma_cm_id id; 137 138 struct rdma_bind_list *bind_list; 139 struct socket *sock; 140 struct hlist_node node; 141 struct list_head list; /* listen_any_list or cma_device.list */ 142 struct list_head listen_list; /* per device listens */ 143 struct cma_device *cma_dev; 144 struct list_head mc_list; 145 146 int internal_id; 147 enum cma_state state; 148 spinlock_t lock; 149 struct mutex qp_mutex; 150 151 struct completion comp; 152 atomic_t refcount; 153 struct mutex handler_mutex; 154 155 int backlog; 156 int timeout_ms; 157 struct ib_sa_query *query; 158 int query_id; 159 union { 160 struct ib_cm_id *ib; 161 struct iw_cm_id *iw; 162 } cm_id; 163 164 u32 seq_num; 165 u32 qkey; 166 u32 qp_num; 167 u8 srq; 168 u8 tos; 169 int unify_ps_tcp; 170}; 171 172struct cma_multicast { 173 struct rdma_id_private *id_priv; 174 union { 175 struct ib_sa_multicast *ib; 176 } multicast; 177 struct list_head list; 178 void *context; 179 struct sockaddr_storage addr; 180 struct kref mcref; 181}; 182 183struct cma_work { 184 struct work_struct work; 185 struct rdma_id_private *id; 186 enum cma_state old_state; 187 enum cma_state new_state; 188 struct rdma_cm_event event; 189}; 190 191struct cma_ndev_work { 192 struct work_struct work; 193 struct rdma_id_private *id; 194 struct rdma_cm_event event; 195}; 196 197struct iboe_mcast_work { 198 struct work_struct work; 199 struct rdma_id_private *id; 200 struct cma_multicast *mc; 201}; 202 203union cma_ip_addr { 204 struct in6_addr ip6; 205 struct { 206 __be32 pad[3]; 207 __be32 addr; 208 } ip4; 209}; 210 211struct cma_hdr { 212 u8 cma_version; 213 u8 ip_version; /* IP version: 7:4 */ 214 __be16 port; 215 union cma_ip_addr src_addr; 216 union cma_ip_addr dst_addr; 217}; 218 219struct sdp_hh { 220 u8 bsdh[16]; 221 u8 sdp_version; /* Major version: 7:4 */ 222 u8 ip_version; /* IP version: 7:4 */ 223 u8 sdp_specific1[10]; 224 __be16 port; 225 __be16 sdp_specific2; 226 union cma_ip_addr src_addr; 227 union cma_ip_addr dst_addr; 228}; 229 230struct sdp_hah { 231 u8 bsdh[16]; 232 u8 sdp_version; 233}; 234 235#define CMA_VERSION 0x00 236#define SDP_MAJ_VERSION 0x2 237 238static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp) 239{ 240 unsigned long flags; 241 int ret; 242 243 spin_lock_irqsave(&id_priv->lock, flags); 244 ret = (id_priv->state == comp); 245 spin_unlock_irqrestore(&id_priv->lock, flags); 246 return ret; 247} 248 249static int cma_comp_exch(struct rdma_id_private *id_priv, 250 enum cma_state comp, enum cma_state exch) 251{ 252 unsigned long flags; 253 int ret; 254 255 spin_lock_irqsave(&id_priv->lock, flags); 256 if ((ret = (id_priv->state == comp))) 257 id_priv->state = exch; 258 spin_unlock_irqrestore(&id_priv->lock, flags); 259 return ret; 260} 261 262static enum cma_state cma_exch(struct rdma_id_private *id_priv, 263 enum cma_state exch) 264{ 265 unsigned long flags; 266 enum cma_state old; 267 268 spin_lock_irqsave(&id_priv->lock, flags); 269 old = id_priv->state; 270 id_priv->state = exch; 271 spin_unlock_irqrestore(&id_priv->lock, flags); 272 return old; 273} 274 275static inline u8 cma_get_ip_ver(struct cma_hdr *hdr) 276{ 277 return hdr->ip_version >> 4; 278} 279 280static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 281{ 282 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 283} 284 285static inline u8 sdp_get_majv(u8 sdp_version) 286{ 287 return sdp_version >> 4; 288} 289 290static inline u8 sdp_get_ip_ver(struct sdp_hh *hh) 291{ 292 return hh->ip_version >> 4; 293} 294 295static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver) 296{ 297 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF); 298} 299 300static inline int cma_is_ud_ps(enum rdma_port_space ps) 301{ 302 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB); 303} 304 305static void cma_attach_to_dev(struct rdma_id_private *id_priv, 306 struct cma_device *cma_dev) 307{ 308 atomic_inc(&cma_dev->refcount); 309 id_priv->cma_dev = cma_dev; 310 id_priv->id.device = cma_dev->device; 311 id_priv->id.route.addr.dev_addr.transport = 312 rdma_node_get_transport(cma_dev->device->node_type); 313 list_add_tail(&id_priv->list, &cma_dev->id_list); 314} 315 316static inline void cma_deref_dev(struct cma_device *cma_dev) 317{ 318 if (atomic_dec_and_test(&cma_dev->refcount)) 319 complete(&cma_dev->comp); 320} 321 322static inline void release_mc(struct kref *kref) 323{ 324 struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); 325 326 kfree(mc->multicast.ib); 327 kfree(mc); 328} 329 330static void cma_detach_from_dev(struct rdma_id_private *id_priv) 331{ 332 list_del(&id_priv->list); 333 cma_deref_dev(id_priv->cma_dev); 334 id_priv->cma_dev = NULL; 335} 336 337static int cma_set_qkey(struct rdma_id_private *id_priv) 338{ 339 struct ib_sa_mcmember_rec rec; 340 int ret = 0; 341 342 if (id_priv->qkey) 343 return 0; 344 345 switch (id_priv->id.ps) { 346 case RDMA_PS_UDP: 347 id_priv->qkey = RDMA_UDP_QKEY; 348 break; 349 case RDMA_PS_IPOIB: 350 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); 351 ret = ib_sa_get_mcmember_rec(id_priv->id.device, 352 id_priv->id.port_num, &rec.mgid, 353 &rec); 354 if (!ret) 355 id_priv->qkey = be32_to_cpu(rec.qkey); 356 break; 357 default: 358 break; 359 } 360 return ret; 361} 362 363static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num) 364{ 365 int i; 366 int err; 367 struct ib_port_attr props; 368 union ib_gid tmp; 369 370 err = ib_query_port(device, port_num, &props); 371 if (err) 372 return 1; 373 374 for (i = 0; i < props.gid_tbl_len; ++i) { 375 err = ib_query_gid(device, port_num, i, &tmp); 376 if (err) 377 return 1; 378 if (!memcmp(&tmp, gid, sizeof tmp)) 379 return 0; 380 } 381 382 return -EAGAIN; 383} 384 385int 386rdma_find_cmid_laddr(struct sockaddr_in *local_addr, unsigned short dev_type, 387 void **cm_id) 388{ 389 int ret; 390 u8 port; 391 int found_dev = 0, found_cmid = 0; 392 struct rdma_id_private *id_priv; 393 struct rdma_id_private *dev_id_priv; 394 struct cma_device *cma_dev; 395 struct rdma_dev_addr dev_addr; 396 union ib_gid gid; 397 enum rdma_link_layer dev_ll = dev_type == ARPHRD_INFINIBAND ? 398 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET; 399 400 memset(&dev_addr, 0, sizeof(dev_addr)); 401 402 ret = rdma_translate_ip((struct sockaddr *)local_addr, 403 &dev_addr); 404 if (ret) 405 goto err; 406 407 /* find rdma device based on MAC address/gid */ 408 mutex_lock(&lock); 409 410 memcpy(&gid, dev_addr.src_dev_addr + 411 rdma_addr_gid_offset(&dev_addr), sizeof(gid)); 412 413 list_for_each_entry(cma_dev, &dev_list, list) 414 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) 415 if ((rdma_port_get_link_layer(cma_dev->device, port) == 416 dev_ll) && 417 (rdma_node_get_transport(cma_dev->device->node_type) == 418 RDMA_TRANSPORT_IWARP)) { 419 ret = find_gid_port(cma_dev->device, 420 &gid, port); 421 if (!ret) { 422 found_dev = 1; 423 goto out; 424 } else if (ret == 1) { 425 mutex_unlock(&lock); 426 goto err; 427 } 428 } 429out: 430 mutex_unlock(&lock); 431 432 if (!found_dev) 433 goto err; 434 435 /* Traverse through the list of listening cm_id's to find the 436 * desired cm_id based on rdma device & port number. 437 */ 438 list_for_each_entry(id_priv, &listen_any_list, list) 439 list_for_each_entry(dev_id_priv, &id_priv->listen_list, 440 listen_list) 441 if (dev_id_priv->cma_dev == cma_dev) 442 if (dev_id_priv->cm_id.iw->local_addr.sin_port 443 == local_addr->sin_port) { 444 *cm_id = (void *)dev_id_priv->cm_id.iw; 445 found_cmid = 1; 446 } 447 return found_cmid ? 0 : -ENODEV; 448 449err: 450 return -ENODEV; 451} 452EXPORT_SYMBOL(rdma_find_cmid_laddr); 453 454static int cma_acquire_dev(struct rdma_id_private *id_priv) 455{ 456 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 457 struct cma_device *cma_dev; 458 union ib_gid gid; 459 int ret = -ENODEV; 460 461 if (dev_addr->dev_type != ARPHRD_INFINIBAND) { 462 iboe_addr_get_sgid(dev_addr, &gid); 463 list_for_each_entry(cma_dev, &dev_list, list) { 464 ret = ib_find_cached_gid(cma_dev->device, &gid, 465 &id_priv->id.port_num, NULL); 466 if (!ret) 467 goto out; 468 } 469 } 470 471 memcpy(&gid, dev_addr->src_dev_addr + 472 rdma_addr_gid_offset(dev_addr), sizeof gid); 473 list_for_each_entry(cma_dev, &dev_list, list) { 474 ret = ib_find_cached_gid(cma_dev->device, &gid, 475 &id_priv->id.port_num, NULL); 476 if (!ret) 477 break; 478 } 479 480out: 481 if (!ret) 482 cma_attach_to_dev(id_priv, cma_dev); 483 484 return ret; 485} 486 487static void cma_deref_id(struct rdma_id_private *id_priv) 488{ 489 if (atomic_dec_and_test(&id_priv->refcount)) 490 complete(&id_priv->comp); 491} 492 493static int cma_disable_callback(struct rdma_id_private *id_priv, 494 enum cma_state state) 495{ 496 mutex_lock(&id_priv->handler_mutex); 497 if (id_priv->state != state) { 498 mutex_unlock(&id_priv->handler_mutex); 499 return -EINVAL; 500 } 501 return 0; 502} 503 504static int cma_has_cm_dev(struct rdma_id_private *id_priv) 505{ 506 return (id_priv->id.device && id_priv->cm_id.ib); 507} 508 509struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler, 510 void *context, enum rdma_port_space ps) 511{ 512 struct rdma_id_private *id_priv; 513 514 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 515 if (!id_priv) 516 return ERR_PTR(-ENOMEM); 517 518 id_priv->state = CMA_IDLE; 519 id_priv->id.context = context; 520 id_priv->id.event_handler = event_handler; 521 id_priv->id.ps = ps; 522 spin_lock_init(&id_priv->lock); 523 mutex_init(&id_priv->qp_mutex); 524 init_completion(&id_priv->comp); 525 atomic_set(&id_priv->refcount, 1); 526 mutex_init(&id_priv->handler_mutex); 527 INIT_LIST_HEAD(&id_priv->listen_list); 528 INIT_LIST_HEAD(&id_priv->mc_list); 529 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 530 531 return &id_priv->id; 532} 533EXPORT_SYMBOL(rdma_create_id); 534 535static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 536{ 537 struct ib_qp_attr qp_attr; 538 int qp_attr_mask, ret; 539 540 qp_attr.qp_state = IB_QPS_INIT; 541 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 542 if (ret) 543 return ret; 544 545 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 546 if (ret) 547 return ret; 548 549 qp_attr.qp_state = IB_QPS_RTR; 550 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 551 if (ret) 552 return ret; 553 554 qp_attr.qp_state = IB_QPS_RTS; 555 qp_attr.sq_psn = 0; 556 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 557 558 return ret; 559} 560 561static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 562{ 563 struct ib_qp_attr qp_attr; 564 int qp_attr_mask, ret; 565 566 qp_attr.qp_state = IB_QPS_INIT; 567 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 568 if (ret) 569 return ret; 570 571 return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 572} 573 574int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 575 struct ib_qp_init_attr *qp_init_attr) 576{ 577 struct rdma_id_private *id_priv; 578 struct ib_qp *qp; 579 int ret; 580 581 id_priv = container_of(id, struct rdma_id_private, id); 582 if (id->device != pd->device) 583 return -EINVAL; 584 585 qp = ib_create_qp(pd, qp_init_attr); 586 if (IS_ERR(qp)) 587 return PTR_ERR(qp); 588 589 if (cma_is_ud_ps(id_priv->id.ps)) 590 ret = cma_init_ud_qp(id_priv, qp); 591 else 592 ret = cma_init_conn_qp(id_priv, qp); 593 if (ret) 594 goto err; 595 596 id->qp = qp; 597 id_priv->qp_num = qp->qp_num; 598 id_priv->srq = (qp->srq != NULL); 599 return 0; 600err: 601 ib_destroy_qp(qp); 602 return ret; 603} 604EXPORT_SYMBOL(rdma_create_qp); 605 606void rdma_destroy_qp(struct rdma_cm_id *id) 607{ 608 struct rdma_id_private *id_priv; 609 610 id_priv = container_of(id, struct rdma_id_private, id); 611 mutex_lock(&id_priv->qp_mutex); 612 ib_destroy_qp(id_priv->id.qp); 613 id_priv->id.qp = NULL; 614 mutex_unlock(&id_priv->qp_mutex); 615} 616EXPORT_SYMBOL(rdma_destroy_qp); 617 618static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, 619 struct rdma_conn_param *conn_param) 620{ 621 struct ib_qp_attr qp_attr; 622 int qp_attr_mask, ret; 623 624 mutex_lock(&id_priv->qp_mutex); 625 if (!id_priv->id.qp) { 626 ret = 0; 627 goto out; 628 } 629 630 /* Need to update QP attributes from default values. */ 631 qp_attr.qp_state = IB_QPS_INIT; 632 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 633 if (ret) 634 goto out; 635 636 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 637 if (ret) 638 goto out; 639 640 qp_attr.qp_state = IB_QPS_RTR; 641 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 642 if (ret) 643 goto out; 644 645 if (conn_param) 646 qp_attr.max_dest_rd_atomic = conn_param->responder_resources; 647 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 648out: 649 mutex_unlock(&id_priv->qp_mutex); 650 return ret; 651} 652 653static int cma_modify_qp_rts(struct rdma_id_private *id_priv, 654 struct rdma_conn_param *conn_param) 655{ 656 struct ib_qp_attr qp_attr; 657 int qp_attr_mask, ret; 658 659 mutex_lock(&id_priv->qp_mutex); 660 if (!id_priv->id.qp) { 661 ret = 0; 662 goto out; 663 } 664 665 qp_attr.qp_state = IB_QPS_RTS; 666 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 667 if (ret) 668 goto out; 669 670 if (conn_param) 671 qp_attr.max_rd_atomic = conn_param->initiator_depth; 672 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 673out: 674 mutex_unlock(&id_priv->qp_mutex); 675 return ret; 676} 677 678static int cma_modify_qp_err(struct rdma_id_private *id_priv) 679{ 680 struct ib_qp_attr qp_attr; 681 int ret; 682 683 mutex_lock(&id_priv->qp_mutex); 684 if (!id_priv->id.qp) { 685 ret = 0; 686 goto out; 687 } 688 689 qp_attr.qp_state = IB_QPS_ERR; 690 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 691out: 692 mutex_unlock(&id_priv->qp_mutex); 693 return ret; 694} 695 696static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 697 struct ib_qp_attr *qp_attr, int *qp_attr_mask) 698{ 699 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 700 int ret; 701 u16 pkey; 702 703 if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) == 704 IB_LINK_LAYER_INFINIBAND) 705 pkey = ib_addr_get_pkey(dev_addr); 706 else 707 pkey = 0xffff; 708 709 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 710 pkey, &qp_attr->pkey_index); 711 if (ret) 712 return ret; 713 714 qp_attr->port_num = id_priv->id.port_num; 715 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 716 717 if (cma_is_ud_ps(id_priv->id.ps)) { 718 ret = cma_set_qkey(id_priv); 719 if (ret) 720 return ret; 721 722 qp_attr->qkey = id_priv->qkey; 723 *qp_attr_mask |= IB_QP_QKEY; 724 } else { 725 qp_attr->qp_access_flags = 0; 726 *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 727 } 728 return 0; 729} 730 731int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 732 int *qp_attr_mask) 733{ 734 struct rdma_id_private *id_priv; 735 int ret = 0; 736 737 id_priv = container_of(id, struct rdma_id_private, id); 738 if (rdma_cap_ib_cm(id->device, id->port_num)) { 739 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps)) 740 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 741 else 742 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 743 qp_attr_mask); 744 745 if (qp_attr->qp_state == IB_QPS_RTR) 746 qp_attr->rq_psn = id_priv->seq_num; 747 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 748 if (!id_priv->cm_id.iw) { 749 qp_attr->qp_access_flags = 0; 750 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 751 } else 752 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 753 qp_attr_mask); 754 } else 755 ret = -ENOSYS; 756 757 return ret; 758} 759EXPORT_SYMBOL(rdma_init_qp_attr); 760 761static inline int cma_zero_addr(struct sockaddr *addr) 762{ 763 struct in6_addr *ip6; 764 765 if (addr->sa_family == AF_INET) 766 return ipv4_is_zeronet( 767 ((struct sockaddr_in *)addr)->sin_addr.s_addr); 768 else { 769 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; 770 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | 771 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; 772 } 773} 774 775static inline int cma_loopback_addr(struct sockaddr *addr) 776{ 777 if (addr->sa_family == AF_INET) 778 return ipv4_is_loopback( 779 ((struct sockaddr_in *) addr)->sin_addr.s_addr); 780 else 781 return ipv6_addr_loopback( 782 &((struct sockaddr_in6 *) addr)->sin6_addr); 783} 784 785static inline int cma_any_addr(struct sockaddr *addr) 786{ 787 return cma_zero_addr(addr) || cma_loopback_addr(addr); 788} 789int 790rdma_cma_any_addr(struct sockaddr *addr) 791{ 792 return cma_any_addr(addr); 793} 794EXPORT_SYMBOL(rdma_cma_any_addr); 795 796static inline __be16 cma_port(struct sockaddr *addr) 797{ 798 if (addr->sa_family == AF_INET) 799 return ((struct sockaddr_in *) addr)->sin_port; 800 else 801 return ((struct sockaddr_in6 *) addr)->sin6_port; 802} 803 804static inline int cma_any_port(struct sockaddr *addr) 805{ 806 return !cma_port(addr); 807} 808 809static int cma_get_net_info(void *hdr, enum rdma_port_space ps, 810 u8 *ip_ver, __be16 *port, 811 union cma_ip_addr **src, union cma_ip_addr **dst) 812{ 813 switch (ps) { 814 case RDMA_PS_SDP: 815 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) != 816 SDP_MAJ_VERSION) 817 return -EINVAL; 818 819 *ip_ver = sdp_get_ip_ver(hdr); 820 *port = ((struct sdp_hh *) hdr)->port; 821 *src = &((struct sdp_hh *) hdr)->src_addr; 822 *dst = &((struct sdp_hh *) hdr)->dst_addr; 823 break; 824 default: 825 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION) 826 return -EINVAL; 827 828 *ip_ver = cma_get_ip_ver(hdr); 829 *port = ((struct cma_hdr *) hdr)->port; 830 *src = &((struct cma_hdr *) hdr)->src_addr; 831 *dst = &((struct cma_hdr *) hdr)->dst_addr; 832 break; 833 } 834 835 if (*ip_ver != 4 && *ip_ver != 6) 836 return -EINVAL; 837 return 0; 838} 839 840static void cma_save_net_info(struct rdma_addr *addr, 841 struct rdma_addr *listen_addr, 842 u8 ip_ver, __be16 port, 843 union cma_ip_addr *src, union cma_ip_addr *dst) 844{ 845 struct sockaddr_in *listen4, *ip4; 846 struct sockaddr_in6 *listen6, *ip6; 847 848 switch (ip_ver) { 849 case 4: 850 listen4 = (struct sockaddr_in *) &listen_addr->src_addr; 851 ip4 = (struct sockaddr_in *) &addr->src_addr; 852 ip4->sin_family = listen4->sin_family; 853 ip4->sin_addr.s_addr = dst->ip4.addr; 854 ip4->sin_port = listen4->sin_port; 855 856 ip4 = (struct sockaddr_in *) &addr->dst_addr; 857 ip4->sin_family = listen4->sin_family; 858 ip4->sin_addr.s_addr = src->ip4.addr; 859 ip4->sin_port = port; 860 break; 861 case 6: 862 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr; 863 ip6 = (struct sockaddr_in6 *) &addr->src_addr; 864 ip6->sin6_family = listen6->sin6_family; 865 ip6->sin6_addr = dst->ip6; 866 ip6->sin6_port = listen6->sin6_port; 867 868 ip6 = (struct sockaddr_in6 *) &addr->dst_addr; 869 ip6->sin6_family = listen6->sin6_family; 870 ip6->sin6_addr = src->ip6; 871 ip6->sin6_port = port; 872 break; 873 default: 874 break; 875 } 876} 877 878static inline int cma_user_data_offset(enum rdma_port_space ps) 879{ 880 switch (ps) { 881 case RDMA_PS_SDP: 882 return 0; 883 default: 884 return sizeof(struct cma_hdr); 885 } 886} 887 888static void cma_cancel_route(struct rdma_id_private *id_priv) 889{ 890 switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) { 891 case IB_LINK_LAYER_INFINIBAND: 892 if (id_priv->query) 893 ib_sa_cancel_query(id_priv->query_id, id_priv->query); 894 break; 895 default: 896 break; 897 } 898} 899 900static void cma_cancel_listens(struct rdma_id_private *id_priv) 901{ 902 struct rdma_id_private *dev_id_priv; 903 904 /* 905 * Remove from listen_any_list to prevent added devices from spawning 906 * additional listen requests. 907 */ 908 mutex_lock(&lock); 909 list_del(&id_priv->list); 910 911 while (!list_empty(&id_priv->listen_list)) { 912 dev_id_priv = list_entry(id_priv->listen_list.next, 913 struct rdma_id_private, listen_list); 914 /* sync with device removal to avoid duplicate destruction */ 915 list_del_init(&dev_id_priv->list); 916 list_del(&dev_id_priv->listen_list); 917 mutex_unlock(&lock); 918 919 rdma_destroy_id(&dev_id_priv->id); 920 mutex_lock(&lock); 921 } 922 mutex_unlock(&lock); 923} 924 925static void cma_cancel_operation(struct rdma_id_private *id_priv, 926 enum cma_state state) 927{ 928 switch (state) { 929 case CMA_ADDR_QUERY: 930 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 931 break; 932 case CMA_ROUTE_QUERY: 933 cma_cancel_route(id_priv); 934 break; 935 case CMA_LISTEN: 936 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr) 937 && !id_priv->cma_dev) 938 cma_cancel_listens(id_priv); 939 break; 940 default: 941 break; 942 } 943} 944 945static void cma_release_port(struct rdma_id_private *id_priv) 946{ 947 struct rdma_bind_list *bind_list = id_priv->bind_list; 948 949 if (!bind_list) 950 return; 951 952 mutex_lock(&lock); 953 hlist_del(&id_priv->node); 954 if (hlist_empty(&bind_list->owners)) { 955 idr_remove(bind_list->ps, bind_list->port); 956 kfree(bind_list); 957 } 958 mutex_unlock(&lock); 959} 960 961static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 962{ 963 struct cma_multicast *mc; 964 965 while (!list_empty(&id_priv->mc_list)) { 966 mc = container_of(id_priv->mc_list.next, 967 struct cma_multicast, list); 968 list_del(&mc->list); 969 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) { 970 case IB_LINK_LAYER_INFINIBAND: 971 ib_sa_free_multicast(mc->multicast.ib); 972 kfree(mc); 973 break; 974 case IB_LINK_LAYER_ETHERNET: 975 kref_put(&mc->mcref, release_mc); 976 break; 977 default: 978 break; 979 } 980 } 981} 982 983void rdma_destroy_id(struct rdma_cm_id *id) 984{ 985 struct rdma_id_private *id_priv; 986 enum cma_state state; 987 988 id_priv = container_of(id, struct rdma_id_private, id); 989 state = cma_exch(id_priv, CMA_DESTROYING); 990 cma_cancel_operation(id_priv, state); 991 992 mutex_lock(&lock); 993 if (id_priv->cma_dev) { 994 mutex_unlock(&lock); 995 if (rdma_cap_ib_cm(id_priv->id.device, 1)) { 996 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) 997 ib_destroy_cm_id(id_priv->cm_id.ib); 998 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) { 999 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw)) 1000 iw_destroy_cm_id(id_priv->cm_id.iw); 1001 } 1002 cma_leave_mc_groups(id_priv); 1003 mutex_lock(&lock); 1004 cma_detach_from_dev(id_priv); 1005 } 1006 mutex_unlock(&lock); 1007 1008 cma_release_port(id_priv); 1009 cma_deref_id(id_priv); 1010 wait_for_completion(&id_priv->comp); 1011 1012 if (id_priv->internal_id) 1013 cma_deref_id(id_priv->id.context); 1014 1015 if (id_priv->sock != NULL && !id_priv->internal_id && 1016 !id_priv->unify_ps_tcp) 1017 sock_release(id_priv->sock); 1018 1019 kfree(id_priv->id.route.path_rec); 1020 kfree(id_priv); 1021} 1022EXPORT_SYMBOL(rdma_destroy_id); 1023 1024static int cma_rep_recv(struct rdma_id_private *id_priv) 1025{ 1026 int ret; 1027 1028 ret = cma_modify_qp_rtr(id_priv, NULL); 1029 if (ret) 1030 goto reject; 1031 1032 ret = cma_modify_qp_rts(id_priv, NULL); 1033 if (ret) 1034 goto reject; 1035 1036 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 1037 if (ret) 1038 goto reject; 1039 1040 return 0; 1041reject: 1042 cma_modify_qp_err(id_priv); 1043 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 1044 NULL, 0, NULL, 0); 1045 return ret; 1046} 1047 1048static int cma_verify_rep(struct rdma_id_private *id_priv, void *data) 1049{ 1050 if (id_priv->id.ps == RDMA_PS_SDP && 1051 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) != 1052 SDP_MAJ_VERSION) 1053 return -EINVAL; 1054 1055 return 0; 1056} 1057 1058static void cma_set_rep_event_data(struct rdma_cm_event *event, 1059 struct ib_cm_rep_event_param *rep_data, 1060 void *private_data) 1061{ 1062 event->param.conn.private_data = private_data; 1063 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 1064 event->param.conn.responder_resources = rep_data->responder_resources; 1065 event->param.conn.initiator_depth = rep_data->initiator_depth; 1066 event->param.conn.flow_control = rep_data->flow_control; 1067 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 1068 event->param.conn.srq = rep_data->srq; 1069 event->param.conn.qp_num = rep_data->remote_qpn; 1070} 1071 1072static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1073{ 1074 struct rdma_id_private *id_priv = cm_id->context; 1075 struct rdma_cm_event event; 1076 int ret = 0; 1077 1078 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && 1079 cma_disable_callback(id_priv, CMA_CONNECT)) || 1080 (ib_event->event == IB_CM_TIMEWAIT_EXIT && 1081 cma_disable_callback(id_priv, CMA_DISCONNECT))) 1082 return 0; 1083 1084 memset(&event, 0, sizeof event); 1085 switch (ib_event->event) { 1086 case IB_CM_REQ_ERROR: 1087 case IB_CM_REP_ERROR: 1088 event.event = RDMA_CM_EVENT_UNREACHABLE; 1089 event.status = -ETIMEDOUT; 1090 break; 1091 case IB_CM_REP_RECEIVED: 1092 event.status = cma_verify_rep(id_priv, ib_event->private_data); 1093 if (event.status) 1094 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1095 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) { 1096 event.status = cma_rep_recv(id_priv); 1097 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 1098 RDMA_CM_EVENT_ESTABLISHED; 1099 } else 1100 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 1101 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 1102 ib_event->private_data); 1103 break; 1104 case IB_CM_RTU_RECEIVED: 1105 case IB_CM_USER_ESTABLISHED: 1106 event.event = RDMA_CM_EVENT_ESTABLISHED; 1107 break; 1108 case IB_CM_DREQ_ERROR: 1109 event.status = -ETIMEDOUT; /* fall through */ 1110 case IB_CM_DREQ_RECEIVED: 1111 case IB_CM_DREP_RECEIVED: 1112 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT)) 1113 goto out; 1114 event.event = RDMA_CM_EVENT_DISCONNECTED; 1115 break; 1116 case IB_CM_TIMEWAIT_EXIT: 1117 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; 1118 break; 1119 case IB_CM_MRA_RECEIVED: 1120 /* ignore event */ 1121 goto out; 1122 case IB_CM_REJ_RECEIVED: 1123 cma_modify_qp_err(id_priv); 1124 event.status = ib_event->param.rej_rcvd.reason; 1125 event.event = RDMA_CM_EVENT_REJECTED; 1126 event.param.conn.private_data = ib_event->private_data; 1127 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 1128 break; 1129 default: 1130 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 1131 ib_event->event); 1132 goto out; 1133 } 1134 1135 ret = id_priv->id.event_handler(&id_priv->id, &event); 1136 if (ret) { 1137 /* Destroy the CM ID by returning a non-zero value. */ 1138 id_priv->cm_id.ib = NULL; 1139 cma_exch(id_priv, CMA_DESTROYING); 1140 mutex_unlock(&id_priv->handler_mutex); 1141 rdma_destroy_id(&id_priv->id); 1142 return ret; 1143 } 1144out: 1145 mutex_unlock(&id_priv->handler_mutex); 1146 return ret; 1147} 1148 1149static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, 1150 struct ib_cm_event *ib_event) 1151{ 1152 struct rdma_id_private *id_priv; 1153 struct rdma_cm_id *id; 1154 struct rdma_route *rt; 1155 union cma_ip_addr *src, *dst; 1156 __be16 port; 1157 u8 ip_ver; 1158 int ret; 1159 1160 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1161 &ip_ver, &port, &src, &dst)) 1162 goto err; 1163 1164 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1165 listen_id->ps); 1166 if (IS_ERR(id)) 1167 goto err; 1168 1169 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1170 ip_ver, port, src, dst); 1171 1172 rt = &id->route; 1173 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 1174 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths, 1175 GFP_KERNEL); 1176 if (!rt->path_rec) 1177 goto destroy_id; 1178 1179 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path; 1180 if (rt->num_paths == 2) 1181 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 1182 1183 if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) { 1184 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; 1185 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 1186 ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey); 1187 } else { 1188 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr, 1189 &rt->addr.dev_addr); 1190 if (ret) 1191 goto destroy_id; 1192 } 1193 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 1194 1195 id_priv = container_of(id, struct rdma_id_private, id); 1196 id_priv->state = CMA_CONNECT; 1197 return id_priv; 1198 1199destroy_id: 1200 rdma_destroy_id(id); 1201err: 1202 return NULL; 1203} 1204 1205static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id, 1206 struct ib_cm_event *ib_event) 1207{ 1208 struct rdma_id_private *id_priv; 1209 struct rdma_cm_id *id; 1210 union cma_ip_addr *src, *dst; 1211 __be16 port; 1212 u8 ip_ver; 1213 int ret; 1214 1215 id = rdma_create_id(listen_id->event_handler, listen_id->context, 1216 listen_id->ps); 1217 if (IS_ERR(id)) 1218 return NULL; 1219 1220 1221 if (cma_get_net_info(ib_event->private_data, listen_id->ps, 1222 &ip_ver, &port, &src, &dst)) 1223 goto err; 1224 1225 cma_save_net_info(&id->route.addr, &listen_id->route.addr, 1226 ip_ver, port, src, dst); 1227 1228 if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) { 1229 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr, 1230 &id->route.addr.dev_addr); 1231 if (ret) 1232 goto err; 1233 } 1234 1235 id_priv = container_of(id, struct rdma_id_private, id); 1236 id_priv->state = CMA_CONNECT; 1237 return id_priv; 1238err: 1239 rdma_destroy_id(id); 1240 return NULL; 1241} 1242 1243static void cma_set_req_event_data(struct rdma_cm_event *event, 1244 struct ib_cm_req_event_param *req_data, 1245 void *private_data, int offset) 1246{ 1247 event->param.conn.private_data = private_data + offset; 1248 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 1249 event->param.conn.responder_resources = req_data->responder_resources; 1250 event->param.conn.initiator_depth = req_data->initiator_depth; 1251 event->param.conn.flow_control = req_data->flow_control; 1252 event->param.conn.retry_count = req_data->retry_count; 1253 event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 1254 event->param.conn.srq = req_data->srq; 1255 event->param.conn.qp_num = req_data->remote_qpn; 1256} 1257 1258static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) 1259{ 1260 struct rdma_id_private *listen_id, *conn_id; 1261 struct rdma_cm_event event; 1262 int offset, ret; 1263 1264 listen_id = cm_id->context; 1265 if (cma_disable_callback(listen_id, CMA_LISTEN)) 1266 return -ECONNABORTED; 1267 1268 memset(&event, 0, sizeof event); 1269 offset = cma_user_data_offset(listen_id->id.ps); 1270 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1271 if (cma_is_ud_ps(listen_id->id.ps)) { 1272 conn_id = cma_new_udp_id(&listen_id->id, ib_event); 1273 event.param.ud.private_data = ib_event->private_data + offset; 1274 event.param.ud.private_data_len = 1275 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 1276 } else { 1277 conn_id = cma_new_conn_id(&listen_id->id, ib_event); 1278 cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 1279 ib_event->private_data, offset); 1280 } 1281 if (!conn_id) { 1282 ret = -ENOMEM; 1283 goto out; 1284 } 1285 1286 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1287 mutex_lock(&lock); 1288 ret = cma_acquire_dev(conn_id); 1289 mutex_unlock(&lock); 1290 if (ret) 1291 goto release_conn_id; 1292 1293 conn_id->cm_id.ib = cm_id; 1294 cm_id->context = conn_id; 1295 cm_id->cm_handler = cma_ib_handler; 1296 1297 ret = conn_id->id.event_handler(&conn_id->id, &event); 1298 if (!ret) { 1299 /* 1300 * Acquire mutex to prevent user executing rdma_destroy_id() 1301 * while we're accessing the cm_id. 1302 */ 1303 mutex_lock(&lock); 1304 if (cma_comp(conn_id, CMA_CONNECT) && 1305 !cma_is_ud_ps(conn_id->id.ps)) 1306 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 1307 mutex_unlock(&lock); 1308 mutex_unlock(&conn_id->handler_mutex); 1309 goto out; 1310 } 1311 1312 /* Destroy the CM ID by returning a non-zero value. */ 1313 conn_id->cm_id.ib = NULL; 1314 1315release_conn_id: 1316 cma_exch(conn_id, CMA_DESTROYING); 1317 mutex_unlock(&conn_id->handler_mutex); 1318 rdma_destroy_id(&conn_id->id); 1319 1320out: 1321 mutex_unlock(&listen_id->handler_mutex); 1322 return ret; 1323} 1324 1325static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr) 1326{ 1327 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr))); 1328} 1329 1330static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr, 1331 struct ib_cm_compare_data *compare) 1332{ 1333 struct cma_hdr *cma_data, *cma_mask; 1334 struct sdp_hh *sdp_data, *sdp_mask; 1335 __be32 ip4_addr; 1336#ifdef INET6 1337 struct in6_addr ip6_addr; 1338#endif 1339 1340 memset(compare, 0, sizeof *compare); 1341 cma_data = (void *) compare->data; 1342 cma_mask = (void *) compare->mask; 1343 sdp_data = (void *) compare->data; 1344 sdp_mask = (void *) compare->mask; 1345 1346 switch (addr->sa_family) { 1347 case AF_INET: 1348 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr; 1349 if (ps == RDMA_PS_SDP) { 1350 sdp_set_ip_ver(sdp_data, 4); 1351 sdp_set_ip_ver(sdp_mask, 0xF); 1352 sdp_data->dst_addr.ip4.addr = ip4_addr; 1353 sdp_mask->dst_addr.ip4.addr = htonl(~0); 1354 } else { 1355 cma_set_ip_ver(cma_data, 4); 1356 cma_set_ip_ver(cma_mask, 0xF); 1357 cma_data->dst_addr.ip4.addr = ip4_addr; 1358 cma_mask->dst_addr.ip4.addr = htonl(~0); 1359 } 1360 break; 1361#ifdef INET6 1362 case AF_INET6: 1363 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr; 1364 if (ps == RDMA_PS_SDP) { 1365 sdp_set_ip_ver(sdp_data, 6); 1366 sdp_set_ip_ver(sdp_mask, 0xF); 1367 sdp_data->dst_addr.ip6 = ip6_addr; 1368 memset(&sdp_mask->dst_addr.ip6, 0xFF, 1369 sizeof sdp_mask->dst_addr.ip6); 1370 } else { 1371 cma_set_ip_ver(cma_data, 6); 1372 cma_set_ip_ver(cma_mask, 0xF); 1373 cma_data->dst_addr.ip6 = ip6_addr; 1374 memset(&cma_mask->dst_addr.ip6, 0xFF, 1375 sizeof cma_mask->dst_addr.ip6); 1376 } 1377 break; 1378#endif 1379 default: 1380 break; 1381 } 1382} 1383 1384static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 1385{ 1386 struct rdma_id_private *id_priv = iw_id->context; 1387 struct rdma_cm_event event; 1388 struct sockaddr_in *sin; 1389 int ret = 0; 1390 1391 if (cma_disable_callback(id_priv, CMA_CONNECT)) 1392 return 0; 1393 1394 memset(&event, 0, sizeof event); 1395 switch (iw_event->event) { 1396 case IW_CM_EVENT_CLOSE: 1397 event.event = RDMA_CM_EVENT_DISCONNECTED; 1398 break; 1399 case IW_CM_EVENT_CONNECT_REPLY: 1400 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1401 *sin = iw_event->local_addr; 1402 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr; 1403 *sin = iw_event->remote_addr; 1404 switch ((int)iw_event->status) { 1405 case 0: 1406 event.event = RDMA_CM_EVENT_ESTABLISHED; 1407 break; 1408 case -ECONNRESET: 1409 case -ECONNREFUSED: 1410 event.event = RDMA_CM_EVENT_REJECTED; 1411 break; 1412 case -ETIMEDOUT: 1413 event.event = RDMA_CM_EVENT_UNREACHABLE; 1414 break; 1415 default: 1416 event.event = RDMA_CM_EVENT_CONNECT_ERROR; 1417 break; 1418 } 1419 break; 1420 case IW_CM_EVENT_ESTABLISHED: 1421 event.event = RDMA_CM_EVENT_ESTABLISHED; 1422 break; 1423 default: 1424 BUG_ON(1); 1425 } 1426 1427 event.status = iw_event->status; 1428 event.param.conn.private_data = iw_event->private_data; 1429 event.param.conn.private_data_len = iw_event->private_data_len; 1430 ret = id_priv->id.event_handler(&id_priv->id, &event); 1431 if (ret) { 1432 /* Destroy the CM ID by returning a non-zero value. */ 1433 id_priv->cm_id.iw = NULL; 1434 cma_exch(id_priv, CMA_DESTROYING); 1435 mutex_unlock(&id_priv->handler_mutex); 1436 rdma_destroy_id(&id_priv->id); 1437 return ret; 1438 } 1439 1440 mutex_unlock(&id_priv->handler_mutex); 1441 return ret; 1442} 1443 1444static int iw_conn_req_handler(struct iw_cm_id *cm_id, 1445 struct iw_cm_event *iw_event) 1446{ 1447 struct rdma_cm_id *new_cm_id; 1448 struct rdma_id_private *listen_id, *conn_id; 1449 struct sockaddr_in *sin; 1450 struct net_device *dev = NULL; 1451 struct rdma_cm_event event; 1452 int ret; 1453 struct ib_device_attr attr; 1454 1455 listen_id = cm_id->context; 1456 if (cma_disable_callback(listen_id, CMA_LISTEN)) 1457 return -ECONNABORTED; 1458 1459 /* Create a new RDMA id for the new IW CM ID */ 1460 new_cm_id = rdma_create_id(listen_id->id.event_handler, 1461 listen_id->id.context, 1462 RDMA_PS_TCP); 1463 if (IS_ERR(new_cm_id)) { 1464 ret = -ENOMEM; 1465 goto out; 1466 } 1467 conn_id = container_of(new_cm_id, struct rdma_id_private, id); 1468 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1469 conn_id->state = CMA_CONNECT; 1470 1471 dev = ip_dev_find(NULL, iw_event->local_addr.sin_addr.s_addr); 1472 if (!dev) { 1473 ret = -EADDRNOTAVAIL; 1474 mutex_unlock(&conn_id->handler_mutex); 1475 rdma_destroy_id(new_cm_id); 1476 goto out; 1477 } 1478 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL); 1479 if (ret) { 1480 mutex_unlock(&conn_id->handler_mutex); 1481 rdma_destroy_id(new_cm_id); 1482 goto out; 1483 } 1484 1485 mutex_lock(&lock); 1486 ret = cma_acquire_dev(conn_id); 1487 mutex_unlock(&lock); 1488 if (ret) { 1489 mutex_unlock(&conn_id->handler_mutex); 1490 rdma_destroy_id(new_cm_id); 1491 goto out; 1492 } 1493 1494 conn_id->cm_id.iw = cm_id; 1495 cm_id->context = conn_id; 1496 cm_id->cm_handler = cma_iw_handler; 1497 1498 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr; 1499 *sin = iw_event->local_addr; 1500 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; 1501 *sin = iw_event->remote_addr; 1502 1503 ret = ib_query_device(conn_id->id.device, &attr); 1504 if (ret) { 1505 mutex_unlock(&conn_id->handler_mutex); 1506 rdma_destroy_id(new_cm_id); 1507 goto out; 1508 } 1509 1510 memset(&event, 0, sizeof event); 1511 event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 1512 event.param.conn.private_data = iw_event->private_data; 1513 event.param.conn.private_data_len = iw_event->private_data_len; 1514 event.param.conn.initiator_depth = attr.max_qp_init_rd_atom; 1515 event.param.conn.responder_resources = attr.max_qp_rd_atom; 1516 ret = conn_id->id.event_handler(&conn_id->id, &event); 1517 if (ret) { 1518 /* User wants to destroy the CM ID */ 1519 conn_id->cm_id.iw = NULL; 1520 cma_exch(conn_id, CMA_DESTROYING); 1521 mutex_unlock(&conn_id->handler_mutex); 1522 rdma_destroy_id(&conn_id->id); 1523 goto out; 1524 } 1525 1526 mutex_unlock(&conn_id->handler_mutex); 1527 1528out: 1529 if (dev) 1530 dev_put(dev); 1531 mutex_unlock(&listen_id->handler_mutex); 1532 return ret; 1533} 1534 1535static int cma_ib_listen(struct rdma_id_private *id_priv) 1536{ 1537 struct ib_cm_compare_data compare_data; 1538 struct sockaddr *addr; 1539 __be64 svc_id; 1540 int ret; 1541 1542 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler, 1543 id_priv); 1544 if (IS_ERR(id_priv->cm_id.ib)) 1545 return PTR_ERR(id_priv->cm_id.ib); 1546 1547 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr; 1548 svc_id = cma_get_service_id(id_priv->id.ps, addr); 1549 if (cma_any_addr(addr)) 1550 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL); 1551 else { 1552 cma_set_compare_data(id_priv->id.ps, addr, &compare_data); 1553 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data); 1554 } 1555 1556 if (ret) { 1557 ib_destroy_cm_id(id_priv->cm_id.ib); 1558 id_priv->cm_id.ib = NULL; 1559 } 1560 1561 return ret; 1562} 1563 1564static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1565{ 1566 int ret; 1567 struct sockaddr_in *sin; 1568 1569 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, 1570 id_priv->sock, 1571 iw_conn_req_handler, 1572 id_priv); 1573 if (IS_ERR(id_priv->cm_id.iw)) 1574 return PTR_ERR(id_priv->cm_id.iw); 1575 1576 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 1577 id_priv->cm_id.iw->local_addr = *sin; 1578 1579 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 1580 1581 if (ret) { 1582 iw_destroy_cm_id(id_priv->cm_id.iw); 1583 id_priv->cm_id.iw = NULL; 1584 } 1585 1586 return ret; 1587} 1588 1589static int cma_listen_handler(struct rdma_cm_id *id, 1590 struct rdma_cm_event *event) 1591{ 1592 struct rdma_id_private *id_priv = id->context; 1593 1594 id->context = id_priv->id.context; 1595 id->event_handler = id_priv->id.event_handler; 1596 return id_priv->id.event_handler(id, event); 1597} 1598 1599static void cma_listen_on_dev(struct rdma_id_private *id_priv, 1600 struct cma_device *cma_dev) 1601{ 1602 struct rdma_id_private *dev_id_priv; 1603 struct rdma_cm_id *id; 1604 int ret; 1605 1606 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps); 1607 if (IS_ERR(id)) 1608 return; 1609 1610 dev_id_priv = container_of(id, struct rdma_id_private, id); 1611 1612 dev_id_priv->state = CMA_ADDR_BOUND; 1613 dev_id_priv->sock = id_priv->sock; 1614 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, 1615 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr)); 1616 1617 cma_attach_to_dev(dev_id_priv, cma_dev); 1618 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 1619 atomic_inc(&id_priv->refcount); 1620 dev_id_priv->internal_id = 1; 1621 1622 ret = rdma_listen(id, id_priv->backlog); 1623 if (ret) 1624 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, " 1625 "listening on device %s\n", ret, cma_dev->device->name); 1626} 1627 1628static void cma_listen_on_all(struct rdma_id_private *id_priv) 1629{ 1630 struct cma_device *cma_dev; 1631 1632 mutex_lock(&lock); 1633 list_add_tail(&id_priv->list, &listen_any_list); 1634 list_for_each_entry(cma_dev, &dev_list, list) 1635 cma_listen_on_dev(id_priv, cma_dev); 1636 mutex_unlock(&lock); 1637} 1638 1639int rdma_listen(struct rdma_cm_id *id, int backlog) 1640{ 1641 struct rdma_id_private *id_priv; 1642 int ret; 1643 1644 id_priv = container_of(id, struct rdma_id_private, id); 1645 if (id_priv->state == CMA_IDLE) { 1646 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET; 1647 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr); 1648 if (ret) 1649 return ret; 1650 } 1651 1652 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN)) 1653 return -EINVAL; 1654 1655 id_priv->backlog = backlog; 1656 if (id->device) { 1657 if (rdma_cap_ib_cm(id->device, 1)) { 1658 ret = cma_ib_listen(id_priv); 1659 if (ret) 1660 goto err; 1661 } else if (rdma_cap_iw_cm(id->device, 1)) { 1662 ret = cma_iw_listen(id_priv, backlog); 1663 if (ret) 1664 goto err; 1665 } else { 1666 ret = -ENOSYS; 1667 goto err; 1668 } 1669 } else 1670 cma_listen_on_all(id_priv); 1671 1672 return 0; 1673err: 1674 id_priv->backlog = 0; 1675 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND); 1676 return ret; 1677} 1678EXPORT_SYMBOL(rdma_listen); 1679 1680void rdma_set_service_type(struct rdma_cm_id *id, int tos) 1681{ 1682 struct rdma_id_private *id_priv; 1683 1684 id_priv = container_of(id, struct rdma_id_private, id); 1685 id_priv->tos = (u8) tos; 1686} 1687EXPORT_SYMBOL(rdma_set_service_type); 1688 1689static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec, 1690 void *context) 1691{ 1692 struct cma_work *work = context; 1693 struct rdma_route *route; 1694 1695 route = &work->id->id.route; 1696 1697 if (!status) { 1698 route->num_paths = 1; 1699 *route->path_rec = *path_rec; 1700 } else { 1701 work->old_state = CMA_ROUTE_QUERY; 1702 work->new_state = CMA_ADDR_RESOLVED; 1703 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 1704 work->event.status = status; 1705 } 1706 1707 queue_work(cma_wq, &work->work); 1708} 1709 1710static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms, 1711 struct cma_work *work) 1712{ 1713 struct rdma_addr *addr = &id_priv->id.route.addr; 1714 struct ib_sa_path_rec path_rec; 1715 ib_sa_comp_mask comp_mask; 1716 struct sockaddr_in6 *sin6; 1717 1718 memset(&path_rec, 0, sizeof path_rec); 1719 rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid); 1720 rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid); 1721 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr)); 1722 path_rec.numb_path = 1; 1723 path_rec.reversible = 1; 1724 path_rec.service_id = cma_get_service_id(id_priv->id.ps, 1725 (struct sockaddr *) &addr->dst_addr); 1726 1727 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 1728 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 1729 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 1730 1731 if (addr->src_addr.ss_family == AF_INET) { 1732 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 1733 comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 1734 } else { 1735 sin6 = (struct sockaddr_in6 *) &addr->src_addr; 1736 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 1737 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 1738 } 1739 1740 if (tavor_quirk) { 1741 path_rec.mtu_selector = IB_SA_LT; 1742 path_rec.mtu = IB_MTU_2048; 1743 } 1744 1745 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 1746 id_priv->id.port_num, &path_rec, 1747 comp_mask, timeout_ms, 1748 GFP_KERNEL, cma_query_handler, 1749 work, &id_priv->query); 1750 1751 return (id_priv->query_id < 0) ? id_priv->query_id : 0; 1752} 1753 1754static void cma_work_handler(struct work_struct *_work) 1755{ 1756 struct cma_work *work = container_of(_work, struct cma_work, work); 1757 struct rdma_id_private *id_priv = work->id; 1758 int destroy = 0; 1759 1760 mutex_lock(&id_priv->handler_mutex); 1761 if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 1762 goto out; 1763 1764 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1765 cma_exch(id_priv, CMA_DESTROYING); 1766 destroy = 1; 1767 } 1768out: 1769 mutex_unlock(&id_priv->handler_mutex); 1770 cma_deref_id(id_priv); 1771 if (destroy) 1772 rdma_destroy_id(&id_priv->id); 1773 kfree(work); 1774} 1775 1776static void cma_ndev_work_handler(struct work_struct *_work) 1777{ 1778 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); 1779 struct rdma_id_private *id_priv = work->id; 1780 int destroy = 0; 1781 1782 mutex_lock(&id_priv->handler_mutex); 1783 if (id_priv->state == CMA_DESTROYING || 1784 id_priv->state == CMA_DEVICE_REMOVAL) 1785 goto out; 1786 1787 if (id_priv->id.event_handler(&id_priv->id, &work->event)) { 1788 cma_exch(id_priv, CMA_DESTROYING); 1789 destroy = 1; 1790 } 1791 1792out: 1793 mutex_unlock(&id_priv->handler_mutex); 1794 cma_deref_id(id_priv); 1795 if (destroy) 1796 rdma_destroy_id(&id_priv->id); 1797 kfree(work); 1798} 1799 1800static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms) 1801{ 1802 struct rdma_route *route = &id_priv->id.route; 1803 struct cma_work *work; 1804 int ret; 1805 1806 work = kzalloc(sizeof *work, GFP_KERNEL); 1807 if (!work) 1808 return -ENOMEM; 1809 1810 work->id = id_priv; 1811 INIT_WORK(&work->work, cma_work_handler); 1812 work->old_state = CMA_ROUTE_QUERY; 1813 work->new_state = CMA_ROUTE_RESOLVED; 1814 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1815 1816 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 1817 if (!route->path_rec) { 1818 ret = -ENOMEM; 1819 goto err1; 1820 } 1821 1822 ret = cma_query_ib_route(id_priv, timeout_ms, work); 1823 if (ret) 1824 goto err2; 1825 1826 return 0; 1827err2: 1828 kfree(route->path_rec); 1829 route->path_rec = NULL; 1830err1: 1831 kfree(work); 1832 return ret; 1833} 1834 1835int rdma_set_ib_paths(struct rdma_cm_id *id, 1836 struct ib_sa_path_rec *path_rec, int num_paths) 1837{ 1838 struct rdma_id_private *id_priv; 1839 int ret; 1840 1841 id_priv = container_of(id, struct rdma_id_private, id); 1842 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED)) 1843 return -EINVAL; 1844 1845 id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL); 1846 if (!id->route.path_rec) { 1847 ret = -ENOMEM; 1848 goto err; 1849 } 1850 1851 memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths); 1852 return 0; 1853err: 1854 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED); 1855 return ret; 1856} 1857EXPORT_SYMBOL(rdma_set_ib_paths); 1858 1859static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms) 1860{ 1861 struct cma_work *work; 1862 1863 work = kzalloc(sizeof *work, GFP_KERNEL); 1864 if (!work) 1865 return -ENOMEM; 1866 1867 work->id = id_priv; 1868 INIT_WORK(&work->work, cma_work_handler); 1869 work->old_state = CMA_ROUTE_QUERY; 1870 work->new_state = CMA_ROUTE_RESOLVED; 1871 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1872 queue_work(cma_wq, &work->work); 1873 return 0; 1874} 1875 1876static u8 tos_to_sl(u8 tos) 1877{ 1878 return def_prec2sl & 7; 1879} 1880 1881static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) 1882{ 1883 struct rdma_route *route = &id_priv->id.route; 1884 struct rdma_addr *addr = &route->addr; 1885 struct cma_work *work; 1886 int ret; 1887 struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr; 1888 struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr; 1889 struct net_device *ndev = NULL; 1890 u16 vid; 1891 1892 if (src_addr->sin_family != dst_addr->sin_family) 1893 return -EINVAL; 1894 1895 work = kzalloc(sizeof *work, GFP_KERNEL); 1896 if (!work) 1897 return -ENOMEM; 1898 1899 work->id = id_priv; 1900 INIT_WORK(&work->work, cma_work_handler); 1901 1902 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); 1903 if (!route->path_rec) { 1904 ret = -ENOMEM; 1905 goto err1; 1906 } 1907 1908 route->num_paths = 1; 1909 1910 if (addr->dev_addr.bound_dev_if) 1911 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if); 1912 if (!ndev) { 1913 ret = -ENODEV; 1914 goto err2; 1915 } 1916 1917 vid = rdma_vlan_dev_vlan_id(ndev); 1918 1919 iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid); 1920 iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid); 1921 1922 route->path_rec->hop_limit = 1; 1923 route->path_rec->reversible = 1; 1924 route->path_rec->pkey = cpu_to_be16(0xffff); 1925 route->path_rec->mtu_selector = IB_SA_EQ; 1926 route->path_rec->sl = tos_to_sl(id_priv->tos); 1927 1928#ifdef __linux__ 1929 route->path_rec->mtu = iboe_get_mtu(ndev->mtu); 1930#else 1931 route->path_rec->mtu = iboe_get_mtu(ndev->if_mtu); 1932#endif 1933 route->path_rec->rate_selector = IB_SA_EQ; 1934 route->path_rec->rate = iboe_get_rate(ndev); 1935 dev_put(ndev); 1936 route->path_rec->packet_life_time_selector = IB_SA_EQ; 1937 route->path_rec->packet_life_time = IBOE_PACKET_LIFETIME; 1938 if (!route->path_rec->mtu) { 1939 ret = -EINVAL; 1940 goto err2; 1941 } 1942 1943 work->old_state = CMA_ROUTE_QUERY; 1944 work->new_state = CMA_ROUTE_RESOLVED; 1945 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1946 work->event.status = 0; 1947 1948 queue_work(cma_wq, &work->work); 1949 1950 return 0; 1951 1952err2: 1953 kfree(route->path_rec); 1954 route->path_rec = NULL; 1955err1: 1956 kfree(work); 1957 return ret; 1958} 1959 1960int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms) 1961{ 1962 struct rdma_id_private *id_priv; 1963 int ret; 1964 1965 id_priv = container_of(id, struct rdma_id_private, id); 1966 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY)) 1967 return -EINVAL; 1968 1969 atomic_inc(&id_priv->refcount); 1970 if (rdma_cap_ib_sa(id->device, id->port_num)) 1971 ret = cma_resolve_ib_route(id_priv, timeout_ms); 1972 else if (rdma_protocol_roce(id->device, id->port_num)) 1973 ret = cma_resolve_iboe_route(id_priv); 1974 else if (rdma_protocol_iwarp(id->device, id->port_num)) 1975 ret = cma_resolve_iw_route(id_priv, timeout_ms); 1976 else 1977 ret = -ENOSYS; 1978 1979 if (ret) 1980 goto err; 1981 1982 return 0; 1983err: 1984 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED); 1985 cma_deref_id(id_priv); 1986 return ret; 1987} 1988EXPORT_SYMBOL(rdma_resolve_route); 1989 1990static int cma_bind_loopback(struct rdma_id_private *id_priv) 1991{ 1992 struct cma_device *cma_dev; 1993 struct ib_port_attr port_attr; 1994 union ib_gid gid; 1995 u16 pkey; 1996 int ret; 1997 u8 p; 1998 1999 mutex_lock(&lock); 2000 if (list_empty(&dev_list)) { 2001 ret = -ENODEV; 2002 goto out; 2003 } 2004 list_for_each_entry(cma_dev, &dev_list, list) 2005 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p) 2006 if (!ib_query_port(cma_dev->device, p, &port_attr) && 2007 port_attr.state == IB_PORT_ACTIVE) 2008 goto port_found; 2009 2010 p = 1; 2011 cma_dev = list_entry(dev_list.next, struct cma_device, list); 2012 2013port_found: 2014 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid); 2015 if (ret) 2016 goto out; 2017 2018 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 2019 if (ret) 2020 goto out; 2021 2022 id_priv->id.route.addr.dev_addr.dev_type = 2023 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ? 2024 ARPHRD_INFINIBAND : ARPHRD_ETHER; 2025 2026 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 2027 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 2028 id_priv->id.port_num = p; 2029 cma_attach_to_dev(id_priv, cma_dev); 2030out: 2031 mutex_unlock(&lock); 2032 return ret; 2033} 2034 2035static void addr_handler(int status, struct sockaddr *src_addr, 2036 struct rdma_dev_addr *dev_addr, void *context) 2037{ 2038 struct rdma_id_private *id_priv = context; 2039 struct rdma_cm_event event; 2040 2041 memset(&event, 0, sizeof event); 2042 mutex_lock(&id_priv->handler_mutex); 2043 2044 /* 2045 * Grab mutex to block rdma_destroy_id() from removing the device while 2046 * we're trying to acquire it. 2047 */ 2048 mutex_lock(&lock); 2049 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) { 2050 mutex_unlock(&lock); 2051 goto out; 2052 } 2053 2054 if (!status && !id_priv->cma_dev) 2055 status = cma_acquire_dev(id_priv); 2056 mutex_unlock(&lock); 2057 2058 if (status) { 2059 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND)) 2060 goto out; 2061 event.event = RDMA_CM_EVENT_ADDR_ERROR; 2062 event.status = status; 2063 } else { 2064 memcpy(&id_priv->id.route.addr.src_addr, src_addr, 2065 ip_addr_size(src_addr)); 2066 event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2067 } 2068 2069 if (id_priv->id.event_handler(&id_priv->id, &event)) { 2070 cma_exch(id_priv, CMA_DESTROYING); 2071 mutex_unlock(&id_priv->handler_mutex); 2072 cma_deref_id(id_priv); 2073 rdma_destroy_id(&id_priv->id); 2074 return; 2075 } 2076out: 2077 mutex_unlock(&id_priv->handler_mutex); 2078 cma_deref_id(id_priv); 2079} 2080 2081static int cma_resolve_loopback(struct rdma_id_private *id_priv) 2082{ 2083 struct cma_work *work; 2084 struct sockaddr *src, *dst; 2085 union ib_gid gid; 2086 int ret; 2087 2088 work = kzalloc(sizeof *work, GFP_KERNEL); 2089 if (!work) 2090 return -ENOMEM; 2091 2092 if (!id_priv->cma_dev) { 2093 ret = cma_bind_loopback(id_priv); 2094 if (ret) 2095 goto err; 2096 } 2097 2098 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 2099 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 2100 2101 src = (struct sockaddr *) &id_priv->id.route.addr.src_addr; 2102 if (cma_zero_addr(src)) { 2103 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr; 2104 if ((src->sa_family = dst->sa_family) == AF_INET) { 2105 ((struct sockaddr_in *) src)->sin_addr.s_addr = 2106 ((struct sockaddr_in *) dst)->sin_addr.s_addr; 2107 } else { 2108 ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr, 2109 &((struct sockaddr_in6 *) dst)->sin6_addr); 2110 } 2111 } 2112 2113 work->id = id_priv; 2114 INIT_WORK(&work->work, cma_work_handler); 2115 work->old_state = CMA_ADDR_QUERY; 2116 work->new_state = CMA_ADDR_RESOLVED; 2117 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2118 queue_work(cma_wq, &work->work); 2119 return 0; 2120err: 2121 kfree(work); 2122 return ret; 2123} 2124 2125static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2126 struct sockaddr *dst_addr) 2127{ 2128 if (!src_addr || !src_addr->sa_family) { 2129 src_addr = (struct sockaddr *) &id->route.addr.src_addr; 2130 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) { 2131 ((struct sockaddr_in6 *) src_addr)->sin6_scope_id = 2132 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id; 2133 } 2134 } 2135 if (!cma_any_addr(src_addr)) 2136 return rdma_bind_addr(id, src_addr); 2137 else { 2138 struct sockaddr_in addr_in; 2139 2140 memset(&addr_in, 0, sizeof addr_in); 2141 addr_in.sin_family = dst_addr->sa_family; 2142 addr_in.sin_len = sizeof addr_in; 2143 return rdma_bind_addr(id, (struct sockaddr *) &addr_in); 2144 } 2145} 2146 2147int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 2148 struct sockaddr *dst_addr, int timeout_ms) 2149{ 2150 struct rdma_id_private *id_priv; 2151 int ret; 2152 2153 id_priv = container_of(id, struct rdma_id_private, id); 2154 if (id_priv->state == CMA_IDLE) { 2155 ret = cma_bind_addr(id, src_addr, dst_addr); 2156 if (ret) 2157 return ret; 2158 } 2159 2160 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY)) 2161 return -EINVAL; 2162 2163 atomic_inc(&id_priv->refcount); 2164 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr)); 2165 if (cma_any_addr(dst_addr)) 2166 ret = cma_resolve_loopback(id_priv); 2167 else 2168 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr, 2169 dst_addr, &id->route.addr.dev_addr, 2170 timeout_ms, addr_handler, id_priv); 2171 if (ret) 2172 goto err; 2173 2174 return 0; 2175err: 2176 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND); 2177 cma_deref_id(id_priv); 2178 return ret; 2179} 2180EXPORT_SYMBOL(rdma_resolve_addr); 2181 2182static void cma_bind_port(struct rdma_bind_list *bind_list, 2183 struct rdma_id_private *id_priv) 2184{ 2185 struct sockaddr_in *sin; 2186 2187 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 2188 sin->sin_port = htons(bind_list->port); 2189 id_priv->bind_list = bind_list; 2190 hlist_add_head(&id_priv->node, &bind_list->owners); 2191} 2192 2193static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv, 2194 unsigned short snum) 2195{ 2196 struct rdma_bind_list *bind_list; 2197 int port, ret; 2198 2199 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 2200 if (!bind_list) 2201 return -ENOMEM; 2202 2203 do { 2204 ret = idr_get_new_above(ps, bind_list, snum, &port); 2205 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 2206 2207 if (ret) 2208 goto err1; 2209 2210 if (port != snum) { 2211 ret = -EADDRNOTAVAIL; 2212 goto err2; 2213 } 2214 2215 bind_list->ps = ps; 2216 bind_list->port = (unsigned short) port; 2217 cma_bind_port(bind_list, id_priv); 2218 return 0; 2219err2: 2220 idr_remove(ps, port); 2221err1: 2222 kfree(bind_list); 2223 return ret; 2224} 2225 2226static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv) 2227{ 2228#if defined(INET) 2229 struct rdma_bind_list *bind_list; 2230 int port, ret, low, high; 2231 2232 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 2233 if (!bind_list) 2234 return -ENOMEM; 2235 2236retry: 2237 /* FIXME: add proper port randomization per like inet_csk_get_port */ 2238 do { 2239 ret = idr_get_new_above(ps, bind_list, next_port, &port); 2240 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL)); 2241 2242 if (ret) 2243 goto err1; 2244 2245 inet_get_local_port_range(&low, &high); 2246 if (port > high) { 2247 if (next_port != low) { 2248 idr_remove(ps, port); 2249 next_port = low; 2250 goto retry; 2251 } 2252 ret = -EADDRNOTAVAIL; 2253 goto err2; 2254 } 2255 2256 if (port == high) 2257 next_port = low; 2258 else 2259 next_port = port + 1; 2260 2261 bind_list->ps = ps; 2262 bind_list->port = (unsigned short) port; 2263 cma_bind_port(bind_list, id_priv); 2264 return 0; 2265err2: 2266 idr_remove(ps, port); 2267err1: 2268 kfree(bind_list); 2269 return ret; 2270#else 2271 return -ENOSPC; 2272#endif 2273} 2274 2275static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv) 2276{ 2277 struct rdma_id_private *cur_id; 2278 struct sockaddr_in *sin, *cur_sin; 2279 struct rdma_bind_list *bind_list; 2280 struct hlist_node *node; 2281 unsigned short snum; 2282 2283 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr; 2284 snum = ntohs(sin->sin_port); 2285#ifdef __linux__ 2286 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 2287 return -EACCES; 2288#endif 2289 2290 bind_list = idr_find(ps, snum); 2291 if (!bind_list) 2292 return cma_alloc_port(ps, id_priv, snum); 2293 2294 /* 2295 * We don't support binding to any address if anyone is bound to 2296 * a specific address on the same port. 2297 */ 2298 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)) 2299 return -EADDRNOTAVAIL; 2300 2301 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) { 2302 if (cma_any_addr((struct sockaddr *) &cur_id->id.route.addr.src_addr)) 2303 return -EADDRNOTAVAIL; 2304 2305 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; 2306 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) 2307 return -EADDRINUSE; 2308 } 2309 2310 cma_bind_port(bind_list, id_priv); 2311 return 0; 2312} 2313 2314static int cma_get_tcp_port(struct rdma_id_private *id_priv) 2315{ 2316 int ret; 2317 int size; 2318 struct socket *sock; 2319 2320 ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); 2321 if (ret) 2322 return ret; 2323#ifdef __linux__ 2324 ret = sock->ops->bind(sock, 2325 (struct sockaddr *) &id_priv->id.route.addr.src_addr, 2326 ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr)); 2327#else 2328 SOCK_LOCK(sock); 2329 sock->so_options |= SO_REUSEADDR; 2330 SOCK_UNLOCK(sock); 2331 2332 ret = -sobind(sock, 2333 (struct sockaddr *)&id_priv->id.route.addr.src_addr, 2334 curthread); 2335#endif 2336 if (ret) { 2337 sock_release(sock); 2338 return ret; 2339 } 2340 2341 size = ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr); 2342 ret = sock_getname(sock, 2343 (struct sockaddr *) &id_priv->id.route.addr.src_addr, 2344 &size, 0); 2345 if (ret) { 2346 sock_release(sock); 2347 return ret; 2348 } 2349 2350 id_priv->sock = sock; 2351 return 0; 2352} 2353 2354static int cma_get_port(struct rdma_id_private *id_priv) 2355{ 2356 struct cma_device *cma_dev; 2357 struct idr *ps; 2358 int ret; 2359 2360 switch (id_priv->id.ps) { 2361 case RDMA_PS_SDP: 2362 ps = &sdp_ps; 2363 break; 2364 case RDMA_PS_TCP: 2365 ps = &tcp_ps; 2366 2367 mutex_lock(&lock); 2368 /* check if there are any iWarp IB devices present */ 2369 list_for_each_entry(cma_dev, &dev_list, list) { 2370 if (rdma_protocol_iwarp(cma_dev->device, 1)) { 2371 id_priv->unify_ps_tcp = 1; 2372 break; 2373 } 2374 } 2375 mutex_unlock(&lock); 2376 2377 if (id_priv->unify_ps_tcp) { 2378 ret = cma_get_tcp_port(id_priv); 2379 if (ret) 2380 goto out; 2381 } 2382 break; 2383 case RDMA_PS_UDP: 2384 ps = &udp_ps; 2385 break; 2386 case RDMA_PS_IPOIB: 2387 ps = &ipoib_ps; 2388 break; 2389 default: 2390 return -EPROTONOSUPPORT; 2391 } 2392 2393 mutex_lock(&lock); 2394 if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr)) 2395 ret = cma_alloc_any_port(ps, id_priv); 2396 else 2397 ret = cma_use_port(ps, id_priv); 2398 mutex_unlock(&lock); 2399out: 2400 return ret; 2401} 2402 2403static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, 2404 struct sockaddr *addr) 2405{ 2406#if defined(INET6) 2407 struct sockaddr_in6 *sin6; 2408 2409 if (addr->sa_family != AF_INET6) 2410 return 0; 2411 2412 sin6 = (struct sockaddr_in6 *) addr; 2413#ifdef __linux__ 2414 if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) && 2415#else 2416 if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) && 2417#endif 2418 !sin6->sin6_scope_id) 2419 return -EINVAL; 2420 2421 dev_addr->bound_dev_if = sin6->sin6_scope_id; 2422#endif 2423 return 0; 2424} 2425 2426int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 2427{ 2428 struct rdma_id_private *id_priv; 2429 int ret; 2430 2431 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6) 2432 return -EAFNOSUPPORT; 2433 2434 id_priv = container_of(id, struct rdma_id_private, id); 2435 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND)) 2436 return -EINVAL; 2437 2438 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 2439 if (ret) 2440 goto err1; 2441 2442 if (!cma_any_addr(addr)) { 2443 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr); 2444 if (ret) 2445 goto err1; 2446 2447 mutex_lock(&lock); 2448 ret = cma_acquire_dev(id_priv); 2449 mutex_unlock(&lock); 2450 if (ret) 2451 goto err1; 2452 } 2453 2454 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr)); 2455 ret = cma_get_port(id_priv); 2456 if (ret) 2457 goto err2; 2458 2459 return 0; 2460err2: 2461 if (id_priv->cma_dev) { 2462 mutex_lock(&lock); 2463 cma_detach_from_dev(id_priv); 2464 mutex_unlock(&lock); 2465 } 2466err1: 2467 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE); 2468 return ret; 2469} 2470EXPORT_SYMBOL(rdma_bind_addr); 2471 2472static int cma_format_hdr(void *hdr, enum rdma_port_space ps, 2473 struct rdma_route *route) 2474{ 2475 struct cma_hdr *cma_hdr; 2476 struct sdp_hh *sdp_hdr; 2477 2478 if (route->addr.src_addr.ss_family == AF_INET) { 2479 struct sockaddr_in *src4, *dst4; 2480 2481 src4 = (struct sockaddr_in *) &route->addr.src_addr; 2482 dst4 = (struct sockaddr_in *) &route->addr.dst_addr; 2483 2484 switch (ps) { 2485 case RDMA_PS_SDP: 2486 sdp_hdr = hdr; 2487 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2488 return -EINVAL; 2489 sdp_set_ip_ver(sdp_hdr, 4); 2490 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2491 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2492 sdp_hdr->port = src4->sin_port; 2493 break; 2494 default: 2495 cma_hdr = hdr; 2496 cma_hdr->cma_version = CMA_VERSION; 2497 cma_set_ip_ver(cma_hdr, 4); 2498 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 2499 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 2500 cma_hdr->port = src4->sin_port; 2501 break; 2502 } 2503 } else { 2504 struct sockaddr_in6 *src6, *dst6; 2505 2506 src6 = (struct sockaddr_in6 *) &route->addr.src_addr; 2507 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr; 2508 2509 switch (ps) { 2510 case RDMA_PS_SDP: 2511 sdp_hdr = hdr; 2512 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION) 2513 return -EINVAL; 2514 sdp_set_ip_ver(sdp_hdr, 6); 2515 sdp_hdr->src_addr.ip6 = src6->sin6_addr; 2516 sdp_hdr->dst_addr.ip6 = dst6->sin6_addr; 2517 sdp_hdr->port = src6->sin6_port; 2518 break; 2519 default: 2520 cma_hdr = hdr; 2521 cma_hdr->cma_version = CMA_VERSION; 2522 cma_set_ip_ver(cma_hdr, 6); 2523 cma_hdr->src_addr.ip6 = src6->sin6_addr; 2524 cma_hdr->dst_addr.ip6 = dst6->sin6_addr; 2525 cma_hdr->port = src6->sin6_port; 2526 break; 2527 } 2528 } 2529 return 0; 2530} 2531 2532static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 2533 struct ib_cm_event *ib_event) 2534{ 2535 struct rdma_id_private *id_priv = cm_id->context; 2536 struct rdma_cm_event event; 2537 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd; 2538 int ret = 0; 2539 2540 if (cma_disable_callback(id_priv, CMA_CONNECT)) 2541 return 0; 2542 2543 memset(&event, 0, sizeof event); 2544 switch (ib_event->event) { 2545 case IB_CM_SIDR_REQ_ERROR: 2546 event.event = RDMA_CM_EVENT_UNREACHABLE; 2547 event.status = -ETIMEDOUT; 2548 break; 2549 case IB_CM_SIDR_REP_RECEIVED: 2550 event.param.ud.private_data = ib_event->private_data; 2551 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 2552 if (rep->status != IB_SIDR_SUCCESS) { 2553 event.event = RDMA_CM_EVENT_UNREACHABLE; 2554 event.status = ib_event->param.sidr_rep_rcvd.status; 2555 break; 2556 } 2557 ret = cma_set_qkey(id_priv); 2558 if (ret) { 2559 event.event = RDMA_CM_EVENT_ADDR_ERROR; 2560 event.status = -EINVAL; 2561 break; 2562 } 2563 if (id_priv->qkey != rep->qkey) { 2564 event.event = RDMA_CM_EVENT_UNREACHABLE; 2565 event.status = -EINVAL; 2566 break; 2567 } 2568 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num, 2569 id_priv->id.route.path_rec, 2570 &event.param.ud.ah_attr); 2571 event.param.ud.qp_num = rep->qpn; 2572 event.param.ud.qkey = rep->qkey; 2573 event.event = RDMA_CM_EVENT_ESTABLISHED; 2574 event.status = 0; 2575 break; 2576 default: 2577 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n", 2578 ib_event->event); 2579 goto out; 2580 } 2581 2582 ret = id_priv->id.event_handler(&id_priv->id, &event); 2583 if (ret) { 2584 /* Destroy the CM ID by returning a non-zero value. */ 2585 id_priv->cm_id.ib = NULL; 2586 cma_exch(id_priv, CMA_DESTROYING); 2587 mutex_unlock(&id_priv->handler_mutex); 2588 rdma_destroy_id(&id_priv->id); 2589 return ret; 2590 } 2591out: 2592 mutex_unlock(&id_priv->handler_mutex); 2593 return ret; 2594} 2595 2596static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 2597 struct rdma_conn_param *conn_param) 2598{ 2599 struct ib_cm_sidr_req_param req; 2600 struct rdma_route *route; 2601 int ret; 2602 2603 req.private_data_len = sizeof(struct cma_hdr) + 2604 conn_param->private_data_len; 2605 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2606 if (!req.private_data) 2607 return -ENOMEM; 2608 2609 if (conn_param->private_data && conn_param->private_data_len) 2610 memcpy((void *) req.private_data + sizeof(struct cma_hdr), 2611 conn_param->private_data, conn_param->private_data_len); 2612 2613 route = &id_priv->id.route; 2614 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route); 2615 if (ret) 2616 goto out; 2617 2618 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, 2619 cma_sidr_rep_handler, id_priv); 2620 if (IS_ERR(id_priv->cm_id.ib)) { 2621 ret = PTR_ERR(id_priv->cm_id.ib); 2622 goto out; 2623 } 2624 2625 req.path = route->path_rec; 2626 req.service_id = cma_get_service_id(id_priv->id.ps, 2627 (struct sockaddr *) &route->addr.dst_addr); 2628 req.timeout_ms = 1 << (cma_response_timeout - 8); 2629 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2630 2631 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 2632 if (ret) { 2633 ib_destroy_cm_id(id_priv->cm_id.ib); 2634 id_priv->cm_id.ib = NULL; 2635 } 2636out: 2637 kfree(req.private_data); 2638 return ret; 2639} 2640 2641static int cma_connect_ib(struct rdma_id_private *id_priv, 2642 struct rdma_conn_param *conn_param) 2643{ 2644 struct ib_cm_req_param req; 2645 struct rdma_route *route; 2646 void *private_data; 2647 int offset, ret; 2648 2649 memset(&req, 0, sizeof req); 2650 offset = cma_user_data_offset(id_priv->id.ps); 2651 req.private_data_len = offset + conn_param->private_data_len; 2652 private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 2653 if (!private_data) 2654 return -ENOMEM; 2655 2656 if (conn_param->private_data && conn_param->private_data_len) 2657 memcpy(private_data + offset, conn_param->private_data, 2658 conn_param->private_data_len); 2659 2660 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler, 2661 id_priv); 2662 if (IS_ERR(id_priv->cm_id.ib)) { 2663 ret = PTR_ERR(id_priv->cm_id.ib); 2664 goto out; 2665 } 2666 2667 route = &id_priv->id.route; 2668 ret = cma_format_hdr(private_data, id_priv->id.ps, route); 2669 if (ret) 2670 goto out; 2671 req.private_data = private_data; 2672 2673 req.primary_path = &route->path_rec[0]; 2674 if (route->num_paths == 2) 2675 req.alternate_path = &route->path_rec[1]; 2676 2677 req.service_id = cma_get_service_id(id_priv->id.ps, 2678 (struct sockaddr *) &route->addr.dst_addr); 2679 req.qp_num = id_priv->qp_num; 2680 req.qp_type = IB_QPT_RC; 2681 req.starting_psn = id_priv->seq_num; 2682 req.responder_resources = conn_param->responder_resources; 2683 req.initiator_depth = conn_param->initiator_depth; 2684 req.flow_control = conn_param->flow_control; 2685 req.retry_count = conn_param->retry_count; 2686 req.rnr_retry_count = conn_param->rnr_retry_count; 2687 req.remote_cm_response_timeout = cma_response_timeout; 2688 req.local_cm_response_timeout = cma_response_timeout; 2689 req.max_cm_retries = CMA_MAX_CM_RETRIES; 2690 req.srq = id_priv->srq ? 1 : 0; 2691 2692 ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 2693out: 2694 if (ret && !IS_ERR(id_priv->cm_id.ib)) { 2695 ib_destroy_cm_id(id_priv->cm_id.ib); 2696 id_priv->cm_id.ib = NULL; 2697 } 2698 2699 kfree(private_data); 2700 return ret; 2701} 2702 2703static int cma_connect_iw(struct rdma_id_private *id_priv, 2704 struct rdma_conn_param *conn_param) 2705{ 2706 struct iw_cm_id *cm_id; 2707 struct sockaddr_in* sin; 2708 int ret; 2709 struct iw_cm_conn_param iw_param; 2710 2711 cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock, 2712 cma_iw_handler, id_priv); 2713 if (IS_ERR(cm_id)) { 2714 ret = PTR_ERR(cm_id); 2715 goto out; 2716 } 2717 2718 id_priv->cm_id.iw = cm_id; 2719 2720 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr; 2721 cm_id->local_addr = *sin; 2722 2723 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr; 2724 cm_id->remote_addr = *sin; 2725 2726 ret = cma_modify_qp_rtr(id_priv, conn_param); 2727 if (ret) 2728 goto out; 2729 2730 iw_param.ord = conn_param->initiator_depth; 2731 iw_param.ird = conn_param->responder_resources; 2732 iw_param.private_data = conn_param->private_data; 2733 iw_param.private_data_len = conn_param->private_data_len; 2734 if (id_priv->id.qp) 2735 iw_param.qpn = id_priv->qp_num; 2736 else 2737 iw_param.qpn = conn_param->qp_num; 2738 ret = iw_cm_connect(cm_id, &iw_param); 2739out: 2740 if (ret && !IS_ERR(cm_id)) { 2741 iw_destroy_cm_id(cm_id); 2742 id_priv->cm_id.iw = NULL; 2743 } 2744 return ret; 2745} 2746 2747int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2748{ 2749 struct rdma_id_private *id_priv; 2750 int ret; 2751 2752 id_priv = container_of(id, struct rdma_id_private, id); 2753 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT)) 2754 return -EINVAL; 2755 2756 if (!id->qp) { 2757 id_priv->qp_num = conn_param->qp_num; 2758 id_priv->srq = conn_param->srq; 2759 } 2760 2761 if (rdma_cap_ib_cm(id->device, id->port_num)) { 2762 if (cma_is_ud_ps(id->ps)) 2763 ret = cma_resolve_ib_udp(id_priv, conn_param); 2764 else 2765 ret = cma_connect_ib(id_priv, conn_param); 2766 } else if (rdma_cap_iw_cm(id->device, id->port_num)) 2767 ret = cma_connect_iw(id_priv, conn_param); 2768 else 2769 ret = -ENOSYS; 2770 if (ret) 2771 goto err; 2772 2773 return 0; 2774err: 2775 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED); 2776 return ret; 2777} 2778EXPORT_SYMBOL(rdma_connect); 2779 2780static int cma_accept_ib(struct rdma_id_private *id_priv, 2781 struct rdma_conn_param *conn_param) 2782{ 2783 struct ib_cm_rep_param rep; 2784 int ret; 2785 2786 ret = cma_modify_qp_rtr(id_priv, conn_param); 2787 if (ret) 2788 goto out; 2789 2790 ret = cma_modify_qp_rts(id_priv, conn_param); 2791 if (ret) 2792 goto out; 2793 2794 memset(&rep, 0, sizeof rep); 2795 rep.qp_num = id_priv->qp_num; 2796 rep.starting_psn = id_priv->seq_num; 2797 rep.private_data = conn_param->private_data; 2798 rep.private_data_len = conn_param->private_data_len; 2799 rep.responder_resources = conn_param->responder_resources; 2800 rep.initiator_depth = conn_param->initiator_depth; 2801 rep.failover_accepted = 0; 2802 rep.flow_control = conn_param->flow_control; 2803 rep.rnr_retry_count = conn_param->rnr_retry_count; 2804 rep.srq = id_priv->srq ? 1 : 0; 2805 2806 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 2807out: 2808 return ret; 2809} 2810 2811static int cma_accept_iw(struct rdma_id_private *id_priv, 2812 struct rdma_conn_param *conn_param) 2813{ 2814 struct iw_cm_conn_param iw_param; 2815 int ret; 2816 2817 ret = cma_modify_qp_rtr(id_priv, conn_param); 2818 if (ret) 2819 return ret; 2820 2821 iw_param.ord = conn_param->initiator_depth; 2822 iw_param.ird = conn_param->responder_resources; 2823 iw_param.private_data = conn_param->private_data; 2824 iw_param.private_data_len = conn_param->private_data_len; 2825 if (id_priv->id.qp) { 2826 iw_param.qpn = id_priv->qp_num; 2827 } else 2828 iw_param.qpn = conn_param->qp_num; 2829 2830 return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 2831} 2832 2833static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 2834 enum ib_cm_sidr_status status, 2835 const void *private_data, int private_data_len) 2836{ 2837 struct ib_cm_sidr_rep_param rep; 2838 int ret; 2839 2840 memset(&rep, 0, sizeof rep); 2841 rep.status = status; 2842 if (status == IB_SIDR_SUCCESS) { 2843 ret = cma_set_qkey(id_priv); 2844 if (ret) 2845 return ret; 2846 rep.qp_num = id_priv->qp_num; 2847 rep.qkey = id_priv->qkey; 2848 } 2849 rep.private_data = private_data; 2850 rep.private_data_len = private_data_len; 2851 2852 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 2853} 2854 2855int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 2856{ 2857 struct rdma_id_private *id_priv; 2858 int ret; 2859 2860 id_priv = container_of(id, struct rdma_id_private, id); 2861 if (!cma_comp(id_priv, CMA_CONNECT)) 2862 return -EINVAL; 2863 2864 if (!id->qp && conn_param) { 2865 id_priv->qp_num = conn_param->qp_num; 2866 id_priv->srq = conn_param->srq; 2867 } 2868 2869 if (rdma_cap_ib_cm(id->device, id->port_num)) { 2870 if (cma_is_ud_ps(id->ps)) 2871 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 2872 conn_param->private_data, 2873 conn_param->private_data_len); 2874 else if (conn_param) 2875 ret = cma_accept_ib(id_priv, conn_param); 2876 else 2877 ret = cma_rep_recv(id_priv); 2878 } else if (rdma_cap_iw_cm(id->device, id->port_num)) 2879 ret = cma_accept_iw(id_priv, conn_param); 2880 else 2881 ret = -ENOSYS; 2882 2883 if (ret) 2884 goto reject; 2885 2886 return 0; 2887reject: 2888 cma_modify_qp_err(id_priv); 2889 rdma_reject(id, NULL, 0); 2890 return ret; 2891} 2892EXPORT_SYMBOL(rdma_accept); 2893 2894int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 2895{ 2896 struct rdma_id_private *id_priv; 2897 int ret; 2898 2899 id_priv = container_of(id, struct rdma_id_private, id); 2900 if (!cma_has_cm_dev(id_priv)) 2901 return -EINVAL; 2902 2903 switch (id->device->node_type) { 2904 case RDMA_NODE_IB_CA: 2905 ret = ib_cm_notify(id_priv->cm_id.ib, event); 2906 break; 2907 default: 2908 ret = 0; 2909 break; 2910 } 2911 return ret; 2912} 2913EXPORT_SYMBOL(rdma_notify); 2914 2915int rdma_reject(struct rdma_cm_id *id, const void *private_data, 2916 u8 private_data_len) 2917{ 2918 struct rdma_id_private *id_priv; 2919 int ret; 2920 2921 id_priv = container_of(id, struct rdma_id_private, id); 2922 if (!cma_has_cm_dev(id_priv)) 2923 return -EINVAL; 2924 2925 if (rdma_cap_ib_cm(id->device, id->port_num)) { 2926 if (cma_is_ud_ps(id->ps)) 2927 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 2928 private_data, private_data_len); 2929 else 2930 ret = ib_send_cm_rej(id_priv->cm_id.ib, 2931 IB_CM_REJ_CONSUMER_DEFINED, NULL, 2932 0, private_data, private_data_len); 2933 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 2934 ret = iw_cm_reject(id_priv->cm_id.iw, 2935 private_data, private_data_len); 2936 } else 2937 ret = -ENOSYS; 2938 2939 return ret; 2940} 2941EXPORT_SYMBOL(rdma_reject); 2942 2943int rdma_disconnect(struct rdma_cm_id *id) 2944{ 2945 struct rdma_id_private *id_priv; 2946 int ret; 2947 2948 id_priv = container_of(id, struct rdma_id_private, id); 2949 if (!cma_has_cm_dev(id_priv)) 2950 return -EINVAL; 2951 2952 if (rdma_cap_ib_cm(id->device, id->port_num)) { 2953 ret = cma_modify_qp_err(id_priv); 2954 if (ret) 2955 goto out; 2956 /* Initiate or respond to a disconnect. */ 2957 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) 2958 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0); 2959 } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 2960 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 2961 } else 2962 ret = -EINVAL; 2963 2964out: 2965 return ret; 2966} 2967EXPORT_SYMBOL(rdma_disconnect); 2968 2969static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 2970{ 2971 struct rdma_id_private *id_priv; 2972 struct cma_multicast *mc = multicast->context; 2973 struct rdma_cm_event event; 2974 int ret; 2975 2976 id_priv = mc->id_priv; 2977 if (cma_disable_callback(id_priv, CMA_ADDR_BOUND) && 2978 cma_disable_callback(id_priv, CMA_ADDR_RESOLVED)) 2979 return 0; 2980 2981 mutex_lock(&id_priv->qp_mutex); 2982 if (!status && id_priv->id.qp) 2983 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 2984 multicast->rec.mlid); 2985 mutex_unlock(&id_priv->qp_mutex); 2986 2987 memset(&event, 0, sizeof event); 2988 event.status = status; 2989 event.param.ud.private_data = mc->context; 2990 if (!status) { 2991 event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 2992 ib_init_ah_from_mcmember(id_priv->id.device, 2993 id_priv->id.port_num, &multicast->rec, 2994 &event.param.ud.ah_attr); 2995 event.param.ud.qp_num = 0xFFFFFF; 2996 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 2997 } else 2998 event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 2999 3000 ret = id_priv->id.event_handler(&id_priv->id, &event); 3001 if (ret) { 3002 cma_exch(id_priv, CMA_DESTROYING); 3003 mutex_unlock(&id_priv->handler_mutex); 3004 rdma_destroy_id(&id_priv->id); 3005 return 0; 3006 } 3007 3008 mutex_unlock(&id_priv->handler_mutex); 3009 return 0; 3010} 3011 3012static void cma_set_mgid(struct rdma_id_private *id_priv, 3013 struct sockaddr *addr, union ib_gid *mgid) 3014{ 3015#if defined(INET) || defined(INET6) 3016 unsigned char mc_map[MAX_ADDR_LEN]; 3017 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3018#endif 3019#ifdef INET 3020 struct sockaddr_in *sin = (struct sockaddr_in *) addr; 3021#endif 3022#ifdef INET6 3023 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 3024#endif 3025 3026 if (cma_any_addr(addr)) { 3027 memset(mgid, 0, sizeof *mgid); 3028#ifdef INET6 3029 } else if ((addr->sa_family == AF_INET6) && 3030 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 3031 0xFF10A01B)) { 3032 /* IPv6 address is an SA assigned MGID. */ 3033 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 3034 } else if (addr->sa_family == AF_INET6) { 3035 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); 3036 if (id_priv->id.ps == RDMA_PS_UDP) 3037 mc_map[7] = 0x01; /* Use RDMA CM signature */ 3038 *mgid = *(union ib_gid *) (mc_map + 4); 3039#endif 3040#ifdef INET 3041 } else { 3042 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); 3043 if (id_priv->id.ps == RDMA_PS_UDP) 3044 mc_map[7] = 0x01; /* Use RDMA CM signature */ 3045 *mgid = *(union ib_gid *) (mc_map + 4); 3046#endif 3047 } 3048} 3049 3050static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 3051 struct cma_multicast *mc) 3052{ 3053 struct ib_sa_mcmember_rec rec; 3054 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3055 ib_sa_comp_mask comp_mask; 3056 int ret; 3057 3058 ib_addr_get_mgid(dev_addr, &rec.mgid); 3059 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 3060 &rec.mgid, &rec); 3061 if (ret) 3062 return ret; 3063 3064 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); 3065 if (id_priv->id.ps == RDMA_PS_UDP) 3066 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 3067 rdma_addr_get_sgid(dev_addr, &rec.port_gid); 3068 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 3069 rec.join_state = 1; 3070 3071 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 3072 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 3073 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 3074 IB_SA_MCMEMBER_REC_FLOW_LABEL | 3075 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 3076 3077 if (id_priv->id.ps == RDMA_PS_IPOIB) 3078 comp_mask |= IB_SA_MCMEMBER_REC_RATE | 3079 IB_SA_MCMEMBER_REC_RATE_SELECTOR; 3080 3081 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 3082 id_priv->id.port_num, &rec, 3083 comp_mask, GFP_KERNEL, 3084 cma_ib_mc_handler, mc); 3085 if (IS_ERR(mc->multicast.ib)) 3086 return PTR_ERR(mc->multicast.ib); 3087 3088 return 0; 3089} 3090 3091 3092static void iboe_mcast_work_handler(struct work_struct *work) 3093{ 3094 struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); 3095 struct cma_multicast *mc = mw->mc; 3096 struct ib_sa_multicast *m = mc->multicast.ib; 3097 3098 mc->multicast.ib->context = mc; 3099 cma_ib_mc_handler(0, m); 3100 kref_put(&mc->mcref, release_mc); 3101 kfree(mw); 3102} 3103 3104static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid) 3105{ 3106 struct sockaddr_in *sin = (struct sockaddr_in *)addr; 3107 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; 3108 3109 if (cma_any_addr(addr)) { 3110 memset(mgid, 0, sizeof *mgid); 3111 } else if (addr->sa_family == AF_INET6) 3112 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 3113 else { 3114 mgid->raw[0] = 0xff; 3115 mgid->raw[1] = 0x0e; 3116 mgid->raw[2] = 0; 3117 mgid->raw[3] = 0; 3118 mgid->raw[4] = 0; 3119 mgid->raw[5] = 0; 3120 mgid->raw[6] = 0; 3121 mgid->raw[7] = 0; 3122 mgid->raw[8] = 0; 3123 mgid->raw[9] = 0; 3124 mgid->raw[10] = 0xff; 3125 mgid->raw[11] = 0xff; 3126 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; 3127 } 3128} 3129 3130static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, 3131 struct cma_multicast *mc) 3132{ 3133 struct iboe_mcast_work *work; 3134 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 3135 int err; 3136 struct sockaddr *addr = (struct sockaddr *)&mc->addr; 3137 struct net_device *ndev = NULL; 3138 3139 if (cma_zero_addr((struct sockaddr *)&mc->addr)) 3140 return -EINVAL; 3141 3142 work = kzalloc(sizeof *work, GFP_KERNEL); 3143 if (!work) 3144 return -ENOMEM; 3145 3146 mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); 3147 if (!mc->multicast.ib) { 3148 err = -ENOMEM; 3149 goto out1; 3150 } 3151 3152 cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid); 3153 3154 mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); 3155 if (id_priv->id.ps == RDMA_PS_UDP) 3156 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 3157 3158 if (dev_addr->bound_dev_if) 3159 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if); 3160 if (!ndev) { 3161 err = -ENODEV; 3162 goto out2; 3163 } 3164 3165 mc->multicast.ib->rec.rate = iboe_get_rate(ndev); 3166 mc->multicast.ib->rec.hop_limit = 1; 3167#ifdef __linux__ 3168 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); 3169#else 3170 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->if_mtu); 3171#endif 3172 dev_put(ndev); 3173 if (!mc->multicast.ib->rec.mtu) { 3174 err = -EINVAL; 3175 goto out2; 3176 } 3177 iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid); 3178 work->id = id_priv; 3179 work->mc = mc; 3180 INIT_WORK(&work->work, iboe_mcast_work_handler); 3181 kref_get(&mc->mcref); 3182 queue_work(cma_wq, &work->work); 3183 3184 return 0; 3185 3186out2: 3187 kfree(mc->multicast.ib); 3188out1: 3189 kfree(work); 3190 return err; 3191} 3192 3193int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 3194 void *context) 3195{ 3196 struct rdma_id_private *id_priv; 3197 struct cma_multicast *mc; 3198 int ret; 3199 3200 id_priv = container_of(id, struct rdma_id_private, id); 3201 if (!cma_comp(id_priv, CMA_ADDR_BOUND) && 3202 !cma_comp(id_priv, CMA_ADDR_RESOLVED)) 3203 return -EINVAL; 3204 3205 mc = kmalloc(sizeof *mc, GFP_KERNEL); 3206 if (!mc) 3207 return -ENOMEM; 3208 3209 memcpy(&mc->addr, addr, ip_addr_size(addr)); 3210 mc->context = context; 3211 mc->id_priv = id_priv; 3212 3213 spin_lock(&id_priv->lock); 3214 list_add(&mc->list, &id_priv->mc_list); 3215 spin_unlock(&id_priv->lock); 3216 3217 switch (rdma_node_get_transport(id->device->node_type)) { 3218 case RDMA_TRANSPORT_IB: 3219 switch (rdma_port_get_link_layer(id->device, id->port_num)) { 3220 case IB_LINK_LAYER_INFINIBAND: 3221 ret = cma_join_ib_multicast(id_priv, mc); 3222 break; 3223 case IB_LINK_LAYER_ETHERNET: 3224 kref_init(&mc->mcref); 3225 ret = cma_iboe_join_multicast(id_priv, mc); 3226 break; 3227 default: 3228 ret = -EINVAL; 3229 } 3230 break; 3231 default: 3232 ret = -ENOSYS; 3233 break; 3234 } 3235 3236 if (ret) { 3237 spin_lock_irq(&id_priv->lock); 3238 list_del(&mc->list); 3239 spin_unlock_irq(&id_priv->lock); 3240 kfree(mc); 3241 } 3242 3243 return ret; 3244} 3245EXPORT_SYMBOL(rdma_join_multicast); 3246 3247void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 3248{ 3249 struct rdma_id_private *id_priv; 3250 struct cma_multicast *mc; 3251 3252 id_priv = container_of(id, struct rdma_id_private, id); 3253 spin_lock_irq(&id_priv->lock); 3254 list_for_each_entry(mc, &id_priv->mc_list, list) { 3255 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) { 3256 list_del(&mc->list); 3257 spin_unlock_irq(&id_priv->lock); 3258 3259 if (id->qp) 3260 ib_detach_mcast(id->qp, 3261 &mc->multicast.ib->rec.mgid, 3262 mc->multicast.ib->rec.mlid); 3263 if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) { 3264 switch (rdma_port_get_link_layer(id->device, id->port_num)) { 3265 case IB_LINK_LAYER_INFINIBAND: 3266 ib_sa_free_multicast(mc->multicast.ib); 3267 kfree(mc); 3268 break; 3269 case IB_LINK_LAYER_ETHERNET: 3270 kref_put(&mc->mcref, release_mc); 3271 break; 3272 default: 3273 break; 3274 } 3275 } 3276 return; 3277 } 3278 } 3279 spin_unlock_irq(&id_priv->lock); 3280} 3281EXPORT_SYMBOL(rdma_leave_multicast); 3282 3283static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) 3284{ 3285 struct rdma_dev_addr *dev_addr; 3286 struct cma_ndev_work *work; 3287 3288 dev_addr = &id_priv->id.route.addr.dev_addr; 3289 3290#ifdef __linux__ 3291 if ((dev_addr->bound_dev_if == ndev->ifindex) && 3292 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { 3293 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", 3294 ndev->name, &id_priv->id); 3295#else 3296 if ((dev_addr->bound_dev_if == ndev->if_index) && 3297 memcmp(dev_addr->src_dev_addr, IF_LLADDR(ndev), ndev->if_addrlen)) { 3298 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n", 3299 ndev->if_xname, &id_priv->id); 3300#endif 3301 work = kzalloc(sizeof *work, GFP_KERNEL); 3302 if (!work) 3303 return -ENOMEM; 3304 3305 INIT_WORK(&work->work, cma_ndev_work_handler); 3306 work->id = id_priv; 3307 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; 3308 atomic_inc(&id_priv->refcount); 3309 queue_work(cma_wq, &work->work); 3310 } 3311 3312 return 0; 3313} 3314 3315static int cma_netdev_callback(struct notifier_block *self, unsigned long event, 3316 void *ctx) 3317{ 3318 struct net_device *ndev = (struct net_device *)ctx; 3319 struct cma_device *cma_dev; 3320 struct rdma_id_private *id_priv; 3321 int ret = NOTIFY_DONE; 3322 3323#ifdef __linux__ 3324 if (dev_net(ndev) != &init_net) 3325 return NOTIFY_DONE; 3326 3327 if (event != NETDEV_BONDING_FAILOVER) 3328 return NOTIFY_DONE; 3329 3330 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING)) 3331 return NOTIFY_DONE; 3332#else 3333 if (event != NETDEV_DOWN && event != NETDEV_UNREGISTER) 3334 return NOTIFY_DONE; 3335#endif 3336 3337 mutex_lock(&lock); 3338 list_for_each_entry(cma_dev, &dev_list, list) 3339 list_for_each_entry(id_priv, &cma_dev->id_list, list) { 3340 ret = cma_netdev_change(ndev, id_priv); 3341 if (ret) 3342 goto out; 3343 } 3344 3345out: 3346 mutex_unlock(&lock); 3347 return ret; 3348} 3349 3350static struct notifier_block cma_nb = { 3351 .notifier_call = cma_netdev_callback 3352}; 3353 3354static void cma_add_one(struct ib_device *device) 3355{ 3356 struct cma_device *cma_dev; 3357 struct rdma_id_private *id_priv; 3358 3359 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 3360 if (!cma_dev) 3361 return; 3362 3363 cma_dev->device = device; 3364 3365 init_completion(&cma_dev->comp); 3366 atomic_set(&cma_dev->refcount, 1); 3367 INIT_LIST_HEAD(&cma_dev->id_list); 3368 ib_set_client_data(device, &cma_client, cma_dev); 3369 3370 mutex_lock(&lock); 3371 list_add_tail(&cma_dev->list, &dev_list); 3372 list_for_each_entry(id_priv, &listen_any_list, list) 3373 cma_listen_on_dev(id_priv, cma_dev); 3374 mutex_unlock(&lock); 3375} 3376 3377static int cma_remove_id_dev(struct rdma_id_private *id_priv) 3378{ 3379 struct rdma_cm_event event; 3380 enum cma_state state; 3381 int ret = 0; 3382 3383 /* Record that we want to remove the device */ 3384 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL); 3385 if (state == CMA_DESTROYING) 3386 return 0; 3387 3388 cma_cancel_operation(id_priv, state); 3389 mutex_lock(&id_priv->handler_mutex); 3390 3391 /* Check for destruction from another callback. */ 3392 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL)) 3393 goto out; 3394 3395 memset(&event, 0, sizeof event); 3396 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 3397 ret = id_priv->id.event_handler(&id_priv->id, &event); 3398out: 3399 mutex_unlock(&id_priv->handler_mutex); 3400 return ret; 3401} 3402 3403static void cma_process_remove(struct cma_device *cma_dev) 3404{ 3405 struct rdma_id_private *id_priv; 3406 int ret; 3407 3408 mutex_lock(&lock); 3409 while (!list_empty(&cma_dev->id_list)) { 3410 id_priv = list_entry(cma_dev->id_list.next, 3411 struct rdma_id_private, list); 3412 3413 list_del(&id_priv->listen_list); 3414 list_del_init(&id_priv->list); 3415 atomic_inc(&id_priv->refcount); 3416 mutex_unlock(&lock); 3417 3418 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); 3419 cma_deref_id(id_priv); 3420 if (ret) 3421 rdma_destroy_id(&id_priv->id); 3422 3423 mutex_lock(&lock); 3424 } 3425 mutex_unlock(&lock); 3426 3427 cma_deref_dev(cma_dev); 3428 wait_for_completion(&cma_dev->comp); 3429} 3430 3431static void cma_remove_one(struct ib_device *device) 3432{ 3433 struct cma_device *cma_dev; 3434 3435 cma_dev = ib_get_client_data(device, &cma_client); 3436 if (!cma_dev) 3437 return; 3438 3439 mutex_lock(&lock); 3440 list_del(&cma_dev->list); 3441 mutex_unlock(&lock); 3442 3443 cma_process_remove(cma_dev); 3444 kfree(cma_dev); 3445} 3446 3447static int cma_init(void) 3448{ 3449 int ret; 3450#if defined(INET) 3451 int low, high, remaining; 3452 3453 get_random_bytes(&next_port, sizeof next_port); 3454 inet_get_local_port_range(&low, &high); 3455 remaining = (high - low) + 1; 3456 next_port = ((unsigned int) next_port % remaining) + low; 3457#endif 3458 3459 cma_wq = create_singlethread_workqueue("rdma_cm"); 3460 if (!cma_wq) 3461 return -ENOMEM; 3462 3463 ib_sa_register_client(&sa_client); 3464 rdma_addr_register_client(&addr_client); 3465 register_netdevice_notifier(&cma_nb); 3466 3467 ret = ib_register_client(&cma_client); 3468 if (ret) 3469 goto err; 3470 return 0; 3471 3472err: 3473 unregister_netdevice_notifier(&cma_nb); 3474 rdma_addr_unregister_client(&addr_client); 3475 ib_sa_unregister_client(&sa_client); 3476 destroy_workqueue(cma_wq); 3477 return ret; 3478} 3479 3480static void cma_cleanup(void) 3481{ 3482 ib_unregister_client(&cma_client); 3483 unregister_netdevice_notifier(&cma_nb); 3484 rdma_addr_unregister_client(&addr_client); 3485 ib_sa_unregister_client(&sa_client); 3486 destroy_workqueue(cma_wq); 3487 idr_destroy(&sdp_ps); 3488 idr_destroy(&tcp_ps); 3489 idr_destroy(&udp_ps); 3490 idr_destroy(&ipoib_ps); 3491} 3492 3493module_init(cma_init); 3494module_exit(cma_cleanup); 3495