1/* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36#include <linux/file.h> 37#include <linux/fs.h> 38 39#include <asm/uaccess.h> 40#include <asm/fcntl.h> 41 42#include "uverbs.h" 43 44static struct lock_class_key pd_lock_key; 45static struct lock_class_key mr_lock_key; 46static struct lock_class_key cq_lock_key; 47static struct lock_class_key qp_lock_key; 48static struct lock_class_key ah_lock_key; 49static struct lock_class_key srq_lock_key; 50 51#define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \ 52 do { \ 53 (udata)->inbuf = (void __user *) (ibuf); \ 54 (udata)->outbuf = (void __user *) (obuf); \ 55 (udata)->inlen = (ilen); \ 56 (udata)->outlen = (olen); \ 57 } while (0) 58 59/* 60 * The ib_uobject locking scheme is as follows: 61 * 62 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 63 * needs to be held during all idr operations. When an object is 64 * looked up, a reference must be taken on the object's kref before 65 * dropping this lock. 66 * 67 * - Each object also has an rwsem. This rwsem must be held for 68 * reading while an operation that uses the object is performed. 69 * For example, while registering an MR, the associated PD's 70 * uobject.mutex must be held for reading. The rwsem must be held 71 * for writing while initializing or destroying an object. 72 * 73 * - In addition, each object has a "live" flag. If this flag is not 74 * set, then lookups of the object will fail even if it is found in 75 * the idr. This handles a reader that blocks and does not acquire 76 * the rwsem until after the object is destroyed. The destroy 77 * operation will set the live flag to 0 and then drop the rwsem; 78 * this will allow the reader to acquire the rwsem, see that the 79 * live flag is 0, and then drop the rwsem and its reference to 80 * object. The underlying storage will not be freed until the last 81 * reference to the object is dropped. 82 */ 83 84static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 85 struct ib_ucontext *context, struct lock_class_key *key) 86{ 87 uobj->user_handle = user_handle; 88 uobj->context = context; 89 kref_init(&uobj->ref); 90 init_rwsem(&uobj->mutex); 91 lockdep_set_class(&uobj->mutex, key); 92 uobj->live = 0; 93} 94 95static void release_uobj(struct kref *kref) 96{ 97 kfree(container_of(kref, struct ib_uobject, ref)); 98} 99 100static void put_uobj(struct ib_uobject *uobj) 101{ 102 kref_put(&uobj->ref, release_uobj); 103} 104 105static void put_uobj_read(struct ib_uobject *uobj) 106{ 107 up_read(&uobj->mutex); 108 put_uobj(uobj); 109} 110 111static void put_uobj_write(struct ib_uobject *uobj) 112{ 113 up_write(&uobj->mutex); 114 put_uobj(uobj); 115} 116 117static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 118{ 119 int ret; 120 121retry: 122 if (!idr_pre_get(idr, GFP_KERNEL)) 123 return -ENOMEM; 124 125 spin_lock(&ib_uverbs_idr_lock); 126 ret = idr_get_new(idr, uobj, &uobj->id); 127 spin_unlock(&ib_uverbs_idr_lock); 128 129 if (ret == -EAGAIN) 130 goto retry; 131 132 return ret; 133} 134 135void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 136{ 137 spin_lock(&ib_uverbs_idr_lock); 138 idr_remove(idr, uobj->id); 139 spin_unlock(&ib_uverbs_idr_lock); 140} 141 142static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 143 struct ib_ucontext *context) 144{ 145 struct ib_uobject *uobj; 146 147 spin_lock(&ib_uverbs_idr_lock); 148 uobj = idr_find(idr, id); 149 if (uobj) { 150 if (uobj->context == context) 151 kref_get(&uobj->ref); 152 else 153 uobj = NULL; 154 } 155 spin_unlock(&ib_uverbs_idr_lock); 156 157 return uobj; 158} 159 160static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 161 struct ib_ucontext *context, int nested) 162{ 163 struct ib_uobject *uobj; 164 165 uobj = __idr_get_uobj(idr, id, context); 166 if (!uobj) 167 return NULL; 168 169 if (nested) 170 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 171 else 172 down_read(&uobj->mutex); 173 if (!uobj->live) { 174 put_uobj_read(uobj); 175 return NULL; 176 } 177 178 return uobj; 179} 180 181static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 182 struct ib_ucontext *context) 183{ 184 struct ib_uobject *uobj; 185 186 uobj = __idr_get_uobj(idr, id, context); 187 if (!uobj) 188 return NULL; 189 190 down_write(&uobj->mutex); 191 if (!uobj->live) { 192 put_uobj_write(uobj); 193 return NULL; 194 } 195 196 return uobj; 197} 198 199static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 200 int nested) 201{ 202 struct ib_uobject *uobj; 203 204 uobj = idr_read_uobj(idr, id, context, nested); 205 return uobj ? uobj->object : NULL; 206} 207 208static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 209{ 210 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 211} 212 213static void put_pd_read(struct ib_pd *pd) 214{ 215 put_uobj_read(pd->uobject); 216} 217 218static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 219{ 220 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 221} 222 223static void put_cq_read(struct ib_cq *cq) 224{ 225 put_uobj_read(cq->uobject); 226} 227 228static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 229{ 230 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 231} 232 233static void put_ah_read(struct ib_ah *ah) 234{ 235 put_uobj_read(ah->uobject); 236} 237 238static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 239{ 240 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 241} 242 243static void put_qp_read(struct ib_qp *qp) 244{ 245 put_uobj_read(qp->uobject); 246} 247 248static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 249{ 250 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 251} 252 253static void put_srq_read(struct ib_srq *srq) 254{ 255 put_uobj_read(srq->uobject); 256} 257 258static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, 259 struct ib_ucontext *context, 260 struct ib_uobject **uobj) 261{ 262 *uobj = idr_read_uobj(&ib_uverbs_xrc_domain_idr, xrcd_handle, 263 context, 0); 264 return *uobj ? (*uobj)->object : NULL; 265} 266 267static void put_xrcd_read(struct ib_uobject *uobj) 268{ 269 put_uobj_read(uobj); 270} 271 272ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 273 const char __user *buf, 274 int in_len, int out_len) 275{ 276 struct ib_uverbs_get_context cmd; 277 struct ib_uverbs_get_context_resp resp; 278 struct ib_udata udata; 279 struct ib_device *ibdev = file->device->ib_dev; 280 struct ib_ucontext *ucontext; 281 struct file *filp; 282 int ret; 283 284 if (out_len < sizeof resp) 285 return -ENOSPC; 286 287 if (copy_from_user(&cmd, buf, sizeof cmd)) 288 return -EFAULT; 289 290 mutex_lock(&file->mutex); 291 292 if (file->ucontext) { 293 ret = -EINVAL; 294 goto err; 295 } 296 297 INIT_UDATA(&udata, buf + sizeof cmd, 298 (unsigned long) cmd.response + sizeof resp, 299 in_len - sizeof cmd, out_len - sizeof resp); 300 301 ucontext = ibdev->alloc_ucontext(ibdev, &udata); 302 if (IS_ERR(ucontext)) { 303 ret = PTR_ERR(file->ucontext); 304 goto err; 305 } 306 307 ucontext->device = ibdev; 308 INIT_LIST_HEAD(&ucontext->pd_list); 309 INIT_LIST_HEAD(&ucontext->mr_list); 310 INIT_LIST_HEAD(&ucontext->mw_list); 311 INIT_LIST_HEAD(&ucontext->cq_list); 312 INIT_LIST_HEAD(&ucontext->qp_list); 313 INIT_LIST_HEAD(&ucontext->srq_list); 314 INIT_LIST_HEAD(&ucontext->ah_list); 315 INIT_LIST_HEAD(&ucontext->xrcd_list); 316 ucontext->closing = 0; 317 318 resp.num_comp_vectors = file->device->num_comp_vectors; 319 320 filp = ib_uverbs_alloc_event_file(file, 1, &resp.async_fd); 321 if (IS_ERR(filp)) { 322 ret = PTR_ERR(filp); 323 goto err_free; 324 } 325 326 if (copy_to_user((void __user *) (unsigned long) cmd.response, 327 &resp, sizeof resp)) { 328 ret = -EFAULT; 329 goto err_file; 330 } 331 332 file->async_file = filp->private_data; 333 334 INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev, 335 ib_uverbs_event_handler); 336 ret = ib_register_event_handler(&file->event_handler); 337 if (ret) 338 goto err_file; 339 340 kref_get(&file->async_file->ref); 341 kref_get(&file->ref); 342 file->ucontext = ucontext; 343 344 fd_install(resp.async_fd, filp); 345 346 mutex_unlock(&file->mutex); 347 348 return in_len; 349 350err_file: 351 put_unused_fd(resp.async_fd); 352 fput(filp); 353 354err_free: 355 ibdev->dealloc_ucontext(ucontext); 356 357err: 358 mutex_unlock(&file->mutex); 359 return ret; 360} 361 362ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 363 const char __user *buf, 364 int in_len, int out_len) 365{ 366 struct ib_uverbs_query_device cmd; 367 struct ib_uverbs_query_device_resp resp; 368 struct ib_device_attr attr; 369 int ret; 370 371 if (out_len < sizeof resp) 372 return -ENOSPC; 373 374 if (copy_from_user(&cmd, buf, sizeof cmd)) 375 return -EFAULT; 376 377 ret = ib_query_device(file->device->ib_dev, &attr); 378 if (ret) 379 return ret; 380 381 memset(&resp, 0, sizeof resp); 382 383 resp.fw_ver = attr.fw_ver; 384 resp.node_guid = file->device->ib_dev->node_guid; 385 resp.sys_image_guid = attr.sys_image_guid; 386 resp.max_mr_size = attr.max_mr_size; 387 resp.page_size_cap = attr.page_size_cap; 388 resp.vendor_id = attr.vendor_id; 389 resp.vendor_part_id = attr.vendor_part_id; 390 resp.hw_ver = attr.hw_ver; 391 resp.max_qp = attr.max_qp; 392 resp.max_qp_wr = attr.max_qp_wr; 393 resp.device_cap_flags = attr.device_cap_flags; 394 resp.max_sge = attr.max_sge; 395 resp.max_sge_rd = attr.max_sge_rd; 396 resp.max_cq = attr.max_cq; 397 resp.max_cqe = attr.max_cqe; 398 resp.max_mr = attr.max_mr; 399 resp.max_pd = attr.max_pd; 400 resp.max_qp_rd_atom = attr.max_qp_rd_atom; 401 resp.max_ee_rd_atom = attr.max_ee_rd_atom; 402 resp.max_res_rd_atom = attr.max_res_rd_atom; 403 resp.max_qp_init_rd_atom = attr.max_qp_init_rd_atom; 404 resp.max_ee_init_rd_atom = attr.max_ee_init_rd_atom; 405 resp.atomic_cap = attr.atomic_cap; 406 resp.max_ee = attr.max_ee; 407 resp.max_rdd = attr.max_rdd; 408 resp.max_mw = attr.max_mw; 409 resp.max_raw_ipv6_qp = attr.max_raw_ipv6_qp; 410 resp.max_raw_ethy_qp = attr.max_raw_ethy_qp; 411 resp.max_mcast_grp = attr.max_mcast_grp; 412 resp.max_mcast_qp_attach = attr.max_mcast_qp_attach; 413 resp.max_total_mcast_qp_attach = attr.max_total_mcast_qp_attach; 414 resp.max_ah = attr.max_ah; 415 resp.max_fmr = attr.max_fmr; 416 resp.max_map_per_fmr = attr.max_map_per_fmr; 417 resp.max_srq = attr.max_srq; 418 resp.max_srq_wr = attr.max_srq_wr; 419 resp.max_srq_sge = attr.max_srq_sge; 420 resp.max_pkeys = attr.max_pkeys; 421 resp.local_ca_ack_delay = attr.local_ca_ack_delay; 422 resp.phys_port_cnt = file->device->ib_dev->phys_port_cnt; 423 424 if (copy_to_user((void __user *) (unsigned long) cmd.response, 425 &resp, sizeof resp)) 426 return -EFAULT; 427 428 return in_len; 429} 430 431ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 432 const char __user *buf, 433 int in_len, int out_len) 434{ 435 struct ib_uverbs_query_port cmd; 436 struct ib_uverbs_query_port_resp resp; 437 struct ib_port_attr attr; 438 int ret; 439 440 if (out_len < sizeof resp) 441 return -ENOSPC; 442 443 if (copy_from_user(&cmd, buf, sizeof cmd)) 444 return -EFAULT; 445 446 ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr); 447 if (ret) 448 return ret; 449 450 memset(&resp, 0, sizeof resp); 451 452 resp.state = attr.state; 453 resp.max_mtu = attr.max_mtu; 454 resp.active_mtu = attr.active_mtu; 455 resp.gid_tbl_len = attr.gid_tbl_len; 456 resp.port_cap_flags = attr.port_cap_flags; 457 resp.max_msg_sz = attr.max_msg_sz; 458 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 459 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 460 resp.pkey_tbl_len = attr.pkey_tbl_len; 461 resp.lid = attr.lid; 462 resp.sm_lid = attr.sm_lid; 463 resp.lmc = attr.lmc; 464 resp.max_vl_num = attr.max_vl_num; 465 resp.sm_sl = attr.sm_sl; 466 resp.subnet_timeout = attr.subnet_timeout; 467 resp.init_type_reply = attr.init_type_reply; 468 resp.active_width = attr.active_width; 469 resp.active_speed = attr.active_speed; 470 resp.phys_state = attr.phys_state; 471 resp.link_layer = attr.link_layer; 472 473 if (copy_to_user((void __user *) (unsigned long) cmd.response, 474 &resp, sizeof resp)) 475 return -EFAULT; 476 477 return in_len; 478} 479 480ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 481 const char __user *buf, 482 int in_len, int out_len) 483{ 484 struct ib_uverbs_alloc_pd cmd; 485 struct ib_uverbs_alloc_pd_resp resp; 486 struct ib_udata udata; 487 struct ib_uobject *uobj; 488 struct ib_pd *pd; 489 int ret; 490 491 if (out_len < sizeof resp) 492 return -ENOSPC; 493 494 if (copy_from_user(&cmd, buf, sizeof cmd)) 495 return -EFAULT; 496 497 INIT_UDATA(&udata, buf + sizeof cmd, 498 (unsigned long) cmd.response + sizeof resp, 499 in_len - sizeof cmd, out_len - sizeof resp); 500 501 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 502 if (!uobj) 503 return -ENOMEM; 504 505 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 506 down_write(&uobj->mutex); 507 508 pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, 509 file->ucontext, &udata); 510 if (IS_ERR(pd)) { 511 ret = PTR_ERR(pd); 512 goto err; 513 } 514 515 pd->device = file->device->ib_dev; 516 pd->uobject = uobj; 517 atomic_set(&pd->usecnt, 0); 518 519 uobj->object = pd; 520 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 521 if (ret) 522 goto err_idr; 523 524 memset(&resp, 0, sizeof resp); 525 resp.pd_handle = uobj->id; 526 527 if (copy_to_user((void __user *) (unsigned long) cmd.response, 528 &resp, sizeof resp)) { 529 ret = -EFAULT; 530 goto err_copy; 531 } 532 533 mutex_lock(&file->mutex); 534 list_add_tail(&uobj->list, &file->ucontext->pd_list); 535 mutex_unlock(&file->mutex); 536 537 uobj->live = 1; 538 539 up_write(&uobj->mutex); 540 541 return in_len; 542 543err_copy: 544 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 545 546err_idr: 547 ib_dealloc_pd(pd); 548 549err: 550 put_uobj_write(uobj); 551 return ret; 552} 553 554ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 555 const char __user *buf, 556 int in_len, int out_len) 557{ 558 struct ib_uverbs_dealloc_pd cmd; 559 struct ib_uobject *uobj; 560 int ret; 561 562 if (copy_from_user(&cmd, buf, sizeof cmd)) 563 return -EFAULT; 564 565 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 566 if (!uobj) 567 return -EINVAL; 568 569 ret = ib_dealloc_pd(uobj->object); 570 if (!ret) 571 uobj->live = 0; 572 573 put_uobj_write(uobj); 574 575 if (ret) 576 return ret; 577 578 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 579 580 mutex_lock(&file->mutex); 581 list_del(&uobj->list); 582 mutex_unlock(&file->mutex); 583 584 put_uobj(uobj); 585 586 return in_len; 587} 588 589ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 590 const char __user *buf, int in_len, 591 int out_len) 592{ 593 struct ib_uverbs_reg_mr cmd; 594 struct ib_uverbs_reg_mr_resp resp; 595 struct ib_udata udata; 596 struct ib_uobject *uobj; 597 struct ib_pd *pd; 598 struct ib_mr *mr; 599 int ret; 600 601 if (out_len < sizeof resp) 602 return -ENOSPC; 603 604 if (copy_from_user(&cmd, buf, sizeof cmd)) 605 return -EFAULT; 606 607 INIT_UDATA(&udata, buf + sizeof cmd, 608 (unsigned long) cmd.response + sizeof resp, 609 in_len - sizeof cmd, out_len - sizeof resp); 610 611 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 612 return -EINVAL; 613 614 /* 615 * Local write permission is required if remote write or 616 * remote atomic permission is also requested. 617 */ 618 if (cmd.access_flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) && 619 !(cmd.access_flags & IB_ACCESS_LOCAL_WRITE)) 620 return -EINVAL; 621 622 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 623 if (!uobj) 624 return -ENOMEM; 625 626 init_uobj(uobj, 0, file->ucontext, &mr_lock_key); 627 down_write(&uobj->mutex); 628 629 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 630 if (!pd) { 631 ret = -EINVAL; 632 goto err_free; 633 } 634 635 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 636 cmd.access_flags, &udata, 0); 637 if (IS_ERR(mr)) { 638 ret = PTR_ERR(mr); 639 goto err_put; 640 } 641 642 mr->device = pd->device; 643 mr->pd = pd; 644 mr->uobject = uobj; 645 atomic_inc(&pd->usecnt); 646 atomic_set(&mr->usecnt, 0); 647 648 uobj->object = mr; 649 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 650 if (ret) 651 goto err_unreg; 652 653 memset(&resp, 0, sizeof resp); 654 resp.lkey = mr->lkey; 655 resp.rkey = mr->rkey; 656 resp.mr_handle = uobj->id; 657 658 if (copy_to_user((void __user *) (unsigned long) cmd.response, 659 &resp, sizeof resp)) { 660 ret = -EFAULT; 661 goto err_copy; 662 } 663 664 put_pd_read(pd); 665 666 mutex_lock(&file->mutex); 667 list_add_tail(&uobj->list, &file->ucontext->mr_list); 668 mutex_unlock(&file->mutex); 669 670 uobj->live = 1; 671 672 up_write(&uobj->mutex); 673 674 return in_len; 675 676err_copy: 677 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 678 679err_unreg: 680 ib_dereg_mr(mr); 681 682err_put: 683 put_pd_read(pd); 684 685err_free: 686 put_uobj_write(uobj); 687 return ret; 688} 689 690ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 691 const char __user *buf, int in_len, 692 int out_len) 693{ 694 struct ib_uverbs_dereg_mr cmd; 695 struct ib_mr *mr; 696 struct ib_uobject *uobj; 697 int ret = -EINVAL; 698 699 if (copy_from_user(&cmd, buf, sizeof cmd)) 700 return -EFAULT; 701 702 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 703 if (!uobj) 704 return -EINVAL; 705 706 mr = uobj->object; 707 708 ret = ib_dereg_mr(mr); 709 if (!ret) 710 uobj->live = 0; 711 712 put_uobj_write(uobj); 713 714 if (ret) 715 return ret; 716 717 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 718 719 mutex_lock(&file->mutex); 720 list_del(&uobj->list); 721 mutex_unlock(&file->mutex); 722 723 put_uobj(uobj); 724 725 return in_len; 726} 727 728ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 729 const char __user *buf, int in_len, 730 int out_len) 731{ 732 struct ib_uverbs_create_comp_channel cmd; 733 struct ib_uverbs_create_comp_channel_resp resp; 734 struct file *filp; 735 736 if (out_len < sizeof resp) 737 return -ENOSPC; 738 739 if (copy_from_user(&cmd, buf, sizeof cmd)) 740 return -EFAULT; 741 742 filp = ib_uverbs_alloc_event_file(file, 0, &resp.fd); 743 if (IS_ERR(filp)) 744 return PTR_ERR(filp); 745 746 if (copy_to_user((void __user *) (unsigned long) cmd.response, 747 &resp, sizeof resp)) { 748 put_unused_fd(resp.fd); 749 fput(filp); 750 return -EFAULT; 751 } 752 753 fd_install(resp.fd, filp); 754 return in_len; 755} 756 757ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 758 const char __user *buf, int in_len, 759 int out_len) 760{ 761 struct ib_uverbs_create_cq cmd; 762 struct ib_uverbs_create_cq_resp resp; 763 struct ib_udata udata; 764 struct ib_ucq_object *obj; 765 struct ib_uverbs_event_file *ev_file = NULL; 766 struct ib_cq *cq; 767 int ret; 768 769 if (out_len < sizeof resp) 770 return -ENOSPC; 771 772 if (copy_from_user(&cmd, buf, sizeof cmd)) 773 return -EFAULT; 774 775 INIT_UDATA(&udata, buf + sizeof cmd, 776 (unsigned long) cmd.response + sizeof resp, 777 in_len - sizeof cmd, out_len - sizeof resp); 778 779 if (cmd.comp_vector >= file->device->num_comp_vectors) 780 return -EINVAL; 781 782 obj = kmalloc(sizeof *obj, GFP_KERNEL); 783 if (!obj) 784 return -ENOMEM; 785 786 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_key); 787 down_write(&obj->uobject.mutex); 788 789 if (cmd.comp_channel >= 0) { 790 ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); 791 if (!ev_file) { 792 ret = -EINVAL; 793 goto err; 794 } 795 } 796 797 obj->uverbs_file = file; 798 obj->comp_events_reported = 0; 799 obj->async_events_reported = 0; 800 INIT_LIST_HEAD(&obj->comp_list); 801 INIT_LIST_HEAD(&obj->async_list); 802 803 cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, 804 cmd.comp_vector, 805 file->ucontext, &udata); 806 if (IS_ERR(cq)) { 807 ret = PTR_ERR(cq); 808 goto err_file; 809 } 810 811 cq->device = file->device->ib_dev; 812 cq->uobject = &obj->uobject; 813 cq->comp_handler = ib_uverbs_comp_handler; 814 cq->event_handler = ib_uverbs_cq_event_handler; 815 cq->cq_context = ev_file; 816 atomic_set(&cq->usecnt, 0); 817 818 obj->uobject.object = cq; 819 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 820 if (ret) 821 goto err_free; 822 823 memset(&resp, 0, sizeof resp); 824 resp.cq_handle = obj->uobject.id; 825 resp.cqe = cq->cqe; 826 827 if (copy_to_user((void __user *) (unsigned long) cmd.response, 828 &resp, sizeof resp)) { 829 ret = -EFAULT; 830 goto err_copy; 831 } 832 833 mutex_lock(&file->mutex); 834 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 835 mutex_unlock(&file->mutex); 836 837 obj->uobject.live = 1; 838 839 up_write(&obj->uobject.mutex); 840 841 return in_len; 842 843err_copy: 844 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 845 846err_free: 847 ib_destroy_cq(cq); 848 849err_file: 850 if (ev_file) 851 ib_uverbs_release_ucq(file, ev_file, obj); 852 853err: 854 put_uobj_write(&obj->uobject); 855 return ret; 856} 857 858ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 859 const char __user *buf, int in_len, 860 int out_len) 861{ 862 struct ib_uverbs_resize_cq cmd; 863 struct ib_uverbs_resize_cq_resp resp; 864 struct ib_udata udata; 865 struct ib_cq *cq; 866 int ret = -EINVAL; 867 868 if (copy_from_user(&cmd, buf, sizeof cmd)) 869 return -EFAULT; 870 871 INIT_UDATA(&udata, buf + sizeof cmd, 872 (unsigned long) cmd.response + sizeof resp, 873 in_len - sizeof cmd, out_len - sizeof resp); 874 875 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 876 if (!cq) 877 return -EINVAL; 878 879 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 880 if (ret) 881 goto out; 882 883 resp.cqe = cq->cqe; 884 885 if (copy_to_user((void __user *) (unsigned long) cmd.response, 886 &resp, sizeof resp.cqe)) 887 ret = -EFAULT; 888 889out: 890 put_cq_read(cq); 891 892 return ret ? ret : in_len; 893} 894 895ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 896 const char __user *buf, int in_len, 897 int out_len) 898{ 899 struct ib_uverbs_poll_cq cmd; 900 struct ib_uverbs_poll_cq_resp *resp; 901 struct ib_cq *cq; 902 struct ib_wc *wc; 903 int ret = 0; 904 int i; 905 int rsize; 906 907 if (copy_from_user(&cmd, buf, sizeof cmd)) 908 return -EFAULT; 909 910 wc = kmalloc(cmd.ne * sizeof *wc, GFP_KERNEL); 911 if (!wc) 912 return -ENOMEM; 913 914 rsize = sizeof *resp + cmd.ne * sizeof(struct ib_uverbs_wc); 915 resp = kmalloc(rsize, GFP_KERNEL); 916 if (!resp) { 917 ret = -ENOMEM; 918 goto out_wc; 919 } 920 921 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 922 if (!cq) { 923 ret = -EINVAL; 924 goto out; 925 } 926 927 resp->count = ib_poll_cq(cq, cmd.ne, wc); 928 929 put_cq_read(cq); 930 931 for (i = 0; i < resp->count; i++) { 932 resp->wc[i].wr_id = wc[i].wr_id; 933 resp->wc[i].status = wc[i].status; 934 resp->wc[i].opcode = wc[i].opcode; 935 resp->wc[i].vendor_err = wc[i].vendor_err; 936 resp->wc[i].byte_len = wc[i].byte_len; 937 resp->wc[i].ex.imm_data = (__u32 __force) wc[i].ex.imm_data; 938 resp->wc[i].qp_num = wc[i].qp->qp_num; 939 resp->wc[i].src_qp = wc[i].src_qp; 940 resp->wc[i].wc_flags = wc[i].wc_flags; 941 resp->wc[i].pkey_index = wc[i].pkey_index; 942 resp->wc[i].slid = wc[i].slid; 943 resp->wc[i].sl = wc[i].sl; 944 resp->wc[i].dlid_path_bits = wc[i].dlid_path_bits; 945 resp->wc[i].port_num = wc[i].port_num; 946 } 947 948 if (copy_to_user((void __user *) (unsigned long) cmd.response, resp, rsize)) 949 ret = -EFAULT; 950 951out: 952 kfree(resp); 953 954out_wc: 955 kfree(wc); 956 return ret ? ret : in_len; 957} 958 959ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 960 const char __user *buf, int in_len, 961 int out_len) 962{ 963 struct ib_uverbs_req_notify_cq cmd; 964 struct ib_cq *cq; 965 966 if (copy_from_user(&cmd, buf, sizeof cmd)) 967 return -EFAULT; 968 969 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 970 if (!cq) 971 return -EINVAL; 972 973 ib_req_notify_cq(cq, cmd.solicited_only ? 974 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 975 976 put_cq_read(cq); 977 978 return in_len; 979} 980 981ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 982 const char __user *buf, int in_len, 983 int out_len) 984{ 985 struct ib_uverbs_destroy_cq cmd; 986 struct ib_uverbs_destroy_cq_resp resp; 987 struct ib_uobject *uobj; 988 struct ib_cq *cq; 989 struct ib_ucq_object *obj; 990 struct ib_uverbs_event_file *ev_file; 991 int ret = -EINVAL; 992 993 if (copy_from_user(&cmd, buf, sizeof cmd)) 994 return -EFAULT; 995 996 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 997 if (!uobj) 998 return -EINVAL; 999 cq = uobj->object; 1000 ev_file = cq->cq_context; 1001 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1002 1003 ret = ib_destroy_cq(cq); 1004 if (!ret) 1005 uobj->live = 0; 1006 1007 put_uobj_write(uobj); 1008 1009 if (ret) 1010 return ret; 1011 1012 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1013 1014 mutex_lock(&file->mutex); 1015 list_del(&uobj->list); 1016 mutex_unlock(&file->mutex); 1017 1018 ib_uverbs_release_ucq(file, ev_file, obj); 1019 1020 memset(&resp, 0, sizeof resp); 1021 resp.comp_events_reported = obj->comp_events_reported; 1022 resp.async_events_reported = obj->async_events_reported; 1023 1024 put_uobj(uobj); 1025 1026 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1027 &resp, sizeof resp)) 1028 return -EFAULT; 1029 1030 return in_len; 1031} 1032 1033ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1034 const char __user *buf, int in_len, 1035 int out_len) 1036{ 1037 struct ib_uverbs_create_qp cmd; 1038 struct ib_uverbs_create_qp_resp resp; 1039 struct ib_udata udata; 1040 struct ib_uqp_object *obj; 1041 struct ib_pd *pd; 1042 struct ib_cq *scq, *rcq; 1043 struct ib_srq *srq; 1044 struct ib_qp *qp; 1045 struct ib_qp_init_attr attr; 1046 struct ib_xrcd *xrcd; 1047 struct ib_uobject *xrcd_uobj; 1048 int ret; 1049 1050 if (out_len < sizeof resp) 1051 return -ENOSPC; 1052 1053 if (copy_from_user(&cmd, buf, sizeof cmd)) 1054 return -EFAULT; 1055 1056 INIT_UDATA(&udata, buf + sizeof cmd, 1057 (unsigned long) cmd.response + sizeof resp, 1058 in_len - sizeof cmd, out_len - sizeof resp); 1059 1060 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1061 if (!obj) 1062 return -ENOMEM; 1063 1064 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key); 1065 down_write(&obj->uevent.uobject.mutex); 1066 1067 srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ? 1068 idr_read_srq(cmd.srq_handle, file->ucontext) : NULL; 1069 xrcd = cmd.qp_type == IB_QPT_XRC ? 1070 idr_read_xrcd(cmd.srq_handle, file->ucontext, &xrcd_uobj) : NULL; 1071 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1072 scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0); 1073 rcq = cmd.recv_cq_handle == cmd.send_cq_handle ? 1074 scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1); 1075 1076 if (!pd || !scq || !rcq || (cmd.is_srq && !srq) || 1077 (cmd.qp_type == IB_QPT_XRC && !xrcd)) { 1078 ret = -EINVAL; 1079 goto err_put; 1080 } 1081 1082 attr.create_flags = 0; 1083 attr.event_handler = ib_uverbs_qp_event_handler; 1084 attr.qp_context = file; 1085 attr.send_cq = scq; 1086 attr.recv_cq = rcq; 1087 attr.srq = srq; 1088 attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 1089 attr.qp_type = cmd.qp_type; 1090 attr.xrcd = xrcd; 1091 attr.create_flags = 0; 1092 1093 attr.cap.max_send_wr = cmd.max_send_wr; 1094 attr.cap.max_recv_wr = cmd.max_recv_wr; 1095 attr.cap.max_send_sge = cmd.max_send_sge; 1096 attr.cap.max_recv_sge = cmd.max_recv_sge; 1097 attr.cap.max_inline_data = cmd.max_inline_data; 1098 1099 obj->uevent.events_reported = 0; 1100 INIT_LIST_HEAD(&obj->uevent.event_list); 1101 INIT_LIST_HEAD(&obj->mcast_list); 1102 1103 qp = pd->device->create_qp(pd, &attr, &udata); 1104 if (IS_ERR(qp)) { 1105 ret = PTR_ERR(qp); 1106 goto err_put; 1107 } 1108 1109 qp->device = pd->device; 1110 qp->pd = pd; 1111 qp->send_cq = attr.send_cq; 1112 qp->recv_cq = attr.recv_cq; 1113 qp->srq = attr.srq; 1114 qp->uobject = &obj->uevent.uobject; 1115 qp->event_handler = attr.event_handler; 1116 qp->qp_context = attr.qp_context; 1117 qp->qp_type = attr.qp_type; 1118 qp->xrcd = attr.xrcd; 1119 atomic_inc(&pd->usecnt); 1120 atomic_inc(&attr.send_cq->usecnt); 1121 atomic_inc(&attr.recv_cq->usecnt); 1122 if (attr.srq) 1123 atomic_inc(&attr.srq->usecnt); 1124 else if (attr.xrcd) 1125 atomic_inc(&attr.xrcd->usecnt); 1126 1127 obj->uevent.uobject.object = qp; 1128 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1129 if (ret) 1130 goto err_destroy; 1131 1132 memset(&resp, 0, sizeof resp); 1133 resp.qpn = qp->qp_num; 1134 resp.qp_handle = obj->uevent.uobject.id; 1135 resp.max_recv_sge = attr.cap.max_recv_sge; 1136 resp.max_send_sge = attr.cap.max_send_sge; 1137 resp.max_recv_wr = attr.cap.max_recv_wr; 1138 resp.max_send_wr = attr.cap.max_send_wr; 1139 resp.max_inline_data = attr.cap.max_inline_data; 1140 1141 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1142 &resp, sizeof resp)) { 1143 ret = -EFAULT; 1144 goto err_copy; 1145 } 1146 1147 put_pd_read(pd); 1148 put_cq_read(scq); 1149 if (rcq != scq) 1150 put_cq_read(rcq); 1151 if (srq) 1152 put_srq_read(srq); 1153 if (xrcd) 1154 put_xrcd_read(xrcd_uobj); 1155 1156 mutex_lock(&file->mutex); 1157 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1158 mutex_unlock(&file->mutex); 1159 1160 obj->uevent.uobject.live = 1; 1161 1162 up_write(&obj->uevent.uobject.mutex); 1163 1164 return in_len; 1165 1166err_copy: 1167 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1168 1169err_destroy: 1170 ib_destroy_qp(qp); 1171 1172err_put: 1173 if (pd) 1174 put_pd_read(pd); 1175 if (scq) 1176 put_cq_read(scq); 1177 if (rcq && rcq != scq) 1178 put_cq_read(rcq); 1179 if (srq) 1180 put_srq_read(srq); 1181 if (xrcd) 1182 put_xrcd_read(xrcd_uobj); 1183 1184 put_uobj_write(&obj->uevent.uobject); 1185 return ret; 1186} 1187 1188ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1189 const char __user *buf, int in_len, 1190 int out_len) 1191{ 1192 struct ib_uverbs_query_qp cmd; 1193 struct ib_uverbs_query_qp_resp resp; 1194 struct ib_qp *qp; 1195 struct ib_qp_attr *attr; 1196 struct ib_qp_init_attr *init_attr; 1197 int ret; 1198 1199 if (copy_from_user(&cmd, buf, sizeof cmd)) 1200 return -EFAULT; 1201 1202 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1203 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1204 if (!attr || !init_attr) { 1205 ret = -ENOMEM; 1206 goto out; 1207 } 1208 1209 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1210 if (!qp) { 1211 ret = -EINVAL; 1212 goto out; 1213 } 1214 1215 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1216 1217 put_qp_read(qp); 1218 1219 if (ret) 1220 goto out; 1221 1222 memset(&resp, 0, sizeof resp); 1223 1224 resp.qp_state = attr->qp_state; 1225 resp.cur_qp_state = attr->cur_qp_state; 1226 resp.path_mtu = attr->path_mtu; 1227 resp.path_mig_state = attr->path_mig_state; 1228 resp.qkey = attr->qkey; 1229 resp.rq_psn = attr->rq_psn; 1230 resp.sq_psn = attr->sq_psn; 1231 resp.dest_qp_num = attr->dest_qp_num; 1232 resp.qp_access_flags = attr->qp_access_flags; 1233 resp.pkey_index = attr->pkey_index; 1234 resp.alt_pkey_index = attr->alt_pkey_index; 1235 resp.sq_draining = attr->sq_draining; 1236 resp.max_rd_atomic = attr->max_rd_atomic; 1237 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1238 resp.min_rnr_timer = attr->min_rnr_timer; 1239 resp.port_num = attr->port_num; 1240 resp.timeout = attr->timeout; 1241 resp.retry_cnt = attr->retry_cnt; 1242 resp.rnr_retry = attr->rnr_retry; 1243 resp.alt_port_num = attr->alt_port_num; 1244 resp.alt_timeout = attr->alt_timeout; 1245 1246 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1247 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 1248 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 1249 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 1250 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 1251 resp.dest.dlid = attr->ah_attr.dlid; 1252 resp.dest.sl = attr->ah_attr.sl; 1253 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 1254 resp.dest.static_rate = attr->ah_attr.static_rate; 1255 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 1256 resp.dest.port_num = attr->ah_attr.port_num; 1257 1258 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1259 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1260 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1261 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1262 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 1263 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 1264 resp.alt_dest.sl = attr->alt_ah_attr.sl; 1265 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1266 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1267 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 1268 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 1269 1270 resp.max_send_wr = init_attr->cap.max_send_wr; 1271 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1272 resp.max_send_sge = init_attr->cap.max_send_sge; 1273 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1274 resp.max_inline_data = init_attr->cap.max_inline_data; 1275 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1276 1277 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1278 &resp, sizeof resp)) 1279 ret = -EFAULT; 1280 1281out: 1282 kfree(attr); 1283 kfree(init_attr); 1284 1285 return ret ? ret : in_len; 1286} 1287 1288ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 1289 const char __user *buf, int in_len, 1290 int out_len) 1291{ 1292 struct ib_uverbs_modify_qp cmd; 1293 struct ib_udata udata; 1294 struct ib_qp *qp; 1295 struct ib_qp_attr *attr; 1296 int ret; 1297 1298 if (copy_from_user(&cmd, buf, sizeof cmd)) 1299 return -EFAULT; 1300 1301 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 1302 out_len); 1303 1304 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1305 if (!attr) 1306 return -ENOMEM; 1307 1308 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1309 if (!qp) { 1310 ret = -EINVAL; 1311 goto out; 1312 } 1313 1314 attr->qp_state = cmd.qp_state; 1315 attr->cur_qp_state = cmd.cur_qp_state; 1316 attr->path_mtu = cmd.path_mtu; 1317 attr->path_mig_state = cmd.path_mig_state; 1318 attr->qkey = cmd.qkey; 1319 attr->rq_psn = cmd.rq_psn; 1320 attr->sq_psn = cmd.sq_psn; 1321 attr->dest_qp_num = cmd.dest_qp_num; 1322 attr->qp_access_flags = cmd.qp_access_flags; 1323 attr->pkey_index = cmd.pkey_index; 1324 attr->alt_pkey_index = cmd.alt_pkey_index; 1325 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 1326 attr->max_rd_atomic = cmd.max_rd_atomic; 1327 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 1328 attr->min_rnr_timer = cmd.min_rnr_timer; 1329 attr->port_num = cmd.port_num; 1330 attr->timeout = cmd.timeout; 1331 attr->retry_cnt = cmd.retry_cnt; 1332 attr->rnr_retry = cmd.rnr_retry; 1333 attr->alt_port_num = cmd.alt_port_num; 1334 attr->alt_timeout = cmd.alt_timeout; 1335 1336 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 1337 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 1338 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 1339 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 1340 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 1341 attr->ah_attr.dlid = cmd.dest.dlid; 1342 attr->ah_attr.sl = cmd.dest.sl; 1343 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 1344 attr->ah_attr.static_rate = cmd.dest.static_rate; 1345 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 1346 attr->ah_attr.port_num = cmd.dest.port_num; 1347 1348 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 1349 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 1350 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 1351 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 1352 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 1353 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 1354 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 1355 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 1356 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 1357 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 1358 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 1359 1360 ret = qp->device->modify_qp(qp, attr, cmd.attr_mask, &udata); 1361 1362 put_qp_read(qp); 1363 1364 if (ret) 1365 goto out; 1366 1367 ret = in_len; 1368 1369out: 1370 kfree(attr); 1371 1372 return ret; 1373} 1374 1375ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 1376 const char __user *buf, int in_len, 1377 int out_len) 1378{ 1379 struct ib_uverbs_destroy_qp cmd; 1380 struct ib_uverbs_destroy_qp_resp resp; 1381 struct ib_uobject *uobj; 1382 struct ib_qp *qp; 1383 struct ib_uqp_object *obj; 1384 int ret = -EINVAL; 1385 1386 if (copy_from_user(&cmd, buf, sizeof cmd)) 1387 return -EFAULT; 1388 1389 memset(&resp, 0, sizeof resp); 1390 1391 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 1392 if (!uobj) 1393 return -EINVAL; 1394 qp = uobj->object; 1395 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 1396 1397 if (!list_empty(&obj->mcast_list)) { 1398 put_uobj_write(uobj); 1399 return -EBUSY; 1400 } 1401 1402 ret = ib_destroy_qp(qp); 1403 if (!ret) 1404 uobj->live = 0; 1405 1406 put_uobj_write(uobj); 1407 1408 if (ret) 1409 return ret; 1410 1411 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 1412 1413 mutex_lock(&file->mutex); 1414 list_del(&uobj->list); 1415 mutex_unlock(&file->mutex); 1416 1417 ib_uverbs_release_uevent(file, &obj->uevent); 1418 1419 resp.events_reported = obj->uevent.events_reported; 1420 1421 put_uobj(uobj); 1422 1423 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1424 &resp, sizeof resp)) 1425 return -EFAULT; 1426 1427 return in_len; 1428} 1429 1430ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 1431 const char __user *buf, int in_len, 1432 int out_len) 1433{ 1434 struct ib_uverbs_post_send cmd; 1435 struct ib_uverbs_post_send_resp resp; 1436 struct ib_uverbs_send_wr *user_wr; 1437 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 1438 struct ib_qp *qp; 1439 int i, sg_ind; 1440 int is_ud; 1441 ssize_t ret = -EINVAL; 1442 1443 if (copy_from_user(&cmd, buf, sizeof cmd)) 1444 return -EFAULT; 1445 1446 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 1447 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 1448 return -EINVAL; 1449 1450 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 1451 return -EINVAL; 1452 1453 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 1454 if (!user_wr) 1455 return -ENOMEM; 1456 1457 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1458 if (!qp) 1459 goto out; 1460 1461 is_ud = qp->qp_type == IB_QPT_UD; 1462 sg_ind = 0; 1463 last = NULL; 1464 for (i = 0; i < cmd.wr_count; ++i) { 1465 if (copy_from_user(user_wr, 1466 buf + sizeof cmd + i * cmd.wqe_size, 1467 cmd.wqe_size)) { 1468 ret = -EFAULT; 1469 goto out_put; 1470 } 1471 1472 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 1473 ret = -EINVAL; 1474 goto out_put; 1475 } 1476 1477 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1478 user_wr->num_sge * sizeof (struct ib_sge), 1479 GFP_KERNEL); 1480 if (!next) { 1481 ret = -ENOMEM; 1482 goto out_put; 1483 } 1484 1485 if (!last) 1486 wr = next; 1487 else 1488 last->next = next; 1489 last = next; 1490 1491 next->next = NULL; 1492 next->wr_id = user_wr->wr_id; 1493 next->num_sge = user_wr->num_sge; 1494 next->opcode = user_wr->opcode; 1495 next->send_flags = user_wr->send_flags; 1496 1497 if (is_ud) { 1498 next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, 1499 file->ucontext); 1500 if (!next->wr.ud.ah) { 1501 ret = -EINVAL; 1502 goto out_put; 1503 } 1504 next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; 1505 next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; 1506 } else { 1507 switch (next->opcode) { 1508 case IB_WR_RDMA_WRITE_WITH_IMM: 1509 next->ex.imm_data = 1510 (__be32 __force) user_wr->ex.imm_data; 1511 case IB_WR_RDMA_WRITE: 1512 case IB_WR_RDMA_READ: 1513 next->wr.rdma.remote_addr = 1514 user_wr->wr.rdma.remote_addr; 1515 next->wr.rdma.rkey = 1516 user_wr->wr.rdma.rkey; 1517 break; 1518 case IB_WR_SEND_WITH_IMM: 1519 next->ex.imm_data = 1520 (__be32 __force) user_wr->ex.imm_data; 1521 break; 1522 case IB_WR_SEND_WITH_INV: 1523 next->ex.invalidate_rkey = 1524 user_wr->ex.invalidate_rkey; 1525 break; 1526 case IB_WR_ATOMIC_CMP_AND_SWP: 1527 case IB_WR_ATOMIC_FETCH_AND_ADD: 1528 next->wr.atomic.remote_addr = 1529 user_wr->wr.atomic.remote_addr; 1530 next->wr.atomic.compare_add = 1531 user_wr->wr.atomic.compare_add; 1532 next->wr.atomic.swap = user_wr->wr.atomic.swap; 1533 next->wr.atomic.rkey = user_wr->wr.atomic.rkey; 1534 break; 1535 default: 1536 break; 1537 } 1538 } 1539 1540 if (next->num_sge) { 1541 next->sg_list = (void *) next + 1542 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1543 if (copy_from_user(next->sg_list, 1544 buf + sizeof cmd + 1545 cmd.wr_count * cmd.wqe_size + 1546 sg_ind * sizeof (struct ib_sge), 1547 next->num_sge * sizeof (struct ib_sge))) { 1548 ret = -EFAULT; 1549 goto out_put; 1550 } 1551 sg_ind += next->num_sge; 1552 } else 1553 next->sg_list = NULL; 1554 } 1555 1556 resp.bad_wr = 0; 1557 ret = qp->device->post_send(qp, wr, &bad_wr); 1558 if (ret) 1559 for (next = wr; next; next = next->next) { 1560 ++resp.bad_wr; 1561 if (next == bad_wr) 1562 break; 1563 } 1564 1565 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1566 &resp, sizeof resp)) 1567 ret = -EFAULT; 1568 1569out_put: 1570 put_qp_read(qp); 1571 1572 while (wr) { 1573 if (is_ud && wr->wr.ud.ah) 1574 put_ah_read(wr->wr.ud.ah); 1575 next = wr->next; 1576 kfree(wr); 1577 wr = next; 1578 } 1579 1580out: 1581 kfree(user_wr); 1582 1583 return ret ? ret : in_len; 1584} 1585 1586static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 1587 int in_len, 1588 u32 wr_count, 1589 u32 sge_count, 1590 u32 wqe_size) 1591{ 1592 struct ib_uverbs_recv_wr *user_wr; 1593 struct ib_recv_wr *wr = NULL, *last, *next; 1594 int sg_ind; 1595 int i; 1596 int ret; 1597 1598 if (in_len < wqe_size * wr_count + 1599 sge_count * sizeof (struct ib_uverbs_sge)) 1600 return ERR_PTR(-EINVAL); 1601 1602 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 1603 return ERR_PTR(-EINVAL); 1604 1605 user_wr = kmalloc(wqe_size, GFP_KERNEL); 1606 if (!user_wr) 1607 return ERR_PTR(-ENOMEM); 1608 1609 sg_ind = 0; 1610 last = NULL; 1611 for (i = 0; i < wr_count; ++i) { 1612 if (copy_from_user(user_wr, buf + i * wqe_size, 1613 wqe_size)) { 1614 ret = -EFAULT; 1615 goto err; 1616 } 1617 1618 if (user_wr->num_sge + sg_ind > sge_count) { 1619 ret = -EINVAL; 1620 goto err; 1621 } 1622 1623 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1624 user_wr->num_sge * sizeof (struct ib_sge), 1625 GFP_KERNEL); 1626 if (!next) { 1627 ret = -ENOMEM; 1628 goto err; 1629 } 1630 1631 if (!last) 1632 wr = next; 1633 else 1634 last->next = next; 1635 last = next; 1636 1637 next->next = NULL; 1638 next->wr_id = user_wr->wr_id; 1639 next->num_sge = user_wr->num_sge; 1640 1641 if (next->num_sge) { 1642 next->sg_list = (void *) next + 1643 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1644 if (copy_from_user(next->sg_list, 1645 buf + wr_count * wqe_size + 1646 sg_ind * sizeof (struct ib_sge), 1647 next->num_sge * sizeof (struct ib_sge))) { 1648 ret = -EFAULT; 1649 goto err; 1650 } 1651 sg_ind += next->num_sge; 1652 } else 1653 next->sg_list = NULL; 1654 } 1655 1656 kfree(user_wr); 1657 return wr; 1658 1659err: 1660 kfree(user_wr); 1661 1662 while (wr) { 1663 next = wr->next; 1664 kfree(wr); 1665 wr = next; 1666 } 1667 1668 return ERR_PTR(ret); 1669} 1670 1671ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 1672 const char __user *buf, int in_len, 1673 int out_len) 1674{ 1675 struct ib_uverbs_post_recv cmd; 1676 struct ib_uverbs_post_recv_resp resp; 1677 struct ib_recv_wr *wr, *next, *bad_wr; 1678 struct ib_qp *qp; 1679 ssize_t ret = -EINVAL; 1680 1681 if (copy_from_user(&cmd, buf, sizeof cmd)) 1682 return -EFAULT; 1683 1684 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1685 in_len - sizeof cmd, cmd.wr_count, 1686 cmd.sge_count, cmd.wqe_size); 1687 if (IS_ERR(wr)) 1688 return PTR_ERR(wr); 1689 1690 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1691 if (!qp) 1692 goto out; 1693 1694 resp.bad_wr = 0; 1695 ret = qp->device->post_recv(qp, wr, &bad_wr); 1696 1697 put_qp_read(qp); 1698 1699 if (ret) 1700 for (next = wr; next; next = next->next) { 1701 ++resp.bad_wr; 1702 if (next == bad_wr) 1703 break; 1704 } 1705 1706 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1707 &resp, sizeof resp)) 1708 ret = -EFAULT; 1709 1710out: 1711 while (wr) { 1712 next = wr->next; 1713 kfree(wr); 1714 wr = next; 1715 } 1716 1717 return ret ? ret : in_len; 1718} 1719 1720ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 1721 const char __user *buf, int in_len, 1722 int out_len) 1723{ 1724 struct ib_uverbs_post_srq_recv cmd; 1725 struct ib_uverbs_post_srq_recv_resp resp; 1726 struct ib_recv_wr *wr, *next, *bad_wr; 1727 struct ib_srq *srq; 1728 ssize_t ret = -EINVAL; 1729 1730 if (copy_from_user(&cmd, buf, sizeof cmd)) 1731 return -EFAULT; 1732 1733 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1734 in_len - sizeof cmd, cmd.wr_count, 1735 cmd.sge_count, cmd.wqe_size); 1736 if (IS_ERR(wr)) 1737 return PTR_ERR(wr); 1738 1739 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 1740 if (!srq) 1741 goto out; 1742 1743 resp.bad_wr = 0; 1744 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 1745 1746 put_srq_read(srq); 1747 1748 if (ret) 1749 for (next = wr; next; next = next->next) { 1750 ++resp.bad_wr; 1751 if (next == bad_wr) 1752 break; 1753 } 1754 1755 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1756 &resp, sizeof resp)) 1757 ret = -EFAULT; 1758 1759out: 1760 while (wr) { 1761 next = wr->next; 1762 kfree(wr); 1763 wr = next; 1764 } 1765 1766 return ret ? ret : in_len; 1767} 1768 1769ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 1770 const char __user *buf, int in_len, 1771 int out_len) 1772{ 1773 struct ib_uverbs_create_ah cmd; 1774 struct ib_uverbs_create_ah_resp resp; 1775 struct ib_uobject *uobj; 1776 struct ib_pd *pd; 1777 struct ib_ah *ah; 1778 struct ib_ah_attr attr; 1779 int ret; 1780 1781 if (out_len < sizeof resp) 1782 return -ENOSPC; 1783 1784 if (copy_from_user(&cmd, buf, sizeof cmd)) 1785 return -EFAULT; 1786 1787 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 1788 if (!uobj) 1789 return -ENOMEM; 1790 1791 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_key); 1792 down_write(&uobj->mutex); 1793 1794 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1795 if (!pd) { 1796 ret = -EINVAL; 1797 goto err; 1798 } 1799 1800 attr.dlid = cmd.attr.dlid; 1801 attr.sl = cmd.attr.sl; 1802 attr.src_path_bits = cmd.attr.src_path_bits; 1803 attr.static_rate = cmd.attr.static_rate; 1804 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 1805 attr.port_num = cmd.attr.port_num; 1806 attr.grh.flow_label = cmd.attr.grh.flow_label; 1807 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 1808 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 1809 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 1810 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 1811 1812 ah = ib_create_ah(pd, &attr); 1813 if (IS_ERR(ah)) { 1814 ret = PTR_ERR(ah); 1815 goto err_put; 1816 } 1817 1818 ah->uobject = uobj; 1819 uobj->object = ah; 1820 1821 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 1822 if (ret) 1823 goto err_destroy; 1824 1825 resp.ah_handle = uobj->id; 1826 1827 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1828 &resp, sizeof resp)) { 1829 ret = -EFAULT; 1830 goto err_copy; 1831 } 1832 1833 put_pd_read(pd); 1834 1835 mutex_lock(&file->mutex); 1836 list_add_tail(&uobj->list, &file->ucontext->ah_list); 1837 mutex_unlock(&file->mutex); 1838 1839 uobj->live = 1; 1840 1841 up_write(&uobj->mutex); 1842 1843 return in_len; 1844 1845err_copy: 1846 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1847 1848err_destroy: 1849 ib_destroy_ah(ah); 1850 1851err_put: 1852 put_pd_read(pd); 1853 1854err: 1855 put_uobj_write(uobj); 1856 return ret; 1857} 1858 1859ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 1860 const char __user *buf, int in_len, int out_len) 1861{ 1862 struct ib_uverbs_destroy_ah cmd; 1863 struct ib_ah *ah; 1864 struct ib_uobject *uobj; 1865 int ret; 1866 1867 if (copy_from_user(&cmd, buf, sizeof cmd)) 1868 return -EFAULT; 1869 1870 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 1871 if (!uobj) 1872 return -EINVAL; 1873 ah = uobj->object; 1874 1875 ret = ib_destroy_ah(ah); 1876 if (!ret) 1877 uobj->live = 0; 1878 1879 put_uobj_write(uobj); 1880 1881 if (ret) 1882 return ret; 1883 1884 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1885 1886 mutex_lock(&file->mutex); 1887 list_del(&uobj->list); 1888 mutex_unlock(&file->mutex); 1889 1890 put_uobj(uobj); 1891 1892 return in_len; 1893} 1894 1895ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 1896 const char __user *buf, int in_len, 1897 int out_len) 1898{ 1899 struct ib_uverbs_attach_mcast cmd; 1900 struct ib_qp *qp; 1901 struct ib_uqp_object *obj; 1902 struct ib_uverbs_mcast_entry *mcast; 1903 int ret; 1904 1905 if (copy_from_user(&cmd, buf, sizeof cmd)) 1906 return -EFAULT; 1907 1908 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1909 if (!qp) 1910 return -EINVAL; 1911 1912 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1913 1914 list_for_each_entry(mcast, &obj->mcast_list, list) 1915 if (cmd.mlid == mcast->lid && 1916 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1917 ret = 0; 1918 goto out_put; 1919 } 1920 1921 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 1922 if (!mcast) { 1923 ret = -ENOMEM; 1924 goto out_put; 1925 } 1926 1927 mcast->lid = cmd.mlid; 1928 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 1929 1930 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 1931 if (!ret) 1932 list_add_tail(&mcast->list, &obj->mcast_list); 1933 else 1934 kfree(mcast); 1935 1936out_put: 1937 put_qp_read(qp); 1938 1939 return ret ? ret : in_len; 1940} 1941 1942ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 1943 const char __user *buf, int in_len, 1944 int out_len) 1945{ 1946 struct ib_uverbs_detach_mcast cmd; 1947 struct ib_uqp_object *obj; 1948 struct ib_qp *qp; 1949 struct ib_uverbs_mcast_entry *mcast; 1950 int ret = -EINVAL; 1951 1952 if (copy_from_user(&cmd, buf, sizeof cmd)) 1953 return -EFAULT; 1954 1955 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1956 if (!qp) 1957 return -EINVAL; 1958 1959 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 1960 if (ret) 1961 goto out_put; 1962 1963 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1964 1965 list_for_each_entry(mcast, &obj->mcast_list, list) 1966 if (cmd.mlid == mcast->lid && 1967 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1968 list_del(&mcast->list); 1969 kfree(mcast); 1970 break; 1971 } 1972 1973out_put: 1974 put_qp_read(qp); 1975 1976 return ret ? ret : in_len; 1977} 1978 1979ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 1980 const char __user *buf, int in_len, 1981 int out_len) 1982{ 1983 struct ib_uverbs_create_srq cmd; 1984 struct ib_uverbs_create_srq_resp resp; 1985 struct ib_udata udata; 1986 struct ib_uevent_object *obj; 1987 struct ib_pd *pd; 1988 struct ib_srq *srq; 1989 struct ib_srq_init_attr attr; 1990 int ret; 1991 1992 if (out_len < sizeof resp) 1993 return -ENOSPC; 1994 1995 if (copy_from_user(&cmd, buf, sizeof cmd)) 1996 return -EFAULT; 1997 1998 INIT_UDATA(&udata, buf + sizeof cmd, 1999 (unsigned long) cmd.response + sizeof resp, 2000 in_len - sizeof cmd, out_len - sizeof resp); 2001 2002 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2003 if (!obj) 2004 return -ENOMEM; 2005 2006 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &srq_lock_key); 2007 down_write(&obj->uobject.mutex); 2008 2009 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2010 if (!pd) { 2011 ret = -EINVAL; 2012 goto err; 2013 } 2014 2015 attr.event_handler = ib_uverbs_srq_event_handler; 2016 attr.srq_context = file; 2017 attr.attr.max_wr = cmd.max_wr; 2018 attr.attr.max_sge = cmd.max_sge; 2019 attr.attr.srq_limit = cmd.srq_limit; 2020 2021 obj->events_reported = 0; 2022 INIT_LIST_HEAD(&obj->event_list); 2023 2024 srq = pd->device->create_srq(pd, &attr, &udata); 2025 if (IS_ERR(srq)) { 2026 ret = PTR_ERR(srq); 2027 goto err_put; 2028 } 2029 2030 srq->device = pd->device; 2031 srq->pd = pd; 2032 srq->uobject = &obj->uobject; 2033 srq->event_handler = attr.event_handler; 2034 srq->srq_context = attr.srq_context; 2035 srq->ext.xrc.cq = NULL; 2036 srq->ext.xrc.xrcd = NULL; 2037 atomic_inc(&pd->usecnt); 2038 atomic_set(&srq->usecnt, 0); 2039 2040 obj->uobject.object = srq; 2041 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2042 if (ret) 2043 goto err_destroy; 2044 2045 memset(&resp, 0, sizeof resp); 2046 resp.srq_handle = obj->uobject.id; 2047 resp.max_wr = attr.attr.max_wr; 2048 resp.max_sge = attr.attr.max_sge; 2049 2050 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2051 &resp, sizeof resp)) { 2052 ret = -EFAULT; 2053 goto err_copy; 2054 } 2055 2056 put_pd_read(pd); 2057 2058 mutex_lock(&file->mutex); 2059 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2060 mutex_unlock(&file->mutex); 2061 2062 obj->uobject.live = 1; 2063 2064 up_write(&obj->uobject.mutex); 2065 2066 return in_len; 2067 2068err_copy: 2069 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2070 2071err_destroy: 2072 ib_destroy_srq(srq); 2073 2074err_put: 2075 put_pd_read(pd); 2076 2077err: 2078 put_uobj_write(&obj->uobject); 2079 return ret; 2080} 2081 2082ssize_t ib_uverbs_create_xrc_srq(struct ib_uverbs_file *file, 2083 const char __user *buf, int in_len, 2084 int out_len) 2085{ 2086 struct ib_uverbs_create_xsrq cmd; 2087 struct ib_uverbs_create_srq_resp resp; 2088 struct ib_udata udata; 2089 struct ib_uevent_object *obj; 2090 struct ib_pd *pd; 2091 struct ib_srq *srq; 2092 struct ib_cq *xrc_cq; 2093 struct ib_xrcd *xrcd; 2094 struct ib_srq_init_attr attr; 2095 struct ib_uobject *xrcd_uobj; 2096 int ret; 2097 2098 if (out_len < sizeof resp) 2099 return -ENOSPC; 2100 2101 if (copy_from_user(&cmd, buf, sizeof cmd)) 2102 return -EFAULT; 2103 2104 INIT_UDATA(&udata, buf + sizeof cmd, 2105 (unsigned long) cmd.response + sizeof resp, 2106 in_len - sizeof cmd, out_len - sizeof resp); 2107 2108 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2109 if (!obj) 2110 return -ENOMEM; 2111 2112 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, 2113 &srq_lock_key); 2114 down_write(&obj->uobject.mutex); 2115 2116 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2117 if (!pd) { 2118 ret = -EINVAL; 2119 goto err; 2120 } 2121 2122 xrc_cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 2123 if (!xrc_cq) { 2124 ret = -EINVAL; 2125 goto err_put_pd; 2126 } 2127 2128 xrcd = idr_read_xrcd(cmd.xrcd_handle, file->ucontext, &xrcd_uobj); 2129 if (!xrcd) { 2130 ret = -EINVAL; 2131 goto err_put_cq; 2132 } 2133 2134 2135 attr.event_handler = ib_uverbs_srq_event_handler; 2136 attr.srq_context = file; 2137 attr.attr.max_wr = cmd.max_wr; 2138 attr.attr.max_sge = cmd.max_sge; 2139 attr.attr.srq_limit = cmd.srq_limit; 2140 2141 obj->events_reported = 0; 2142 INIT_LIST_HEAD(&obj->event_list); 2143 2144 srq = pd->device->create_xrc_srq(pd, xrc_cq, xrcd, &attr, &udata); 2145 if (IS_ERR(srq)) { 2146 ret = PTR_ERR(srq); 2147 goto err_put; 2148 } 2149 2150 srq->device = pd->device; 2151 srq->pd = pd; 2152 srq->uobject = &obj->uobject; 2153 srq->event_handler = attr.event_handler; 2154 srq->srq_context = attr.srq_context; 2155 srq->ext.xrc.cq = xrc_cq; 2156 srq->ext.xrc.xrcd = xrcd; 2157 atomic_inc(&pd->usecnt); 2158 atomic_inc(&xrc_cq->usecnt); 2159 atomic_inc(&xrcd->usecnt); 2160 2161 atomic_set(&srq->usecnt, 0); 2162 2163 obj->uobject.object = srq; 2164 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2165 if (ret) 2166 goto err_destroy; 2167 2168 memset(&resp, 0, sizeof resp); 2169 resp.srq_handle = obj->uobject.id; 2170 resp.max_wr = attr.attr.max_wr; 2171 resp.max_sge = attr.attr.max_sge; 2172 2173 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2174 &resp, sizeof resp)) { 2175 ret = -EFAULT; 2176 goto err_copy; 2177 } 2178 2179 put_xrcd_read(xrcd_uobj); 2180 put_cq_read(xrc_cq); 2181 put_pd_read(pd); 2182 2183 mutex_lock(&file->mutex); 2184 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2185 mutex_unlock(&file->mutex); 2186 2187 obj->uobject.live = 1; 2188 2189 up_write(&obj->uobject.mutex); 2190 2191 return in_len; 2192 2193err_copy: 2194 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2195 2196err_destroy: 2197 ib_destroy_srq(srq); 2198 2199err_put: 2200 put_xrcd_read(xrcd_uobj); 2201 2202err_put_cq: 2203 put_cq_read(xrc_cq); 2204 2205err_put_pd: 2206 put_pd_read(pd); 2207 2208err: 2209 put_uobj_write(&obj->uobject); 2210 return ret; 2211} 2212 2213ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 2214 const char __user *buf, int in_len, 2215 int out_len) 2216{ 2217 struct ib_uverbs_modify_srq cmd; 2218 struct ib_udata udata; 2219 struct ib_srq *srq; 2220 struct ib_srq_attr attr; 2221 int ret; 2222 2223 if (copy_from_user(&cmd, buf, sizeof cmd)) 2224 return -EFAULT; 2225 2226 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 2227 out_len); 2228 2229 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2230 if (!srq) 2231 return -EINVAL; 2232 2233 attr.max_wr = cmd.max_wr; 2234 attr.srq_limit = cmd.srq_limit; 2235 2236 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 2237 2238 put_srq_read(srq); 2239 2240 return ret ? ret : in_len; 2241} 2242 2243ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 2244 const char __user *buf, 2245 int in_len, int out_len) 2246{ 2247 struct ib_uverbs_query_srq cmd; 2248 struct ib_uverbs_query_srq_resp resp; 2249 struct ib_srq_attr attr; 2250 struct ib_srq *srq; 2251 int ret; 2252 2253 if (out_len < sizeof resp) 2254 return -ENOSPC; 2255 2256 if (copy_from_user(&cmd, buf, sizeof cmd)) 2257 return -EFAULT; 2258 2259 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2260 if (!srq) 2261 return -EINVAL; 2262 2263 ret = ib_query_srq(srq, &attr); 2264 2265 put_srq_read(srq); 2266 2267 if (ret) 2268 return ret; 2269 2270 memset(&resp, 0, sizeof resp); 2271 2272 resp.max_wr = attr.max_wr; 2273 resp.max_sge = attr.max_sge; 2274 resp.srq_limit = attr.srq_limit; 2275 2276 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2277 &resp, sizeof resp)) 2278 return -EFAULT; 2279 2280 return in_len; 2281} 2282 2283ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 2284 const char __user *buf, int in_len, 2285 int out_len) 2286{ 2287 struct ib_uverbs_destroy_srq cmd; 2288 struct ib_uverbs_destroy_srq_resp resp; 2289 struct ib_uobject *uobj; 2290 struct ib_srq *srq; 2291 struct ib_uevent_object *obj; 2292 int ret = -EINVAL; 2293 2294 if (copy_from_user(&cmd, buf, sizeof cmd)) 2295 return -EFAULT; 2296 2297 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 2298 if (!uobj) 2299 return -EINVAL; 2300 srq = uobj->object; 2301 obj = container_of(uobj, struct ib_uevent_object, uobject); 2302 2303 ret = ib_destroy_srq(srq); 2304 if (!ret) 2305 uobj->live = 0; 2306 2307 put_uobj_write(uobj); 2308 2309 if (ret) 2310 return ret; 2311 2312 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 2313 2314 mutex_lock(&file->mutex); 2315 list_del(&uobj->list); 2316 mutex_unlock(&file->mutex); 2317 2318 ib_uverbs_release_uevent(file, obj); 2319 2320 memset(&resp, 0, sizeof resp); 2321 resp.events_reported = obj->events_reported; 2322 2323 put_uobj(uobj); 2324 2325 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2326 &resp, sizeof resp)) 2327 ret = -EFAULT; 2328 2329 return ret ? ret : in_len; 2330} 2331 2332static struct inode *xrc_file2inode(struct file *f) 2333{ 2334 return f->f_dentry->d_inode; 2335} 2336 2337struct xrcd_table_entry { 2338 struct rb_node node; 2339 struct inode *inode; 2340 struct ib_xrcd *xrcd; 2341}; 2342 2343static int xrcd_table_insert(struct ib_device *dev, 2344 struct inode *i_n, 2345 struct ib_xrcd *xrcd) 2346{ 2347 struct xrcd_table_entry *entry, *scan; 2348 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2349 struct rb_node *parent = NULL; 2350 2351 entry = kmalloc(sizeof(struct xrcd_table_entry), GFP_KERNEL); 2352 if (!entry) 2353 return -ENOMEM; 2354 2355 entry->inode = i_n; 2356 entry->xrcd = xrcd; 2357 2358 while (*p) { 2359 parent = *p; 2360 scan = rb_entry(parent, struct xrcd_table_entry, node); 2361 2362 if (i_n < scan->inode) 2363 p = &(*p)->rb_left; 2364 else if (i_n > scan->inode) 2365 p = &(*p)->rb_right; 2366 else { 2367 kfree(entry); 2368 return -EEXIST; 2369 } 2370 } 2371 2372 rb_link_node(&entry->node, parent, p); 2373 rb_insert_color(&entry->node, &dev->ib_uverbs_xrcd_table); 2374 igrab(i_n); 2375 return 0; 2376} 2377 2378static struct xrcd_table_entry *xrcd_table_search(struct ib_device *dev, 2379 struct inode *i_n) 2380{ 2381 struct xrcd_table_entry *scan; 2382 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2383 struct rb_node *parent = NULL; 2384 2385 while (*p) { 2386 parent = *p; 2387 scan = rb_entry(parent, struct xrcd_table_entry, node); 2388 2389 if (i_n < scan->inode) 2390 p = &(*p)->rb_left; 2391 else if (i_n > scan->inode) 2392 p = &(*p)->rb_right; 2393 else 2394 return scan; 2395 } 2396 return NULL; 2397} 2398 2399static int find_xrcd(struct ib_device *dev, struct inode *i_n, 2400 struct ib_xrcd **xrcd) 2401{ 2402 struct xrcd_table_entry *entry; 2403 2404 entry = xrcd_table_search(dev, i_n); 2405 if (!entry) 2406 return -EINVAL; 2407 2408 *xrcd = entry->xrcd; 2409 return 0; 2410} 2411 2412 2413static void xrcd_table_delete(struct ib_device *dev, 2414 struct inode *i_n) 2415{ 2416 struct xrcd_table_entry *entry = xrcd_table_search(dev, i_n); 2417 2418 if (entry) { 2419 iput(i_n); 2420 rb_erase(&entry->node, &dev->ib_uverbs_xrcd_table); 2421 kfree(entry); 2422 } 2423} 2424 2425ssize_t ib_uverbs_open_xrc_domain(struct ib_uverbs_file *file, 2426 const char __user *buf, int in_len, 2427 int out_len) 2428{ 2429 struct ib_uverbs_open_xrc_domain cmd; 2430 struct ib_uverbs_open_xrc_domain_resp resp; 2431 struct ib_udata udata; 2432 struct ib_uobject *uobj; 2433 struct ib_uxrcd_object *xrcd_uobj; 2434 struct ib_xrcd *xrcd = NULL; 2435 struct file *f = NULL; 2436 struct inode *inode = NULL; 2437 int ret = 0; 2438 int new_xrcd = 0; 2439 2440 if (out_len < sizeof resp) 2441 return -ENOSPC; 2442 2443 if (copy_from_user(&cmd, buf, sizeof cmd)) 2444 return -EFAULT; 2445 2446 INIT_UDATA(&udata, buf + sizeof cmd, 2447 (unsigned long) cmd.response + sizeof resp, 2448 in_len - sizeof cmd, out_len - sizeof resp); 2449 2450 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2451 if (cmd.fd != (u32) (-1)) { 2452 /* search for file descriptor */ 2453 f = fget(cmd.fd); 2454 if (!f) { 2455 ret = -EBADF; 2456 goto err_table_mutex_unlock; 2457 } 2458 2459 inode = xrc_file2inode(f); 2460 if (!inode) { 2461 ret = -EBADF; 2462 goto err_table_mutex_unlock; 2463 } 2464 2465 ret = find_xrcd(file->device->ib_dev, inode, &xrcd); 2466 if (ret && !(cmd.oflags & O_CREAT)) { 2467 /* no file descriptor. Need CREATE flag */ 2468 ret = -EAGAIN; 2469 goto err_table_mutex_unlock; 2470 } 2471 2472 if (xrcd && cmd.oflags & O_EXCL) { 2473 ret = -EINVAL; 2474 goto err_table_mutex_unlock; 2475 } 2476 } 2477 2478 xrcd_uobj = kmalloc(sizeof *xrcd_uobj, GFP_KERNEL); 2479 if (!xrcd_uobj) { 2480 ret = -ENOMEM; 2481 goto err_table_mutex_unlock; 2482 } 2483 2484 uobj = &xrcd_uobj->uobject; 2485 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 2486 down_write(&uobj->mutex); 2487 2488 if (!xrcd) { 2489 xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev, 2490 file->ucontext, &udata); 2491 if (IS_ERR(xrcd)) { 2492 ret = PTR_ERR(xrcd); 2493 goto err; 2494 } 2495 xrcd->uobject = (cmd.fd == -1) ? uobj : NULL; 2496 xrcd->inode = inode; 2497 xrcd->device = file->device->ib_dev; 2498 atomic_set(&xrcd->usecnt, 0); 2499 new_xrcd = 1; 2500 } 2501 2502 uobj->object = xrcd; 2503 ret = idr_add_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2504 if (ret) 2505 goto err_idr; 2506 2507 memset(&resp, 0, sizeof resp); 2508 resp.xrcd_handle = uobj->id; 2509 2510 if (inode) { 2511 if (new_xrcd) { 2512 /* create new inode/xrcd table entry */ 2513 ret = xrcd_table_insert(file->device->ib_dev, inode, xrcd); 2514 if (ret) 2515 goto err_insert_xrcd; 2516 } 2517 atomic_inc(&xrcd->usecnt); 2518 } 2519 if (f) 2520 fput(f); 2521 2522 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2523 &resp, sizeof resp)) { 2524 ret = -EFAULT; 2525 goto err_copy; 2526 } 2527 2528 INIT_LIST_HEAD(&xrcd_uobj->xrc_reg_qp_list); 2529 2530 mutex_lock(&file->mutex); 2531 list_add_tail(&uobj->list, &file->ucontext->xrcd_list); 2532 mutex_unlock(&file->mutex); 2533 2534 uobj->live = 1; 2535 2536 up_write(&uobj->mutex); 2537 2538 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2539 return in_len; 2540 2541err_copy: 2542 2543 if (inode) { 2544 if (new_xrcd) 2545 xrcd_table_delete(file->device->ib_dev, inode); 2546 atomic_dec(&xrcd->usecnt); 2547 } 2548 2549err_insert_xrcd: 2550 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2551 2552err_idr: 2553 ib_dealloc_xrcd(xrcd); 2554 2555err: 2556 put_uobj_write(uobj); 2557 2558err_table_mutex_unlock: 2559 2560 if (f) 2561 fput(f); 2562 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2563 return ret; 2564} 2565 2566ssize_t ib_uverbs_close_xrc_domain(struct ib_uverbs_file *file, 2567 const char __user *buf, int in_len, 2568 int out_len) 2569{ 2570 struct ib_uverbs_close_xrc_domain cmd; 2571 struct ib_uobject *uobj, *t_uobj; 2572 struct ib_uxrcd_object *xrcd_uobj; 2573 struct ib_xrcd *xrcd = NULL; 2574 struct inode *inode = NULL; 2575 int ret = 0; 2576 2577 if (copy_from_user(&cmd, buf, sizeof cmd)) 2578 return -EFAULT; 2579 2580 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2581 uobj = idr_write_uobj(&ib_uverbs_xrc_domain_idr, cmd.xrcd_handle, 2582 file->ucontext); 2583 if (!uobj) { 2584 ret = -EINVAL; 2585 goto err_unlock_mutex; 2586 } 2587 2588 mutex_lock(&file->mutex); 2589 if (!ret) { 2590 list_for_each_entry(t_uobj, &file->ucontext->qp_list, list) { 2591 struct ib_qp *qp = t_uobj->object; 2592 if (qp->xrcd && qp->xrcd == uobj->object) { 2593 ret = -EBUSY; 2594 break; 2595 } 2596 } 2597 } 2598 if (!ret) { 2599 list_for_each_entry(t_uobj, &file->ucontext->srq_list, list) { 2600 struct ib_srq *srq = t_uobj->object; 2601 if (srq->ext.xrc.xrcd && srq->ext.xrc.xrcd == uobj->object) { 2602 ret = -EBUSY; 2603 break; 2604 } 2605 } 2606 } 2607 mutex_unlock(&file->mutex); 2608 if (ret) { 2609 put_uobj_write(uobj); 2610 goto err_unlock_mutex; 2611 } 2612 2613 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2614 if (!list_empty(&xrcd_uobj->xrc_reg_qp_list)) { 2615 ret = -EBUSY; 2616 put_uobj_write(uobj); 2617 goto err_unlock_mutex; 2618 } 2619 2620 xrcd = (struct ib_xrcd *) (uobj->object); 2621 inode = xrcd->inode; 2622 2623 if (inode) 2624 atomic_dec(&xrcd->usecnt); 2625 2626 ret = ib_dealloc_xrcd(uobj->object); 2627 if (!ret) 2628 uobj->live = 0; 2629 2630 put_uobj_write(uobj); 2631 2632 if (ret && !inode) 2633 goto err_unlock_mutex; 2634 2635 if (!ret && inode) 2636 xrcd_table_delete(file->device->ib_dev, inode); 2637 2638 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2639 2640 mutex_lock(&file->mutex); 2641 list_del(&uobj->list); 2642 mutex_unlock(&file->mutex); 2643 2644 put_uobj(uobj); 2645 2646 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2647 return in_len; 2648 2649err_unlock_mutex: 2650 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2651 return ret; 2652} 2653 2654void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev, 2655 struct ib_xrcd *xrcd) 2656{ 2657 struct inode *inode = NULL; 2658 int ret = 0; 2659 2660 inode = xrcd->inode; 2661 if (inode) 2662 atomic_dec(&xrcd->usecnt); 2663 2664 ret = ib_dealloc_xrcd(xrcd); 2665 if (!ret && inode) 2666 xrcd_table_delete(ib_dev, inode); 2667} 2668 2669ssize_t ib_uverbs_create_xrc_rcv_qp(struct ib_uverbs_file *file, 2670 const char __user *buf, int in_len, 2671 int out_len) 2672{ 2673 struct ib_uverbs_create_xrc_rcv_qp cmd; 2674 struct ib_uverbs_create_xrc_rcv_qp_resp resp; 2675 struct ib_uxrc_rcv_object *obj; 2676 struct ib_qp_init_attr init_attr; 2677 struct ib_xrcd *xrcd; 2678 struct ib_uobject *uobj; 2679 struct ib_uxrcd_object *xrcd_uobj; 2680 u32 qp_num; 2681 int err; 2682 2683 if (out_len < sizeof resp) 2684 return -ENOSPC; 2685 2686 if (copy_from_user(&cmd, buf, sizeof cmd)) 2687 return -EFAULT; 2688 2689 obj = kzalloc(sizeof *obj, GFP_KERNEL); 2690 if (!obj) 2691 return -ENOMEM; 2692 2693 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2694 if (!xrcd) { 2695 err = -EINVAL; 2696 goto err_out; 2697 } 2698 2699 init_attr.event_handler = ib_uverbs_xrc_rcv_qp_event_handler; 2700 init_attr.qp_context = file; 2701 init_attr.srq = NULL; 2702 init_attr.sq_sig_type = 2703 cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2704 init_attr.qp_type = IB_QPT_XRC; 2705 init_attr.xrcd = xrcd; 2706 2707 init_attr.cap.max_send_wr = 1; 2708 init_attr.cap.max_recv_wr = 0; 2709 init_attr.cap.max_send_sge = 1; 2710 init_attr.cap.max_recv_sge = 0; 2711 init_attr.cap.max_inline_data = 0; 2712 2713 err = xrcd->device->create_xrc_rcv_qp(&init_attr, &qp_num); 2714 if (err) 2715 goto err_put; 2716 2717 memset(&resp, 0, sizeof resp); 2718 resp.qpn = qp_num; 2719 2720 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2721 &resp, sizeof resp)) { 2722 err = -EFAULT; 2723 goto err_destroy; 2724 } 2725 2726 atomic_inc(&xrcd->usecnt); 2727 put_xrcd_read(uobj); 2728 obj->qp_num = qp_num; 2729 obj->domain_handle = cmd.xrc_domain_handle; 2730 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2731 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2732 list_add_tail(&obj->list, &xrcd_uobj->xrc_reg_qp_list); 2733 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2734 2735 return in_len; 2736 2737err_destroy: 2738 xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2739err_put: 2740 put_xrcd_read(uobj); 2741err_out: 2742 kfree(obj); 2743 return err; 2744} 2745 2746ssize_t ib_uverbs_modify_xrc_rcv_qp(struct ib_uverbs_file *file, 2747 const char __user *buf, int in_len, 2748 int out_len) 2749{ 2750 struct ib_uverbs_modify_xrc_rcv_qp cmd; 2751 struct ib_qp_attr *attr; 2752 struct ib_xrcd *xrcd; 2753 struct ib_uobject *uobj; 2754 int err; 2755 2756 if (copy_from_user(&cmd, buf, sizeof cmd)) 2757 return -EFAULT; 2758 2759 attr = kzalloc(sizeof *attr, GFP_KERNEL); 2760 if (!attr) 2761 return -ENOMEM; 2762 2763 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2764 if (!xrcd) { 2765 kfree(attr); 2766 return -EINVAL; 2767 } 2768 2769 attr->qp_state = cmd.qp_state; 2770 attr->cur_qp_state = cmd.cur_qp_state; 2771 attr->qp_access_flags = cmd.qp_access_flags; 2772 attr->pkey_index = cmd.pkey_index; 2773 attr->port_num = cmd.port_num; 2774 attr->path_mtu = cmd.path_mtu; 2775 attr->path_mig_state = cmd.path_mig_state; 2776 attr->qkey = cmd.qkey; 2777 attr->rq_psn = cmd.rq_psn; 2778 attr->sq_psn = cmd.sq_psn; 2779 attr->dest_qp_num = cmd.dest_qp_num; 2780 attr->alt_pkey_index = cmd.alt_pkey_index; 2781 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 2782 attr->max_rd_atomic = cmd.max_rd_atomic; 2783 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 2784 attr->min_rnr_timer = cmd.min_rnr_timer; 2785 attr->port_num = cmd.port_num; 2786 attr->timeout = cmd.timeout; 2787 attr->retry_cnt = cmd.retry_cnt; 2788 attr->rnr_retry = cmd.rnr_retry; 2789 attr->alt_port_num = cmd.alt_port_num; 2790 attr->alt_timeout = cmd.alt_timeout; 2791 2792 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 2793 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 2794 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 2795 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 2796 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 2797 attr->ah_attr.dlid = cmd.dest.dlid; 2798 attr->ah_attr.sl = cmd.dest.sl; 2799 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 2800 attr->ah_attr.static_rate = cmd.dest.static_rate; 2801 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 2802 attr->ah_attr.port_num = cmd.dest.port_num; 2803 2804 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 2805 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 2806 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 2807 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 2808 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 2809 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 2810 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 2811 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 2812 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 2813 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 2814 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 2815 2816 err = xrcd->device->modify_xrc_rcv_qp(xrcd, cmd.qp_num, attr, cmd.attr_mask); 2817 put_xrcd_read(uobj); 2818 kfree(attr); 2819 return err ? err : in_len; 2820} 2821 2822ssize_t ib_uverbs_query_xrc_rcv_qp(struct ib_uverbs_file *file, 2823 const char __user *buf, int in_len, 2824 int out_len) 2825{ 2826 struct ib_uverbs_query_xrc_rcv_qp cmd; 2827 struct ib_uverbs_query_qp_resp resp; 2828 struct ib_qp_attr *attr; 2829 struct ib_qp_init_attr *init_attr; 2830 struct ib_xrcd *xrcd; 2831 struct ib_uobject *uobj; 2832 int ret; 2833 2834 if (copy_from_user(&cmd, buf, sizeof cmd)) 2835 return -EFAULT; 2836 2837 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2838 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2839 if (!attr || !init_attr) { 2840 ret = -ENOMEM; 2841 goto out; 2842 } 2843 2844 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2845 if (!xrcd) { 2846 ret = -EINVAL; 2847 goto out; 2848 } 2849 2850 ret = xrcd->device->query_xrc_rcv_qp(xrcd, cmd.qp_num, attr, 2851 cmd.attr_mask, init_attr); 2852 2853 put_xrcd_read(uobj); 2854 2855 if (ret) 2856 goto out; 2857 2858 memset(&resp, 0, sizeof resp); 2859 resp.qp_state = attr->qp_state; 2860 resp.cur_qp_state = attr->cur_qp_state; 2861 resp.path_mtu = attr->path_mtu; 2862 resp.path_mig_state = attr->path_mig_state; 2863 resp.qkey = attr->qkey; 2864 resp.rq_psn = attr->rq_psn; 2865 resp.sq_psn = attr->sq_psn; 2866 resp.dest_qp_num = attr->dest_qp_num; 2867 resp.qp_access_flags = attr->qp_access_flags; 2868 resp.pkey_index = attr->pkey_index; 2869 resp.alt_pkey_index = attr->alt_pkey_index; 2870 resp.sq_draining = attr->sq_draining; 2871 resp.max_rd_atomic = attr->max_rd_atomic; 2872 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2873 resp.min_rnr_timer = attr->min_rnr_timer; 2874 resp.port_num = attr->port_num; 2875 resp.timeout = attr->timeout; 2876 resp.retry_cnt = attr->retry_cnt; 2877 resp.rnr_retry = attr->rnr_retry; 2878 resp.alt_port_num = attr->alt_port_num; 2879 resp.alt_timeout = attr->alt_timeout; 2880 2881 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2882 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2883 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2884 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2885 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2886 resp.dest.dlid = attr->ah_attr.dlid; 2887 resp.dest.sl = attr->ah_attr.sl; 2888 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2889 resp.dest.static_rate = attr->ah_attr.static_rate; 2890 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2891 resp.dest.port_num = attr->ah_attr.port_num; 2892 2893 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2894 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2895 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2896 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2897 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2898 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2899 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2900 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2901 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2902 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2903 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2904 2905 resp.max_send_wr = init_attr->cap.max_send_wr; 2906 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2907 resp.max_send_sge = init_attr->cap.max_send_sge; 2908 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2909 resp.max_inline_data = init_attr->cap.max_inline_data; 2910 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2911 2912 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2913 &resp, sizeof resp)) 2914 ret = -EFAULT; 2915 2916out: 2917 kfree(attr); 2918 kfree(init_attr); 2919 2920 return ret ? ret : in_len; 2921} 2922 2923ssize_t ib_uverbs_reg_xrc_rcv_qp(struct ib_uverbs_file *file, 2924 const char __user *buf, int in_len, 2925 int out_len) 2926{ 2927 struct ib_uverbs_reg_xrc_rcv_qp cmd; 2928 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2929 struct ib_xrcd *xrcd; 2930 struct ib_uobject *uobj; 2931 struct ib_uxrcd_object *xrcd_uobj; 2932 int ret; 2933 2934 if (copy_from_user(&cmd, buf, sizeof cmd)) 2935 return -EFAULT; 2936 2937 qp_obj = kmalloc(sizeof *qp_obj, GFP_KERNEL); 2938 if (!qp_obj) 2939 return -ENOMEM; 2940 2941 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2942 if (!xrcd) { 2943 ret = -EINVAL; 2944 goto err_out; 2945 } 2946 2947 ret = xrcd->device->reg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 2948 if (ret) 2949 goto err_put; 2950 2951 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2952 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2953 list_for_each_entry(tmp, &xrcd_uobj->xrc_reg_qp_list, list) 2954 if (cmd.qp_num == tmp->qp_num) { 2955 kfree(qp_obj); 2956 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2957 put_xrcd_read(uobj); 2958 return in_len; 2959 } 2960 qp_obj->qp_num = cmd.qp_num; 2961 qp_obj->domain_handle = cmd.xrc_domain_handle; 2962 list_add_tail(&qp_obj->list, &xrcd_uobj->xrc_reg_qp_list); 2963 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2964 atomic_inc(&xrcd->usecnt); 2965 put_xrcd_read(uobj); 2966 return in_len; 2967 2968err_put: 2969 put_xrcd_read(uobj); 2970err_out: 2971 2972 kfree(qp_obj); 2973 return ret; 2974} 2975 2976int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file, 2977 struct ib_xrcd *xrcd, u32 qp_num) 2978{ 2979 int err; 2980 err = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2981 if (!err) 2982 atomic_dec(&xrcd->usecnt); 2983 return err; 2984} 2985 2986ssize_t ib_uverbs_unreg_xrc_rcv_qp(struct ib_uverbs_file *file, 2987 const char __user *buf, int in_len, 2988 int out_len) 2989{ 2990 struct ib_uverbs_unreg_xrc_rcv_qp cmd; 2991 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2992 struct ib_xrcd *xrcd; 2993 struct ib_uobject *uobj; 2994 struct ib_uxrcd_object *xrcd_uobj; 2995 int ret; 2996 2997 if (copy_from_user(&cmd, buf, sizeof cmd)) 2998 return -EFAULT; 2999 3000 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 3001 if (!xrcd) 3002 return -EINVAL; 3003 3004 ret = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 3005 if (ret) { 3006 put_xrcd_read(uobj); 3007 return -EINVAL; 3008 } 3009 atomic_dec(&xrcd->usecnt); 3010 3011 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 3012 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 3013 list_for_each_entry_safe(qp_obj, tmp, &xrcd_uobj->xrc_reg_qp_list, list) 3014 if (cmd.qp_num == qp_obj->qp_num) { 3015 list_del(&qp_obj->list); 3016 kfree(qp_obj); 3017 break; 3018 } 3019 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 3020 put_xrcd_read(uobj); 3021 return in_len; 3022} 3023