uverbs_cmd.c revision 271127
1/* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36#include <linux/file.h> 37#include <linux/fs.h> 38#include <linux/lockdep.h> 39 40#include <asm/uaccess.h> 41#include <asm/fcntl.h> 42 43#include "uverbs.h" 44 45static struct lock_class_key pd_lock_key; 46static struct lock_class_key mr_lock_key; 47static struct lock_class_key cq_lock_key; 48static struct lock_class_key qp_lock_key; 49static struct lock_class_key ah_lock_key; 50static struct lock_class_key srq_lock_key; 51 52#define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \ 53 do { \ 54 (udata)->inbuf = (void __user *) (ibuf); \ 55 (udata)->outbuf = (void __user *) (obuf); \ 56 (udata)->inlen = (ilen); \ 57 (udata)->outlen = (olen); \ 58 } while (0) 59 60/* 61 * The ib_uobject locking scheme is as follows: 62 * 63 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 64 * needs to be held during all idr operations. When an object is 65 * looked up, a reference must be taken on the object's kref before 66 * dropping this lock. 67 * 68 * - Each object also has an rwsem. This rwsem must be held for 69 * reading while an operation that uses the object is performed. 70 * For example, while registering an MR, the associated PD's 71 * uobject.mutex must be held for reading. The rwsem must be held 72 * for writing while initializing or destroying an object. 73 * 74 * - In addition, each object has a "live" flag. If this flag is not 75 * set, then lookups of the object will fail even if it is found in 76 * the idr. This handles a reader that blocks and does not acquire 77 * the rwsem until after the object is destroyed. The destroy 78 * operation will set the live flag to 0 and then drop the rwsem; 79 * this will allow the reader to acquire the rwsem, see that the 80 * live flag is 0, and then drop the rwsem and its reference to 81 * object. The underlying storage will not be freed until the last 82 * reference to the object is dropped. 83 */ 84 85static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 86 struct ib_ucontext *context, struct lock_class_key *key) 87{ 88 uobj->user_handle = user_handle; 89 uobj->context = context; 90 kref_init(&uobj->ref); 91 init_rwsem(&uobj->mutex); 92 lockdep_set_class(&uobj->mutex, key); 93 uobj->live = 0; 94} 95 96static void release_uobj(struct kref *kref) 97{ 98 kfree(container_of(kref, struct ib_uobject, ref)); 99} 100 101static void put_uobj(struct ib_uobject *uobj) 102{ 103 kref_put(&uobj->ref, release_uobj); 104} 105 106static void put_uobj_read(struct ib_uobject *uobj) 107{ 108 up_read(&uobj->mutex); 109 put_uobj(uobj); 110} 111 112static void put_uobj_write(struct ib_uobject *uobj) 113{ 114 up_write(&uobj->mutex); 115 put_uobj(uobj); 116} 117 118static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 119{ 120 int ret; 121 122retry: 123 if (!idr_pre_get(idr, GFP_KERNEL)) 124 return -ENOMEM; 125 126 spin_lock(&ib_uverbs_idr_lock); 127 ret = idr_get_new(idr, uobj, &uobj->id); 128 spin_unlock(&ib_uverbs_idr_lock); 129 130 if (ret == -EAGAIN) 131 goto retry; 132 133 return ret; 134} 135 136void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 137{ 138 spin_lock(&ib_uverbs_idr_lock); 139 idr_remove(idr, uobj->id); 140 spin_unlock(&ib_uverbs_idr_lock); 141} 142 143static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 144 struct ib_ucontext *context) 145{ 146 struct ib_uobject *uobj; 147 148 spin_lock(&ib_uverbs_idr_lock); 149 uobj = idr_find(idr, id); 150 if (uobj) { 151 if (uobj->context == context) 152 kref_get(&uobj->ref); 153 else 154 uobj = NULL; 155 } 156 spin_unlock(&ib_uverbs_idr_lock); 157 158 return uobj; 159} 160 161static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 162 struct ib_ucontext *context, int nested) 163{ 164 struct ib_uobject *uobj; 165 166 uobj = __idr_get_uobj(idr, id, context); 167 if (!uobj) 168 return NULL; 169 170 if (nested) 171 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 172 else 173 down_read(&uobj->mutex); 174 if (!uobj->live) { 175 put_uobj_read(uobj); 176 return NULL; 177 } 178 179 return uobj; 180} 181 182static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 183 struct ib_ucontext *context) 184{ 185 struct ib_uobject *uobj; 186 187 uobj = __idr_get_uobj(idr, id, context); 188 if (!uobj) 189 return NULL; 190 191 down_write(&uobj->mutex); 192 if (!uobj->live) { 193 put_uobj_write(uobj); 194 return NULL; 195 } 196 197 return uobj; 198} 199 200static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 201 int nested) 202{ 203 struct ib_uobject *uobj; 204 205 uobj = idr_read_uobj(idr, id, context, nested); 206 return uobj ? uobj->object : NULL; 207} 208 209static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 210{ 211 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 212} 213 214static void put_pd_read(struct ib_pd *pd) 215{ 216 put_uobj_read(pd->uobject); 217} 218 219static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 220{ 221 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 222} 223 224static void put_cq_read(struct ib_cq *cq) 225{ 226 put_uobj_read(cq->uobject); 227} 228 229static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 230{ 231 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 232} 233 234static void put_ah_read(struct ib_ah *ah) 235{ 236 put_uobj_read(ah->uobject); 237} 238 239static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 240{ 241 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 242} 243 244static void put_qp_read(struct ib_qp *qp) 245{ 246 put_uobj_read(qp->uobject); 247} 248 249static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 250{ 251 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 252} 253 254static void put_srq_read(struct ib_srq *srq) 255{ 256 put_uobj_read(srq->uobject); 257} 258 259static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, 260 struct ib_ucontext *context, 261 struct ib_uobject **uobj) 262{ 263 *uobj = idr_read_uobj(&ib_uverbs_xrc_domain_idr, xrcd_handle, 264 context, 0); 265 return *uobj ? (*uobj)->object : NULL; 266} 267 268static void put_xrcd_read(struct ib_uobject *uobj) 269{ 270 put_uobj_read(uobj); 271} 272 273ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 274 const char __user *buf, 275 int in_len, int out_len) 276{ 277 struct ib_uverbs_get_context cmd; 278 struct ib_uverbs_get_context_resp resp; 279 struct ib_udata udata; 280 struct ib_device *ibdev = file->device->ib_dev; 281 struct ib_ucontext *ucontext; 282 struct file *filp; 283 int ret; 284 285 if (out_len < sizeof resp) 286 return -ENOSPC; 287 288 if (copy_from_user(&cmd, buf, sizeof cmd)) 289 return -EFAULT; 290 291 mutex_lock(&file->mutex); 292 293 if (file->ucontext) { 294 ret = -EINVAL; 295 goto err; 296 } 297 298 INIT_UDATA(&udata, buf + sizeof cmd, 299 (unsigned long) cmd.response + sizeof resp, 300 in_len - sizeof cmd, out_len - sizeof resp); 301 302 ucontext = ibdev->alloc_ucontext(ibdev, &udata); 303 if (IS_ERR(ucontext)) { 304 ret = PTR_ERR(file->ucontext); 305 goto err; 306 } 307 308 ucontext->device = ibdev; 309 INIT_LIST_HEAD(&ucontext->pd_list); 310 INIT_LIST_HEAD(&ucontext->mr_list); 311 INIT_LIST_HEAD(&ucontext->mw_list); 312 INIT_LIST_HEAD(&ucontext->cq_list); 313 INIT_LIST_HEAD(&ucontext->qp_list); 314 INIT_LIST_HEAD(&ucontext->srq_list); 315 INIT_LIST_HEAD(&ucontext->ah_list); 316 INIT_LIST_HEAD(&ucontext->xrcd_list); 317 ucontext->closing = 0; 318 319 resp.num_comp_vectors = file->device->num_comp_vectors; 320 321 filp = ib_uverbs_alloc_event_file(file, 1, &resp.async_fd); 322 if (IS_ERR(filp)) { 323 ret = PTR_ERR(filp); 324 goto err_free; 325 } 326 327 if (copy_to_user((void __user *) (unsigned long) cmd.response, 328 &resp, sizeof resp)) { 329 ret = -EFAULT; 330 goto err_file; 331 } 332 333 file->async_file = filp->private_data; 334 335 INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev, 336 ib_uverbs_event_handler); 337 ret = ib_register_event_handler(&file->event_handler); 338 if (ret) 339 goto err_file; 340 341 kref_get(&file->async_file->ref); 342 kref_get(&file->ref); 343 file->ucontext = ucontext; 344 345 fd_install(resp.async_fd, filp); 346 347 mutex_unlock(&file->mutex); 348 349 return in_len; 350 351err_file: 352 put_unused_fd(resp.async_fd); 353 fput(filp); 354 355err_free: 356 ibdev->dealloc_ucontext(ucontext); 357 358err: 359 mutex_unlock(&file->mutex); 360 return ret; 361} 362 363ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 364 const char __user *buf, 365 int in_len, int out_len) 366{ 367 struct ib_uverbs_query_device cmd; 368 struct ib_uverbs_query_device_resp resp; 369 struct ib_device_attr attr; 370 int ret; 371 372 if (out_len < sizeof resp) 373 return -ENOSPC; 374 375 if (copy_from_user(&cmd, buf, sizeof cmd)) 376 return -EFAULT; 377 378 ret = ib_query_device(file->device->ib_dev, &attr); 379 if (ret) 380 return ret; 381 382 memset(&resp, 0, sizeof resp); 383 384 resp.fw_ver = attr.fw_ver; 385 resp.node_guid = file->device->ib_dev->node_guid; 386 resp.sys_image_guid = attr.sys_image_guid; 387 resp.max_mr_size = attr.max_mr_size; 388 resp.page_size_cap = attr.page_size_cap; 389 resp.vendor_id = attr.vendor_id; 390 resp.vendor_part_id = attr.vendor_part_id; 391 resp.hw_ver = attr.hw_ver; 392 resp.max_qp = attr.max_qp; 393 resp.max_qp_wr = attr.max_qp_wr; 394 resp.device_cap_flags = attr.device_cap_flags; 395 resp.max_sge = attr.max_sge; 396 resp.max_sge_rd = attr.max_sge_rd; 397 resp.max_cq = attr.max_cq; 398 resp.max_cqe = attr.max_cqe; 399 resp.max_mr = attr.max_mr; 400 resp.max_pd = attr.max_pd; 401 resp.max_qp_rd_atom = attr.max_qp_rd_atom; 402 resp.max_ee_rd_atom = attr.max_ee_rd_atom; 403 resp.max_res_rd_atom = attr.max_res_rd_atom; 404 resp.max_qp_init_rd_atom = attr.max_qp_init_rd_atom; 405 resp.max_ee_init_rd_atom = attr.max_ee_init_rd_atom; 406 resp.atomic_cap = attr.atomic_cap; 407 resp.max_ee = attr.max_ee; 408 resp.max_rdd = attr.max_rdd; 409 resp.max_mw = attr.max_mw; 410 resp.max_raw_ipv6_qp = attr.max_raw_ipv6_qp; 411 resp.max_raw_ethy_qp = attr.max_raw_ethy_qp; 412 resp.max_mcast_grp = attr.max_mcast_grp; 413 resp.max_mcast_qp_attach = attr.max_mcast_qp_attach; 414 resp.max_total_mcast_qp_attach = attr.max_total_mcast_qp_attach; 415 resp.max_ah = attr.max_ah; 416 resp.max_fmr = attr.max_fmr; 417 resp.max_map_per_fmr = attr.max_map_per_fmr; 418 resp.max_srq = attr.max_srq; 419 resp.max_srq_wr = attr.max_srq_wr; 420 resp.max_srq_sge = attr.max_srq_sge; 421 resp.max_pkeys = attr.max_pkeys; 422 resp.local_ca_ack_delay = attr.local_ca_ack_delay; 423 resp.phys_port_cnt = file->device->ib_dev->phys_port_cnt; 424 425 if (copy_to_user((void __user *) (unsigned long) cmd.response, 426 &resp, sizeof resp)) 427 return -EFAULT; 428 429 return in_len; 430} 431 432ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 433 const char __user *buf, 434 int in_len, int out_len) 435{ 436 struct ib_uverbs_query_port cmd; 437 struct ib_uverbs_query_port_resp resp; 438 struct ib_port_attr attr; 439 int ret; 440 441 if (out_len < sizeof resp) 442 return -ENOSPC; 443 444 if (copy_from_user(&cmd, buf, sizeof cmd)) 445 return -EFAULT; 446 447 ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr); 448 if (ret) 449 return ret; 450 451 memset(&resp, 0, sizeof resp); 452 453 resp.state = attr.state; 454 resp.max_mtu = attr.max_mtu; 455 resp.active_mtu = attr.active_mtu; 456 resp.gid_tbl_len = attr.gid_tbl_len; 457 resp.port_cap_flags = attr.port_cap_flags; 458 resp.max_msg_sz = attr.max_msg_sz; 459 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 460 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 461 resp.pkey_tbl_len = attr.pkey_tbl_len; 462 resp.lid = attr.lid; 463 resp.sm_lid = attr.sm_lid; 464 resp.lmc = attr.lmc; 465 resp.max_vl_num = attr.max_vl_num; 466 resp.sm_sl = attr.sm_sl; 467 resp.subnet_timeout = attr.subnet_timeout; 468 resp.init_type_reply = attr.init_type_reply; 469 resp.active_width = attr.active_width; 470 resp.active_speed = attr.active_speed; 471 resp.phys_state = attr.phys_state; 472 resp.link_layer = attr.link_layer; 473 474 if (copy_to_user((void __user *) (unsigned long) cmd.response, 475 &resp, sizeof resp)) 476 return -EFAULT; 477 478 return in_len; 479} 480 481ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 482 const char __user *buf, 483 int in_len, int out_len) 484{ 485 struct ib_uverbs_alloc_pd cmd; 486 struct ib_uverbs_alloc_pd_resp resp; 487 struct ib_udata udata; 488 struct ib_uobject *uobj; 489 struct ib_pd *pd; 490 int ret; 491 492 if (out_len < sizeof resp) 493 return -ENOSPC; 494 495 if (copy_from_user(&cmd, buf, sizeof cmd)) 496 return -EFAULT; 497 498 INIT_UDATA(&udata, buf + sizeof cmd, 499 (unsigned long) cmd.response + sizeof resp, 500 in_len - sizeof cmd, out_len - sizeof resp); 501 502 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 503 if (!uobj) 504 return -ENOMEM; 505 506 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 507 down_write(&uobj->mutex); 508 509 pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, 510 file->ucontext, &udata); 511 if (IS_ERR(pd)) { 512 ret = PTR_ERR(pd); 513 goto err; 514 } 515 516 pd->device = file->device->ib_dev; 517 pd->uobject = uobj; 518 atomic_set(&pd->usecnt, 0); 519 520 uobj->object = pd; 521 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 522 if (ret) 523 goto err_idr; 524 525 memset(&resp, 0, sizeof resp); 526 resp.pd_handle = uobj->id; 527 528 if (copy_to_user((void __user *) (unsigned long) cmd.response, 529 &resp, sizeof resp)) { 530 ret = -EFAULT; 531 goto err_copy; 532 } 533 534 mutex_lock(&file->mutex); 535 list_add_tail(&uobj->list, &file->ucontext->pd_list); 536 mutex_unlock(&file->mutex); 537 538 uobj->live = 1; 539 540 up_write(&uobj->mutex); 541 542 return in_len; 543 544err_copy: 545 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 546 547err_idr: 548 ib_dealloc_pd(pd); 549 550err: 551 put_uobj_write(uobj); 552 return ret; 553} 554 555ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 556 const char __user *buf, 557 int in_len, int out_len) 558{ 559 struct ib_uverbs_dealloc_pd cmd; 560 struct ib_uobject *uobj; 561 int ret; 562 563 if (copy_from_user(&cmd, buf, sizeof cmd)) 564 return -EFAULT; 565 566 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 567 if (!uobj) 568 return -EINVAL; 569 570 ret = ib_dealloc_pd(uobj->object); 571 if (!ret) 572 uobj->live = 0; 573 574 put_uobj_write(uobj); 575 576 if (ret) 577 return ret; 578 579 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 580 581 mutex_lock(&file->mutex); 582 list_del(&uobj->list); 583 mutex_unlock(&file->mutex); 584 585 put_uobj(uobj); 586 587 return in_len; 588} 589 590ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 591 const char __user *buf, int in_len, 592 int out_len) 593{ 594 struct ib_uverbs_reg_mr cmd; 595 struct ib_uverbs_reg_mr_resp resp; 596 struct ib_udata udata; 597 struct ib_uobject *uobj; 598 struct ib_pd *pd; 599 struct ib_mr *mr; 600 int ret; 601 602 if (out_len < sizeof resp) 603 return -ENOSPC; 604 605 if (copy_from_user(&cmd, buf, sizeof cmd)) 606 return -EFAULT; 607 608 INIT_UDATA(&udata, buf + sizeof cmd, 609 (unsigned long) cmd.response + sizeof resp, 610 in_len - sizeof cmd, out_len - sizeof resp); 611 612 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 613 return -EINVAL; 614 615 /* 616 * Local write permission is required if remote write or 617 * remote atomic permission is also requested. 618 */ 619 if (cmd.access_flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) && 620 !(cmd.access_flags & IB_ACCESS_LOCAL_WRITE)) 621 return -EINVAL; 622 623 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 624 if (!uobj) 625 return -ENOMEM; 626 627 init_uobj(uobj, 0, file->ucontext, &mr_lock_key); 628 down_write(&uobj->mutex); 629 630 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 631 if (!pd) { 632 ret = -EINVAL; 633 goto err_free; 634 } 635 636 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 637 cmd.access_flags, &udata, 0); 638 if (IS_ERR(mr)) { 639 ret = PTR_ERR(mr); 640 goto err_put; 641 } 642 643 mr->device = pd->device; 644 mr->pd = pd; 645 mr->uobject = uobj; 646 atomic_inc(&pd->usecnt); 647 atomic_set(&mr->usecnt, 0); 648 649 uobj->object = mr; 650 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 651 if (ret) 652 goto err_unreg; 653 654 memset(&resp, 0, sizeof resp); 655 resp.lkey = mr->lkey; 656 resp.rkey = mr->rkey; 657 resp.mr_handle = uobj->id; 658 659 if (copy_to_user((void __user *) (unsigned long) cmd.response, 660 &resp, sizeof resp)) { 661 ret = -EFAULT; 662 goto err_copy; 663 } 664 665 put_pd_read(pd); 666 667 mutex_lock(&file->mutex); 668 list_add_tail(&uobj->list, &file->ucontext->mr_list); 669 mutex_unlock(&file->mutex); 670 671 uobj->live = 1; 672 673 up_write(&uobj->mutex); 674 675 return in_len; 676 677err_copy: 678 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 679 680err_unreg: 681 ib_dereg_mr(mr); 682 683err_put: 684 put_pd_read(pd); 685 686err_free: 687 put_uobj_write(uobj); 688 return ret; 689} 690 691ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 692 const char __user *buf, int in_len, 693 int out_len) 694{ 695 struct ib_uverbs_dereg_mr cmd; 696 struct ib_mr *mr; 697 struct ib_uobject *uobj; 698 int ret = -EINVAL; 699 700 if (copy_from_user(&cmd, buf, sizeof cmd)) 701 return -EFAULT; 702 703 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 704 if (!uobj) 705 return -EINVAL; 706 707 mr = uobj->object; 708 709 ret = ib_dereg_mr(mr); 710 if (!ret) 711 uobj->live = 0; 712 713 put_uobj_write(uobj); 714 715 if (ret) 716 return ret; 717 718 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 719 720 mutex_lock(&file->mutex); 721 list_del(&uobj->list); 722 mutex_unlock(&file->mutex); 723 724 put_uobj(uobj); 725 726 return in_len; 727} 728 729ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 730 const char __user *buf, int in_len, 731 int out_len) 732{ 733 struct ib_uverbs_create_comp_channel cmd; 734 struct ib_uverbs_create_comp_channel_resp resp; 735 struct file *filp; 736 737 if (out_len < sizeof resp) 738 return -ENOSPC; 739 740 if (copy_from_user(&cmd, buf, sizeof cmd)) 741 return -EFAULT; 742 743 filp = ib_uverbs_alloc_event_file(file, 0, &resp.fd); 744 if (IS_ERR(filp)) 745 return PTR_ERR(filp); 746 747 if (copy_to_user((void __user *) (unsigned long) cmd.response, 748 &resp, sizeof resp)) { 749 put_unused_fd(resp.fd); 750 fput(filp); 751 return -EFAULT; 752 } 753 754 fd_install(resp.fd, filp); 755 return in_len; 756} 757 758ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 759 const char __user *buf, int in_len, 760 int out_len) 761{ 762 struct ib_uverbs_create_cq cmd; 763 struct ib_uverbs_create_cq_resp resp; 764 struct ib_udata udata; 765 struct ib_ucq_object *obj; 766 struct ib_uverbs_event_file *ev_file = NULL; 767 struct ib_cq *cq; 768 int ret; 769 770 if (out_len < sizeof resp) 771 return -ENOSPC; 772 773 if (copy_from_user(&cmd, buf, sizeof cmd)) 774 return -EFAULT; 775 776 INIT_UDATA(&udata, buf + sizeof cmd, 777 (unsigned long) cmd.response + sizeof resp, 778 in_len - sizeof cmd, out_len - sizeof resp); 779 780 if (cmd.comp_vector >= file->device->num_comp_vectors) 781 return -EINVAL; 782 783 obj = kmalloc(sizeof *obj, GFP_KERNEL); 784 if (!obj) 785 return -ENOMEM; 786 787 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_key); 788 down_write(&obj->uobject.mutex); 789 790 if (cmd.comp_channel >= 0) { 791 ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); 792 if (!ev_file) { 793 ret = -EINVAL; 794 goto err; 795 } 796 } 797 798 obj->uverbs_file = file; 799 obj->comp_events_reported = 0; 800 obj->async_events_reported = 0; 801 INIT_LIST_HEAD(&obj->comp_list); 802 INIT_LIST_HEAD(&obj->async_list); 803 804 cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, 805 cmd.comp_vector, 806 file->ucontext, &udata); 807 if (IS_ERR(cq)) { 808 ret = PTR_ERR(cq); 809 goto err_file; 810 } 811 812 cq->device = file->device->ib_dev; 813 cq->uobject = &obj->uobject; 814 cq->comp_handler = ib_uverbs_comp_handler; 815 cq->event_handler = ib_uverbs_cq_event_handler; 816 cq->cq_context = ev_file; 817 atomic_set(&cq->usecnt, 0); 818 819 obj->uobject.object = cq; 820 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 821 if (ret) 822 goto err_free; 823 824 memset(&resp, 0, sizeof resp); 825 resp.cq_handle = obj->uobject.id; 826 resp.cqe = cq->cqe; 827 828 if (copy_to_user((void __user *) (unsigned long) cmd.response, 829 &resp, sizeof resp)) { 830 ret = -EFAULT; 831 goto err_copy; 832 } 833 834 mutex_lock(&file->mutex); 835 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 836 mutex_unlock(&file->mutex); 837 838 obj->uobject.live = 1; 839 840 up_write(&obj->uobject.mutex); 841 842 return in_len; 843 844err_copy: 845 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 846 847err_free: 848 ib_destroy_cq(cq); 849 850err_file: 851 if (ev_file) 852 ib_uverbs_release_ucq(file, ev_file, obj); 853 854err: 855 put_uobj_write(&obj->uobject); 856 return ret; 857} 858 859ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 860 const char __user *buf, int in_len, 861 int out_len) 862{ 863 struct ib_uverbs_resize_cq cmd; 864 struct ib_uverbs_resize_cq_resp resp; 865 struct ib_udata udata; 866 struct ib_cq *cq; 867 int ret = -EINVAL; 868 869 if (copy_from_user(&cmd, buf, sizeof cmd)) 870 return -EFAULT; 871 872 INIT_UDATA(&udata, buf + sizeof cmd, 873 (unsigned long) cmd.response + sizeof resp, 874 in_len - sizeof cmd, out_len - sizeof resp); 875 876 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 877 if (!cq) 878 return -EINVAL; 879 880 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 881 if (ret) 882 goto out; 883 884 resp.cqe = cq->cqe; 885 886 if (copy_to_user((void __user *) (unsigned long) cmd.response, 887 &resp, sizeof resp.cqe)) 888 ret = -EFAULT; 889 890out: 891 put_cq_read(cq); 892 893 return ret ? ret : in_len; 894} 895 896ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 897 const char __user *buf, int in_len, 898 int out_len) 899{ 900 struct ib_uverbs_poll_cq cmd; 901 struct ib_uverbs_poll_cq_resp *resp; 902 struct ib_cq *cq; 903 struct ib_wc *wc; 904 int ret = 0; 905 int i; 906 int rsize; 907 908 if (copy_from_user(&cmd, buf, sizeof cmd)) 909 return -EFAULT; 910 911 wc = kmalloc(cmd.ne * sizeof *wc, GFP_KERNEL); 912 if (!wc) 913 return -ENOMEM; 914 915 rsize = sizeof *resp + cmd.ne * sizeof(struct ib_uverbs_wc); 916 resp = kmalloc(rsize, GFP_KERNEL); 917 if (!resp) { 918 ret = -ENOMEM; 919 goto out_wc; 920 } 921 922 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 923 if (!cq) { 924 ret = -EINVAL; 925 goto out; 926 } 927 928 resp->count = ib_poll_cq(cq, cmd.ne, wc); 929 930 put_cq_read(cq); 931 932 for (i = 0; i < resp->count; i++) { 933 resp->wc[i].wr_id = wc[i].wr_id; 934 resp->wc[i].status = wc[i].status; 935 resp->wc[i].opcode = wc[i].opcode; 936 resp->wc[i].vendor_err = wc[i].vendor_err; 937 resp->wc[i].byte_len = wc[i].byte_len; 938 resp->wc[i].ex.imm_data = (__u32 __force) wc[i].ex.imm_data; 939 resp->wc[i].qp_num = wc[i].qp->qp_num; 940 resp->wc[i].src_qp = wc[i].src_qp; 941 resp->wc[i].wc_flags = wc[i].wc_flags; 942 resp->wc[i].pkey_index = wc[i].pkey_index; 943 resp->wc[i].slid = wc[i].slid; 944 resp->wc[i].sl = wc[i].sl; 945 resp->wc[i].dlid_path_bits = wc[i].dlid_path_bits; 946 resp->wc[i].port_num = wc[i].port_num; 947 } 948 949 if (copy_to_user((void __user *) (unsigned long) cmd.response, resp, rsize)) 950 ret = -EFAULT; 951 952out: 953 kfree(resp); 954 955out_wc: 956 kfree(wc); 957 return ret ? ret : in_len; 958} 959 960ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 961 const char __user *buf, int in_len, 962 int out_len) 963{ 964 struct ib_uverbs_req_notify_cq cmd; 965 struct ib_cq *cq; 966 967 if (copy_from_user(&cmd, buf, sizeof cmd)) 968 return -EFAULT; 969 970 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 971 if (!cq) 972 return -EINVAL; 973 974 ib_req_notify_cq(cq, cmd.solicited_only ? 975 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 976 977 put_cq_read(cq); 978 979 return in_len; 980} 981 982ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 983 const char __user *buf, int in_len, 984 int out_len) 985{ 986 struct ib_uverbs_destroy_cq cmd; 987 struct ib_uverbs_destroy_cq_resp resp; 988 struct ib_uobject *uobj; 989 struct ib_cq *cq; 990 struct ib_ucq_object *obj; 991 struct ib_uverbs_event_file *ev_file; 992 int ret = -EINVAL; 993 994 if (copy_from_user(&cmd, buf, sizeof cmd)) 995 return -EFAULT; 996 997 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 998 if (!uobj) 999 return -EINVAL; 1000 cq = uobj->object; 1001 ev_file = cq->cq_context; 1002 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1003 1004 ret = ib_destroy_cq(cq); 1005 if (!ret) 1006 uobj->live = 0; 1007 1008 put_uobj_write(uobj); 1009 1010 if (ret) 1011 return ret; 1012 1013 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1014 1015 mutex_lock(&file->mutex); 1016 list_del(&uobj->list); 1017 mutex_unlock(&file->mutex); 1018 1019 ib_uverbs_release_ucq(file, ev_file, obj); 1020 1021 memset(&resp, 0, sizeof resp); 1022 resp.comp_events_reported = obj->comp_events_reported; 1023 resp.async_events_reported = obj->async_events_reported; 1024 1025 put_uobj(uobj); 1026 1027 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1028 &resp, sizeof resp)) 1029 return -EFAULT; 1030 1031 return in_len; 1032} 1033 1034ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1035 const char __user *buf, int in_len, 1036 int out_len) 1037{ 1038 struct ib_uverbs_create_qp cmd; 1039 struct ib_uverbs_create_qp_resp resp; 1040 struct ib_udata udata; 1041 struct ib_uqp_object *obj; 1042 struct ib_pd *pd; 1043 struct ib_cq *scq, *rcq; 1044 struct ib_srq *srq; 1045 struct ib_qp *qp; 1046 struct ib_qp_init_attr attr; 1047 struct ib_xrcd *xrcd; 1048 struct ib_uobject *xrcd_uobj; 1049 int ret; 1050 1051 if (out_len < sizeof resp) 1052 return -ENOSPC; 1053 1054 if (copy_from_user(&cmd, buf, sizeof cmd)) 1055 return -EFAULT; 1056 1057 INIT_UDATA(&udata, buf + sizeof cmd, 1058 (unsigned long) cmd.response + sizeof resp, 1059 in_len - sizeof cmd, out_len - sizeof resp); 1060 1061 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1062 if (!obj) 1063 return -ENOMEM; 1064 1065 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key); 1066 down_write(&obj->uevent.uobject.mutex); 1067 1068 srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ? 1069 idr_read_srq(cmd.srq_handle, file->ucontext) : NULL; 1070 xrcd = cmd.qp_type == IB_QPT_XRC ? 1071 idr_read_xrcd(cmd.srq_handle, file->ucontext, &xrcd_uobj) : NULL; 1072 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1073 scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0); 1074 rcq = cmd.recv_cq_handle == cmd.send_cq_handle ? 1075 scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1); 1076 1077 if (!pd || !scq || !rcq || (cmd.is_srq && !srq) || 1078 (cmd.qp_type == IB_QPT_XRC && !xrcd)) { 1079 ret = -EINVAL; 1080 goto err_put; 1081 } 1082 1083 attr.create_flags = 0; 1084 attr.event_handler = ib_uverbs_qp_event_handler; 1085 attr.qp_context = file; 1086 attr.send_cq = scq; 1087 attr.recv_cq = rcq; 1088 attr.srq = srq; 1089 attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 1090 attr.qp_type = cmd.qp_type; 1091 attr.xrcd = xrcd; 1092 attr.create_flags = 0; 1093 1094 attr.cap.max_send_wr = cmd.max_send_wr; 1095 attr.cap.max_recv_wr = cmd.max_recv_wr; 1096 attr.cap.max_send_sge = cmd.max_send_sge; 1097 attr.cap.max_recv_sge = cmd.max_recv_sge; 1098 attr.cap.max_inline_data = cmd.max_inline_data; 1099 1100 obj->uevent.events_reported = 0; 1101 INIT_LIST_HEAD(&obj->uevent.event_list); 1102 INIT_LIST_HEAD(&obj->mcast_list); 1103 1104 qp = pd->device->create_qp(pd, &attr, &udata); 1105 if (IS_ERR(qp)) { 1106 ret = PTR_ERR(qp); 1107 goto err_put; 1108 } 1109 1110 qp->device = pd->device; 1111 qp->pd = pd; 1112 qp->send_cq = attr.send_cq; 1113 qp->recv_cq = attr.recv_cq; 1114 qp->srq = attr.srq; 1115 qp->uobject = &obj->uevent.uobject; 1116 qp->event_handler = attr.event_handler; 1117 qp->qp_context = attr.qp_context; 1118 qp->qp_type = attr.qp_type; 1119 qp->xrcd = attr.xrcd; 1120 atomic_inc(&pd->usecnt); 1121 atomic_inc(&attr.send_cq->usecnt); 1122 atomic_inc(&attr.recv_cq->usecnt); 1123 if (attr.srq) 1124 atomic_inc(&attr.srq->usecnt); 1125 else if (attr.xrcd) 1126 atomic_inc(&attr.xrcd->usecnt); 1127 1128 obj->uevent.uobject.object = qp; 1129 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1130 if (ret) 1131 goto err_destroy; 1132 1133 memset(&resp, 0, sizeof resp); 1134 resp.qpn = qp->qp_num; 1135 resp.qp_handle = obj->uevent.uobject.id; 1136 resp.max_recv_sge = attr.cap.max_recv_sge; 1137 resp.max_send_sge = attr.cap.max_send_sge; 1138 resp.max_recv_wr = attr.cap.max_recv_wr; 1139 resp.max_send_wr = attr.cap.max_send_wr; 1140 resp.max_inline_data = attr.cap.max_inline_data; 1141 1142 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1143 &resp, sizeof resp)) { 1144 ret = -EFAULT; 1145 goto err_copy; 1146 } 1147 1148 put_pd_read(pd); 1149 put_cq_read(scq); 1150 if (rcq != scq) 1151 put_cq_read(rcq); 1152 if (srq) 1153 put_srq_read(srq); 1154 if (xrcd) 1155 put_xrcd_read(xrcd_uobj); 1156 1157 mutex_lock(&file->mutex); 1158 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1159 mutex_unlock(&file->mutex); 1160 1161 obj->uevent.uobject.live = 1; 1162 1163 up_write(&obj->uevent.uobject.mutex); 1164 1165 return in_len; 1166 1167err_copy: 1168 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1169 1170err_destroy: 1171 ib_destroy_qp(qp); 1172 1173err_put: 1174 if (pd) 1175 put_pd_read(pd); 1176 if (scq) 1177 put_cq_read(scq); 1178 if (rcq && rcq != scq) 1179 put_cq_read(rcq); 1180 if (srq) 1181 put_srq_read(srq); 1182 if (xrcd) 1183 put_xrcd_read(xrcd_uobj); 1184 1185 put_uobj_write(&obj->uevent.uobject); 1186 return ret; 1187} 1188 1189ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1190 const char __user *buf, int in_len, 1191 int out_len) 1192{ 1193 struct ib_uverbs_query_qp cmd; 1194 struct ib_uverbs_query_qp_resp resp; 1195 struct ib_qp *qp; 1196 struct ib_qp_attr *attr; 1197 struct ib_qp_init_attr *init_attr; 1198 int ret; 1199 1200 if (copy_from_user(&cmd, buf, sizeof cmd)) 1201 return -EFAULT; 1202 1203 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1204 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1205 if (!attr || !init_attr) { 1206 ret = -ENOMEM; 1207 goto out; 1208 } 1209 1210 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1211 if (!qp) { 1212 ret = -EINVAL; 1213 goto out; 1214 } 1215 1216 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1217 1218 put_qp_read(qp); 1219 1220 if (ret) 1221 goto out; 1222 1223 memset(&resp, 0, sizeof resp); 1224 1225 resp.qp_state = attr->qp_state; 1226 resp.cur_qp_state = attr->cur_qp_state; 1227 resp.path_mtu = attr->path_mtu; 1228 resp.path_mig_state = attr->path_mig_state; 1229 resp.qkey = attr->qkey; 1230 resp.rq_psn = attr->rq_psn; 1231 resp.sq_psn = attr->sq_psn; 1232 resp.dest_qp_num = attr->dest_qp_num; 1233 resp.qp_access_flags = attr->qp_access_flags; 1234 resp.pkey_index = attr->pkey_index; 1235 resp.alt_pkey_index = attr->alt_pkey_index; 1236 resp.sq_draining = attr->sq_draining; 1237 resp.max_rd_atomic = attr->max_rd_atomic; 1238 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1239 resp.min_rnr_timer = attr->min_rnr_timer; 1240 resp.port_num = attr->port_num; 1241 resp.timeout = attr->timeout; 1242 resp.retry_cnt = attr->retry_cnt; 1243 resp.rnr_retry = attr->rnr_retry; 1244 resp.alt_port_num = attr->alt_port_num; 1245 resp.alt_timeout = attr->alt_timeout; 1246 1247 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1248 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 1249 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 1250 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 1251 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 1252 resp.dest.dlid = attr->ah_attr.dlid; 1253 resp.dest.sl = attr->ah_attr.sl; 1254 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 1255 resp.dest.static_rate = attr->ah_attr.static_rate; 1256 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 1257 resp.dest.port_num = attr->ah_attr.port_num; 1258 1259 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1260 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1261 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1262 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1263 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 1264 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 1265 resp.alt_dest.sl = attr->alt_ah_attr.sl; 1266 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1267 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1268 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 1269 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 1270 1271 resp.max_send_wr = init_attr->cap.max_send_wr; 1272 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1273 resp.max_send_sge = init_attr->cap.max_send_sge; 1274 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1275 resp.max_inline_data = init_attr->cap.max_inline_data; 1276 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1277 1278 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1279 &resp, sizeof resp)) 1280 ret = -EFAULT; 1281 1282out: 1283 kfree(attr); 1284 kfree(init_attr); 1285 1286 return ret ? ret : in_len; 1287} 1288 1289ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 1290 const char __user *buf, int in_len, 1291 int out_len) 1292{ 1293 struct ib_uverbs_modify_qp cmd; 1294 struct ib_udata udata; 1295 struct ib_qp *qp; 1296 struct ib_qp_attr *attr; 1297 int ret; 1298 1299 if (copy_from_user(&cmd, buf, sizeof cmd)) 1300 return -EFAULT; 1301 1302 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 1303 out_len); 1304 1305 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1306 if (!attr) 1307 return -ENOMEM; 1308 1309 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1310 if (!qp) { 1311 ret = -EINVAL; 1312 goto out; 1313 } 1314 1315 attr->qp_state = cmd.qp_state; 1316 attr->cur_qp_state = cmd.cur_qp_state; 1317 attr->path_mtu = cmd.path_mtu; 1318 attr->path_mig_state = cmd.path_mig_state; 1319 attr->qkey = cmd.qkey; 1320 attr->rq_psn = cmd.rq_psn; 1321 attr->sq_psn = cmd.sq_psn; 1322 attr->dest_qp_num = cmd.dest_qp_num; 1323 attr->qp_access_flags = cmd.qp_access_flags; 1324 attr->pkey_index = cmd.pkey_index; 1325 attr->alt_pkey_index = cmd.alt_pkey_index; 1326 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 1327 attr->max_rd_atomic = cmd.max_rd_atomic; 1328 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 1329 attr->min_rnr_timer = cmd.min_rnr_timer; 1330 attr->port_num = cmd.port_num; 1331 attr->timeout = cmd.timeout; 1332 attr->retry_cnt = cmd.retry_cnt; 1333 attr->rnr_retry = cmd.rnr_retry; 1334 attr->alt_port_num = cmd.alt_port_num; 1335 attr->alt_timeout = cmd.alt_timeout; 1336 1337 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 1338 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 1339 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 1340 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 1341 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 1342 attr->ah_attr.dlid = cmd.dest.dlid; 1343 attr->ah_attr.sl = cmd.dest.sl; 1344 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 1345 attr->ah_attr.static_rate = cmd.dest.static_rate; 1346 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 1347 attr->ah_attr.port_num = cmd.dest.port_num; 1348 1349 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 1350 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 1351 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 1352 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 1353 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 1354 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 1355 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 1356 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 1357 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 1358 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 1359 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 1360 1361 ret = qp->device->modify_qp(qp, attr, cmd.attr_mask, &udata); 1362 1363 put_qp_read(qp); 1364 1365 if (ret) 1366 goto out; 1367 1368 ret = in_len; 1369 1370out: 1371 kfree(attr); 1372 1373 return ret; 1374} 1375 1376ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 1377 const char __user *buf, int in_len, 1378 int out_len) 1379{ 1380 struct ib_uverbs_destroy_qp cmd; 1381 struct ib_uverbs_destroy_qp_resp resp; 1382 struct ib_uobject *uobj; 1383 struct ib_qp *qp; 1384 struct ib_uqp_object *obj; 1385 int ret = -EINVAL; 1386 1387 if (copy_from_user(&cmd, buf, sizeof cmd)) 1388 return -EFAULT; 1389 1390 memset(&resp, 0, sizeof resp); 1391 1392 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 1393 if (!uobj) 1394 return -EINVAL; 1395 qp = uobj->object; 1396 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 1397 1398 if (!list_empty(&obj->mcast_list)) { 1399 put_uobj_write(uobj); 1400 return -EBUSY; 1401 } 1402 1403 ret = ib_destroy_qp(qp); 1404 if (!ret) 1405 uobj->live = 0; 1406 1407 put_uobj_write(uobj); 1408 1409 if (ret) 1410 return ret; 1411 1412 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 1413 1414 mutex_lock(&file->mutex); 1415 list_del(&uobj->list); 1416 mutex_unlock(&file->mutex); 1417 1418 ib_uverbs_release_uevent(file, &obj->uevent); 1419 1420 resp.events_reported = obj->uevent.events_reported; 1421 1422 put_uobj(uobj); 1423 1424 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1425 &resp, sizeof resp)) 1426 return -EFAULT; 1427 1428 return in_len; 1429} 1430 1431ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 1432 const char __user *buf, int in_len, 1433 int out_len) 1434{ 1435 struct ib_uverbs_post_send cmd; 1436 struct ib_uverbs_post_send_resp resp; 1437 struct ib_uverbs_send_wr *user_wr; 1438 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 1439 struct ib_qp *qp; 1440 int i, sg_ind; 1441 int is_ud; 1442 ssize_t ret = -EINVAL; 1443 1444 if (copy_from_user(&cmd, buf, sizeof cmd)) 1445 return -EFAULT; 1446 1447 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 1448 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 1449 return -EINVAL; 1450 1451 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 1452 return -EINVAL; 1453 1454 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 1455 if (!user_wr) 1456 return -ENOMEM; 1457 1458 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1459 if (!qp) 1460 goto out; 1461 1462 is_ud = qp->qp_type == IB_QPT_UD; 1463 sg_ind = 0; 1464 last = NULL; 1465 for (i = 0; i < cmd.wr_count; ++i) { 1466 if (copy_from_user(user_wr, 1467 buf + sizeof cmd + i * cmd.wqe_size, 1468 cmd.wqe_size)) { 1469 ret = -EFAULT; 1470 goto out_put; 1471 } 1472 1473 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 1474 ret = -EINVAL; 1475 goto out_put; 1476 } 1477 1478 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1479 user_wr->num_sge * sizeof (struct ib_sge), 1480 GFP_KERNEL); 1481 if (!next) { 1482 ret = -ENOMEM; 1483 goto out_put; 1484 } 1485 1486 if (!last) 1487 wr = next; 1488 else 1489 last->next = next; 1490 last = next; 1491 1492 next->next = NULL; 1493 next->wr_id = user_wr->wr_id; 1494 next->num_sge = user_wr->num_sge; 1495 next->opcode = user_wr->opcode; 1496 next->send_flags = user_wr->send_flags; 1497 1498 if (is_ud) { 1499 next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, 1500 file->ucontext); 1501 if (!next->wr.ud.ah) { 1502 ret = -EINVAL; 1503 goto out_put; 1504 } 1505 next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; 1506 next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; 1507 } else { 1508 switch (next->opcode) { 1509 case IB_WR_RDMA_WRITE_WITH_IMM: 1510 next->ex.imm_data = 1511 (__be32 __force) user_wr->ex.imm_data; 1512 case IB_WR_RDMA_WRITE: 1513 case IB_WR_RDMA_READ: 1514 next->wr.rdma.remote_addr = 1515 user_wr->wr.rdma.remote_addr; 1516 next->wr.rdma.rkey = 1517 user_wr->wr.rdma.rkey; 1518 break; 1519 case IB_WR_SEND_WITH_IMM: 1520 next->ex.imm_data = 1521 (__be32 __force) user_wr->ex.imm_data; 1522 break; 1523 case IB_WR_SEND_WITH_INV: 1524 next->ex.invalidate_rkey = 1525 user_wr->ex.invalidate_rkey; 1526 break; 1527 case IB_WR_ATOMIC_CMP_AND_SWP: 1528 case IB_WR_ATOMIC_FETCH_AND_ADD: 1529 next->wr.atomic.remote_addr = 1530 user_wr->wr.atomic.remote_addr; 1531 next->wr.atomic.compare_add = 1532 user_wr->wr.atomic.compare_add; 1533 next->wr.atomic.swap = user_wr->wr.atomic.swap; 1534 next->wr.atomic.rkey = user_wr->wr.atomic.rkey; 1535 break; 1536 default: 1537 break; 1538 } 1539 } 1540 1541 if (next->num_sge) { 1542 next->sg_list = (void *) next + 1543 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1544 if (copy_from_user(next->sg_list, 1545 buf + sizeof cmd + 1546 cmd.wr_count * cmd.wqe_size + 1547 sg_ind * sizeof (struct ib_sge), 1548 next->num_sge * sizeof (struct ib_sge))) { 1549 ret = -EFAULT; 1550 goto out_put; 1551 } 1552 sg_ind += next->num_sge; 1553 } else 1554 next->sg_list = NULL; 1555 } 1556 1557 resp.bad_wr = 0; 1558 ret = qp->device->post_send(qp, wr, &bad_wr); 1559 if (ret) 1560 for (next = wr; next; next = next->next) { 1561 ++resp.bad_wr; 1562 if (next == bad_wr) 1563 break; 1564 } 1565 1566 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1567 &resp, sizeof resp)) 1568 ret = -EFAULT; 1569 1570out_put: 1571 put_qp_read(qp); 1572 1573 while (wr) { 1574 if (is_ud && wr->wr.ud.ah) 1575 put_ah_read(wr->wr.ud.ah); 1576 next = wr->next; 1577 kfree(wr); 1578 wr = next; 1579 } 1580 1581out: 1582 kfree(user_wr); 1583 1584 return ret ? ret : in_len; 1585} 1586 1587static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 1588 int in_len, 1589 u32 wr_count, 1590 u32 sge_count, 1591 u32 wqe_size) 1592{ 1593 struct ib_uverbs_recv_wr *user_wr; 1594 struct ib_recv_wr *wr = NULL, *last, *next; 1595 int sg_ind; 1596 int i; 1597 int ret; 1598 1599 if (in_len < wqe_size * wr_count + 1600 sge_count * sizeof (struct ib_uverbs_sge)) 1601 return ERR_PTR(-EINVAL); 1602 1603 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 1604 return ERR_PTR(-EINVAL); 1605 1606 user_wr = kmalloc(wqe_size, GFP_KERNEL); 1607 if (!user_wr) 1608 return ERR_PTR(-ENOMEM); 1609 1610 sg_ind = 0; 1611 last = NULL; 1612 for (i = 0; i < wr_count; ++i) { 1613 if (copy_from_user(user_wr, buf + i * wqe_size, 1614 wqe_size)) { 1615 ret = -EFAULT; 1616 goto err; 1617 } 1618 1619 if (user_wr->num_sge + sg_ind > sge_count) { 1620 ret = -EINVAL; 1621 goto err; 1622 } 1623 1624 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1625 user_wr->num_sge * sizeof (struct ib_sge), 1626 GFP_KERNEL); 1627 if (!next) { 1628 ret = -ENOMEM; 1629 goto err; 1630 } 1631 1632 if (!last) 1633 wr = next; 1634 else 1635 last->next = next; 1636 last = next; 1637 1638 next->next = NULL; 1639 next->wr_id = user_wr->wr_id; 1640 next->num_sge = user_wr->num_sge; 1641 1642 if (next->num_sge) { 1643 next->sg_list = (void *) next + 1644 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1645 if (copy_from_user(next->sg_list, 1646 buf + wr_count * wqe_size + 1647 sg_ind * sizeof (struct ib_sge), 1648 next->num_sge * sizeof (struct ib_sge))) { 1649 ret = -EFAULT; 1650 goto err; 1651 } 1652 sg_ind += next->num_sge; 1653 } else 1654 next->sg_list = NULL; 1655 } 1656 1657 kfree(user_wr); 1658 return wr; 1659 1660err: 1661 kfree(user_wr); 1662 1663 while (wr) { 1664 next = wr->next; 1665 kfree(wr); 1666 wr = next; 1667 } 1668 1669 return ERR_PTR(ret); 1670} 1671 1672ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 1673 const char __user *buf, int in_len, 1674 int out_len) 1675{ 1676 struct ib_uverbs_post_recv cmd; 1677 struct ib_uverbs_post_recv_resp resp; 1678 struct ib_recv_wr *wr, *next, *bad_wr; 1679 struct ib_qp *qp; 1680 ssize_t ret = -EINVAL; 1681 1682 if (copy_from_user(&cmd, buf, sizeof cmd)) 1683 return -EFAULT; 1684 1685 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1686 in_len - sizeof cmd, cmd.wr_count, 1687 cmd.sge_count, cmd.wqe_size); 1688 if (IS_ERR(wr)) 1689 return PTR_ERR(wr); 1690 1691 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1692 if (!qp) 1693 goto out; 1694 1695 resp.bad_wr = 0; 1696 ret = qp->device->post_recv(qp, wr, &bad_wr); 1697 1698 put_qp_read(qp); 1699 1700 if (ret) 1701 for (next = wr; next; next = next->next) { 1702 ++resp.bad_wr; 1703 if (next == bad_wr) 1704 break; 1705 } 1706 1707 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1708 &resp, sizeof resp)) 1709 ret = -EFAULT; 1710 1711out: 1712 while (wr) { 1713 next = wr->next; 1714 kfree(wr); 1715 wr = next; 1716 } 1717 1718 return ret ? ret : in_len; 1719} 1720 1721ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 1722 const char __user *buf, int in_len, 1723 int out_len) 1724{ 1725 struct ib_uverbs_post_srq_recv cmd; 1726 struct ib_uverbs_post_srq_recv_resp resp; 1727 struct ib_recv_wr *wr, *next, *bad_wr; 1728 struct ib_srq *srq; 1729 ssize_t ret = -EINVAL; 1730 1731 if (copy_from_user(&cmd, buf, sizeof cmd)) 1732 return -EFAULT; 1733 1734 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1735 in_len - sizeof cmd, cmd.wr_count, 1736 cmd.sge_count, cmd.wqe_size); 1737 if (IS_ERR(wr)) 1738 return PTR_ERR(wr); 1739 1740 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 1741 if (!srq) 1742 goto out; 1743 1744 resp.bad_wr = 0; 1745 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 1746 1747 put_srq_read(srq); 1748 1749 if (ret) 1750 for (next = wr; next; next = next->next) { 1751 ++resp.bad_wr; 1752 if (next == bad_wr) 1753 break; 1754 } 1755 1756 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1757 &resp, sizeof resp)) 1758 ret = -EFAULT; 1759 1760out: 1761 while (wr) { 1762 next = wr->next; 1763 kfree(wr); 1764 wr = next; 1765 } 1766 1767 return ret ? ret : in_len; 1768} 1769 1770ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 1771 const char __user *buf, int in_len, 1772 int out_len) 1773{ 1774 struct ib_uverbs_create_ah cmd; 1775 struct ib_uverbs_create_ah_resp resp; 1776 struct ib_uobject *uobj; 1777 struct ib_pd *pd; 1778 struct ib_ah *ah; 1779 struct ib_ah_attr attr; 1780 int ret; 1781 1782 if (out_len < sizeof resp) 1783 return -ENOSPC; 1784 1785 if (copy_from_user(&cmd, buf, sizeof cmd)) 1786 return -EFAULT; 1787 1788 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 1789 if (!uobj) 1790 return -ENOMEM; 1791 1792 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_key); 1793 down_write(&uobj->mutex); 1794 1795 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1796 if (!pd) { 1797 ret = -EINVAL; 1798 goto err; 1799 } 1800 1801 attr.dlid = cmd.attr.dlid; 1802 attr.sl = cmd.attr.sl; 1803 attr.src_path_bits = cmd.attr.src_path_bits; 1804 attr.static_rate = cmd.attr.static_rate; 1805 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 1806 attr.port_num = cmd.attr.port_num; 1807 attr.grh.flow_label = cmd.attr.grh.flow_label; 1808 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 1809 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 1810 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 1811 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 1812 1813 ah = ib_create_ah(pd, &attr); 1814 if (IS_ERR(ah)) { 1815 ret = PTR_ERR(ah); 1816 goto err_put; 1817 } 1818 1819 ah->uobject = uobj; 1820 uobj->object = ah; 1821 1822 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 1823 if (ret) 1824 goto err_destroy; 1825 1826 resp.ah_handle = uobj->id; 1827 1828 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1829 &resp, sizeof resp)) { 1830 ret = -EFAULT; 1831 goto err_copy; 1832 } 1833 1834 put_pd_read(pd); 1835 1836 mutex_lock(&file->mutex); 1837 list_add_tail(&uobj->list, &file->ucontext->ah_list); 1838 mutex_unlock(&file->mutex); 1839 1840 uobj->live = 1; 1841 1842 up_write(&uobj->mutex); 1843 1844 return in_len; 1845 1846err_copy: 1847 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1848 1849err_destroy: 1850 ib_destroy_ah(ah); 1851 1852err_put: 1853 put_pd_read(pd); 1854 1855err: 1856 put_uobj_write(uobj); 1857 return ret; 1858} 1859 1860ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 1861 const char __user *buf, int in_len, int out_len) 1862{ 1863 struct ib_uverbs_destroy_ah cmd; 1864 struct ib_ah *ah; 1865 struct ib_uobject *uobj; 1866 int ret; 1867 1868 if (copy_from_user(&cmd, buf, sizeof cmd)) 1869 return -EFAULT; 1870 1871 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 1872 if (!uobj) 1873 return -EINVAL; 1874 ah = uobj->object; 1875 1876 ret = ib_destroy_ah(ah); 1877 if (!ret) 1878 uobj->live = 0; 1879 1880 put_uobj_write(uobj); 1881 1882 if (ret) 1883 return ret; 1884 1885 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1886 1887 mutex_lock(&file->mutex); 1888 list_del(&uobj->list); 1889 mutex_unlock(&file->mutex); 1890 1891 put_uobj(uobj); 1892 1893 return in_len; 1894} 1895 1896ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 1897 const char __user *buf, int in_len, 1898 int out_len) 1899{ 1900 struct ib_uverbs_attach_mcast cmd; 1901 struct ib_qp *qp; 1902 struct ib_uqp_object *obj; 1903 struct ib_uverbs_mcast_entry *mcast; 1904 int ret; 1905 1906 if (copy_from_user(&cmd, buf, sizeof cmd)) 1907 return -EFAULT; 1908 1909 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1910 if (!qp) 1911 return -EINVAL; 1912 1913 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1914 1915 list_for_each_entry(mcast, &obj->mcast_list, list) 1916 if (cmd.mlid == mcast->lid && 1917 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1918 ret = 0; 1919 goto out_put; 1920 } 1921 1922 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 1923 if (!mcast) { 1924 ret = -ENOMEM; 1925 goto out_put; 1926 } 1927 1928 mcast->lid = cmd.mlid; 1929 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 1930 1931 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 1932 if (!ret) 1933 list_add_tail(&mcast->list, &obj->mcast_list); 1934 else 1935 kfree(mcast); 1936 1937out_put: 1938 put_qp_read(qp); 1939 1940 return ret ? ret : in_len; 1941} 1942 1943ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 1944 const char __user *buf, int in_len, 1945 int out_len) 1946{ 1947 struct ib_uverbs_detach_mcast cmd; 1948 struct ib_uqp_object *obj; 1949 struct ib_qp *qp; 1950 struct ib_uverbs_mcast_entry *mcast; 1951 int ret = -EINVAL; 1952 1953 if (copy_from_user(&cmd, buf, sizeof cmd)) 1954 return -EFAULT; 1955 1956 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1957 if (!qp) 1958 return -EINVAL; 1959 1960 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 1961 if (ret) 1962 goto out_put; 1963 1964 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1965 1966 list_for_each_entry(mcast, &obj->mcast_list, list) 1967 if (cmd.mlid == mcast->lid && 1968 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1969 list_del(&mcast->list); 1970 kfree(mcast); 1971 break; 1972 } 1973 1974out_put: 1975 put_qp_read(qp); 1976 1977 return ret ? ret : in_len; 1978} 1979 1980ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 1981 const char __user *buf, int in_len, 1982 int out_len) 1983{ 1984 struct ib_uverbs_create_srq cmd; 1985 struct ib_uverbs_create_srq_resp resp; 1986 struct ib_udata udata; 1987 struct ib_uevent_object *obj; 1988 struct ib_pd *pd; 1989 struct ib_srq *srq; 1990 struct ib_srq_init_attr attr; 1991 int ret; 1992 1993 if (out_len < sizeof resp) 1994 return -ENOSPC; 1995 1996 if (copy_from_user(&cmd, buf, sizeof cmd)) 1997 return -EFAULT; 1998 1999 INIT_UDATA(&udata, buf + sizeof cmd, 2000 (unsigned long) cmd.response + sizeof resp, 2001 in_len - sizeof cmd, out_len - sizeof resp); 2002 2003 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2004 if (!obj) 2005 return -ENOMEM; 2006 2007 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &srq_lock_key); 2008 down_write(&obj->uobject.mutex); 2009 2010 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2011 if (!pd) { 2012 ret = -EINVAL; 2013 goto err; 2014 } 2015 2016 attr.event_handler = ib_uverbs_srq_event_handler; 2017 attr.srq_context = file; 2018 attr.attr.max_wr = cmd.max_wr; 2019 attr.attr.max_sge = cmd.max_sge; 2020 attr.attr.srq_limit = cmd.srq_limit; 2021 2022 obj->events_reported = 0; 2023 INIT_LIST_HEAD(&obj->event_list); 2024 2025 srq = pd->device->create_srq(pd, &attr, &udata); 2026 if (IS_ERR(srq)) { 2027 ret = PTR_ERR(srq); 2028 goto err_put; 2029 } 2030 2031 srq->device = pd->device; 2032 srq->pd = pd; 2033 srq->uobject = &obj->uobject; 2034 srq->event_handler = attr.event_handler; 2035 srq->srq_context = attr.srq_context; 2036 srq->ext.xrc.cq = NULL; 2037 srq->ext.xrc.xrcd = NULL; 2038 atomic_inc(&pd->usecnt); 2039 atomic_set(&srq->usecnt, 0); 2040 2041 obj->uobject.object = srq; 2042 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2043 if (ret) 2044 goto err_destroy; 2045 2046 memset(&resp, 0, sizeof resp); 2047 resp.srq_handle = obj->uobject.id; 2048 resp.max_wr = attr.attr.max_wr; 2049 resp.max_sge = attr.attr.max_sge; 2050 2051 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2052 &resp, sizeof resp)) { 2053 ret = -EFAULT; 2054 goto err_copy; 2055 } 2056 2057 put_pd_read(pd); 2058 2059 mutex_lock(&file->mutex); 2060 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2061 mutex_unlock(&file->mutex); 2062 2063 obj->uobject.live = 1; 2064 2065 up_write(&obj->uobject.mutex); 2066 2067 return in_len; 2068 2069err_copy: 2070 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2071 2072err_destroy: 2073 ib_destroy_srq(srq); 2074 2075err_put: 2076 put_pd_read(pd); 2077 2078err: 2079 put_uobj_write(&obj->uobject); 2080 return ret; 2081} 2082 2083ssize_t ib_uverbs_create_xrc_srq(struct ib_uverbs_file *file, 2084 const char __user *buf, int in_len, 2085 int out_len) 2086{ 2087 struct ib_uverbs_create_xsrq cmd; 2088 struct ib_uverbs_create_srq_resp resp; 2089 struct ib_udata udata; 2090 struct ib_uevent_object *obj; 2091 struct ib_pd *pd; 2092 struct ib_srq *srq; 2093 struct ib_cq *xrc_cq; 2094 struct ib_xrcd *xrcd; 2095 struct ib_srq_init_attr attr; 2096 struct ib_uobject *xrcd_uobj; 2097 int ret; 2098 2099 if (out_len < sizeof resp) 2100 return -ENOSPC; 2101 2102 if (copy_from_user(&cmd, buf, sizeof cmd)) 2103 return -EFAULT; 2104 2105 INIT_UDATA(&udata, buf + sizeof cmd, 2106 (unsigned long) cmd.response + sizeof resp, 2107 in_len - sizeof cmd, out_len - sizeof resp); 2108 2109 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2110 if (!obj) 2111 return -ENOMEM; 2112 2113 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, 2114 &srq_lock_key); 2115 down_write(&obj->uobject.mutex); 2116 2117 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2118 if (!pd) { 2119 ret = -EINVAL; 2120 goto err; 2121 } 2122 2123 xrc_cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 2124 if (!xrc_cq) { 2125 ret = -EINVAL; 2126 goto err_put_pd; 2127 } 2128 2129 xrcd = idr_read_xrcd(cmd.xrcd_handle, file->ucontext, &xrcd_uobj); 2130 if (!xrcd) { 2131 ret = -EINVAL; 2132 goto err_put_cq; 2133 } 2134 2135 2136 attr.event_handler = ib_uverbs_srq_event_handler; 2137 attr.srq_context = file; 2138 attr.attr.max_wr = cmd.max_wr; 2139 attr.attr.max_sge = cmd.max_sge; 2140 attr.attr.srq_limit = cmd.srq_limit; 2141 2142 obj->events_reported = 0; 2143 INIT_LIST_HEAD(&obj->event_list); 2144 2145 srq = pd->device->create_xrc_srq(pd, xrc_cq, xrcd, &attr, &udata); 2146 if (IS_ERR(srq)) { 2147 ret = PTR_ERR(srq); 2148 goto err_put; 2149 } 2150 2151 srq->device = pd->device; 2152 srq->pd = pd; 2153 srq->uobject = &obj->uobject; 2154 srq->event_handler = attr.event_handler; 2155 srq->srq_context = attr.srq_context; 2156 srq->ext.xrc.cq = xrc_cq; 2157 srq->ext.xrc.xrcd = xrcd; 2158 atomic_inc(&pd->usecnt); 2159 atomic_inc(&xrc_cq->usecnt); 2160 atomic_inc(&xrcd->usecnt); 2161 2162 atomic_set(&srq->usecnt, 0); 2163 2164 obj->uobject.object = srq; 2165 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2166 if (ret) 2167 goto err_destroy; 2168 2169 memset(&resp, 0, sizeof resp); 2170 resp.srq_handle = obj->uobject.id; 2171 resp.max_wr = attr.attr.max_wr; 2172 resp.max_sge = attr.attr.max_sge; 2173 2174 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2175 &resp, sizeof resp)) { 2176 ret = -EFAULT; 2177 goto err_copy; 2178 } 2179 2180 put_xrcd_read(xrcd_uobj); 2181 put_cq_read(xrc_cq); 2182 put_pd_read(pd); 2183 2184 mutex_lock(&file->mutex); 2185 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2186 mutex_unlock(&file->mutex); 2187 2188 obj->uobject.live = 1; 2189 2190 up_write(&obj->uobject.mutex); 2191 2192 return in_len; 2193 2194err_copy: 2195 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2196 2197err_destroy: 2198 ib_destroy_srq(srq); 2199 2200err_put: 2201 put_xrcd_read(xrcd_uobj); 2202 2203err_put_cq: 2204 put_cq_read(xrc_cq); 2205 2206err_put_pd: 2207 put_pd_read(pd); 2208 2209err: 2210 put_uobj_write(&obj->uobject); 2211 return ret; 2212} 2213 2214ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 2215 const char __user *buf, int in_len, 2216 int out_len) 2217{ 2218 struct ib_uverbs_modify_srq cmd; 2219 struct ib_udata udata; 2220 struct ib_srq *srq; 2221 struct ib_srq_attr attr; 2222 int ret; 2223 2224 if (copy_from_user(&cmd, buf, sizeof cmd)) 2225 return -EFAULT; 2226 2227 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 2228 out_len); 2229 2230 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2231 if (!srq) 2232 return -EINVAL; 2233 2234 attr.max_wr = cmd.max_wr; 2235 attr.srq_limit = cmd.srq_limit; 2236 2237 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 2238 2239 put_srq_read(srq); 2240 2241 return ret ? ret : in_len; 2242} 2243 2244ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 2245 const char __user *buf, 2246 int in_len, int out_len) 2247{ 2248 struct ib_uverbs_query_srq cmd; 2249 struct ib_uverbs_query_srq_resp resp; 2250 struct ib_srq_attr attr; 2251 struct ib_srq *srq; 2252 int ret; 2253 2254 if (out_len < sizeof resp) 2255 return -ENOSPC; 2256 2257 if (copy_from_user(&cmd, buf, sizeof cmd)) 2258 return -EFAULT; 2259 2260 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2261 if (!srq) 2262 return -EINVAL; 2263 2264 ret = ib_query_srq(srq, &attr); 2265 2266 put_srq_read(srq); 2267 2268 if (ret) 2269 return ret; 2270 2271 memset(&resp, 0, sizeof resp); 2272 2273 resp.max_wr = attr.max_wr; 2274 resp.max_sge = attr.max_sge; 2275 resp.srq_limit = attr.srq_limit; 2276 2277 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2278 &resp, sizeof resp)) 2279 return -EFAULT; 2280 2281 return in_len; 2282} 2283 2284ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 2285 const char __user *buf, int in_len, 2286 int out_len) 2287{ 2288 struct ib_uverbs_destroy_srq cmd; 2289 struct ib_uverbs_destroy_srq_resp resp; 2290 struct ib_uobject *uobj; 2291 struct ib_srq *srq; 2292 struct ib_uevent_object *obj; 2293 int ret = -EINVAL; 2294 2295 if (copy_from_user(&cmd, buf, sizeof cmd)) 2296 return -EFAULT; 2297 2298 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 2299 if (!uobj) 2300 return -EINVAL; 2301 srq = uobj->object; 2302 obj = container_of(uobj, struct ib_uevent_object, uobject); 2303 2304 ret = ib_destroy_srq(srq); 2305 if (!ret) 2306 uobj->live = 0; 2307 2308 put_uobj_write(uobj); 2309 2310 if (ret) 2311 return ret; 2312 2313 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 2314 2315 mutex_lock(&file->mutex); 2316 list_del(&uobj->list); 2317 mutex_unlock(&file->mutex); 2318 2319 ib_uverbs_release_uevent(file, obj); 2320 2321 memset(&resp, 0, sizeof resp); 2322 resp.events_reported = obj->events_reported; 2323 2324 put_uobj(uobj); 2325 2326 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2327 &resp, sizeof resp)) 2328 ret = -EFAULT; 2329 2330 return ret ? ret : in_len; 2331} 2332 2333static struct inode *xrc_file2inode(struct file *f) 2334{ 2335 return f->f_dentry->d_inode; 2336} 2337 2338struct xrcd_table_entry { 2339 struct rb_node node; 2340 struct inode *inode; 2341 struct ib_xrcd *xrcd; 2342}; 2343 2344static int xrcd_table_insert(struct ib_device *dev, 2345 struct inode *i_n, 2346 struct ib_xrcd *xrcd) 2347{ 2348 struct xrcd_table_entry *entry, *scan; 2349 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2350 struct rb_node *parent = NULL; 2351 2352 entry = kmalloc(sizeof(struct xrcd_table_entry), GFP_KERNEL); 2353 if (!entry) 2354 return -ENOMEM; 2355 2356 entry->inode = i_n; 2357 entry->xrcd = xrcd; 2358 2359 while (*p) { 2360 parent = *p; 2361 scan = rb_entry(parent, struct xrcd_table_entry, node); 2362 2363 if (i_n < scan->inode) 2364 p = &(*p)->rb_left; 2365 else if (i_n > scan->inode) 2366 p = &(*p)->rb_right; 2367 else { 2368 kfree(entry); 2369 return -EEXIST; 2370 } 2371 } 2372 2373 rb_link_node(&entry->node, parent, p); 2374 rb_insert_color(&entry->node, &dev->ib_uverbs_xrcd_table); 2375 igrab(i_n); 2376 return 0; 2377} 2378 2379static struct xrcd_table_entry *xrcd_table_search(struct ib_device *dev, 2380 struct inode *i_n) 2381{ 2382 struct xrcd_table_entry *scan; 2383 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2384 struct rb_node *parent = NULL; 2385 2386 while (*p) { 2387 parent = *p; 2388 scan = rb_entry(parent, struct xrcd_table_entry, node); 2389 2390 if (i_n < scan->inode) 2391 p = &(*p)->rb_left; 2392 else if (i_n > scan->inode) 2393 p = &(*p)->rb_right; 2394 else 2395 return scan; 2396 } 2397 return NULL; 2398} 2399 2400static int find_xrcd(struct ib_device *dev, struct inode *i_n, 2401 struct ib_xrcd **xrcd) 2402{ 2403 struct xrcd_table_entry *entry; 2404 2405 entry = xrcd_table_search(dev, i_n); 2406 if (!entry) 2407 return -EINVAL; 2408 2409 *xrcd = entry->xrcd; 2410 return 0; 2411} 2412 2413 2414static void xrcd_table_delete(struct ib_device *dev, 2415 struct inode *i_n) 2416{ 2417 struct xrcd_table_entry *entry = xrcd_table_search(dev, i_n); 2418 2419 if (entry) { 2420 iput(i_n); 2421 rb_erase(&entry->node, &dev->ib_uverbs_xrcd_table); 2422 kfree(entry); 2423 } 2424} 2425 2426ssize_t ib_uverbs_open_xrc_domain(struct ib_uverbs_file *file, 2427 const char __user *buf, int in_len, 2428 int out_len) 2429{ 2430 struct ib_uverbs_open_xrc_domain cmd; 2431 struct ib_uverbs_open_xrc_domain_resp resp; 2432 struct ib_udata udata; 2433 struct ib_uobject *uobj; 2434 struct ib_uxrcd_object *xrcd_uobj; 2435 struct ib_xrcd *xrcd = NULL; 2436 struct file *f = NULL; 2437 struct inode *inode = NULL; 2438 int ret = 0; 2439 int new_xrcd = 0; 2440 2441 if (out_len < sizeof resp) 2442 return -ENOSPC; 2443 2444 if (copy_from_user(&cmd, buf, sizeof cmd)) 2445 return -EFAULT; 2446 2447 INIT_UDATA(&udata, buf + sizeof cmd, 2448 (unsigned long) cmd.response + sizeof resp, 2449 in_len - sizeof cmd, out_len - sizeof resp); 2450 2451 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2452 if (cmd.fd != (u32) (-1)) { 2453 /* search for file descriptor */ 2454 f = fget(cmd.fd); 2455 if (!f) { 2456 ret = -EBADF; 2457 goto err_table_mutex_unlock; 2458 } 2459 2460 inode = xrc_file2inode(f); 2461 if (!inode) { 2462 ret = -EBADF; 2463 goto err_table_mutex_unlock; 2464 } 2465 2466 ret = find_xrcd(file->device->ib_dev, inode, &xrcd); 2467 if (ret && !(cmd.oflags & O_CREAT)) { 2468 /* no file descriptor. Need CREATE flag */ 2469 ret = -EAGAIN; 2470 goto err_table_mutex_unlock; 2471 } 2472 2473 if (xrcd && cmd.oflags & O_EXCL) { 2474 ret = -EINVAL; 2475 goto err_table_mutex_unlock; 2476 } 2477 } 2478 2479 xrcd_uobj = kmalloc(sizeof *xrcd_uobj, GFP_KERNEL); 2480 if (!xrcd_uobj) { 2481 ret = -ENOMEM; 2482 goto err_table_mutex_unlock; 2483 } 2484 2485 uobj = &xrcd_uobj->uobject; 2486 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 2487 down_write(&uobj->mutex); 2488 2489 if (!xrcd) { 2490 xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev, 2491 file->ucontext, &udata); 2492 if (IS_ERR(xrcd)) { 2493 ret = PTR_ERR(xrcd); 2494 goto err; 2495 } 2496 xrcd->uobject = (cmd.fd == -1) ? uobj : NULL; 2497 xrcd->inode = inode; 2498 xrcd->device = file->device->ib_dev; 2499 atomic_set(&xrcd->usecnt, 0); 2500 new_xrcd = 1; 2501 } 2502 2503 uobj->object = xrcd; 2504 ret = idr_add_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2505 if (ret) 2506 goto err_idr; 2507 2508 memset(&resp, 0, sizeof resp); 2509 resp.xrcd_handle = uobj->id; 2510 2511 if (inode) { 2512 if (new_xrcd) { 2513 /* create new inode/xrcd table entry */ 2514 ret = xrcd_table_insert(file->device->ib_dev, inode, xrcd); 2515 if (ret) 2516 goto err_insert_xrcd; 2517 } 2518 atomic_inc(&xrcd->usecnt); 2519 } 2520 if (f) 2521 fput(f); 2522 2523 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2524 &resp, sizeof resp)) { 2525 ret = -EFAULT; 2526 goto err_copy; 2527 } 2528 2529 INIT_LIST_HEAD(&xrcd_uobj->xrc_reg_qp_list); 2530 2531 mutex_lock(&file->mutex); 2532 list_add_tail(&uobj->list, &file->ucontext->xrcd_list); 2533 mutex_unlock(&file->mutex); 2534 2535 uobj->live = 1; 2536 2537 up_write(&uobj->mutex); 2538 2539 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2540 return in_len; 2541 2542err_copy: 2543 2544 if (inode) { 2545 if (new_xrcd) 2546 xrcd_table_delete(file->device->ib_dev, inode); 2547 atomic_dec(&xrcd->usecnt); 2548 } 2549 2550err_insert_xrcd: 2551 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2552 2553err_idr: 2554 ib_dealloc_xrcd(xrcd); 2555 2556err: 2557 put_uobj_write(uobj); 2558 2559err_table_mutex_unlock: 2560 2561 if (f) 2562 fput(f); 2563 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2564 return ret; 2565} 2566 2567ssize_t ib_uverbs_close_xrc_domain(struct ib_uverbs_file *file, 2568 const char __user *buf, int in_len, 2569 int out_len) 2570{ 2571 struct ib_uverbs_close_xrc_domain cmd; 2572 struct ib_uobject *uobj, *t_uobj; 2573 struct ib_uxrcd_object *xrcd_uobj; 2574 struct ib_xrcd *xrcd = NULL; 2575 struct inode *inode = NULL; 2576 int ret = 0; 2577 2578 if (copy_from_user(&cmd, buf, sizeof cmd)) 2579 return -EFAULT; 2580 2581 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2582 uobj = idr_write_uobj(&ib_uverbs_xrc_domain_idr, cmd.xrcd_handle, 2583 file->ucontext); 2584 if (!uobj) { 2585 ret = -EINVAL; 2586 goto err_unlock_mutex; 2587 } 2588 2589 mutex_lock(&file->mutex); 2590 if (!ret) { 2591 list_for_each_entry(t_uobj, &file->ucontext->qp_list, list) { 2592 struct ib_qp *qp = t_uobj->object; 2593 if (qp->xrcd && qp->xrcd == uobj->object) { 2594 ret = -EBUSY; 2595 break; 2596 } 2597 } 2598 } 2599 if (!ret) { 2600 list_for_each_entry(t_uobj, &file->ucontext->srq_list, list) { 2601 struct ib_srq *srq = t_uobj->object; 2602 if (srq->ext.xrc.xrcd && srq->ext.xrc.xrcd == uobj->object) { 2603 ret = -EBUSY; 2604 break; 2605 } 2606 } 2607 } 2608 mutex_unlock(&file->mutex); 2609 if (ret) { 2610 put_uobj_write(uobj); 2611 goto err_unlock_mutex; 2612 } 2613 2614 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2615 if (!list_empty(&xrcd_uobj->xrc_reg_qp_list)) { 2616 ret = -EBUSY; 2617 put_uobj_write(uobj); 2618 goto err_unlock_mutex; 2619 } 2620 2621 xrcd = (struct ib_xrcd *) (uobj->object); 2622 inode = xrcd->inode; 2623 2624 if (inode) 2625 atomic_dec(&xrcd->usecnt); 2626 2627 ret = ib_dealloc_xrcd(uobj->object); 2628 if (!ret) 2629 uobj->live = 0; 2630 2631 put_uobj_write(uobj); 2632 2633 if (ret && !inode) 2634 goto err_unlock_mutex; 2635 2636 if (!ret && inode) 2637 xrcd_table_delete(file->device->ib_dev, inode); 2638 2639 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2640 2641 mutex_lock(&file->mutex); 2642 list_del(&uobj->list); 2643 mutex_unlock(&file->mutex); 2644 2645 put_uobj(uobj); 2646 2647 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2648 return in_len; 2649 2650err_unlock_mutex: 2651 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2652 return ret; 2653} 2654 2655void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev, 2656 struct ib_xrcd *xrcd) 2657{ 2658 struct inode *inode = NULL; 2659 int ret = 0; 2660 2661 inode = xrcd->inode; 2662 if (inode) 2663 atomic_dec(&xrcd->usecnt); 2664 2665 ret = ib_dealloc_xrcd(xrcd); 2666 if (!ret && inode) 2667 xrcd_table_delete(ib_dev, inode); 2668} 2669 2670ssize_t ib_uverbs_create_xrc_rcv_qp(struct ib_uverbs_file *file, 2671 const char __user *buf, int in_len, 2672 int out_len) 2673{ 2674 struct ib_uverbs_create_xrc_rcv_qp cmd; 2675 struct ib_uverbs_create_xrc_rcv_qp_resp resp; 2676 struct ib_uxrc_rcv_object *obj; 2677 struct ib_qp_init_attr init_attr; 2678 struct ib_xrcd *xrcd; 2679 struct ib_uobject *uobj; 2680 struct ib_uxrcd_object *xrcd_uobj; 2681 u32 qp_num; 2682 int err; 2683 2684 if (out_len < sizeof resp) 2685 return -ENOSPC; 2686 2687 if (copy_from_user(&cmd, buf, sizeof cmd)) 2688 return -EFAULT; 2689 2690 obj = kzalloc(sizeof *obj, GFP_KERNEL); 2691 if (!obj) 2692 return -ENOMEM; 2693 2694 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2695 if (!xrcd) { 2696 err = -EINVAL; 2697 goto err_out; 2698 } 2699 2700 init_attr.event_handler = ib_uverbs_xrc_rcv_qp_event_handler; 2701 init_attr.qp_context = file; 2702 init_attr.srq = NULL; 2703 init_attr.sq_sig_type = 2704 cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2705 init_attr.qp_type = IB_QPT_XRC; 2706 init_attr.xrcd = xrcd; 2707 2708 init_attr.cap.max_send_wr = 1; 2709 init_attr.cap.max_recv_wr = 0; 2710 init_attr.cap.max_send_sge = 1; 2711 init_attr.cap.max_recv_sge = 0; 2712 init_attr.cap.max_inline_data = 0; 2713 2714 err = xrcd->device->create_xrc_rcv_qp(&init_attr, &qp_num); 2715 if (err) 2716 goto err_put; 2717 2718 memset(&resp, 0, sizeof resp); 2719 resp.qpn = qp_num; 2720 2721 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2722 &resp, sizeof resp)) { 2723 err = -EFAULT; 2724 goto err_destroy; 2725 } 2726 2727 atomic_inc(&xrcd->usecnt); 2728 put_xrcd_read(uobj); 2729 obj->qp_num = qp_num; 2730 obj->domain_handle = cmd.xrc_domain_handle; 2731 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2732 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2733 list_add_tail(&obj->list, &xrcd_uobj->xrc_reg_qp_list); 2734 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2735 2736 return in_len; 2737 2738err_destroy: 2739 xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2740err_put: 2741 put_xrcd_read(uobj); 2742err_out: 2743 kfree(obj); 2744 return err; 2745} 2746 2747ssize_t ib_uverbs_modify_xrc_rcv_qp(struct ib_uverbs_file *file, 2748 const char __user *buf, int in_len, 2749 int out_len) 2750{ 2751 struct ib_uverbs_modify_xrc_rcv_qp cmd; 2752 struct ib_qp_attr *attr; 2753 struct ib_xrcd *xrcd; 2754 struct ib_uobject *uobj; 2755 int err; 2756 2757 if (copy_from_user(&cmd, buf, sizeof cmd)) 2758 return -EFAULT; 2759 2760 attr = kzalloc(sizeof *attr, GFP_KERNEL); 2761 if (!attr) 2762 return -ENOMEM; 2763 2764 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2765 if (!xrcd) { 2766 kfree(attr); 2767 return -EINVAL; 2768 } 2769 2770 attr->qp_state = cmd.qp_state; 2771 attr->cur_qp_state = cmd.cur_qp_state; 2772 attr->qp_access_flags = cmd.qp_access_flags; 2773 attr->pkey_index = cmd.pkey_index; 2774 attr->port_num = cmd.port_num; 2775 attr->path_mtu = cmd.path_mtu; 2776 attr->path_mig_state = cmd.path_mig_state; 2777 attr->qkey = cmd.qkey; 2778 attr->rq_psn = cmd.rq_psn; 2779 attr->sq_psn = cmd.sq_psn; 2780 attr->dest_qp_num = cmd.dest_qp_num; 2781 attr->alt_pkey_index = cmd.alt_pkey_index; 2782 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 2783 attr->max_rd_atomic = cmd.max_rd_atomic; 2784 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 2785 attr->min_rnr_timer = cmd.min_rnr_timer; 2786 attr->port_num = cmd.port_num; 2787 attr->timeout = cmd.timeout; 2788 attr->retry_cnt = cmd.retry_cnt; 2789 attr->rnr_retry = cmd.rnr_retry; 2790 attr->alt_port_num = cmd.alt_port_num; 2791 attr->alt_timeout = cmd.alt_timeout; 2792 2793 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 2794 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 2795 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 2796 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 2797 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 2798 attr->ah_attr.dlid = cmd.dest.dlid; 2799 attr->ah_attr.sl = cmd.dest.sl; 2800 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 2801 attr->ah_attr.static_rate = cmd.dest.static_rate; 2802 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 2803 attr->ah_attr.port_num = cmd.dest.port_num; 2804 2805 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 2806 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 2807 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 2808 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 2809 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 2810 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 2811 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 2812 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 2813 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 2814 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 2815 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 2816 2817 err = xrcd->device->modify_xrc_rcv_qp(xrcd, cmd.qp_num, attr, cmd.attr_mask); 2818 put_xrcd_read(uobj); 2819 kfree(attr); 2820 return err ? err : in_len; 2821} 2822 2823ssize_t ib_uverbs_query_xrc_rcv_qp(struct ib_uverbs_file *file, 2824 const char __user *buf, int in_len, 2825 int out_len) 2826{ 2827 struct ib_uverbs_query_xrc_rcv_qp cmd; 2828 struct ib_uverbs_query_qp_resp resp; 2829 struct ib_qp_attr *attr; 2830 struct ib_qp_init_attr *init_attr; 2831 struct ib_xrcd *xrcd; 2832 struct ib_uobject *uobj; 2833 int ret; 2834 2835 if (copy_from_user(&cmd, buf, sizeof cmd)) 2836 return -EFAULT; 2837 2838 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2839 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2840 if (!attr || !init_attr) { 2841 ret = -ENOMEM; 2842 goto out; 2843 } 2844 2845 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2846 if (!xrcd) { 2847 ret = -EINVAL; 2848 goto out; 2849 } 2850 2851 ret = xrcd->device->query_xrc_rcv_qp(xrcd, cmd.qp_num, attr, 2852 cmd.attr_mask, init_attr); 2853 2854 put_xrcd_read(uobj); 2855 2856 if (ret) 2857 goto out; 2858 2859 memset(&resp, 0, sizeof resp); 2860 resp.qp_state = attr->qp_state; 2861 resp.cur_qp_state = attr->cur_qp_state; 2862 resp.path_mtu = attr->path_mtu; 2863 resp.path_mig_state = attr->path_mig_state; 2864 resp.qkey = attr->qkey; 2865 resp.rq_psn = attr->rq_psn; 2866 resp.sq_psn = attr->sq_psn; 2867 resp.dest_qp_num = attr->dest_qp_num; 2868 resp.qp_access_flags = attr->qp_access_flags; 2869 resp.pkey_index = attr->pkey_index; 2870 resp.alt_pkey_index = attr->alt_pkey_index; 2871 resp.sq_draining = attr->sq_draining; 2872 resp.max_rd_atomic = attr->max_rd_atomic; 2873 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2874 resp.min_rnr_timer = attr->min_rnr_timer; 2875 resp.port_num = attr->port_num; 2876 resp.timeout = attr->timeout; 2877 resp.retry_cnt = attr->retry_cnt; 2878 resp.rnr_retry = attr->rnr_retry; 2879 resp.alt_port_num = attr->alt_port_num; 2880 resp.alt_timeout = attr->alt_timeout; 2881 2882 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2883 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2884 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2885 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2886 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2887 resp.dest.dlid = attr->ah_attr.dlid; 2888 resp.dest.sl = attr->ah_attr.sl; 2889 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2890 resp.dest.static_rate = attr->ah_attr.static_rate; 2891 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2892 resp.dest.port_num = attr->ah_attr.port_num; 2893 2894 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2895 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2896 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2897 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2898 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2899 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2900 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2901 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2902 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2903 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2904 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2905 2906 resp.max_send_wr = init_attr->cap.max_send_wr; 2907 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2908 resp.max_send_sge = init_attr->cap.max_send_sge; 2909 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2910 resp.max_inline_data = init_attr->cap.max_inline_data; 2911 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2912 2913 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2914 &resp, sizeof resp)) 2915 ret = -EFAULT; 2916 2917out: 2918 kfree(attr); 2919 kfree(init_attr); 2920 2921 return ret ? ret : in_len; 2922} 2923 2924ssize_t ib_uverbs_reg_xrc_rcv_qp(struct ib_uverbs_file *file, 2925 const char __user *buf, int in_len, 2926 int out_len) 2927{ 2928 struct ib_uverbs_reg_xrc_rcv_qp cmd; 2929 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2930 struct ib_xrcd *xrcd; 2931 struct ib_uobject *uobj; 2932 struct ib_uxrcd_object *xrcd_uobj; 2933 int ret; 2934 2935 if (copy_from_user(&cmd, buf, sizeof cmd)) 2936 return -EFAULT; 2937 2938 qp_obj = kmalloc(sizeof *qp_obj, GFP_KERNEL); 2939 if (!qp_obj) 2940 return -ENOMEM; 2941 2942 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2943 if (!xrcd) { 2944 ret = -EINVAL; 2945 goto err_out; 2946 } 2947 2948 ret = xrcd->device->reg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 2949 if (ret) 2950 goto err_put; 2951 2952 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2953 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2954 list_for_each_entry(tmp, &xrcd_uobj->xrc_reg_qp_list, list) 2955 if (cmd.qp_num == tmp->qp_num) { 2956 kfree(qp_obj); 2957 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2958 put_xrcd_read(uobj); 2959 return in_len; 2960 } 2961 qp_obj->qp_num = cmd.qp_num; 2962 qp_obj->domain_handle = cmd.xrc_domain_handle; 2963 list_add_tail(&qp_obj->list, &xrcd_uobj->xrc_reg_qp_list); 2964 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2965 atomic_inc(&xrcd->usecnt); 2966 put_xrcd_read(uobj); 2967 return in_len; 2968 2969err_put: 2970 put_xrcd_read(uobj); 2971err_out: 2972 2973 kfree(qp_obj); 2974 return ret; 2975} 2976 2977int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file, 2978 struct ib_xrcd *xrcd, u32 qp_num) 2979{ 2980 int err; 2981 err = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2982 if (!err) 2983 atomic_dec(&xrcd->usecnt); 2984 return err; 2985} 2986 2987ssize_t ib_uverbs_unreg_xrc_rcv_qp(struct ib_uverbs_file *file, 2988 const char __user *buf, int in_len, 2989 int out_len) 2990{ 2991 struct ib_uverbs_unreg_xrc_rcv_qp cmd; 2992 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2993 struct ib_xrcd *xrcd; 2994 struct ib_uobject *uobj; 2995 struct ib_uxrcd_object *xrcd_uobj; 2996 int ret; 2997 2998 if (copy_from_user(&cmd, buf, sizeof cmd)) 2999 return -EFAULT; 3000 3001 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 3002 if (!xrcd) 3003 return -EINVAL; 3004 3005 ret = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 3006 if (ret) { 3007 put_xrcd_read(uobj); 3008 return -EINVAL; 3009 } 3010 atomic_dec(&xrcd->usecnt); 3011 3012 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 3013 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 3014 list_for_each_entry_safe(qp_obj, tmp, &xrcd_uobj->xrc_reg_qp_list, list) 3015 if (cmd.qp_num == qp_obj->qp_num) { 3016 list_del(&qp_obj->list); 3017 kfree(qp_obj); 3018 break; 3019 } 3020 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 3021 put_xrcd_read(uobj); 3022 return in_len; 3023} 3024