mr.c revision 271127
1/* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/errno.h> 36#include <linux/slab.h> 37#include <linux/kernel.h> 38#include <linux/vmalloc.h> 39 40#include <linux/mlx4/cmd.h> 41 42#include "mlx4.h" 43#include "icm.h" 44 45#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) 46#define MLX4_MPT_FLAG_FREE (0x3UL << 28) 47#define MLX4_MPT_FLAG_MIO (1 << 17) 48#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) 49#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) 50#define MLX4_MPT_FLAG_REGION (1 << 8) 51 52#define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27) 53#define MLX4_MPT_PD_FLAG_RAE (1 << 28) 54#define MLX4_MPT_PD_FLAG_EN_INV (3 << 24) 55 56#define MLX4_MPT_STATUS_SW 0xF0 57#define MLX4_MPT_STATUS_HW 0x00 58 59static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 60{ 61 int o; 62 int m; 63 u32 seg; 64 65 spin_lock(&buddy->lock); 66 67 for (o = order; o <= buddy->max_order; ++o) 68 if (buddy->num_free[o]) { 69 m = 1 << (buddy->max_order - o); 70 seg = find_first_bit(buddy->bits[o], m); 71 if (seg < m) 72 goto found; 73 } 74 75 spin_unlock(&buddy->lock); 76 return -1; 77 78 found: 79 clear_bit(seg, buddy->bits[o]); 80 --buddy->num_free[o]; 81 82 while (o > order) { 83 --o; 84 seg <<= 1; 85 set_bit(seg ^ 1, buddy->bits[o]); 86 ++buddy->num_free[o]; 87 } 88 89 spin_unlock(&buddy->lock); 90 91 seg <<= order; 92 93 return seg; 94} 95 96static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 97{ 98 seg >>= order; 99 100 spin_lock(&buddy->lock); 101 102 while (test_bit(seg ^ 1, buddy->bits[order])) { 103 clear_bit(seg ^ 1, buddy->bits[order]); 104 --buddy->num_free[order]; 105 seg >>= 1; 106 ++order; 107 } 108 109 set_bit(seg, buddy->bits[order]); 110 ++buddy->num_free[order]; 111 112 spin_unlock(&buddy->lock); 113} 114 115static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 116{ 117 int i, s; 118 119 buddy->max_order = max_order; 120 spin_lock_init(&buddy->lock); 121 122 buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), 123 GFP_KERNEL); 124 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 125 GFP_KERNEL); 126 if (!buddy->bits || !buddy->num_free) 127 goto err_out; 128 129 for (i = 0; i <= buddy->max_order; ++i) { 130 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 131 buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); 132 if (!buddy->bits[i]) { 133 goto err_out_free; 134 } 135 } 136 137 set_bit(0, buddy->bits[buddy->max_order]); 138 buddy->num_free[buddy->max_order] = 1; 139 140 return 0; 141 142err_out_free: 143 for (i = 0; i <= buddy->max_order; ++i) 144 if ( buddy->bits[i] ) 145 kfree(buddy->bits[i]); 146 147err_out: 148 kfree(buddy->bits); 149 kfree(buddy->num_free); 150 151 return -ENOMEM; 152} 153 154static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 155{ 156 int i; 157 158 for (i = 0; i <= buddy->max_order; ++i) 159 kfree(buddy->bits[i]); 160 161 kfree(buddy->bits); 162 kfree(buddy->num_free); 163} 164 165u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 166{ 167 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 168 u32 seg; 169 int seg_order; 170 u32 offset; 171 172 seg_order = max_t(int, order - log_mtts_per_seg, 0); 173 174 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); 175 if (seg == -1) 176 return -1; 177 178 offset = seg * (1 << log_mtts_per_seg); 179 180 if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, 181 offset + (1 << order) - 1)) { 182 mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); 183 return -1; 184 } 185 186 return offset; 187} 188 189static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 190{ 191 u64 in_param = 0; 192 u64 out_param; 193 int err; 194 195 if (mlx4_is_mfunc(dev)) { 196 set_param_l(&in_param, order); 197 err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, 198 RES_OP_RESERVE_AND_MAP, 199 MLX4_CMD_ALLOC_RES, 200 MLX4_CMD_TIME_CLASS_A, 201 MLX4_CMD_WRAPPED); 202 if (err) 203 return -1; 204 return get_param_l(&out_param); 205 } 206 return __mlx4_alloc_mtt_range(dev, order); 207} 208 209int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 210 struct mlx4_mtt *mtt) 211{ 212 int i; 213 214 if (!npages) { 215 mtt->order = -1; 216 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 217 return 0; 218 } else 219 mtt->page_shift = page_shift; 220 221 for (mtt->order = 0, i = 1; i < npages; i <<= 1) 222 ++mtt->order; 223 224 mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); 225 if (mtt->offset == -1) { 226 mlx4_err(dev, "Failed to allocate mtts for %d pages(order %d)\n", 227 npages, mtt->order); 228 return -ENOMEM; 229 } 230 231 return 0; 232} 233EXPORT_SYMBOL_GPL(mlx4_mtt_init); 234 235void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 236{ 237 u32 first_seg; 238 int seg_order; 239 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 240 241 seg_order = max_t(int, order - log_mtts_per_seg, 0); 242 first_seg = offset / (1 << log_mtts_per_seg); 243 244 mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); 245 mlx4_table_put_range(dev, &mr_table->mtt_table, offset, 246 offset + (1 << order) - 1); 247} 248 249static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 250{ 251 u64 in_param = 0; 252 int err; 253 254 if (mlx4_is_mfunc(dev)) { 255 set_param_l(&in_param, offset); 256 set_param_h(&in_param, order); 257 err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, 258 MLX4_CMD_FREE_RES, 259 MLX4_CMD_TIME_CLASS_A, 260 MLX4_CMD_WRAPPED); 261 if (err) 262 mlx4_warn(dev, "Failed to free mtt range at:" 263 "%d order:%d\n", offset, order); 264 return; 265 } 266 __mlx4_free_mtt_range(dev, offset, order); 267} 268 269void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 270{ 271 if (mtt->order < 0) 272 return; 273 274 mlx4_free_mtt_range(dev, mtt->offset, mtt->order); 275} 276EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 277 278u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 279{ 280 return (u64) mtt->offset * dev->caps.mtt_entry_sz; 281} 282EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 283 284static u32 hw_index_to_key(u32 ind) 285{ 286 return (ind >> 24) | (ind << 8); 287} 288 289static u32 key_to_hw_index(u32 key) 290{ 291 return (key << 24) | (key >> 8); 292} 293 294static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 295 int mpt_index) 296{ 297 return mlx4_cmd(dev, mailbox->dma, mpt_index, 298 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, 299 MLX4_CMD_WRAPPED); 300} 301 302static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 303 int mpt_index) 304{ 305 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 306 !mailbox, MLX4_CMD_HW2SW_MPT, 307 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 308} 309 310static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, 311 u64 iova, u64 size, u32 access, int npages, 312 int page_shift, struct mlx4_mr *mr) 313{ 314 mr->iova = iova; 315 mr->size = size; 316 mr->pd = pd; 317 mr->access = access; 318 mr->enabled = MLX4_MR_DISABLED; 319 mr->key = hw_index_to_key(mridx); 320 321 return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 322} 323 324static int mlx4_WRITE_MTT(struct mlx4_dev *dev, 325 struct mlx4_cmd_mailbox *mailbox, 326 int num_entries) 327{ 328 return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, 329 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 330} 331 332int __mlx4_mr_reserve(struct mlx4_dev *dev) 333{ 334 struct mlx4_priv *priv = mlx4_priv(dev); 335 336 return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 337} 338 339static int mlx4_mr_reserve(struct mlx4_dev *dev) 340{ 341 u64 out_param; 342 343 if (mlx4_is_mfunc(dev)) { 344 if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, 345 MLX4_CMD_ALLOC_RES, 346 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 347 return -1; 348 return get_param_l(&out_param); 349 } 350 return __mlx4_mr_reserve(dev); 351} 352 353void __mlx4_mr_release(struct mlx4_dev *dev, u32 index) 354{ 355 struct mlx4_priv *priv = mlx4_priv(dev); 356 357 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); 358} 359 360static void mlx4_mr_release(struct mlx4_dev *dev, u32 index) 361{ 362 u64 in_param = 0; 363 364 if (mlx4_is_mfunc(dev)) { 365 set_param_l(&in_param, index); 366 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, 367 MLX4_CMD_FREE_RES, 368 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 369 mlx4_warn(dev, "Failed to release mr index:%d\n", 370 index); 371 return; 372 } 373 __mlx4_mr_release(dev, index); 374} 375 376int __mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index) 377{ 378 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 379 380 return mlx4_table_get(dev, &mr_table->dmpt_table, index); 381} 382 383static int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index) 384{ 385 u64 param = 0; 386 387 if (mlx4_is_mfunc(dev)) { 388 set_param_l(¶m, index); 389 return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, 390 MLX4_CMD_ALLOC_RES, 391 MLX4_CMD_TIME_CLASS_A, 392 MLX4_CMD_WRAPPED); 393 } 394 return __mlx4_mr_alloc_icm(dev, index); 395} 396 397void __mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index) 398{ 399 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 400 401 mlx4_table_put(dev, &mr_table->dmpt_table, index); 402} 403 404static void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index) 405{ 406 u64 in_param = 0; 407 408 if (mlx4_is_mfunc(dev)) { 409 set_param_l(&in_param, index); 410 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, 411 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, 412 MLX4_CMD_WRAPPED)) 413 mlx4_warn(dev, "Failed to free icm of mr index:%d\n", 414 index); 415 return; 416 } 417 return __mlx4_mr_free_icm(dev, index); 418} 419 420int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 421 int npages, int page_shift, struct mlx4_mr *mr) 422{ 423 u32 index; 424 int err; 425 426 index = mlx4_mr_reserve(dev); 427 if (index == -1) 428 return -ENOMEM; 429 430 err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, 431 access, npages, page_shift, mr); 432 if (err) 433 mlx4_mr_release(dev, index); 434 435 return err; 436} 437EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 438 439static void mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) 440{ 441 int err; 442 443 if (mr->enabled == MLX4_MR_EN_HW) { 444 err = mlx4_HW2SW_MPT(dev, NULL, 445 key_to_hw_index(mr->key) & 446 (dev->caps.num_mpts - 1)); 447 if (err) 448 mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); 449 450 mr->enabled = MLX4_MR_EN_SW; 451 } 452 mlx4_mtt_cleanup(dev, &mr->mtt); 453} 454 455void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 456{ 457 mlx4_mr_free_reserved(dev, mr); 458 if (mr->enabled) 459 mlx4_mr_free_icm(dev, key_to_hw_index(mr->key)); 460 mlx4_mr_release(dev, key_to_hw_index(mr->key)); 461} 462EXPORT_SYMBOL_GPL(mlx4_mr_free); 463 464int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 465{ 466 struct mlx4_cmd_mailbox *mailbox; 467 struct mlx4_mpt_entry *mpt_entry; 468 int err; 469 470 err = mlx4_mr_alloc_icm(dev, key_to_hw_index(mr->key)); 471 if (err) 472 return err; 473 474 mailbox = mlx4_alloc_cmd_mailbox(dev); 475 if (IS_ERR(mailbox)) { 476 err = PTR_ERR(mailbox); 477 goto err_table; 478 } 479 mpt_entry = mailbox->buf; 480 481 memset(mpt_entry, 0, sizeof *mpt_entry); 482 483 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | 484 MLX4_MPT_FLAG_REGION | 485 mr->access); 486 487 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 488 mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); 489 mpt_entry->start = cpu_to_be64(mr->iova); 490 mpt_entry->length = cpu_to_be64(mr->size); 491 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 492 493 if (mr->mtt.order < 0) { 494 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 495 mpt_entry->mtt_addr = 0; 496 } else { 497 mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, 498 &mr->mtt)); 499 } 500 501 if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { 502 /* fast register MR in free state */ 503 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 504 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | 505 MLX4_MPT_PD_FLAG_RAE); 506 mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); 507 } else { 508 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); 509 } 510 511 err = mlx4_SW2HW_MPT(dev, mailbox, 512 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 513 if (err) { 514 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 515 goto err_cmd; 516 } 517 mr->enabled = MLX4_MR_EN_HW; 518 519 mlx4_free_cmd_mailbox(dev, mailbox); 520 521 return 0; 522 523err_cmd: 524 mlx4_free_cmd_mailbox(dev, mailbox); 525 526err_table: 527 mlx4_mr_free_icm(dev, key_to_hw_index(mr->key)); 528 return err; 529} 530EXPORT_SYMBOL_GPL(mlx4_mr_enable); 531 532static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 533 int start_index, int npages, u64 *page_list) 534{ 535 struct mlx4_priv *priv = mlx4_priv(dev); 536 __be64 *mtts; 537 dma_addr_t dma_handle; 538 int i; 539 540 mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + 541 start_index, &dma_handle); 542 543 if (!mtts) 544 return -ENOMEM; 545 546 dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, 547 npages * sizeof (u64), DMA_TO_DEVICE); 548 549 for (i = 0; i < npages; ++i) 550 mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 551 552 dma_sync_single_for_device(&dev->pdev->dev, dma_handle, 553 npages * sizeof (u64), DMA_TO_DEVICE); 554 555 return 0; 556} 557 558int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 559 int start_index, int npages, u64 *page_list) 560{ 561 int err = 0; 562 int chunk; 563 int mtts_per_page; 564 int max_mtts_first_page; 565 566 /* compute how may mtts fit in the first page */ 567 mtts_per_page = PAGE_SIZE / sizeof(u64); 568 max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) 569 % mtts_per_page; 570 571 chunk = min_t(int, max_mtts_first_page, npages); 572 573 while (npages > 0) { 574 err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); 575 if (err) 576 return err; 577 npages -= chunk; 578 start_index += chunk; 579 page_list += chunk; 580 581 chunk = min_t(int, mtts_per_page, npages); 582 } 583 return err; 584} 585 586int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 587 int start_index, int npages, u64 *page_list) 588{ 589 struct mlx4_cmd_mailbox *mailbox = NULL; 590 __be64 *inbox = NULL; 591 int chunk; 592 int err = 0; 593 int i; 594 595 if (mtt->order < 0) 596 return -EINVAL; 597 598 if (mlx4_is_mfunc(dev)) { 599 mailbox = mlx4_alloc_cmd_mailbox(dev); 600 if (IS_ERR(mailbox)) 601 return PTR_ERR(mailbox); 602 inbox = mailbox->buf; 603 604 while (npages > 0) { 605 chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, 606 npages); 607 inbox[0] = cpu_to_be64(mtt->offset + start_index); 608 inbox[1] = 0; 609 for (i = 0; i < chunk; ++i) 610 inbox[i + 2] = cpu_to_be64(page_list[i] | 611 MLX4_MTT_FLAG_PRESENT); 612 err = mlx4_WRITE_MTT(dev, mailbox, chunk); 613 if (err) { 614 mlx4_free_cmd_mailbox(dev, mailbox); 615 return err; 616 } 617 618 npages -= chunk; 619 start_index += chunk; 620 page_list += chunk; 621 } 622 mlx4_free_cmd_mailbox(dev, mailbox); 623 return err; 624 } 625 626 return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); 627} 628EXPORT_SYMBOL_GPL(mlx4_write_mtt); 629 630int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 631 struct mlx4_buf *buf) 632{ 633 u64 *page_list; 634 int err; 635 int i; 636 637 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 638 if (!page_list) 639 return -ENOMEM; 640 641 for (i = 0; i < buf->npages; ++i) 642 if (buf->nbufs == 1) 643 page_list[i] = buf->direct.map + (i << buf->page_shift); 644 else 645 page_list[i] = buf->page_list[i].map; 646 647 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 648 649 kfree(page_list); 650 return err; 651} 652EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 653 654int mlx4_init_mr_table(struct mlx4_dev *dev) 655{ 656 struct mlx4_priv *priv = mlx4_priv(dev); 657 struct mlx4_mr_table *mr_table = &priv->mr_table; 658 int err; 659 660 /* Nothing to do for slaves - all MR handling is forwarded 661 * to the master */ 662 if (mlx4_is_slave(dev)) 663 return 0; 664 665 if (!is_power_of_2(dev->caps.num_mpts)) 666 return -EINVAL; 667 668 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 669 ~0, dev->caps.reserved_mrws, 0); 670 if (err) 671 return err; 672 673 err = mlx4_buddy_init(&mr_table->mtt_buddy, 674 ilog2((u32)dev->caps.num_mtts / 675 (1 << log_mtts_per_seg))); 676 if (err) 677 goto err_buddy; 678 679 if (dev->caps.reserved_mtts) { 680 priv->reserved_mtts = 681 mlx4_alloc_mtt_range(dev, 682 fls(dev->caps.reserved_mtts - 1)); 683 if (priv->reserved_mtts < 0) { 684 mlx4_warn(dev, "MTT table of order %u is too small.\n", 685 mr_table->mtt_buddy.max_order); 686 err = -ENOMEM; 687 goto err_reserve_mtts; 688 } 689 } 690 691 return 0; 692 693err_reserve_mtts: 694 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 695 696err_buddy: 697 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 698 699 return err; 700} 701 702void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 703{ 704 struct mlx4_priv *priv = mlx4_priv(dev); 705 struct mlx4_mr_table *mr_table = &priv->mr_table; 706 707 if (mlx4_is_slave(dev)) 708 return; 709 if (priv->reserved_mtts >= 0) 710 mlx4_free_mtt_range(dev, priv->reserved_mtts, 711 fls(dev->caps.reserved_mtts - 1)); 712 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 713 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 714} 715 716static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, 717 int npages, u64 iova) 718{ 719 int i, page_mask; 720 721 if (npages > fmr->max_pages) 722 return -EINVAL; 723 724 page_mask = (1 << fmr->page_shift) - 1; 725 726 /* We are getting page lists, so va must be page aligned. */ 727 if (iova & page_mask) 728 return -EINVAL; 729 730 /* Trust the user not to pass misaligned data in page_list */ 731 if (0) 732 for (i = 0; i < npages; ++i) { 733 if (page_list[i] & ~page_mask) 734 return -EINVAL; 735 } 736 737 if (fmr->maps >= fmr->max_maps) 738 return -EINVAL; 739 740 return 0; 741} 742 743int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, 744 int npages, u64 iova, u32 *lkey, u32 *rkey) 745{ 746 u32 key; 747 int i, err; 748 749 err = mlx4_check_fmr(fmr, page_list, npages, iova); 750 if (err) 751 return err; 752 753 ++fmr->maps; 754 755 key = key_to_hw_index(fmr->mr.key); 756 key += dev->caps.num_mpts; 757 *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); 758 759 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; 760 761 /* Make sure MPT status is visible before writing MTT entries */ 762 wmb(); 763 764 dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, 765 npages * sizeof(u64), DMA_TO_DEVICE); 766 767 for (i = 0; i < npages; ++i) 768 fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 769 770 dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, 771 npages * sizeof(u64), DMA_TO_DEVICE); 772 773 fmr->mpt->key = cpu_to_be32(key); 774 fmr->mpt->lkey = cpu_to_be32(key); 775 fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); 776 fmr->mpt->start = cpu_to_be64(iova); 777 778 /* Make MTT entries are visible before setting MPT status */ 779 wmb(); 780 781 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; 782 783 /* Make sure MPT status is visible before consumer can use FMR */ 784 wmb(); 785 786 return 0; 787} 788EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); 789 790int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, 791 int max_maps, u8 page_shift, struct mlx4_fmr *fmr) 792{ 793 struct mlx4_priv *priv = mlx4_priv(dev); 794 int err = -ENOMEM; 795 796 if (max_maps > dev->caps.max_fmr_maps) 797 return -EINVAL; 798 799 if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) 800 return -EINVAL; 801 802 /* All MTTs must fit in the same page */ 803 if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) 804 return -EINVAL; 805 806 fmr->page_shift = page_shift; 807 fmr->max_pages = max_pages; 808 fmr->max_maps = max_maps; 809 fmr->maps = 0; 810 811 err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, 812 page_shift, &fmr->mr); 813 if (err) 814 return err; 815 816 fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, 817 fmr->mr.mtt.offset, 818 &fmr->dma_handle); 819 820 if (!fmr->mtts) { 821 err = -ENOMEM; 822 goto err_free; 823 } 824 825 return 0; 826 827err_free: 828 mlx4_mr_free(dev, &fmr->mr); 829 return err; 830} 831EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); 832 833int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 834{ 835 struct mlx4_priv *priv = mlx4_priv(dev); 836 int err; 837 838 err = mlx4_mr_enable(dev, &fmr->mr); 839 if (err) 840 return err; 841 842 fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, 843 key_to_hw_index(fmr->mr.key), NULL); 844 if (!fmr->mpt) 845 return -ENOMEM; 846 847 return 0; 848} 849EXPORT_SYMBOL_GPL(mlx4_fmr_enable); 850 851void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, 852 u32 *lkey, u32 *rkey) 853{ 854 struct mlx4_cmd_mailbox *mailbox; 855 int err; 856 857 if (!fmr->maps) 858 return; 859 860 fmr->maps = 0; 861 862 mailbox = mlx4_alloc_cmd_mailbox(dev); 863 if (IS_ERR(mailbox)) { 864 err = PTR_ERR(mailbox); 865 mlx4_warn(dev, "mlx4_alloc_cmd_mailbox failed (%d)\n", err); 866 return; 867 } 868 869 err = mlx4_HW2SW_MPT(dev, NULL, 870 key_to_hw_index(fmr->mr.key) & 871 (dev->caps.num_mpts - 1)); 872 mlx4_free_cmd_mailbox(dev, mailbox); 873 if (err) { 874 mlx4_warn(dev, "mlx4_HW2SW_MPT failed (%d)\n", err); 875 return; 876 } 877 fmr->mr.enabled = MLX4_MR_EN_SW; 878} 879EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); 880 881int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 882{ 883 if (fmr->maps) 884 return -EBUSY; 885 886 mlx4_mr_free(dev, &fmr->mr); 887 fmr->mr.enabled = MLX4_MR_DISABLED; 888 889 return 0; 890} 891EXPORT_SYMBOL_GPL(mlx4_fmr_free); 892 893int mlx4_SYNC_TPT(struct mlx4_dev *dev) 894{ 895 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, 896 MLX4_CMD_NATIVE); 897} 898EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); 899