1/* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/init.h> 36#include <linux/errno.h> 37 38#include <linux/mlx4/cmd.h> 39 40#include "mlx4.h" 41#include "icm.h" 42 43/* 44 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 45 */ 46struct mlx4_mpt_entry { 47 __be32 flags; 48 __be32 qpn; 49 __be32 key; 50 __be32 pd; 51 __be64 start; 52 __be64 length; 53 __be32 lkey; 54 __be32 win_cnt; 55 u8 reserved1[3]; 56 u8 mtt_rep; 57 __be64 mtt_seg; 58 __be32 mtt_sz; 59 __be32 entity_size; 60 __be32 first_byte_offset; 61} __attribute__((packed)); 62 63#define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28) 64#define MLX4_MPT_FLAG_MIO (1 << 17) 65#define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15) 66#define MLX4_MPT_FLAG_PHYSICAL (1 << 9) 67#define MLX4_MPT_FLAG_REGION (1 << 8) 68 69#define MLX4_MTT_FLAG_PRESENT 1 70 71static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 72{ 73 int o; 74 int m; 75 u32 seg; 76 77 spin_lock(&buddy->lock); 78 79 for (o = order; o <= buddy->max_order; ++o) { 80 m = 1 << (buddy->max_order - o); 81 seg = find_first_bit(buddy->bits[o], m); 82 if (seg < m) 83 goto found; 84 } 85 86 spin_unlock(&buddy->lock); 87 return -1; 88 89 found: 90 clear_bit(seg, buddy->bits[o]); 91 92 while (o > order) { 93 --o; 94 seg <<= 1; 95 set_bit(seg ^ 1, buddy->bits[o]); 96 } 97 98 spin_unlock(&buddy->lock); 99 100 seg <<= order; 101 102 return seg; 103} 104 105static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 106{ 107 seg >>= order; 108 109 spin_lock(&buddy->lock); 110 111 while (test_bit(seg ^ 1, buddy->bits[order])) { 112 clear_bit(seg ^ 1, buddy->bits[order]); 113 seg >>= 1; 114 ++order; 115 } 116 117 set_bit(seg, buddy->bits[order]); 118 119 spin_unlock(&buddy->lock); 120} 121 122static int __devinit mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 123{ 124 int i, s; 125 126 buddy->max_order = max_order; 127 spin_lock_init(&buddy->lock); 128 129 buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), 130 GFP_KERNEL); 131 if (!buddy->bits) 132 goto err_out; 133 134 for (i = 0; i <= buddy->max_order; ++i) { 135 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 136 buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); 137 if (!buddy->bits[i]) 138 goto err_out_free; 139 bitmap_zero(buddy->bits[i], 1 << (buddy->max_order - i)); 140 } 141 142 set_bit(0, buddy->bits[buddy->max_order]); 143 144 return 0; 145 146err_out_free: 147 for (i = 0; i <= buddy->max_order; ++i) 148 kfree(buddy->bits[i]); 149 150 kfree(buddy->bits); 151 152err_out: 153 return -ENOMEM; 154} 155 156static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 157{ 158 int i; 159 160 for (i = 0; i <= buddy->max_order; ++i) 161 kfree(buddy->bits[i]); 162 163 kfree(buddy->bits); 164} 165 166static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 167{ 168 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 169 u32 seg; 170 171 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, order); 172 if (seg == -1) 173 return -1; 174 175 if (mlx4_table_get_range(dev, &mr_table->mtt_table, seg, 176 seg + (1 << order) - 1)) { 177 mlx4_buddy_free(&mr_table->mtt_buddy, seg, order); 178 return -1; 179 } 180 181 return seg; 182} 183 184int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 185 struct mlx4_mtt *mtt) 186{ 187 int i; 188 189 if (!npages) { 190 mtt->order = -1; 191 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 192 return 0; 193 } else 194 mtt->page_shift = page_shift; 195 196 for (mtt->order = 0, i = MLX4_MTT_ENTRY_PER_SEG; i < npages; i <<= 1) 197 ++mtt->order; 198 199 mtt->first_seg = mlx4_alloc_mtt_range(dev, mtt->order); 200 if (mtt->first_seg == -1) 201 return -ENOMEM; 202 203 return 0; 204} 205EXPORT_SYMBOL_GPL(mlx4_mtt_init); 206 207void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 208{ 209 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 210 211 if (mtt->order < 0) 212 return; 213 214 mlx4_buddy_free(&mr_table->mtt_buddy, mtt->first_seg, mtt->order); 215 mlx4_table_put_range(dev, &mr_table->mtt_table, mtt->first_seg, 216 mtt->first_seg + (1 << mtt->order) - 1); 217} 218EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 219 220u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 221{ 222 return (u64) mtt->first_seg * dev->caps.mtt_entry_sz; 223} 224EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 225 226static u32 hw_index_to_key(u32 ind) 227{ 228 return (ind >> 24) | (ind << 8); 229} 230 231static u32 key_to_hw_index(u32 key) 232{ 233 return (key << 24) | (key >> 8); 234} 235 236static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 237 int mpt_index) 238{ 239 return mlx4_cmd(dev, mailbox->dma, mpt_index, 0, MLX4_CMD_SW2HW_MPT, 240 MLX4_CMD_TIME_CLASS_B); 241} 242 243static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 244 int mpt_index) 245{ 246 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 247 !mailbox, MLX4_CMD_HW2SW_MPT, MLX4_CMD_TIME_CLASS_B); 248} 249 250int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 251 int npages, int page_shift, struct mlx4_mr *mr) 252{ 253 struct mlx4_priv *priv = mlx4_priv(dev); 254 u32 index; 255 int err; 256 257 index = mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 258 if (index == -1) { 259 err = -ENOMEM; 260 goto err; 261 } 262 263 mr->iova = iova; 264 mr->size = size; 265 mr->pd = pd; 266 mr->access = access; 267 mr->enabled = 0; 268 mr->key = hw_index_to_key(index); 269 270 err = mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 271 if (err) 272 goto err_index; 273 274 return 0; 275 276err_index: 277 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); 278 279err: 280 kfree(mr); 281 return err; 282} 283EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 284 285void mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 286{ 287 struct mlx4_priv *priv = mlx4_priv(dev); 288 int err; 289 290 if (mr->enabled) { 291 err = mlx4_HW2SW_MPT(dev, NULL, 292 key_to_hw_index(mr->key) & 293 (dev->caps.num_mpts - 1)); 294 if (err) 295 mlx4_warn(dev, "HW2SW_MPT failed (%d)\n", err); 296 } 297 298 mlx4_mtt_cleanup(dev, &mr->mtt); 299 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, key_to_hw_index(mr->key)); 300} 301EXPORT_SYMBOL_GPL(mlx4_mr_free); 302 303int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 304{ 305 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 306 struct mlx4_cmd_mailbox *mailbox; 307 struct mlx4_mpt_entry *mpt_entry; 308 int err; 309 310 err = mlx4_table_get(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key)); 311 if (err) 312 return err; 313 314 mailbox = mlx4_alloc_cmd_mailbox(dev); 315 if (IS_ERR(mailbox)) { 316 err = PTR_ERR(mailbox); 317 goto err_table; 318 } 319 mpt_entry = mailbox->buf; 320 321 memset(mpt_entry, 0, sizeof *mpt_entry); 322 323 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS | 324 MLX4_MPT_FLAG_MIO | 325 MLX4_MPT_FLAG_REGION | 326 mr->access); 327 328 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 329 mpt_entry->pd = cpu_to_be32(mr->pd); 330 mpt_entry->start = cpu_to_be64(mr->iova); 331 mpt_entry->length = cpu_to_be64(mr->size); 332 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 333 if (mr->mtt.order < 0) { 334 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 335 mpt_entry->mtt_seg = 0; 336 } else 337 mpt_entry->mtt_seg = cpu_to_be64(mlx4_mtt_addr(dev, &mr->mtt)); 338 339 err = mlx4_SW2HW_MPT(dev, mailbox, 340 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 341 if (err) { 342 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 343 goto err_cmd; 344 } 345 346 mr->enabled = 1; 347 348 mlx4_free_cmd_mailbox(dev, mailbox); 349 350 return 0; 351 352err_cmd: 353 mlx4_free_cmd_mailbox(dev, mailbox); 354 355err_table: 356 mlx4_table_put(dev, &mr_table->dmpt_table, key_to_hw_index(mr->key)); 357 return err; 358} 359EXPORT_SYMBOL_GPL(mlx4_mr_enable); 360 361static int mlx4_WRITE_MTT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 362 int num_mtt) 363{ 364 return mlx4_cmd(dev, mailbox->dma, num_mtt, 0, MLX4_CMD_WRITE_MTT, 365 MLX4_CMD_TIME_CLASS_B); 366} 367 368int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 369 int start_index, int npages, u64 *page_list) 370{ 371 struct mlx4_cmd_mailbox *mailbox; 372 __be64 *mtt_entry; 373 int i; 374 int err = 0; 375 376 if (mtt->order < 0) 377 return -EINVAL; 378 379 mailbox = mlx4_alloc_cmd_mailbox(dev); 380 if (IS_ERR(mailbox)) 381 return PTR_ERR(mailbox); 382 383 mtt_entry = mailbox->buf; 384 385 while (npages > 0) { 386 mtt_entry[0] = cpu_to_be64(mlx4_mtt_addr(dev, mtt) + start_index * 8); 387 mtt_entry[1] = 0; 388 389 for (i = 0; i < npages && i < MLX4_MAILBOX_SIZE / 8 - 2; ++i) 390 mtt_entry[i + 2] = cpu_to_be64(page_list[i] | 391 MLX4_MTT_FLAG_PRESENT); 392 393 /* 394 * If we have an odd number of entries to write, add 395 * one more dummy entry for firmware efficiency. 396 */ 397 if (i & 1) 398 mtt_entry[i + 2] = 0; 399 400 err = mlx4_WRITE_MTT(dev, mailbox, (i + 1) & ~1); 401 if (err) 402 goto out; 403 404 npages -= i; 405 start_index += i; 406 page_list += i; 407 } 408 409out: 410 mlx4_free_cmd_mailbox(dev, mailbox); 411 412 return err; 413} 414EXPORT_SYMBOL_GPL(mlx4_write_mtt); 415 416int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 417 struct mlx4_buf *buf) 418{ 419 u64 *page_list; 420 int err; 421 int i; 422 423 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 424 if (!page_list) 425 return -ENOMEM; 426 427 for (i = 0; i < buf->npages; ++i) 428 if (buf->nbufs == 1) 429 page_list[i] = buf->u.direct.map + (i << buf->page_shift); 430 else 431 page_list[i] = buf->u.page_list[i].map; 432 433 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 434 435 kfree(page_list); 436 return err; 437} 438EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 439 440int __devinit mlx4_init_mr_table(struct mlx4_dev *dev) 441{ 442 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 443 int err; 444 445 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 446 ~0, dev->caps.reserved_mrws); 447 if (err) 448 return err; 449 450 err = mlx4_buddy_init(&mr_table->mtt_buddy, 451 ilog2(dev->caps.num_mtt_segs)); 452 if (err) 453 goto err_buddy; 454 455 if (dev->caps.reserved_mtts) { 456 if (mlx4_alloc_mtt_range(dev, ilog2(dev->caps.reserved_mtts)) == -1) { 457 mlx4_warn(dev, "MTT table of order %d is too small.\n", 458 mr_table->mtt_buddy.max_order); 459 err = -ENOMEM; 460 goto err_reserve_mtts; 461 } 462 } 463 464 return 0; 465 466err_reserve_mtts: 467 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 468 469err_buddy: 470 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 471 472 return err; 473} 474 475void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 476{ 477 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 478 479 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 480 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 481} 482