mthca_memfree.c revision 273246
1/* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#include <linux/mm.h> 36#include <linux/scatterlist.h> 37#include <linux/sched.h> 38 39#include <linux/page.h> 40 41#include "mthca_memfree.h" 42#include "mthca_dev.h" 43#include "mthca_cmd.h" 44 45/* 46 * We allocate in as big chunks as we can, up to a maximum of 256 KB 47 * per chunk. 48 */ 49enum { 50 MTHCA_ICM_ALLOC_SIZE = 1 << 18, 51 MTHCA_TABLE_CHUNK_SIZE = 1 << 18 52}; 53 54struct mthca_user_db_table { 55 struct mutex mutex; 56 struct { 57 u64 uvirt; 58 struct scatterlist mem; 59 int refcount; 60 } page[0]; 61}; 62 63static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) 64{ 65 int i; 66 67 if (chunk->nsg > 0) 68 pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, 69 PCI_DMA_BIDIRECTIONAL); 70 71 for (i = 0; i < chunk->npages; ++i) 72 __free_pages(sg_page(&chunk->mem[i]), 73 get_order(chunk->mem[i].length)); 74} 75 76static void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) 77{ 78 int i; 79 80 for (i = 0; i < chunk->npages; ++i) { 81 dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, 82 lowmem_page_address(sg_page(&chunk->mem[i])), 83 sg_dma_address(&chunk->mem[i])); 84 } 85} 86 87void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent) 88{ 89 struct mthca_icm_chunk *chunk, *tmp; 90 91 if (!icm) 92 return; 93 94 list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { 95 if (coherent) 96 mthca_free_icm_coherent(dev, chunk); 97 else 98 mthca_free_icm_pages(dev, chunk); 99 100 kfree(chunk); 101 } 102 103 kfree(icm); 104} 105 106static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) 107{ 108 struct page *page; 109 110 /* 111 * Use __GFP_ZERO because buggy firmware assumes ICM pages are 112 * cleared, and subtle failures are seen if they aren't. 113 */ 114 page = alloc_pages(gfp_mask | __GFP_ZERO, order); 115 if (!page) 116 return -ENOMEM; 117 118 sg_set_page(mem, page, PAGE_SIZE << order, 0); 119 return 0; 120} 121 122static int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, 123 int order, gfp_t gfp_mask) 124{ 125 void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), 126 gfp_mask); 127 if (!buf) 128 return -ENOMEM; 129 130 sg_set_buf(mem, buf, PAGE_SIZE << order); 131 BUG_ON(mem->offset); 132 sg_dma_len(mem) = PAGE_SIZE << order; 133 return 0; 134} 135 136struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, 137 gfp_t gfp_mask, int coherent) 138{ 139 struct mthca_icm *icm; 140 struct mthca_icm_chunk *chunk = NULL; 141 int cur_order; 142 int ret; 143 144 /* We use sg_set_buf for coherent allocs, which assumes low memory */ 145 BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); 146 147 icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 148 if (!icm) 149 return icm; 150 151 icm->refcount = 0; 152 INIT_LIST_HEAD(&icm->chunk_list); 153 154 cur_order = get_order(MTHCA_ICM_ALLOC_SIZE); 155 156 while (npages > 0) { 157 if (!chunk) { 158 chunk = kmalloc(sizeof *chunk, 159 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 160 if (!chunk) 161 goto fail; 162 163 sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN); 164 chunk->npages = 0; 165 chunk->nsg = 0; 166 list_add_tail(&chunk->list, &icm->chunk_list); 167 } 168 169 while (1 << cur_order > npages) 170 --cur_order; 171 172 if (coherent) 173 ret = mthca_alloc_icm_coherent(&dev->pdev->dev, 174 &chunk->mem[chunk->npages], 175 cur_order, gfp_mask); 176 else 177 ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages], 178 cur_order, gfp_mask); 179 180 if (!ret) { 181 ++chunk->npages; 182 183 if (coherent) 184 ++chunk->nsg; 185 else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) { 186 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 187 chunk->npages, 188 PCI_DMA_BIDIRECTIONAL); 189 190 if (chunk->nsg <= 0) 191 goto fail; 192 } 193 194 if (chunk->npages == MTHCA_ICM_CHUNK_LEN) 195 chunk = NULL; 196 197 npages -= 1 << cur_order; 198 } else { 199 --cur_order; 200 if (cur_order < 0) 201 goto fail; 202 } 203 } 204 205 if (!coherent && chunk) { 206 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, 207 chunk->npages, 208 PCI_DMA_BIDIRECTIONAL); 209 210 if (chunk->nsg <= 0) 211 goto fail; 212 } 213 214 return icm; 215 216fail: 217 mthca_free_icm(dev, icm, coherent); 218 return NULL; 219} 220 221int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 222{ 223 int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 224 int ret = 0; 225 u8 status; 226 227 mutex_lock(&table->mutex); 228 229 if (table->icm[i]) { 230 ++table->icm[i]->refcount; 231 goto out; 232 } 233 234 table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, 235 (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 236 __GFP_NOWARN, table->coherent); 237 if (!table->icm[i]) { 238 ret = -ENOMEM; 239 goto out; 240 } 241 242 if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 243 &status) || status) { 244 mthca_free_icm(dev, table->icm[i], table->coherent); 245 table->icm[i] = NULL; 246 ret = -ENOMEM; 247 goto out; 248 } 249 250 ++table->icm[i]->refcount; 251 252out: 253 mutex_unlock(&table->mutex); 254 return ret; 255} 256 257void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) 258{ 259 int i; 260 u8 status; 261 262 if (!mthca_is_memfree(dev)) 263 return; 264 265 i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; 266 267 mutex_lock(&table->mutex); 268 269 if (--table->icm[i]->refcount == 0) { 270 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 271 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, 272 &status); 273 mthca_free_icm(dev, table->icm[i], table->coherent); 274 table->icm[i] = NULL; 275 } 276 277 mutex_unlock(&table->mutex); 278} 279 280void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle) 281{ 282 int idx, offset, dma_offset, i; 283 struct mthca_icm_chunk *chunk; 284 struct mthca_icm *icm; 285 struct page *page = NULL; 286 287 if (!table->lowmem) 288 return NULL; 289 290 mutex_lock(&table->mutex); 291 292 idx = (obj & (table->num_obj - 1)) * table->obj_size; 293 icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE]; 294 dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE; 295 296 if (!icm) 297 goto out; 298 299 list_for_each_entry(chunk, &icm->chunk_list, list) { 300 for (i = 0; i < chunk->npages; ++i) { 301 if (dma_handle && dma_offset >= 0) { 302 if (sg_dma_len(&chunk->mem[i]) > dma_offset) 303 *dma_handle = sg_dma_address(&chunk->mem[i]) + 304 dma_offset; 305 dma_offset -= sg_dma_len(&chunk->mem[i]); 306 } 307 /* DMA mapping can merge pages but not split them, 308 * so if we found the page, dma_handle has already 309 * been assigned to. */ 310 if (chunk->mem[i].length > offset) { 311 page = sg_page(&chunk->mem[i]); 312 goto out; 313 } 314 offset -= chunk->mem[i].length; 315 } 316 } 317 318out: 319 mutex_unlock(&table->mutex); 320 return page ? lowmem_page_address(page) + offset : NULL; 321} 322 323int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, 324 int start, int end) 325{ 326 int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; 327 int i, err; 328 329 for (i = start; i <= end; i += inc) { 330 err = mthca_table_get(dev, table, i); 331 if (err) 332 goto fail; 333 } 334 335 return 0; 336 337fail: 338 while (i > start) { 339 i -= inc; 340 mthca_table_put(dev, table, i); 341 } 342 343 return err; 344} 345 346void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, 347 int start, int end) 348{ 349 int i; 350 351 if (!mthca_is_memfree(dev)) 352 return; 353 354 for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) 355 mthca_table_put(dev, table, i); 356} 357 358struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, 359 u64 virt, int obj_size, 360 int nobj, int reserved, 361 int use_lowmem, int use_coherent) 362{ 363 struct mthca_icm_table *table; 364 int obj_per_chunk; 365 int num_icm; 366 unsigned chunk_size; 367 int i; 368 u8 status; 369 370 obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size; 371 num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); 372 373 table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); 374 if (!table) 375 return NULL; 376 377 table->virt = virt; 378 table->num_icm = num_icm; 379 table->num_obj = nobj; 380 table->obj_size = obj_size; 381 table->lowmem = use_lowmem; 382 table->coherent = use_coherent; 383 mutex_init(&table->mutex); 384 385 for (i = 0; i < num_icm; ++i) 386 table->icm[i] = NULL; 387 388 for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { 389 chunk_size = MTHCA_TABLE_CHUNK_SIZE; 390 if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size) 391 chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE; 392 393 table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT, 394 (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | 395 __GFP_NOWARN, use_coherent); 396 if (!table->icm[i]) 397 goto err; 398 if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE, 399 &status) || status) { 400 mthca_free_icm(dev, table->icm[i], table->coherent); 401 table->icm[i] = NULL; 402 goto err; 403 } 404 405 /* 406 * Add a reference to this ICM chunk so that it never 407 * gets freed (since it contains reserved firmware objects). 408 */ 409 ++table->icm[i]->refcount; 410 } 411 412 return table; 413 414err: 415 for (i = 0; i < num_icm; ++i) 416 if (table->icm[i]) { 417 mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, 418 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, 419 &status); 420 mthca_free_icm(dev, table->icm[i], table->coherent); 421 } 422 423 kfree(table); 424 425 return NULL; 426} 427 428void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) 429{ 430 int i; 431 u8 status; 432 433 for (i = 0; i < table->num_icm; ++i) 434 if (table->icm[i]) { 435 mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, 436 MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, 437 &status); 438 mthca_free_icm(dev, table->icm[i], table->coherent); 439 } 440 441 kfree(table); 442} 443 444static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page) 445{ 446 return dev->uar_table.uarc_base + 447 uar->index * dev->uar_table.uarc_size + 448 page * MTHCA_ICM_PAGE_SIZE; 449} 450 451 452#include <vm/vm_map.h> 453#include <vm/vm_pageout.h> 454#include <vm/pmap.h> 455 456#include <sys/resource.h> 457#include <sys/resourcevar.h> 458 459int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, 460 struct mthca_user_db_table *db_tab, int index, u64 uaddr) 461{ 462#ifdef __linux__ 463 struct page *pages[1]; 464 int ret = 0; 465 u8 status; 466 int i; 467 468 if (!mthca_is_memfree(dev)) 469 return 0; 470 471 if (index < 0 || index > dev->uar_table.uarc_size / 8) 472 return -EINVAL; 473 474 mutex_lock(&db_tab->mutex); 475 476 i = index / MTHCA_DB_REC_PER_PAGE; 477 478 if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || 479 (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || 480 (uaddr & 4095)) { 481 ret = -EINVAL; 482 goto out; 483 } 484 485 if (db_tab->page[i].refcount) { 486 ++db_tab->page[i].refcount; 487 goto out; 488 } 489 490 ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0, 491 pages, NULL); 492 if (ret < 0) 493 goto out; 494 495 sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE, 496 uaddr & ~PAGE_MASK); 497 498 ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 499 if (ret < 0) { 500 put_page(pages[0]); 501 goto out; 502 } 503 504 ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), 505 mthca_uarc_virt(dev, uar, i), &status); 506 if (!ret && status) 507 ret = -EINVAL; 508 if (ret) { 509 pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 510 put_page(sg_page(&db_tab->page[i].mem)); 511 goto out; 512 } 513 514 db_tab->page[i].uvirt = uaddr; 515 db_tab->page[i].refcount = 1; 516 517out: 518 mutex_unlock(&db_tab->mutex); 519 return ret; 520#else 521 struct proc *proc; 522 vm_offset_t start; 523 vm_paddr_t paddr; 524 pmap_t pmap; 525 vm_page_t m; 526 int ret = 0; 527 u8 status; 528 int i; 529 530 if (!mthca_is_memfree(dev)) 531 return 0; 532 533 if (index < 0 || index > dev->uar_table.uarc_size / 8) 534 return -EINVAL; 535 536 mutex_lock(&db_tab->mutex); 537 538 i = index / MTHCA_DB_REC_PER_PAGE; 539 start = 0; 540 541 if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || 542 (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || 543 (uaddr & 4095)) { 544 ret = -EINVAL; 545 goto out; 546 } 547 548 if (db_tab->page[i].refcount) { 549 ++db_tab->page[i].refcount; 550 goto out; 551 } 552 553 proc = curproc; 554 pmap = vm_map_pmap(&proc->p_vmspace->vm_map); 555 PROC_LOCK(proc); 556 if (ptoa(pmap_wired_count(pmap) + 1) > lim_cur(proc, RLIMIT_MEMLOCK)) { 557 PROC_UNLOCK(proc); 558 ret = -ENOMEM; 559 goto out; 560 } 561 PROC_UNLOCK(proc); 562 if (cnt.v_wire_count + 1 > vm_page_max_wired) { 563 ret = -EAGAIN; 564 goto out; 565 } 566 start = uaddr & PAGE_MASK; 567 ret = vm_map_wire(&proc->p_vmspace->vm_map, start, start + PAGE_SIZE, 568 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES | VM_MAP_WIRE_WRITE); 569 if (ret != KERN_SUCCESS) { 570 start = 0; 571 ret = -ENOMEM; 572 goto out; 573 } 574 paddr = pmap_extract(pmap, uaddr); 575 if (paddr == 0) { 576 ret = -EFAULT; 577 goto out; 578 } 579 m = PHYS_TO_VM_PAGE(paddr); 580 581 sg_set_page(&db_tab->page[i].mem, m, MTHCA_ICM_PAGE_SIZE, 582 uaddr & ~PAGE_MASK); 583 584 ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 585 if (ret < 0) 586 goto out; 587 588 ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), 589 mthca_uarc_virt(dev, uar, i), &status); 590 if (!ret && status) 591 ret = -EINVAL; 592 if (ret) { 593 pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 594 goto out; 595 } 596 597 db_tab->page[i].uvirt = uaddr; 598 db_tab->page[i].refcount = 1; 599 600out: 601 if (ret < 0 && start) 602 vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, 603 start, start + PAGE_SIZE, 604 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 605 mutex_unlock(&db_tab->mutex); 606 return ret; 607#endif 608} 609 610void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar, 611 struct mthca_user_db_table *db_tab, int index) 612{ 613 if (!mthca_is_memfree(dev)) 614 return; 615 616 /* 617 * To make our bookkeeping simpler, we don't unmap DB 618 * pages until we clean up the whole db table. 619 */ 620 621 mutex_lock(&db_tab->mutex); 622 623 --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount; 624 625 mutex_unlock(&db_tab->mutex); 626} 627 628struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) 629{ 630 struct mthca_user_db_table *db_tab; 631 int npages; 632 int i; 633 634 if (!mthca_is_memfree(dev)) 635 return NULL; 636 637 npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; 638 db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL); 639 if (!db_tab) 640 return ERR_PTR(-ENOMEM); 641 642 mutex_init(&db_tab->mutex); 643 for (i = 0; i < npages; ++i) { 644 db_tab->page[i].refcount = 0; 645 db_tab->page[i].uvirt = 0; 646 sg_init_table(&db_tab->page[i].mem, 1); 647 } 648 649 return db_tab; 650} 651 652void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, 653 struct mthca_user_db_table *db_tab) 654{ 655 int i; 656 u8 status; 657 658 if (!mthca_is_memfree(dev)) 659 return; 660 661 for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) { 662 if (db_tab->page[i].uvirt) { 663 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status); 664 pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); 665#ifdef __linux__ 666 put_page(sg_page(&db_tab->page[i].mem)); 667#else 668 vm_offset_t start; 669 670 start = db_tab->page[i].uvirt & PAGE_MASK; 671 vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, 672 start, start + PAGE_SIZE, 673 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 674#endif 675 } 676 } 677 678 kfree(db_tab); 679} 680 681int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, 682 u32 qn, __be32 **db) 683{ 684 int group; 685 int start, end, dir; 686 int i, j; 687 struct mthca_db_page *page; 688 int ret = 0; 689 u8 status; 690 691 mutex_lock(&dev->db_tab->mutex); 692 693 switch (type) { 694 case MTHCA_DB_TYPE_CQ_ARM: 695 case MTHCA_DB_TYPE_SQ: 696 group = 0; 697 start = 0; 698 end = dev->db_tab->max_group1; 699 dir = 1; 700 break; 701 702 case MTHCA_DB_TYPE_CQ_SET_CI: 703 case MTHCA_DB_TYPE_RQ: 704 case MTHCA_DB_TYPE_SRQ: 705 group = 1; 706 start = dev->db_tab->npages - 1; 707 end = dev->db_tab->min_group2; 708 dir = -1; 709 break; 710 711 default: 712 ret = -EINVAL; 713 goto out; 714 } 715 716 for (i = start; i != end; i += dir) 717 if (dev->db_tab->page[i].db_rec && 718 !bitmap_full(dev->db_tab->page[i].used, 719 MTHCA_DB_REC_PER_PAGE)) { 720 page = dev->db_tab->page + i; 721 goto found; 722 } 723 724 for (i = start; i != end; i += dir) 725 if (!dev->db_tab->page[i].db_rec) { 726 page = dev->db_tab->page + i; 727 goto alloc; 728 } 729 730 if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) { 731 ret = -ENOMEM; 732 goto out; 733 } 734 735 if (group == 0) 736 ++dev->db_tab->max_group1; 737 else 738 --dev->db_tab->min_group2; 739 740 page = dev->db_tab->page + end; 741 742alloc: 743 page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 744 &page->mapping, GFP_KERNEL); 745 if (!page->db_rec) { 746 ret = -ENOMEM; 747 goto out; 748 } 749 memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE); 750 751 ret = mthca_MAP_ICM_page(dev, page->mapping, 752 mthca_uarc_virt(dev, &dev->driver_uar, i), &status); 753 if (!ret && status) 754 ret = -EINVAL; 755 if (ret) { 756 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 757 page->db_rec, page->mapping); 758 goto out; 759 } 760 761 bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE); 762 763found: 764 j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE); 765 set_bit(j, page->used); 766 767 if (group == 1) 768 j = MTHCA_DB_REC_PER_PAGE - 1 - j; 769 770 ret = i * MTHCA_DB_REC_PER_PAGE + j; 771 772 page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5)); 773 774 *db = (__be32 *) &page->db_rec[j]; 775 776out: 777 mutex_unlock(&dev->db_tab->mutex); 778 779 return ret; 780} 781 782void mthca_free_db(struct mthca_dev *dev, int type, int db_index) 783{ 784 int i, j; 785 struct mthca_db_page *page; 786 u8 status; 787 788 i = db_index / MTHCA_DB_REC_PER_PAGE; 789 j = db_index % MTHCA_DB_REC_PER_PAGE; 790 791 page = dev->db_tab->page + i; 792 793 mutex_lock(&dev->db_tab->mutex); 794 795 page->db_rec[j] = 0; 796 if (i >= dev->db_tab->min_group2) 797 j = MTHCA_DB_REC_PER_PAGE - 1 - j; 798 clear_bit(j, page->used); 799 800 if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && 801 i >= dev->db_tab->max_group1 - 1) { 802 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); 803 804 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 805 page->db_rec, page->mapping); 806 page->db_rec = NULL; 807 808 if (i == dev->db_tab->max_group1) { 809 --dev->db_tab->max_group1; 810 /* XXX may be able to unmap more pages now */ 811 } 812 if (i == dev->db_tab->min_group2) 813 ++dev->db_tab->min_group2; 814 } 815 816 mutex_unlock(&dev->db_tab->mutex); 817} 818 819int mthca_init_db_tab(struct mthca_dev *dev) 820{ 821 int i; 822 823 if (!mthca_is_memfree(dev)) 824 return 0; 825 826 dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL); 827 if (!dev->db_tab) 828 return -ENOMEM; 829 830 mutex_init(&dev->db_tab->mutex); 831 832 dev->db_tab->npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; 833 dev->db_tab->max_group1 = 0; 834 dev->db_tab->min_group2 = dev->db_tab->npages - 1; 835 836 dev->db_tab->page = kmalloc(dev->db_tab->npages * 837 sizeof *dev->db_tab->page, 838 GFP_KERNEL); 839 if (!dev->db_tab->page) { 840 kfree(dev->db_tab); 841 return -ENOMEM; 842 } 843 844 for (i = 0; i < dev->db_tab->npages; ++i) 845 dev->db_tab->page[i].db_rec = NULL; 846 847 return 0; 848} 849 850void mthca_cleanup_db_tab(struct mthca_dev *dev) 851{ 852 int i; 853 u8 status; 854 855 if (!mthca_is_memfree(dev)) 856 return; 857 858 /* 859 * Because we don't always free our UARC pages when they 860 * become empty to make mthca_free_db() simpler we need to 861 * make a sweep through the doorbell pages and free any 862 * leftover pages now. 863 */ 864 for (i = 0; i < dev->db_tab->npages; ++i) { 865 if (!dev->db_tab->page[i].db_rec) 866 continue; 867 868 if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) 869 mthca_warn(dev, "Kernel UARC page %d not empty\n", i); 870 871 mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); 872 873 dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, 874 dev->db_tab->page[i].db_rec, 875 dev->db_tab->page[i].mapping); 876 } 877 878 kfree(dev->db_tab->page); 879 kfree(dev->db_tab); 880} 881