1353944Sdim/* 2353944Sdim * Copyright (c) 2004 Topspin Communications. All rights reserved. 3353944Sdim * 4353944Sdim * This software is available to you under a choice of one of two 5353944Sdim * licenses. You may choose to be licensed under the terms of the GNU 6353944Sdim * General Public License (GPL) Version 2, available from the file 7353944Sdim * COPYING in the main directory of this source tree, or the 8353944Sdim * OpenIB.org BSD license below: 9353944Sdim * 10353944Sdim * Redistribution and use in source and binary forms, with or 11353944Sdim * without modification, are permitted provided that the following 12353944Sdim * conditions are met: 13353944Sdim * 14353944Sdim * - Redistributions of source code must retain the above 15353944Sdim * copyright notice, this list of conditions and the following 16353944Sdim * disclaimer. 17353944Sdim * 18353944Sdim * - Redistributions in binary form must reproduce the above 19353944Sdim * copyright notice, this list of conditions and the following 20353944Sdim * disclaimer in the documentation and/or other materials 21353944Sdim * provided with the distribution. 22353944Sdim * 23353944Sdim * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24353944Sdim * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25353944Sdim * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26353944Sdim * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27353944Sdim * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28353944Sdim * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29353944Sdim * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30353944Sdim * SOFTWARE. 31353944Sdim */ 32353944Sdim 33353944Sdim#include <linux/errno.h> 34353944Sdim#include <linux/slab.h> 35353944Sdim#include <linux/bitmap.h> 36353944Sdim 37353944Sdim#include "mthca_dev.h" 38353944Sdim 39353944Sdim/* Trivial bitmap-based allocator */ 40353944Sdimu32 mthca_alloc(struct mthca_alloc *alloc) 41353944Sdim{ 42353944Sdim unsigned long flags; 43353944Sdim u32 obj; 44353944Sdim 45353944Sdim spin_lock_irqsave(&alloc->lock, flags); 46353944Sdim 47353944Sdim obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last); 48353944Sdim if (obj >= alloc->max) { 49353944Sdim alloc->top = (alloc->top + alloc->max) & alloc->mask; 50353944Sdim obj = find_first_zero_bit(alloc->table, alloc->max); 51353944Sdim } 52353944Sdim 53353944Sdim if (obj < alloc->max) { 54353944Sdim __set_bit(obj, alloc->table); 55353944Sdim obj |= alloc->top; 56353944Sdim } else 57353944Sdim obj = -1; 58353944Sdim 59353944Sdim spin_unlock_irqrestore(&alloc->lock, flags); 60353944Sdim 61353944Sdim return obj; 62353944Sdim} 63353944Sdim 64353944Sdimvoid mthca_free(struct mthca_alloc *alloc, u32 obj) 65353944Sdim{ 66353944Sdim unsigned long flags; 67353944Sdim 68353944Sdim obj &= alloc->max - 1; 69353944Sdim 70353944Sdim spin_lock_irqsave(&alloc->lock, flags); 71353944Sdim 72353944Sdim __clear_bit(obj, alloc->table); 73353944Sdim alloc->last = min(alloc->last, obj); 74353944Sdim alloc->top = (alloc->top + alloc->max) & alloc->mask; 75353944Sdim 76353944Sdim spin_unlock_irqrestore(&alloc->lock, flags); 77353944Sdim} 78353944Sdim 79353944Sdimint mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, 80353944Sdim u32 reserved) 81353944Sdim{ 82353944Sdim /* num must be a power of 2 */ 83353944Sdim if (num != 1 << (ffs(num) - 1)) 84353944Sdim return -EINVAL; 85353944Sdim 86353944Sdim alloc->last = 0; 87353944Sdim alloc->top = 0; 88353944Sdim alloc->max = num; 89353944Sdim alloc->mask = mask; 90353944Sdim spin_lock_init(&alloc->lock); 91353944Sdim alloc->table = bitmap_zalloc(num, GFP_KERNEL); 92353944Sdim if (!alloc->table) 93353944Sdim return -ENOMEM; 94353944Sdim 95353944Sdim bitmap_set(alloc->table, 0, reserved); 96353944Sdim 97353944Sdim return 0; 98353944Sdim} 99353944Sdim 100353944Sdimvoid mthca_alloc_cleanup(struct mthca_alloc *alloc) 101353944Sdim{ 102353944Sdim bitmap_free(alloc->table); 103353944Sdim} 104353944Sdim 105353944Sdim/* 106353944Sdim * Array of pointers with lazy allocation of leaf pages. Callers of 107353944Sdim * _get, _set and _clear methods must use a lock or otherwise 108353944Sdim * serialize access to the array. 109353944Sdim */ 110353944Sdim 111353944Sdim#define MTHCA_ARRAY_MASK (PAGE_SIZE / sizeof (void *) - 1) 112353944Sdim 113353944Sdimvoid *mthca_array_get(struct mthca_array *array, int index) 114353944Sdim{ 115353944Sdim int p = (index * sizeof (void *)) >> PAGE_SHIFT; 116353944Sdim 117353944Sdim if (array->page_list[p].page) 118353944Sdim return array->page_list[p].page[index & MTHCA_ARRAY_MASK]; 119353944Sdim else 120353944Sdim return NULL; 121353944Sdim} 122353944Sdim 123353944Sdimint mthca_array_set(struct mthca_array *array, int index, void *value) 124353944Sdim{ 125353944Sdim int p = (index * sizeof (void *)) >> PAGE_SHIFT; 126353944Sdim 127353944Sdim /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ 128353944Sdim if (!array->page_list[p].page) 129353944Sdim array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); 130353944Sdim 131 if (!array->page_list[p].page) 132 return -ENOMEM; 133 134 array->page_list[p].page[index & MTHCA_ARRAY_MASK] = value; 135 ++array->page_list[p].used; 136 137 return 0; 138} 139 140void mthca_array_clear(struct mthca_array *array, int index) 141{ 142 int p = (index * sizeof (void *)) >> PAGE_SHIFT; 143 144 if (--array->page_list[p].used == 0) { 145 free_page((unsigned long) array->page_list[p].page); 146 array->page_list[p].page = NULL; 147 } else 148 array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; 149 150 if (array->page_list[p].used < 0) 151 pr_debug("Array %p index %d page %d with ref count %d < 0\n", 152 array, index, p, array->page_list[p].used); 153} 154 155int mthca_array_init(struct mthca_array *array, int nent) 156{ 157 int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; 158 int i; 159 160 array->page_list = kmalloc_array(npage, sizeof(*array->page_list), 161 GFP_KERNEL); 162 if (!array->page_list) 163 return -ENOMEM; 164 165 for (i = 0; i < npage; ++i) { 166 array->page_list[i].page = NULL; 167 array->page_list[i].used = 0; 168 } 169 170 return 0; 171} 172 173void mthca_array_cleanup(struct mthca_array *array, int nent) 174{ 175 int i; 176 177 for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 178 free_page((unsigned long) array->page_list[i].page); 179 180 kfree(array->page_list); 181} 182 183/* 184 * Handling for queue buffers -- we allocate a bunch of memory and 185 * register it in a memory region at HCA virtual address 0. If the 186 * requested size is > max_direct, we split the allocation into 187 * multiple pages, so we don't require too much contiguous memory. 188 */ 189 190int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, 191 union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, 192 int hca_write, struct mthca_mr *mr) 193{ 194 int err = -ENOMEM; 195 int npages, shift; 196 u64 *dma_list = NULL; 197 dma_addr_t t; 198 int i; 199 200 if (size <= max_direct) { 201 *is_direct = 1; 202 npages = 1; 203 shift = get_order(size) + PAGE_SHIFT; 204 205 buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, 206 size, &t, GFP_KERNEL); 207 if (!buf->direct.buf) 208 return -ENOMEM; 209 210 dma_unmap_addr_set(&buf->direct, mapping, t); 211 212 while (t & ((1 << shift) - 1)) { 213 --shift; 214 npages *= 2; 215 } 216 217 dma_list = kmalloc_array(npages, sizeof(*dma_list), 218 GFP_KERNEL); 219 if (!dma_list) 220 goto err_free; 221 222 for (i = 0; i < npages; ++i) 223 dma_list[i] = t + i * (1 << shift); 224 } else { 225 *is_direct = 0; 226 npages = (size + PAGE_SIZE - 1) / PAGE_SIZE; 227 shift = PAGE_SHIFT; 228 229 dma_list = kmalloc_array(npages, sizeof(*dma_list), 230 GFP_KERNEL); 231 if (!dma_list) 232 return -ENOMEM; 233 234 buf->page_list = kmalloc_array(npages, 235 sizeof(*buf->page_list), 236 GFP_KERNEL); 237 if (!buf->page_list) 238 goto err_out; 239 240 for (i = 0; i < npages; ++i) 241 buf->page_list[i].buf = NULL; 242 243 for (i = 0; i < npages; ++i) { 244 buf->page_list[i].buf = 245 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, 246 &t, GFP_KERNEL); 247 if (!buf->page_list[i].buf) 248 goto err_free; 249 250 dma_list[i] = t; 251 dma_unmap_addr_set(&buf->page_list[i], mapping, t); 252 253 clear_page(buf->page_list[i].buf); 254 } 255 } 256 257 err = mthca_mr_alloc_phys(dev, pd->pd_num, 258 dma_list, shift, npages, 259 0, size, 260 MTHCA_MPT_FLAG_LOCAL_READ | 261 (hca_write ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0), 262 mr); 263 if (err) 264 goto err_free; 265 266 kfree(dma_list); 267 268 return 0; 269 270err_free: 271 mthca_buf_free(dev, size, buf, *is_direct, NULL); 272 273err_out: 274 kfree(dma_list); 275 276 return err; 277} 278 279void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, 280 int is_direct, struct mthca_mr *mr) 281{ 282 int i; 283 284 if (mr) 285 mthca_free_mr(dev, mr); 286 287 if (is_direct) 288 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, 289 dma_unmap_addr(&buf->direct, mapping)); 290 else { 291 for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) 292 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, 293 buf->page_list[i].buf, 294 dma_unmap_addr(&buf->page_list[i], 295 mapping)); 296 kfree(buf->page_list); 297 } 298} 299