1139804Simp/*- 2117624Sharti * Copyright (c) 2003 3117624Sharti * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4117624Sharti * All rights reserved. 5117624Sharti * 6117624Sharti * Redistribution and use in source and binary forms, with or without 7117624Sharti * modification, are permitted provided that the following conditions 8117624Sharti * are met: 9117624Sharti * 1. Redistributions of source code must retain the above copyright 10117624Sharti * notice, this list of conditions and the following disclaimer. 11117624Sharti * 2. Redistributions in binary form must reproduce the above copyright 12117624Sharti * notice, this list of conditions and the following disclaimer in the 13117624Sharti * documentation and/or other materials provided with the distribution. 14117624Sharti * 15117624Sharti * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16117624Sharti * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17117624Sharti * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18117624Sharti * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19117624Sharti * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20117624Sharti * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21117624Sharti * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22117624Sharti * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23117624Sharti * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24117624Sharti * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25117624Sharti * SUCH DAMAGE. 26117624Sharti * 27117624Sharti * Author: Hartmut Brandt <harti@freebsd.org> 28117624Sharti */ 29117624Sharti 30117624Sharti#include <sys/cdefs.h> 31117624Sharti__FBSDID("$FreeBSD$"); 32117624Sharti 33117624Sharti#include <sys/param.h> 34117624Sharti#include <sys/lock.h> 35117624Sharti#include <sys/mutex.h> 36117624Sharti#include <sys/kernel.h> 37117624Sharti#include <sys/systm.h> 38117624Sharti#include <sys/malloc.h> 39117624Sharti#include <sys/module.h> 40117624Sharti 41117624Sharti#include <machine/bus.h> 42117624Sharti 43117624Sharti#include <sys/mbpool.h> 44117624Sharti 45117624ShartiMODULE_VERSION(libmbpool, 1); 46117624Sharti 47117624Sharti/* 48117624Sharti * Memory is allocated as DMA-able pages. Each page is divided into a number 49117624Sharti * of equal chunks where the last 4 bytes of each chunk are occupied by 50117624Sharti * the page number and the chunk number. The caller must take these four 51117624Sharti * bytes into account when specifying the chunk size. Each page is mapped by 52117624Sharti * its own DMA map using the user specified DMA tag. 53117624Sharti * 54117624Sharti * Each chunk has a used and a card bit in the high bits of its page number. 55117624Sharti * 0 0 chunk is free and may be allocated 56117624Sharti * 1 1 chunk has been given to the interface 57117624Sharti * 0 1 chunk is traveling through the system 58117624Sharti * 1 0 illegal 59117624Sharti */ 60117624Shartistruct mbtrail { 61117624Sharti uint16_t chunk; 62117624Sharti uint16_t page; 63117624Sharti}; 64117624Sharti#define MBP_CARD 0x8000 65117624Sharti#define MBP_USED 0x4000 66117624Sharti#define MBP_PMSK 0x3fff /* page number mask */ 67117624Sharti#define MBP_CMSK 0x01ff /* chunk number mask */ 68117624Sharti 69117624Shartistruct mbfree { 70117624Sharti SLIST_ENTRY(mbfree) link; /* link on free list */ 71117624Sharti}; 72117624Sharti 73117624Shartistruct mbpage { 74117624Sharti bus_dmamap_t map; /* map for this page */ 75117624Sharti bus_addr_t phy; /* physical address */ 76117624Sharti void *va; /* the memory */ 77117624Sharti}; 78117624Sharti 79117624Shartistruct mbpool { 80117624Sharti const char *name; /* a name for this pool */ 81117624Sharti bus_dma_tag_t dmat; /* tag for mapping */ 82117624Sharti u_int max_pages; /* maximum number of pages */ 83117624Sharti size_t page_size; /* size of each allocation */ 84117624Sharti size_t chunk_size; /* size of each external mbuf */ 85117624Sharti 86117624Sharti struct mtx free_lock; /* lock of free list */ 87117624Sharti SLIST_HEAD(, mbfree) free_list; /* free list */ 88117624Sharti u_int npages; /* current number of pages */ 89117624Sharti u_int nchunks; /* chunks per page */ 90117624Sharti struct mbpage pages[]; /* pages */ 91117624Sharti}; 92117624Sharti 93117624Shartistatic MALLOC_DEFINE(M_MBPOOL, "mbpools", "mbuf pools"); 94117624Sharti 95117624Sharti/* 96117624Sharti * Make a trail pointer from a chunk pointer 97117624Sharti */ 98117624Sharti#define C2T(P, C) ((struct mbtrail *)((char *)(C) + (P)->chunk_size - \ 99117624Sharti sizeof(struct mbtrail))) 100117624Sharti 101117624Sharti/* 102117624Sharti * Make a free chunk pointer from a chunk number 103117624Sharti */ 104117624Sharti#define N2C(P, PG, C) ((struct mbfree *)((char *)(PG)->va + \ 105117624Sharti (C) * (P)->chunk_size)) 106117624Sharti 107117624Sharti/* 108117624Sharti * Make/parse handles 109117624Sharti */ 110117624Sharti#define HMAKE(P, C) ((((P) & MBP_PMSK) << 16) | ((C) << 7)) 111117624Sharti#define HPAGE(H) (((H) >> 16) & MBP_PMSK) 112117624Sharti#define HCHUNK(H) (((H) >> 7) & MBP_CMSK) 113117624Sharti 114117624Sharti/* 115117624Sharti * initialize a pool 116117624Sharti */ 117117624Shartiint 118117624Shartimbp_create(struct mbpool **pp, const char *name, bus_dma_tag_t dmat, 119117624Sharti u_int max_pages, size_t page_size, size_t chunk_size) 120117624Sharti{ 121117624Sharti u_int nchunks; 122117624Sharti 123117624Sharti if (max_pages > MBPOOL_MAX_MAXPAGES || chunk_size == 0) 124117624Sharti return (EINVAL); 125117624Sharti nchunks = page_size / chunk_size; 126117624Sharti if (nchunks == 0 || nchunks > MBPOOL_MAX_CHUNKS) 127117624Sharti return (EINVAL); 128117624Sharti 129117624Sharti (*pp) = malloc(sizeof(struct mbpool) + 130117624Sharti max_pages * sizeof(struct mbpage), 131117624Sharti M_MBPOOL, M_WAITOK | M_ZERO); 132117624Sharti 133117624Sharti (*pp)->name = name; 134117624Sharti (*pp)->dmat = dmat; 135117624Sharti (*pp)->max_pages = max_pages; 136117624Sharti (*pp)->page_size = page_size; 137117624Sharti (*pp)->chunk_size = chunk_size; 138117624Sharti (*pp)->nchunks = nchunks; 139117624Sharti 140117624Sharti SLIST_INIT(&(*pp)->free_list); 141123263Struckman mtx_init(&(*pp)->free_lock, name, NULL, MTX_DEF); 142117624Sharti 143117624Sharti return (0); 144117624Sharti} 145117624Sharti 146117624Sharti/* 147117624Sharti * destroy a pool 148117624Sharti */ 149117624Shartivoid 150117624Shartimbp_destroy(struct mbpool *p) 151117624Sharti{ 152117624Sharti u_int i; 153117624Sharti struct mbpage *pg; 154117624Sharti#ifdef DIAGNOSTIC 155117624Sharti struct mbtrail *tr; 156117624Sharti u_int b; 157117624Sharti#endif 158117624Sharti 159117624Sharti for (i = 0; i < p->npages; i++) { 160117624Sharti pg = &p->pages[i]; 161117624Sharti#ifdef DIAGNOSTIC 162117624Sharti for (b = 0; b < p->nchunks; b++) { 163117624Sharti tr = C2T(p, N2C(p, pg, b)); 164117624Sharti if (tr->page & MBP_CARD) 165117624Sharti printf("%s: (%s) buf still on card" 166117624Sharti " %u/%u\n", __func__, p->name, i, b); 167117624Sharti if (tr->page & MBP_USED) 168117624Sharti printf("%s: (%s) sbuf still in use" 169117624Sharti " %u/%u\n", __func__, p->name, i, b); 170117624Sharti } 171117624Sharti#endif 172117624Sharti bus_dmamap_unload(p->dmat, pg->map); 173117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 174117624Sharti } 175117624Sharti mtx_destroy(&p->free_lock); 176117624Sharti 177117624Sharti free(p, M_MBPOOL); 178117624Sharti} 179117624Sharti 180117624Sharti/* 181117624Sharti * Helper function when loading a one segment DMA buffer. 182117624Sharti */ 183117624Shartistatic void 184117624Shartimbp_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 185117624Sharti{ 186117624Sharti if (error == 0) 187117624Sharti *(bus_addr_t *)arg = segs[0].ds_addr; 188117624Sharti} 189117624Sharti 190117624Sharti/* 191117624Sharti * Allocate a new page 192117624Sharti */ 193117624Shartistatic void 194117624Shartimbp_alloc_page(struct mbpool *p) 195117624Sharti{ 196117624Sharti int error; 197117624Sharti struct mbpage *pg; 198117624Sharti u_int i; 199117624Sharti struct mbfree *f; 200117624Sharti struct mbtrail *t; 201117624Sharti 202117624Sharti if (p->npages == p->max_pages) { 203117624Sharti#ifdef DIAGNOSTIC 204117624Sharti printf("%s: (%s) page limit reached %u\n", __func__, 205117624Sharti p->name, p->max_pages); 206117624Sharti#endif 207117624Sharti return; 208117624Sharti } 209117624Sharti pg = &p->pages[p->npages]; 210117624Sharti 211117624Sharti error = bus_dmamem_alloc(p->dmat, &pg->va, BUS_DMA_NOWAIT, &pg->map); 212117624Sharti if (error != 0) { 213117624Sharti free(pg, M_MBPOOL); 214117624Sharti return; 215117624Sharti } 216117624Sharti 217117624Sharti error = bus_dmamap_load(p->dmat, pg->map, pg->va, p->page_size, 218117624Sharti mbp_callback, &pg->phy, 0); 219117624Sharti if (error != 0) { 220117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 221117624Sharti free(pg, M_MBPOOL); 222117624Sharti return; 223117624Sharti } 224117624Sharti 225117624Sharti for (i = 0; i < p->nchunks; i++) { 226117624Sharti f = N2C(p, pg, i); 227117624Sharti t = C2T(p, f); 228117624Sharti t->page = p->npages; 229117624Sharti t->chunk = i; 230117624Sharti SLIST_INSERT_HEAD(&p->free_list, f, link); 231117624Sharti } 232117624Sharti 233117624Sharti p->npages++; 234117624Sharti} 235117624Sharti 236117624Sharti/* 237117624Sharti * allocate a chunk 238117624Sharti */ 239117624Shartivoid * 240117624Shartimbp_alloc(struct mbpool *p, bus_addr_t *pap, uint32_t *hp) 241117624Sharti{ 242117624Sharti struct mbfree *cf; 243117624Sharti struct mbtrail *t; 244117624Sharti 245117624Sharti mtx_lock(&p->free_lock); 246117624Sharti if ((cf = SLIST_FIRST(&p->free_list)) == NULL) { 247117624Sharti mbp_alloc_page(p); 248117624Sharti cf = SLIST_FIRST(&p->free_list); 249117624Sharti } 250117624Sharti if (cf == NULL) { 251117624Sharti mtx_unlock(&p->free_lock); 252117624Sharti return (NULL); 253117624Sharti } 254117624Sharti SLIST_REMOVE_HEAD(&p->free_list, link); 255117624Sharti mtx_unlock(&p->free_lock); 256117624Sharti 257117624Sharti t = C2T(p, cf); 258117624Sharti 259117624Sharti *pap = p->pages[t->page].phy + t->chunk * p->chunk_size; 260117624Sharti *hp = HMAKE(t->page, t->chunk); 261117624Sharti 262117624Sharti t->page |= MBP_CARD | MBP_USED; 263117624Sharti 264117624Sharti return (cf); 265117624Sharti} 266117624Sharti 267117624Sharti/* 268117624Sharti * Free a chunk 269117624Sharti */ 270117624Shartivoid 271117624Shartimbp_free(struct mbpool *p, void *ptr) 272117624Sharti{ 273117624Sharti struct mbtrail *t; 274117624Sharti 275117624Sharti mtx_lock(&p->free_lock); 276117624Sharti t = C2T(p, ptr); 277117624Sharti t->page &= ~(MBP_USED | MBP_CARD); 278117624Sharti SLIST_INSERT_HEAD(&p->free_list, (struct mbfree *)ptr, link); 279117624Sharti mtx_unlock(&p->free_lock); 280117624Sharti} 281117624Sharti 282117624Sharti/* 283117624Sharti * Mbuf system external mbuf free routine 284117624Sharti */ 285117624Shartivoid 286117624Shartimbp_ext_free(void *buf, void *arg) 287117624Sharti{ 288117624Sharti mbp_free(arg, buf); 289117624Sharti} 290117624Sharti 291117624Sharti/* 292117624Sharti * Free all buffers that are marked as beeing on the card 293117624Sharti */ 294117624Shartivoid 295117624Shartimbp_card_free(struct mbpool *p) 296117624Sharti{ 297117624Sharti u_int i, b; 298117624Sharti struct mbpage *pg; 299117624Sharti struct mbtrail *tr; 300117624Sharti struct mbfree *cf; 301117624Sharti 302117624Sharti mtx_lock(&p->free_lock); 303117624Sharti for (i = 0; i < p->npages; i++) { 304117624Sharti pg = &p->pages[i]; 305117624Sharti for (b = 0; b < p->nchunks; b++) { 306117624Sharti cf = N2C(p, pg, b); 307117624Sharti tr = C2T(p, cf); 308117624Sharti if (tr->page & MBP_CARD) { 309117624Sharti tr->page &= MBP_PMSK; 310117624Sharti SLIST_INSERT_HEAD(&p->free_list, cf, link); 311117624Sharti } 312117624Sharti } 313117624Sharti } 314117624Sharti mtx_unlock(&p->free_lock); 315117624Sharti} 316117624Sharti 317117624Sharti/* 318117624Sharti * Count buffers 319117624Sharti */ 320117624Shartivoid 321117624Shartimbp_count(struct mbpool *p, u_int *used, u_int *card, u_int *free) 322117624Sharti{ 323117624Sharti u_int i, b; 324117624Sharti struct mbpage *pg; 325117624Sharti struct mbtrail *tr; 326117624Sharti struct mbfree *cf; 327117624Sharti 328117624Sharti *used = *card = *free = 0; 329117624Sharti for (i = 0; i < p->npages; i++) { 330117624Sharti pg = &p->pages[i]; 331117624Sharti for (b = 0; b < p->nchunks; b++) { 332117624Sharti tr = C2T(p, N2C(p, pg, b)); 333117624Sharti if (tr->page & MBP_CARD) 334117624Sharti (*card)++; 335117624Sharti if (tr->page & MBP_USED) 336117624Sharti (*used)++; 337117624Sharti } 338117624Sharti } 339117624Sharti mtx_lock(&p->free_lock); 340117624Sharti SLIST_FOREACH(cf, &p->free_list, link) 341170023Srwatson (*free)++; 342117624Sharti mtx_unlock(&p->free_lock); 343117624Sharti} 344117624Sharti 345117624Sharti/* 346117624Sharti * Get the buffer from a handle and clear the card flag. 347117624Sharti */ 348117624Shartivoid * 349117624Shartimbp_get(struct mbpool *p, uint32_t h) 350117624Sharti{ 351117624Sharti struct mbfree *cf; 352117624Sharti struct mbtrail *tr; 353117624Sharti 354117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 355117624Sharti tr = C2T(p, cf); 356117624Sharti 357117624Sharti#ifdef DIAGNOSTIC 358117624Sharti if (!(tr->page & MBP_CARD)) 359117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 360117624Sharti p->name, HCHUNK(h), HPAGE(h)); 361117624Sharti#endif 362117624Sharti 363117624Sharti tr->page &= ~MBP_CARD; 364117624Sharti return (cf); 365117624Sharti} 366117624Sharti 367117624Sharti/* 368117624Sharti * Get the buffer from a handle and keep the card flag. 369117624Sharti */ 370117624Shartivoid * 371117624Shartimbp_get_keep(struct mbpool *p, uint32_t h) 372117624Sharti{ 373117624Sharti struct mbfree *cf; 374117624Sharti struct mbtrail *tr; 375117624Sharti 376117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 377117624Sharti tr = C2T(p, cf); 378117624Sharti 379117624Sharti#ifdef DIAGNOSTIC 380117624Sharti if (!(tr->page & MBP_CARD)) 381117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 382117624Sharti p->name, HCHUNK(h), HPAGE(h)); 383117624Sharti#endif 384117624Sharti 385117624Sharti return (cf); 386117624Sharti} 387117624Sharti 388117624Sharti/* 389117624Sharti * sync the chunk 390117624Sharti */ 391117624Shartivoid 392117624Shartimbp_sync(struct mbpool *p, uint32_t h, bus_addr_t off, bus_size_t len, u_int op) 393117624Sharti{ 394117624Sharti 395117624Sharti#if 0 396117624Sharti bus_dmamap_sync_size(p->dmat, p->pages[HPAGE(h)].map, 397117624Sharti HCHUNK(h) * p->chunk_size + off, len, op); 398117624Sharti#endif 399117624Sharti} 400