1139804Simp/*- 2117624Sharti * Copyright (c) 2003 3117624Sharti * Fraunhofer Institute for Open Communication Systems (FhG Fokus). 4117624Sharti * All rights reserved. 5117624Sharti * 6117624Sharti * Redistribution and use in source and binary forms, with or without 7117624Sharti * modification, are permitted provided that the following conditions 8117624Sharti * are met: 9117624Sharti * 1. Redistributions of source code must retain the above copyright 10117624Sharti * notice, this list of conditions and the following disclaimer. 11117624Sharti * 2. Redistributions in binary form must reproduce the above copyright 12117624Sharti * notice, this list of conditions and the following disclaimer in the 13117624Sharti * documentation and/or other materials provided with the distribution. 14117624Sharti * 15117624Sharti * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16117624Sharti * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17117624Sharti * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18117624Sharti * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19117624Sharti * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20117624Sharti * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21117624Sharti * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22117624Sharti * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23117624Sharti * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24117624Sharti * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25117624Sharti * SUCH DAMAGE. 26117624Sharti * 27117624Sharti * Author: Hartmut Brandt <harti@freebsd.org> 28117624Sharti */ 29117624Sharti 30117624Sharti#include <sys/cdefs.h> 31117624Sharti__FBSDID("$FreeBSD$"); 32117624Sharti 33117624Sharti#include <sys/param.h> 34117624Sharti#include <sys/lock.h> 35117624Sharti#include <sys/mutex.h> 36117624Sharti#include <sys/kernel.h> 37117624Sharti#include <sys/systm.h> 38117624Sharti#include <sys/malloc.h> 39117624Sharti#include <sys/module.h> 40117624Sharti 41117624Sharti#include <machine/bus.h> 42117624Sharti 43254842Sandre#include <sys/mbuf.h> 44117624Sharti#include <sys/mbpool.h> 45117624Sharti 46117624ShartiMODULE_VERSION(libmbpool, 1); 47117624Sharti 48117624Sharti/* 49117624Sharti * Memory is allocated as DMA-able pages. Each page is divided into a number 50117624Sharti * of equal chunks where the last 4 bytes of each chunk are occupied by 51117624Sharti * the page number and the chunk number. The caller must take these four 52117624Sharti * bytes into account when specifying the chunk size. Each page is mapped by 53117624Sharti * its own DMA map using the user specified DMA tag. 54117624Sharti * 55117624Sharti * Each chunk has a used and a card bit in the high bits of its page number. 56117624Sharti * 0 0 chunk is free and may be allocated 57117624Sharti * 1 1 chunk has been given to the interface 58117624Sharti * 0 1 chunk is traveling through the system 59117624Sharti * 1 0 illegal 60117624Sharti */ 61117624Shartistruct mbtrail { 62117624Sharti uint16_t chunk; 63117624Sharti uint16_t page; 64117624Sharti}; 65117624Sharti#define MBP_CARD 0x8000 66117624Sharti#define MBP_USED 0x4000 67117624Sharti#define MBP_PMSK 0x3fff /* page number mask */ 68117624Sharti#define MBP_CMSK 0x01ff /* chunk number mask */ 69117624Sharti 70117624Shartistruct mbfree { 71117624Sharti SLIST_ENTRY(mbfree) link; /* link on free list */ 72117624Sharti}; 73117624Sharti 74117624Shartistruct mbpage { 75117624Sharti bus_dmamap_t map; /* map for this page */ 76117624Sharti bus_addr_t phy; /* physical address */ 77117624Sharti void *va; /* the memory */ 78117624Sharti}; 79117624Sharti 80117624Shartistruct mbpool { 81117624Sharti const char *name; /* a name for this pool */ 82117624Sharti bus_dma_tag_t dmat; /* tag for mapping */ 83117624Sharti u_int max_pages; /* maximum number of pages */ 84117624Sharti size_t page_size; /* size of each allocation */ 85117624Sharti size_t chunk_size; /* size of each external mbuf */ 86117624Sharti 87117624Sharti struct mtx free_lock; /* lock of free list */ 88117624Sharti SLIST_HEAD(, mbfree) free_list; /* free list */ 89117624Sharti u_int npages; /* current number of pages */ 90117624Sharti u_int nchunks; /* chunks per page */ 91117624Sharti struct mbpage pages[]; /* pages */ 92117624Sharti}; 93117624Sharti 94117624Shartistatic MALLOC_DEFINE(M_MBPOOL, "mbpools", "mbuf pools"); 95117624Sharti 96117624Sharti/* 97117624Sharti * Make a trail pointer from a chunk pointer 98117624Sharti */ 99117624Sharti#define C2T(P, C) ((struct mbtrail *)((char *)(C) + (P)->chunk_size - \ 100117624Sharti sizeof(struct mbtrail))) 101117624Sharti 102117624Sharti/* 103117624Sharti * Make a free chunk pointer from a chunk number 104117624Sharti */ 105117624Sharti#define N2C(P, PG, C) ((struct mbfree *)((char *)(PG)->va + \ 106117624Sharti (C) * (P)->chunk_size)) 107117624Sharti 108117624Sharti/* 109117624Sharti * Make/parse handles 110117624Sharti */ 111117624Sharti#define HMAKE(P, C) ((((P) & MBP_PMSK) << 16) | ((C) << 7)) 112117624Sharti#define HPAGE(H) (((H) >> 16) & MBP_PMSK) 113117624Sharti#define HCHUNK(H) (((H) >> 7) & MBP_CMSK) 114117624Sharti 115117624Sharti/* 116117624Sharti * initialize a pool 117117624Sharti */ 118117624Shartiint 119117624Shartimbp_create(struct mbpool **pp, const char *name, bus_dma_tag_t dmat, 120117624Sharti u_int max_pages, size_t page_size, size_t chunk_size) 121117624Sharti{ 122117624Sharti u_int nchunks; 123117624Sharti 124117624Sharti if (max_pages > MBPOOL_MAX_MAXPAGES || chunk_size == 0) 125117624Sharti return (EINVAL); 126117624Sharti nchunks = page_size / chunk_size; 127117624Sharti if (nchunks == 0 || nchunks > MBPOOL_MAX_CHUNKS) 128117624Sharti return (EINVAL); 129117624Sharti 130117624Sharti (*pp) = malloc(sizeof(struct mbpool) + 131117624Sharti max_pages * sizeof(struct mbpage), 132117624Sharti M_MBPOOL, M_WAITOK | M_ZERO); 133117624Sharti 134117624Sharti (*pp)->name = name; 135117624Sharti (*pp)->dmat = dmat; 136117624Sharti (*pp)->max_pages = max_pages; 137117624Sharti (*pp)->page_size = page_size; 138117624Sharti (*pp)->chunk_size = chunk_size; 139117624Sharti (*pp)->nchunks = nchunks; 140117624Sharti 141117624Sharti SLIST_INIT(&(*pp)->free_list); 142123263Struckman mtx_init(&(*pp)->free_lock, name, NULL, MTX_DEF); 143117624Sharti 144117624Sharti return (0); 145117624Sharti} 146117624Sharti 147117624Sharti/* 148117624Sharti * destroy a pool 149117624Sharti */ 150117624Shartivoid 151117624Shartimbp_destroy(struct mbpool *p) 152117624Sharti{ 153117624Sharti u_int i; 154117624Sharti struct mbpage *pg; 155117624Sharti#ifdef DIAGNOSTIC 156117624Sharti struct mbtrail *tr; 157117624Sharti u_int b; 158117624Sharti#endif 159117624Sharti 160117624Sharti for (i = 0; i < p->npages; i++) { 161117624Sharti pg = &p->pages[i]; 162117624Sharti#ifdef DIAGNOSTIC 163117624Sharti for (b = 0; b < p->nchunks; b++) { 164117624Sharti tr = C2T(p, N2C(p, pg, b)); 165117624Sharti if (tr->page & MBP_CARD) 166117624Sharti printf("%s: (%s) buf still on card" 167117624Sharti " %u/%u\n", __func__, p->name, i, b); 168117624Sharti if (tr->page & MBP_USED) 169117624Sharti printf("%s: (%s) sbuf still in use" 170117624Sharti " %u/%u\n", __func__, p->name, i, b); 171117624Sharti } 172117624Sharti#endif 173117624Sharti bus_dmamap_unload(p->dmat, pg->map); 174117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 175117624Sharti } 176117624Sharti mtx_destroy(&p->free_lock); 177117624Sharti 178117624Sharti free(p, M_MBPOOL); 179117624Sharti} 180117624Sharti 181117624Sharti/* 182117624Sharti * Helper function when loading a one segment DMA buffer. 183117624Sharti */ 184117624Shartistatic void 185117624Shartimbp_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 186117624Sharti{ 187117624Sharti if (error == 0) 188117624Sharti *(bus_addr_t *)arg = segs[0].ds_addr; 189117624Sharti} 190117624Sharti 191117624Sharti/* 192117624Sharti * Allocate a new page 193117624Sharti */ 194117624Shartistatic void 195117624Shartimbp_alloc_page(struct mbpool *p) 196117624Sharti{ 197117624Sharti int error; 198117624Sharti struct mbpage *pg; 199117624Sharti u_int i; 200117624Sharti struct mbfree *f; 201117624Sharti struct mbtrail *t; 202117624Sharti 203117624Sharti if (p->npages == p->max_pages) { 204117624Sharti#ifdef DIAGNOSTIC 205117624Sharti printf("%s: (%s) page limit reached %u\n", __func__, 206117624Sharti p->name, p->max_pages); 207117624Sharti#endif 208117624Sharti return; 209117624Sharti } 210117624Sharti pg = &p->pages[p->npages]; 211117624Sharti 212117624Sharti error = bus_dmamem_alloc(p->dmat, &pg->va, BUS_DMA_NOWAIT, &pg->map); 213299625Sngie if (error != 0) 214117624Sharti return; 215117624Sharti 216117624Sharti error = bus_dmamap_load(p->dmat, pg->map, pg->va, p->page_size, 217117624Sharti mbp_callback, &pg->phy, 0); 218117624Sharti if (error != 0) { 219117624Sharti bus_dmamem_free(p->dmat, pg->va, pg->map); 220117624Sharti return; 221117624Sharti } 222117624Sharti 223117624Sharti for (i = 0; i < p->nchunks; i++) { 224117624Sharti f = N2C(p, pg, i); 225117624Sharti t = C2T(p, f); 226117624Sharti t->page = p->npages; 227117624Sharti t->chunk = i; 228117624Sharti SLIST_INSERT_HEAD(&p->free_list, f, link); 229117624Sharti } 230117624Sharti 231117624Sharti p->npages++; 232117624Sharti} 233117624Sharti 234117624Sharti/* 235117624Sharti * allocate a chunk 236117624Sharti */ 237117624Shartivoid * 238117624Shartimbp_alloc(struct mbpool *p, bus_addr_t *pap, uint32_t *hp) 239117624Sharti{ 240117624Sharti struct mbfree *cf; 241117624Sharti struct mbtrail *t; 242117624Sharti 243117624Sharti mtx_lock(&p->free_lock); 244117624Sharti if ((cf = SLIST_FIRST(&p->free_list)) == NULL) { 245117624Sharti mbp_alloc_page(p); 246117624Sharti cf = SLIST_FIRST(&p->free_list); 247117624Sharti } 248117624Sharti if (cf == NULL) { 249117624Sharti mtx_unlock(&p->free_lock); 250117624Sharti return (NULL); 251117624Sharti } 252117624Sharti SLIST_REMOVE_HEAD(&p->free_list, link); 253117624Sharti mtx_unlock(&p->free_lock); 254117624Sharti 255117624Sharti t = C2T(p, cf); 256117624Sharti 257117624Sharti *pap = p->pages[t->page].phy + t->chunk * p->chunk_size; 258117624Sharti *hp = HMAKE(t->page, t->chunk); 259117624Sharti 260117624Sharti t->page |= MBP_CARD | MBP_USED; 261117624Sharti 262117624Sharti return (cf); 263117624Sharti} 264117624Sharti 265117624Sharti/* 266117624Sharti * Free a chunk 267117624Sharti */ 268117624Shartivoid 269117624Shartimbp_free(struct mbpool *p, void *ptr) 270117624Sharti{ 271117624Sharti struct mbtrail *t; 272117624Sharti 273117624Sharti mtx_lock(&p->free_lock); 274117624Sharti t = C2T(p, ptr); 275117624Sharti t->page &= ~(MBP_USED | MBP_CARD); 276117624Sharti SLIST_INSERT_HEAD(&p->free_list, (struct mbfree *)ptr, link); 277117624Sharti mtx_unlock(&p->free_lock); 278117624Sharti} 279117624Sharti 280117624Sharti/* 281117624Sharti * Mbuf system external mbuf free routine 282117624Sharti */ 283254842Sandreint 284254799Sandrembp_ext_free(struct mbuf *m, void *buf, void *arg) 285117624Sharti{ 286117624Sharti mbp_free(arg, buf); 287254842Sandre 288254842Sandre return (EXT_FREE_OK); 289117624Sharti} 290117624Sharti 291117624Sharti/* 292302234Sbdrewery * Free all buffers that are marked as being on the card 293117624Sharti */ 294117624Shartivoid 295117624Shartimbp_card_free(struct mbpool *p) 296117624Sharti{ 297117624Sharti u_int i, b; 298117624Sharti struct mbpage *pg; 299117624Sharti struct mbtrail *tr; 300117624Sharti struct mbfree *cf; 301117624Sharti 302117624Sharti mtx_lock(&p->free_lock); 303117624Sharti for (i = 0; i < p->npages; i++) { 304117624Sharti pg = &p->pages[i]; 305117624Sharti for (b = 0; b < p->nchunks; b++) { 306117624Sharti cf = N2C(p, pg, b); 307117624Sharti tr = C2T(p, cf); 308117624Sharti if (tr->page & MBP_CARD) { 309117624Sharti tr->page &= MBP_PMSK; 310117624Sharti SLIST_INSERT_HEAD(&p->free_list, cf, link); 311117624Sharti } 312117624Sharti } 313117624Sharti } 314117624Sharti mtx_unlock(&p->free_lock); 315117624Sharti} 316117624Sharti 317117624Sharti/* 318117624Sharti * Count buffers 319117624Sharti */ 320117624Shartivoid 321117624Shartimbp_count(struct mbpool *p, u_int *used, u_int *card, u_int *free) 322117624Sharti{ 323117624Sharti u_int i, b; 324117624Sharti struct mbpage *pg; 325117624Sharti struct mbtrail *tr; 326117624Sharti struct mbfree *cf; 327117624Sharti 328117624Sharti *used = *card = *free = 0; 329117624Sharti for (i = 0; i < p->npages; i++) { 330117624Sharti pg = &p->pages[i]; 331117624Sharti for (b = 0; b < p->nchunks; b++) { 332117624Sharti tr = C2T(p, N2C(p, pg, b)); 333117624Sharti if (tr->page & MBP_CARD) 334117624Sharti (*card)++; 335117624Sharti if (tr->page & MBP_USED) 336117624Sharti (*used)++; 337117624Sharti } 338117624Sharti } 339117624Sharti mtx_lock(&p->free_lock); 340117624Sharti SLIST_FOREACH(cf, &p->free_list, link) 341170023Srwatson (*free)++; 342117624Sharti mtx_unlock(&p->free_lock); 343117624Sharti} 344117624Sharti 345117624Sharti/* 346117624Sharti * Get the buffer from a handle and clear the card flag. 347117624Sharti */ 348117624Shartivoid * 349117624Shartimbp_get(struct mbpool *p, uint32_t h) 350117624Sharti{ 351117624Sharti struct mbfree *cf; 352117624Sharti struct mbtrail *tr; 353117624Sharti 354117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 355117624Sharti tr = C2T(p, cf); 356117624Sharti 357117624Sharti#ifdef DIAGNOSTIC 358117624Sharti if (!(tr->page & MBP_CARD)) 359117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 360117624Sharti p->name, HCHUNK(h), HPAGE(h)); 361117624Sharti#endif 362117624Sharti 363117624Sharti tr->page &= ~MBP_CARD; 364117624Sharti return (cf); 365117624Sharti} 366117624Sharti 367117624Sharti/* 368117624Sharti * Get the buffer from a handle and keep the card flag. 369117624Sharti */ 370117624Shartivoid * 371117624Shartimbp_get_keep(struct mbpool *p, uint32_t h) 372117624Sharti{ 373117624Sharti struct mbfree *cf; 374117624Sharti struct mbtrail *tr; 375117624Sharti 376117624Sharti cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h)); 377117624Sharti tr = C2T(p, cf); 378117624Sharti 379117624Sharti#ifdef DIAGNOSTIC 380117624Sharti if (!(tr->page & MBP_CARD)) 381117624Sharti printf("%s: (%s) chunk %u page %u not on card\n", __func__, 382117624Sharti p->name, HCHUNK(h), HPAGE(h)); 383117624Sharti#endif 384117624Sharti 385117624Sharti return (cf); 386117624Sharti} 387117624Sharti 388117624Sharti/* 389117624Sharti * sync the chunk 390117624Sharti */ 391117624Shartivoid 392117624Shartimbp_sync(struct mbpool *p, uint32_t h, bus_addr_t off, bus_size_t len, u_int op) 393117624Sharti{ 394117624Sharti 395117624Sharti#if 0 396117624Sharti bus_dmamap_sync_size(p->dmat, p->pages[HPAGE(h)].map, 397117624Sharti HCHUNK(h) * p->chunk_size + off, len, op); 398117624Sharti#endif 399117624Sharti} 400