1/*-
2 * Copyright (c) 2003
3 *	Fraunhofer Institute for Open Communication Systems (FhG Fokus).
4 * 	All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * Author: Hartmut Brandt <harti@freebsd.org>
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/lock.h>
35#include <sys/mutex.h>
36#include <sys/kernel.h>
37#include <sys/systm.h>
38#include <sys/malloc.h>
39#include <sys/module.h>
40
41#include <machine/bus.h>
42
43#include <sys/mbuf.h>
44#include <sys/mbpool.h>
45
46MODULE_VERSION(libmbpool, 1);
47
48/*
49 * Memory is allocated as DMA-able pages. Each page is divided into a number
50 * of equal chunks where the last 4 bytes of each chunk are occupied by
51 * the page number and the chunk number. The caller must take these four
52 * bytes into account when specifying the chunk size. Each page is mapped by
53 * its own DMA map using the user specified DMA tag.
54 *
55 * Each chunk has a used and a card bit in the high bits of its page number.
56 *  0    0	chunk is free and may be allocated
57 *  1    1	chunk has been given to the interface
58 *  0    1	chunk is traveling through the system
59 *  1    0	illegal
60 */
61struct mbtrail {
62	uint16_t	chunk;
63	uint16_t	page;
64};
65#define	MBP_CARD	0x8000
66#define	MBP_USED	0x4000
67#define	MBP_PMSK	0x3fff		/* page number mask */
68#define	MBP_CMSK	0x01ff		/* chunk number mask */
69
70struct mbfree {
71	SLIST_ENTRY(mbfree) link;	/* link on free list */
72};
73
74struct mbpage {
75	bus_dmamap_t	map;		/* map for this page */
76	bus_addr_t	phy;		/* physical address */
77	void		*va;		/* the memory */
78};
79
80struct mbpool {
81	const char	*name;		/* a name for this pool */
82	bus_dma_tag_t	dmat;		/* tag for mapping */
83	u_int		max_pages;	/* maximum number of pages */
84	size_t		page_size;	/* size of each allocation */
85	size_t		chunk_size;	/* size of each external mbuf */
86
87	struct mtx	free_lock;	/* lock of free list */
88	SLIST_HEAD(, mbfree) free_list;	/* free list */
89	u_int		npages;		/* current number of pages */
90	u_int		nchunks;	/* chunks per page */
91	struct mbpage	pages[];	/* pages */
92};
93
94static MALLOC_DEFINE(M_MBPOOL, "mbpools", "mbuf pools");
95
96/*
97 * Make a trail pointer from a chunk pointer
98 */
99#define	C2T(P, C)	((struct mbtrail *)((char *)(C) + (P)->chunk_size - \
100			    sizeof(struct mbtrail)))
101
102/*
103 * Make a free chunk pointer from a chunk number
104 */
105#define	N2C(P, PG, C)	((struct mbfree *)((char *)(PG)->va + \
106			    (C) * (P)->chunk_size))
107
108/*
109 * Make/parse handles
110 */
111#define	HMAKE(P, C)	((((P) & MBP_PMSK) << 16) | ((C) << 7))
112#define	HPAGE(H)	(((H) >> 16) & MBP_PMSK)
113#define	HCHUNK(H)	(((H) >>  7) & MBP_CMSK)
114
115/*
116 * initialize a pool
117 */
118int
119mbp_create(struct mbpool **pp, const char *name, bus_dma_tag_t dmat,
120    u_int max_pages, size_t page_size, size_t chunk_size)
121{
122	u_int nchunks;
123
124	if (max_pages > MBPOOL_MAX_MAXPAGES || chunk_size == 0)
125		return (EINVAL);
126	nchunks = page_size / chunk_size;
127	if (nchunks == 0 || nchunks > MBPOOL_MAX_CHUNKS)
128		return (EINVAL);
129
130	(*pp) = malloc(sizeof(struct mbpool) +
131	    max_pages * sizeof(struct mbpage),
132	    M_MBPOOL, M_WAITOK | M_ZERO);
133
134	(*pp)->name = name;
135	(*pp)->dmat = dmat;
136	(*pp)->max_pages = max_pages;
137	(*pp)->page_size = page_size;
138	(*pp)->chunk_size = chunk_size;
139	(*pp)->nchunks = nchunks;
140
141	SLIST_INIT(&(*pp)->free_list);
142	mtx_init(&(*pp)->free_lock, name, NULL, MTX_DEF);
143
144	return (0);
145}
146
147/*
148 * destroy a pool
149 */
150void
151mbp_destroy(struct mbpool *p)
152{
153	u_int i;
154	struct mbpage *pg;
155#ifdef DIAGNOSTIC
156	struct mbtrail *tr;
157	u_int b;
158#endif
159
160	for (i = 0; i < p->npages; i++) {
161		pg = &p->pages[i];
162#ifdef DIAGNOSTIC
163		for (b = 0; b < p->nchunks; b++) {
164			tr = C2T(p, N2C(p, pg, b));
165			if (tr->page & MBP_CARD)
166				printf("%s: (%s) buf still on card"
167				    " %u/%u\n", __func__, p->name, i, b);
168			if (tr->page & MBP_USED)
169				printf("%s: (%s) sbuf still in use"
170				    " %u/%u\n", __func__, p->name, i, b);
171		}
172#endif
173		bus_dmamap_unload(p->dmat, pg->map);
174		bus_dmamem_free(p->dmat, pg->va, pg->map);
175	}
176	mtx_destroy(&p->free_lock);
177
178	free(p, M_MBPOOL);
179}
180
181/*
182 * Helper function when loading a one segment DMA buffer.
183 */
184static void
185mbp_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
186{
187	if (error == 0)
188		*(bus_addr_t *)arg = segs[0].ds_addr;
189}
190
191/*
192 * Allocate a new page
193 */
194static void
195mbp_alloc_page(struct mbpool *p)
196{
197	int error;
198	struct mbpage *pg;
199	u_int i;
200	struct mbfree *f;
201	struct mbtrail *t;
202
203	if (p->npages == p->max_pages) {
204#ifdef DIAGNOSTIC
205		printf("%s: (%s) page limit reached %u\n", __func__,
206		    p->name, p->max_pages);
207#endif
208		return;
209	}
210	pg = &p->pages[p->npages];
211
212	error = bus_dmamem_alloc(p->dmat, &pg->va, BUS_DMA_NOWAIT, &pg->map);
213	if (error != 0) {
214		free(pg, M_MBPOOL);
215		return;
216	}
217
218	error = bus_dmamap_load(p->dmat, pg->map, pg->va, p->page_size,
219	    mbp_callback, &pg->phy, 0);
220	if (error != 0) {
221		bus_dmamem_free(p->dmat, pg->va, pg->map);
222		free(pg, M_MBPOOL);
223		return;
224	}
225
226	for (i = 0; i < p->nchunks; i++) {
227		f = N2C(p, pg, i);
228		t = C2T(p, f);
229		t->page = p->npages;
230		t->chunk = i;
231		SLIST_INSERT_HEAD(&p->free_list, f, link);
232	}
233
234	p->npages++;
235}
236
237/*
238 * allocate a chunk
239 */
240void *
241mbp_alloc(struct mbpool *p, bus_addr_t *pap, uint32_t *hp)
242{
243	struct mbfree *cf;
244	struct mbtrail *t;
245
246	mtx_lock(&p->free_lock);
247	if ((cf = SLIST_FIRST(&p->free_list)) == NULL) {
248		mbp_alloc_page(p);
249		cf = SLIST_FIRST(&p->free_list);
250	}
251	if (cf == NULL) {
252		mtx_unlock(&p->free_lock);
253		return (NULL);
254	}
255	SLIST_REMOVE_HEAD(&p->free_list, link);
256	mtx_unlock(&p->free_lock);
257
258	t = C2T(p, cf);
259
260	*pap = p->pages[t->page].phy + t->chunk * p->chunk_size;
261	*hp = HMAKE(t->page, t->chunk);
262
263	t->page |= MBP_CARD | MBP_USED;
264
265	return (cf);
266}
267
268/*
269 * Free a chunk
270 */
271void
272mbp_free(struct mbpool *p, void *ptr)
273{
274	struct mbtrail *t;
275
276	mtx_lock(&p->free_lock);
277	t = C2T(p, ptr);
278	t->page &= ~(MBP_USED | MBP_CARD);
279	SLIST_INSERT_HEAD(&p->free_list, (struct mbfree *)ptr, link);
280	mtx_unlock(&p->free_lock);
281}
282
283/*
284 * Mbuf system external mbuf free routine
285 */
286int
287mbp_ext_free(struct mbuf *m, void *buf, void *arg)
288{
289	mbp_free(arg, buf);
290
291	return (EXT_FREE_OK);
292}
293
294/*
295 * Free all buffers that are marked as beeing on the card
296 */
297void
298mbp_card_free(struct mbpool *p)
299{
300	u_int i, b;
301	struct mbpage *pg;
302	struct mbtrail *tr;
303	struct mbfree *cf;
304
305	mtx_lock(&p->free_lock);
306	for (i = 0; i < p->npages; i++) {
307		pg = &p->pages[i];
308		for (b = 0; b < p->nchunks; b++) {
309			cf = N2C(p, pg, b);
310			tr = C2T(p, cf);
311			if (tr->page & MBP_CARD) {
312				tr->page &= MBP_PMSK;
313				SLIST_INSERT_HEAD(&p->free_list, cf, link);
314			}
315		}
316	}
317	mtx_unlock(&p->free_lock);
318}
319
320/*
321 * Count buffers
322 */
323void
324mbp_count(struct mbpool *p, u_int *used, u_int *card, u_int *free)
325{
326	u_int i, b;
327	struct mbpage *pg;
328	struct mbtrail *tr;
329	struct mbfree *cf;
330
331	*used = *card = *free = 0;
332	for (i = 0; i < p->npages; i++) {
333		pg = &p->pages[i];
334		for (b = 0; b < p->nchunks; b++) {
335			tr = C2T(p, N2C(p, pg, b));
336			if (tr->page & MBP_CARD)
337				(*card)++;
338			if (tr->page & MBP_USED)
339				(*used)++;
340		}
341	}
342	mtx_lock(&p->free_lock);
343	SLIST_FOREACH(cf, &p->free_list, link)
344		(*free)++;
345	mtx_unlock(&p->free_lock);
346}
347
348/*
349 * Get the buffer from a handle and clear the card flag.
350 */
351void *
352mbp_get(struct mbpool *p, uint32_t h)
353{
354	struct mbfree *cf;
355	struct mbtrail *tr;
356
357	cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h));
358	tr = C2T(p, cf);
359
360#ifdef DIAGNOSTIC
361	if (!(tr->page & MBP_CARD))
362		printf("%s: (%s) chunk %u page %u not on card\n", __func__,
363		    p->name, HCHUNK(h), HPAGE(h));
364#endif
365
366	tr->page &= ~MBP_CARD;
367	return (cf);
368}
369
370/*
371 * Get the buffer from a handle and keep the card flag.
372 */
373void *
374mbp_get_keep(struct mbpool *p, uint32_t h)
375{
376	struct mbfree *cf;
377	struct mbtrail *tr;
378
379	cf = N2C(p, &p->pages[HPAGE(h)], HCHUNK(h));
380	tr = C2T(p, cf);
381
382#ifdef DIAGNOSTIC
383	if (!(tr->page & MBP_CARD))
384		printf("%s: (%s) chunk %u page %u not on card\n", __func__,
385		    p->name, HCHUNK(h), HPAGE(h));
386#endif
387
388	return (cf);
389}
390
391/*
392 * sync the chunk
393 */
394void
395mbp_sync(struct mbpool *p, uint32_t h, bus_addr_t off, bus_size_t len, u_int op)
396{
397
398#if 0
399	bus_dmamap_sync_size(p->dmat, p->pages[HPAGE(h)].map,
400	    HCHUNK(h) * p->chunk_size + off, len, op);
401#endif
402}
403