1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1997, 1998 Justin T. Gibbs. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions, and the following disclaimer, 12 * without modification, immediately at the beginning of the file. 13 * 2. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29/* 30 * Common code for managing bounce pages for bus_dma backends. As 31 * this code currently assumes it can access internal members of 32 * opaque types like bus_dma_tag_t and bus_dmamap it is #include'd in 33 * backends rather than being compiled standalone. 34 * 35 * Prerequisites: 36 * 37 * - M_BUSDMA malloc type 38 * - struct bus_dmamap 39 * - hw_busdma SYSCTL_NODE 40 * - macros to access the following fields of bus_dma_tag_t: 41 * - dmat_alignment() 42 * - dmat_flags() 43 * - dmat_lowaddr() 44 * - dmat_lockfunc() 45 * - dmat_lockarg() 46 */ 47 48#include <sys/kthread.h> 49#include <sys/sched.h> 50 51struct bounce_page { 52 vm_offset_t vaddr; /* kva of bounce buffer */ 53 bus_addr_t busaddr; /* Physical address */ 54 vm_offset_t datavaddr; /* kva of client data */ 55#if defined(__amd64__) || defined(__i386__) 56 vm_page_t datapage[2]; /* physical page(s) of client data */ 57#else 58 vm_page_t datapage; /* physical page of client data */ 59#endif 60 vm_offset_t dataoffs; /* page offset of client data */ 61 bus_size_t datacount; /* client data count */ 62 STAILQ_ENTRY(bounce_page) links; 63}; 64 65struct bounce_zone { 66 STAILQ_ENTRY(bounce_zone) links; 67 STAILQ_HEAD(, bounce_page) bounce_page_list; 68 STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 69 int total_bpages; 70 int free_bpages; 71 int reserved_bpages; 72 int active_bpages; 73 int total_bounced; 74 int total_deferred; 75 int map_count; 76#ifdef dmat_domain 77 int domain; 78#endif 79 sbintime_t total_deferred_time; 80 bus_size_t alignment; 81 bus_addr_t lowaddr; 82 char zoneid[8]; 83 char lowaddrid[20]; 84 struct sysctl_ctx_list sysctl_tree; 85 struct sysctl_oid *sysctl_tree_top; 86}; 87 88static struct mtx bounce_lock; 89static int total_bpages; 90static int busdma_zonecount; 91 92static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 93static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 94 95static MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages"); 96 97SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 98 "Total bounce pages"); 99 100static void busdma_thread(void *); 101static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 102 int commit); 103 104static int 105_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 106{ 107 struct bounce_zone *bz; 108 109 /* Reserve Necessary Bounce Pages */ 110 mtx_lock(&bounce_lock); 111 if (flags & BUS_DMA_NOWAIT) { 112 if (reserve_bounce_pages(dmat, map, 0) != 0) { 113 map->pagesneeded = 0; 114 mtx_unlock(&bounce_lock); 115 return (ENOMEM); 116 } 117 } else { 118 if (reserve_bounce_pages(dmat, map, 1) != 0) { 119 /* Queue us for resources */ 120 bz = dmat->bounce_zone; 121 STAILQ_INSERT_TAIL(&bz->bounce_map_waitinglist, map, 122 links); 123 map->queued_time = sbinuptime(); 124 mtx_unlock(&bounce_lock); 125 return (EINPROGRESS); 126 } 127 } 128 mtx_unlock(&bounce_lock); 129 130 return (0); 131} 132 133static void 134init_bounce_pages(void *dummy __unused) 135{ 136 137 total_bpages = 0; 138 STAILQ_INIT(&bounce_zone_list); 139 STAILQ_INIT(&bounce_map_callbacklist); 140 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 141} 142SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 143 144static struct sysctl_ctx_list * 145busdma_sysctl_tree(struct bounce_zone *bz) 146{ 147 148 return (&bz->sysctl_tree); 149} 150 151static struct sysctl_oid * 152busdma_sysctl_tree_top(struct bounce_zone *bz) 153{ 154 155 return (bz->sysctl_tree_top); 156} 157 158/* 159 * Returns true if the address falls within the tag's exclusion window, or 160 * fails to meet its alignment requirements. 161 */ 162static bool 163addr_needs_bounce(bus_dma_tag_t dmat, bus_addr_t paddr) 164{ 165 166 if (paddr > dmat_lowaddr(dmat) && paddr <= dmat_highaddr(dmat)) 167 return (true); 168 if (!vm_addr_align_ok(paddr, dmat_alignment(dmat))) 169 return (true); 170 171 return (false); 172} 173 174static int 175alloc_bounce_zone(bus_dma_tag_t dmat) 176{ 177 struct bounce_zone *bz; 178 bool start_thread; 179 180 /* Check to see if we already have a suitable zone */ 181 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 182 if ((dmat_alignment(dmat) <= bz->alignment) && 183#ifdef dmat_domain 184 dmat_domain(dmat) == bz->domain && 185#endif 186 (dmat_lowaddr(dmat) >= bz->lowaddr)) { 187 dmat->bounce_zone = bz; 188 return (0); 189 } 190 } 191 192 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA, 193 M_NOWAIT | M_ZERO)) == NULL) 194 return (ENOMEM); 195 196 STAILQ_INIT(&bz->bounce_page_list); 197 STAILQ_INIT(&bz->bounce_map_waitinglist); 198 bz->free_bpages = 0; 199 bz->reserved_bpages = 0; 200 bz->active_bpages = 0; 201 bz->lowaddr = dmat_lowaddr(dmat); 202 bz->alignment = MAX(dmat_alignment(dmat), PAGE_SIZE); 203 bz->map_count = 0; 204#ifdef dmat_domain 205 bz->domain = dmat_domain(dmat); 206#endif 207 snprintf(bz->zoneid, sizeof(bz->zoneid), "zone%d", busdma_zonecount); 208 busdma_zonecount++; 209 snprintf(bz->lowaddrid, sizeof(bz->lowaddrid), "%#jx", 210 (uintmax_t)bz->lowaddr); 211 start_thread = STAILQ_EMPTY(&bounce_zone_list); 212 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 213 dmat->bounce_zone = bz; 214 215 sysctl_ctx_init(&bz->sysctl_tree); 216 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 217 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 218 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 219 if (bz->sysctl_tree_top == NULL) { 220 sysctl_ctx_free(&bz->sysctl_tree); 221 return (0); /* XXX error code? */ 222 } 223 224 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 225 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 226 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 227 "Total bounce pages"); 228 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 229 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 230 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 231 "Free bounce pages"); 232 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 233 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 234 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 235 "Reserved bounce pages"); 236 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 237 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 238 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 239 "Active bounce pages"); 240 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 241 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 242 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 243 "Total bounce requests (pages bounced)"); 244 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 245 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 246 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 247 "Total bounce requests that were deferred"); 248 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 249 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 250 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 251 SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz), 252 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 253 "alignment", CTLFLAG_RD, &bz->alignment, ""); 254#ifdef dmat_domain 255 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 256 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 257 "domain", CTLFLAG_RD, &bz->domain, 0, 258 "memory domain"); 259#endif 260 SYSCTL_ADD_SBINTIME_USEC(busdma_sysctl_tree(bz), 261 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 262 "total_deferred_time", CTLFLAG_RD, &bz->total_deferred_time, 263 "Cumulative time busdma requests are deferred (us)"); 264 if (start_thread) { 265 if (kproc_create(busdma_thread, NULL, NULL, 0, 0, "busdma") != 266 0) 267 printf("failed to create busdma thread"); 268 } 269 return (0); 270} 271 272static int 273alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 274{ 275 struct bounce_zone *bz; 276 int count; 277 278 bz = dmat->bounce_zone; 279 count = 0; 280 while (numpages > 0) { 281 struct bounce_page *bpage; 282 283#ifdef dmat_domain 284 bpage = malloc_domainset(sizeof(*bpage), M_BUSDMA, 285 DOMAINSET_PREF(bz->domain), M_NOWAIT | M_ZERO); 286#else 287 bpage = malloc(sizeof(*bpage), M_BUSDMA, M_NOWAIT | M_ZERO); 288#endif 289 290 if (bpage == NULL) 291 break; 292#ifdef dmat_domain 293 bpage->vaddr = (vm_offset_t)contigmalloc_domainset(PAGE_SIZE, 294 M_BOUNCE, DOMAINSET_PREF(bz->domain), M_NOWAIT, 295 0ul, bz->lowaddr, PAGE_SIZE, 0); 296#else 297 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE, 298 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 299#endif 300 if (bpage->vaddr == 0) { 301 free(bpage, M_BUSDMA); 302 break; 303 } 304 bpage->busaddr = pmap_kextract(bpage->vaddr); 305 mtx_lock(&bounce_lock); 306 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 307 total_bpages++; 308 bz->total_bpages++; 309 bz->free_bpages++; 310 mtx_unlock(&bounce_lock); 311 count++; 312 numpages--; 313 } 314 return (count); 315} 316 317static int 318reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 319{ 320 struct bounce_zone *bz; 321 int pages; 322 323 mtx_assert(&bounce_lock, MA_OWNED); 324 bz = dmat->bounce_zone; 325 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 326 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 327 return (map->pagesneeded - (map->pagesreserved + pages)); 328 bz->free_bpages -= pages; 329 bz->reserved_bpages += pages; 330 map->pagesreserved += pages; 331 pages = map->pagesneeded - map->pagesreserved; 332 333 return (pages); 334} 335 336#if defined(__amd64__) || defined(__i386__) 337static bus_addr_t 338add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 339 vm_paddr_t addr1, vm_paddr_t addr2, bus_size_t size) 340#else 341static bus_addr_t 342add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 343 bus_addr_t addr, bus_size_t size) 344#endif 345{ 346 struct bounce_zone *bz; 347 struct bounce_page *bpage; 348 349 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 350 KASSERT(map != NULL, ("add_bounce_page: bad map %p", map)); 351#if defined(__amd64__) || defined(__i386__) 352 KASSERT(map != &nobounce_dmamap, ("add_bounce_page: bad map %p", map)); 353#endif 354#ifdef __riscv 355 KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0, 356 ("add_bounce_page: bad map %p", map)); 357#endif 358 359 bz = dmat->bounce_zone; 360 if (map->pagesneeded == 0) 361 panic("add_bounce_page: map doesn't need any pages"); 362 map->pagesneeded--; 363 364 if (map->pagesreserved == 0) 365 panic("add_bounce_page: map doesn't need any pages"); 366 map->pagesreserved--; 367 368 mtx_lock(&bounce_lock); 369 bpage = STAILQ_FIRST(&bz->bounce_page_list); 370 if (bpage == NULL) 371 panic("add_bounce_page: free page list is empty"); 372 373 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 374 bz->reserved_bpages--; 375 bz->active_bpages++; 376 mtx_unlock(&bounce_lock); 377 378 if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) { 379 /* Page offset needs to be preserved. */ 380#if defined(__amd64__) || defined(__i386__) 381 bpage->vaddr |= addr1 & PAGE_MASK; 382 bpage->busaddr |= addr1 & PAGE_MASK; 383 KASSERT(addr2 == 0, 384 ("Trying to bounce multiple pages with BUS_DMA_KEEP_PG_OFFSET")); 385#else 386 bpage->vaddr |= addr & PAGE_MASK; 387 bpage->busaddr |= addr & PAGE_MASK; 388#endif 389 } 390 bpage->datavaddr = vaddr; 391#if defined(__amd64__) || defined(__i386__) 392 bpage->datapage[0] = PHYS_TO_VM_PAGE(addr1); 393 KASSERT((addr2 & PAGE_MASK) == 0, ("Second page is not aligned")); 394 bpage->datapage[1] = PHYS_TO_VM_PAGE(addr2); 395 bpage->dataoffs = addr1 & PAGE_MASK; 396#else 397 bpage->datapage = PHYS_TO_VM_PAGE(addr); 398 bpage->dataoffs = addr & PAGE_MASK; 399#endif 400 bpage->datacount = size; 401 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 402 return (bpage->busaddr); 403} 404 405static void 406free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 407{ 408 struct bounce_page *bpage; 409 struct bounce_zone *bz; 410 bool schedule_thread; 411 u_int count; 412 413 if (STAILQ_EMPTY(&map->bpages)) 414 return; 415 416 bz = dmat->bounce_zone; 417 count = 0; 418 schedule_thread = false; 419 STAILQ_FOREACH(bpage, &map->bpages, links) { 420 bpage->datavaddr = 0; 421 bpage->datacount = 0; 422 423 if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) { 424 /* 425 * Reset the bounce page to start at offset 0. 426 * Other uses of this bounce page may need to 427 * store a full page of data and/or assume it 428 * starts on a page boundary. 429 */ 430 bpage->vaddr &= ~PAGE_MASK; 431 bpage->busaddr &= ~PAGE_MASK; 432 } 433 count++; 434 } 435 436 mtx_lock(&bounce_lock); 437 STAILQ_CONCAT(&bz->bounce_page_list, &map->bpages); 438 bz->free_bpages += count; 439 bz->active_bpages -= count; 440 while ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) { 441 if (reserve_bounce_pages(map->dmat, map, 1) != 0) 442 break; 443 444 STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); 445 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); 446 bz->total_deferred++; 447 schedule_thread = true; 448 } 449 mtx_unlock(&bounce_lock); 450 if (schedule_thread) 451 wakeup(&bounce_map_callbacklist); 452} 453 454/* 455 * Add a single contiguous physical range to the segment list. 456 */ 457static bus_size_t 458_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 459 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 460{ 461 int seg; 462 463 KASSERT(curaddr <= BUS_SPACE_MAXADDR, 464 ("ds_addr %#jx > BUS_SPACE_MAXADDR %#jx; dmat %p fl %#x low %#jx " 465 "hi %#jx", 466 (uintmax_t)curaddr, (uintmax_t)BUS_SPACE_MAXADDR, 467 dmat, dmat_bounce_flags(dmat), (uintmax_t)dmat_lowaddr(dmat), 468 (uintmax_t)dmat_highaddr(dmat))); 469 470 /* 471 * Make sure we don't cross any boundaries. 472 */ 473 if (!vm_addr_bound_ok(curaddr, sgsize, dmat_boundary(dmat))) 474 sgsize = roundup2(curaddr, dmat_boundary(dmat)) - curaddr; 475 476 /* 477 * Insert chunk into a segment, coalescing with 478 * previous segment if possible. 479 */ 480 seg = *segp; 481 if (seg == -1) { 482 seg = 0; 483 segs[seg].ds_addr = curaddr; 484 segs[seg].ds_len = sgsize; 485 } else { 486 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 487 (segs[seg].ds_len + sgsize) <= dmat_maxsegsz(dmat) && 488 vm_addr_bound_ok(segs[seg].ds_addr, 489 segs[seg].ds_len + sgsize, dmat_boundary(dmat))) 490 segs[seg].ds_len += sgsize; 491 else { 492 if (++seg >= dmat_nsegments(dmat)) 493 return (0); 494 segs[seg].ds_addr = curaddr; 495 segs[seg].ds_len = sgsize; 496 } 497 } 498 *segp = seg; 499 return (sgsize); 500} 501 502/* 503 * Add a contiguous physical range to the segment list, respecting the tag's 504 * maximum segment size and splitting it into multiple segments as necessary. 505 */ 506static bool 507_bus_dmamap_addsegs(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 508 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 509{ 510 bus_size_t done, todo; 511 512 while (sgsize > 0) { 513 todo = MIN(sgsize, dmat_maxsegsz(dmat)); 514 done = _bus_dmamap_addseg(dmat, map, curaddr, todo, segs, 515 segp); 516 if (done == 0) 517 return (false); 518 curaddr += done; 519 sgsize -= done; 520 } 521 return (true); 522} 523 524static void 525busdma_thread(void *dummy __unused) 526{ 527 STAILQ_HEAD(, bus_dmamap) callbacklist; 528 bus_dma_tag_t dmat; 529 struct bus_dmamap *map, *nmap; 530 struct bounce_zone *bz; 531 532 thread_lock(curthread); 533 sched_class(curthread, PRI_ITHD); 534 sched_ithread_prio(curthread, PI_SWI(SWI_BUSDMA)); 535 thread_unlock(curthread); 536 for (;;) { 537 mtx_lock(&bounce_lock); 538 while (STAILQ_EMPTY(&bounce_map_callbacklist)) 539 mtx_sleep(&bounce_map_callbacklist, &bounce_lock, 0, 540 "-", 0); 541 STAILQ_INIT(&callbacklist); 542 STAILQ_CONCAT(&callbacklist, &bounce_map_callbacklist); 543 mtx_unlock(&bounce_lock); 544 545 STAILQ_FOREACH_SAFE(map, &callbacklist, links, nmap) { 546 dmat = map->dmat; 547 bz = dmat->bounce_zone; 548 dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat), 549 BUS_DMA_LOCK); 550 bz->total_deferred_time += (sbinuptime() - map->queued_time); 551 bus_dmamap_load_mem(map->dmat, map, &map->mem, 552 map->callback, map->callback_arg, BUS_DMA_WAITOK); 553 dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat), 554 BUS_DMA_UNLOCK); 555 } 556 } 557} 558