192108Sphk/*- 292108Sphk * Copyright (c) 2002 Poul-Henning Kamp 392108Sphk * Copyright (c) 2002 Networks Associates Technology, Inc. 4248508Skib * Copyright (c) 2013 The FreeBSD Foundation 592108Sphk * All rights reserved. 692108Sphk * 792108Sphk * This software was developed for the FreeBSD Project by Poul-Henning Kamp 892108Sphk * and NAI Labs, the Security Research Division of Network Associates, Inc. 992108Sphk * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 1092108Sphk * DARPA CHATS research program. 1192108Sphk * 12248508Skib * Portions of this software were developed by Konstantin Belousov 13248508Skib * under sponsorship from the FreeBSD Foundation. 14248508Skib * 1592108Sphk * Redistribution and use in source and binary forms, with or without 1692108Sphk * modification, are permitted provided that the following conditions 1792108Sphk * are met: 1892108Sphk * 1. Redistributions of source code must retain the above copyright 1992108Sphk * notice, this list of conditions and the following disclaimer. 2092108Sphk * 2. Redistributions in binary form must reproduce the above copyright 2192108Sphk * notice, this list of conditions and the following disclaimer in the 2292108Sphk * documentation and/or other materials provided with the distribution. 2392108Sphk * 3. The names of the authors may not be used to endorse or promote 2492108Sphk * products derived from this software without specific prior written 2592108Sphk * permission. 2692108Sphk * 2792108Sphk * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2892108Sphk * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2992108Sphk * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 3092108Sphk * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 3192108Sphk * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 3292108Sphk * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3392108Sphk * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3492108Sphk * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3592108Sphk * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3692108Sphk * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3792108Sphk * SUCH DAMAGE. 3892108Sphk */ 3992108Sphk 40116196Sobrien#include <sys/cdefs.h> 41116196Sobrien__FBSDID("$FreeBSD$"); 4292108Sphk 4392108Sphk#include <sys/param.h> 4492108Sphk#include <sys/systm.h> 4592108Sphk#include <sys/kernel.h> 4692108Sphk#include <sys/malloc.h> 4792108Sphk#include <sys/bio.h> 48136755Srwatson#include <sys/ktr.h> 49150177Sjhb#include <sys/proc.h> 50149576Spjd#include <sys/stack.h> 51248508Skib#include <sys/sysctl.h> 52252330Sjeff#include <sys/vmem.h> 5392108Sphk 5492108Sphk#include <sys/errno.h> 5592108Sphk#include <geom/geom.h> 5693250Sphk#include <geom/geom_int.h> 57112370Sphk#include <sys/devicestat.h> 5892108Sphk 59114526Sphk#include <vm/uma.h> 60248508Skib#include <vm/vm.h> 61248508Skib#include <vm/vm_param.h> 62248508Skib#include <vm/vm_kern.h> 63248508Skib#include <vm/vm_page.h> 64248508Skib#include <vm/vm_object.h> 65248508Skib#include <vm/vm_extern.h> 66248508Skib#include <vm/vm_map.h> 67114526Sphk 68260385Sscottlstatic int g_io_transient_map_bio(struct bio *bp); 69260385Sscottl 7092108Sphkstatic struct g_bioq g_bio_run_down; 7192108Sphkstatic struct g_bioq g_bio_run_up; 72125137Sphkstatic struct g_bioq g_bio_run_task; 7392108Sphk 74287569Simp/* 75287569Simp * Pace is a hint that we've had some trouble recently allocating 76287569Simp * bios, so we should back off trying to send I/O down the stack 77287569Simp * a bit to let the problem resolve. When pacing, we also turn 78287569Simp * off direct dispatch to also reduce memory pressure from I/Os 79287569Simp * there, at the expxense of some added latency while the memory 80287569Simp * pressures exist. See g_io_schedule_down() for more details 81287569Simp * and limitations. 82287569Simp */ 83287569Simpstatic volatile u_int pace; 84287569Simp 85114526Sphkstatic uma_zone_t biozone; 86106338Sphk 87193981Sluigi/* 88193981Sluigi * The head of the list of classifiers used in g_io_request. 89193981Sluigi * Use g_register_classifier() and g_unregister_classifier() 90193981Sluigi * to add/remove entries to the list. 91193981Sluigi * Classifiers are invoked in registration order. 92193981Sluigi */ 93193981Sluigistatic TAILQ_HEAD(g_classifier_tailq, g_classifier_hook) 94193981Sluigi g_classifier_tailq = TAILQ_HEAD_INITIALIZER(g_classifier_tailq); 95193981Sluigi 9692108Sphk#include <machine/atomic.h> 9792108Sphk 9892108Sphkstatic void 9992108Sphkg_bioq_lock(struct g_bioq *bq) 10092108Sphk{ 10192108Sphk 10292108Sphk mtx_lock(&bq->bio_queue_lock); 10392108Sphk} 10492108Sphk 10592108Sphkstatic void 10692108Sphkg_bioq_unlock(struct g_bioq *bq) 10792108Sphk{ 10892108Sphk 10992108Sphk mtx_unlock(&bq->bio_queue_lock); 11092108Sphk} 11192108Sphk 11292108Sphk#if 0 11392108Sphkstatic void 11492108Sphkg_bioq_destroy(struct g_bioq *bq) 11592108Sphk{ 11692108Sphk 11792108Sphk mtx_destroy(&bq->bio_queue_lock); 11892108Sphk} 11992108Sphk#endif 12092108Sphk 12192108Sphkstatic void 12292108Sphkg_bioq_init(struct g_bioq *bq) 12392108Sphk{ 12492108Sphk 12592108Sphk TAILQ_INIT(&bq->bio_queue); 12693818Sjhb mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 12792108Sphk} 12892108Sphk 12992108Sphkstatic struct bio * 13092108Sphkg_bioq_first(struct g_bioq *bq) 13192108Sphk{ 13292108Sphk struct bio *bp; 13392108Sphk 13492108Sphk bp = TAILQ_FIRST(&bq->bio_queue); 13592108Sphk if (bp != NULL) { 136134519Sphk KASSERT((bp->bio_flags & BIO_ONQUEUE), 137134519Sphk ("Bio not on queue bp=%p target %p", bp, bq)); 138134519Sphk bp->bio_flags &= ~BIO_ONQUEUE; 13992108Sphk TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 14092108Sphk bq->bio_queue_length--; 14192108Sphk } 14292108Sphk return (bp); 14392108Sphk} 14492108Sphk 14592108Sphkstruct bio * 14692108Sphkg_new_bio(void) 14792108Sphk{ 14892108Sphk struct bio *bp; 14992108Sphk 150114526Sphk bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 151149576Spjd#ifdef KTR 152173001Spjd if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 153149576Spjd struct stack st; 154149576Spjd 155149576Spjd CTR1(KTR_GEOM, "g_new_bio(): %p", bp); 156149576Spjd stack_save(&st); 157149576Spjd CTRSTACK(KTR_GEOM, &st, 3, 0); 158149576Spjd } 159149576Spjd#endif 16092108Sphk return (bp); 16192108Sphk} 16292108Sphk 163134379Sphkstruct bio * 164134379Sphkg_alloc_bio(void) 165134379Sphk{ 166134379Sphk struct bio *bp; 167134379Sphk 168134379Sphk bp = uma_zalloc(biozone, M_WAITOK | M_ZERO); 169149576Spjd#ifdef KTR 170173001Spjd if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 171149576Spjd struct stack st; 172149576Spjd 173149576Spjd CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp); 174149576Spjd stack_save(&st); 175149576Spjd CTRSTACK(KTR_GEOM, &st, 3, 0); 176149576Spjd } 177149576Spjd#endif 178134379Sphk return (bp); 179134379Sphk} 180134379Sphk 18192108Sphkvoid 18292108Sphkg_destroy_bio(struct bio *bp) 18392108Sphk{ 184149576Spjd#ifdef KTR 185173001Spjd if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 186149576Spjd struct stack st; 18792108Sphk 188149576Spjd CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp); 189149576Spjd stack_save(&st); 190149576Spjd CTRSTACK(KTR_GEOM, &st, 3, 0); 191149576Spjd } 192149576Spjd#endif 193114526Sphk uma_zfree(biozone, bp); 19492108Sphk} 19592108Sphk 19692108Sphkstruct bio * 19792108Sphkg_clone_bio(struct bio *bp) 19892108Sphk{ 19992108Sphk struct bio *bp2; 20092108Sphk 201114526Sphk bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 202104058Sphk if (bp2 != NULL) { 203110517Sphk bp2->bio_parent = bp; 204104058Sphk bp2->bio_cmd = bp->bio_cmd; 205239132Sjimharris /* 206239132Sjimharris * BIO_ORDERED flag may be used by disk drivers to enforce 207239132Sjimharris * ordering restrictions, so this flag needs to be cloned. 208292348Sken * BIO_UNMAPPED and BIO_VLIST should be inherited, to properly 209292348Sken * indicate which way the buffer is passed. 210239132Sjimharris * Other bio flags are not suitable for cloning. 211239132Sjimharris */ 212292348Sken bp2->bio_flags = bp->bio_flags & 213292348Sken (BIO_ORDERED | BIO_UNMAPPED | BIO_VLIST); 214104058Sphk bp2->bio_length = bp->bio_length; 215104058Sphk bp2->bio_offset = bp->bio_offset; 216104058Sphk bp2->bio_data = bp->bio_data; 217248508Skib bp2->bio_ma = bp->bio_ma; 218248508Skib bp2->bio_ma_n = bp->bio_ma_n; 219248508Skib bp2->bio_ma_offset = bp->bio_ma_offset; 220104058Sphk bp2->bio_attribute = bp->bio_attribute; 221193981Sluigi /* Inherit classification info from the parent */ 222193981Sluigi bp2->bio_classifier1 = bp->bio_classifier1; 223193981Sluigi bp2->bio_classifier2 = bp->bio_classifier2; 224110523Sphk bp->bio_children++; 225104058Sphk } 226149576Spjd#ifdef KTR 227173001Spjd if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 228149576Spjd struct stack st; 229149576Spjd 230156686Sru CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2); 231149576Spjd stack_save(&st); 232149576Spjd CTRSTACK(KTR_GEOM, &st, 3, 0); 233149576Spjd } 234149576Spjd#endif 23592108Sphk return(bp2); 23692108Sphk} 23792108Sphk 238159304Spjdstruct bio * 239159304Spjdg_duplicate_bio(struct bio *bp) 240159304Spjd{ 241159304Spjd struct bio *bp2; 242159304Spjd 243159304Spjd bp2 = uma_zalloc(biozone, M_WAITOK | M_ZERO); 244292348Sken bp2->bio_flags = bp->bio_flags & (BIO_UNMAPPED | BIO_VLIST); 245159304Spjd bp2->bio_parent = bp; 246159304Spjd bp2->bio_cmd = bp->bio_cmd; 247159304Spjd bp2->bio_length = bp->bio_length; 248159304Spjd bp2->bio_offset = bp->bio_offset; 249159304Spjd bp2->bio_data = bp->bio_data; 250248508Skib bp2->bio_ma = bp->bio_ma; 251248508Skib bp2->bio_ma_n = bp->bio_ma_n; 252248508Skib bp2->bio_ma_offset = bp->bio_ma_offset; 253159304Spjd bp2->bio_attribute = bp->bio_attribute; 254159304Spjd bp->bio_children++; 255159304Spjd#ifdef KTR 256173001Spjd if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 257159304Spjd struct stack st; 258159304Spjd 259159304Spjd CTR2(KTR_GEOM, "g_duplicate_bio(%p): %p", bp, bp2); 260159304Spjd stack_save(&st); 261159304Spjd CTRSTACK(KTR_GEOM, &st, 3, 0); 262159304Spjd } 263159304Spjd#endif 264159304Spjd return(bp2); 265159304Spjd} 266159304Spjd 26792108Sphkvoid 26892108Sphkg_io_init() 26992108Sphk{ 27092108Sphk 27192108Sphk g_bioq_init(&g_bio_run_down); 27292108Sphk g_bioq_init(&g_bio_run_up); 273125137Sphk g_bioq_init(&g_bio_run_task); 274114526Sphk biozone = uma_zcreate("g_bio", sizeof (struct bio), 275114526Sphk NULL, NULL, 276114526Sphk NULL, NULL, 277114526Sphk 0, 0); 27892108Sphk} 27992108Sphk 28092108Sphkint 28194283Sphkg_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 28292108Sphk{ 28392108Sphk struct bio *bp; 28492108Sphk int error; 28592108Sphk 28692108Sphk g_trace(G_T_BIO, "bio_getattr(%s)", attr); 287134379Sphk bp = g_alloc_bio(); 288104665Sphk bp->bio_cmd = BIO_GETATTR; 289104665Sphk bp->bio_done = NULL; 290104665Sphk bp->bio_attribute = attr; 291104665Sphk bp->bio_length = *len; 292104665Sphk bp->bio_data = ptr; 293104665Sphk g_io_request(bp, cp); 294104665Sphk error = biowait(bp, "ggetattr"); 295104665Sphk *len = bp->bio_completed; 296104665Sphk g_destroy_bio(bp); 29792108Sphk return (error); 29892108Sphk} 29992108Sphk 300163832Spjdint 301163832Spjdg_io_flush(struct g_consumer *cp) 302163832Spjd{ 303163832Spjd struct bio *bp; 304163832Spjd int error; 305163832Spjd 306163832Spjd g_trace(G_T_BIO, "bio_flush(%s)", cp->provider->name); 307163832Spjd bp = g_alloc_bio(); 308163832Spjd bp->bio_cmd = BIO_FLUSH; 309212160Sgibbs bp->bio_flags |= BIO_ORDERED; 310163832Spjd bp->bio_done = NULL; 311163832Spjd bp->bio_attribute = NULL; 312163832Spjd bp->bio_offset = cp->provider->mediasize; 313163832Spjd bp->bio_length = 0; 314163832Spjd bp->bio_data = NULL; 315163832Spjd g_io_request(bp, cp); 316163832Spjd error = biowait(bp, "gflush"); 317163832Spjd g_destroy_bio(bp); 318163832Spjd return (error); 319163832Spjd} 320163832Spjd 321110471Sphkstatic int 322110471Sphkg_io_check(struct bio *bp) 32392108Sphk{ 324110471Sphk struct g_consumer *cp; 325110471Sphk struct g_provider *pp; 326260385Sscottl off_t excess; 327260385Sscottl int error; 32892108Sphk 329110471Sphk cp = bp->bio_from; 330110471Sphk pp = bp->bio_to; 33192108Sphk 332110471Sphk /* Fail if access counters dont allow the operation */ 33393778Sphk switch(bp->bio_cmd) { 33493778Sphk case BIO_READ: 33593778Sphk case BIO_GETATTR: 336110471Sphk if (cp->acr == 0) 337110471Sphk return (EPERM); 33893778Sphk break; 33993778Sphk case BIO_WRITE: 34093778Sphk case BIO_DELETE: 341163832Spjd case BIO_FLUSH: 342110471Sphk if (cp->acw == 0) 343110471Sphk return (EPERM); 34493778Sphk break; 34593778Sphk default: 346110471Sphk return (EPERM); 34793778Sphk } 34893778Sphk /* if provider is marked for error, don't disturb. */ 349110471Sphk if (pp->error) 350110471Sphk return (pp->error); 351238886Smav if (cp->flags & G_CF_ORPHAN) 352238886Smav return (ENXIO); 353110471Sphk 35493778Sphk switch(bp->bio_cmd) { 35593778Sphk case BIO_READ: 35693778Sphk case BIO_WRITE: 35793778Sphk case BIO_DELETE: 358206650Savg /* Zero sectorsize or mediasize is probably a lack of media. */ 359206650Savg if (pp->sectorsize == 0 || pp->mediasize == 0) 360121323Sphk return (ENXIO); 361108051Sphk /* Reject I/O not on sector boundary */ 362110471Sphk if (bp->bio_offset % pp->sectorsize) 363110471Sphk return (EINVAL); 364108051Sphk /* Reject I/O not integral sector long */ 365110471Sphk if (bp->bio_length % pp->sectorsize) 366110471Sphk return (EINVAL); 367121253Sphk /* Reject requests before or past the end of media. */ 368121253Sphk if (bp->bio_offset < 0) 369121253Sphk return (EIO); 370110471Sphk if (bp->bio_offset > pp->mediasize) 371110471Sphk return (EIO); 372260385Sscottl 373260385Sscottl /* Truncate requests to the end of providers media. */ 374260385Sscottl excess = bp->bio_offset + bp->bio_length; 375260385Sscottl if (excess > bp->bio_to->mediasize) { 376260385Sscottl KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 || 377260385Sscottl round_page(bp->bio_ma_offset + 378260385Sscottl bp->bio_length) / PAGE_SIZE == bp->bio_ma_n, 379260385Sscottl ("excess bio %p too short", bp)); 380260385Sscottl excess -= bp->bio_to->mediasize; 381260385Sscottl bp->bio_length -= excess; 382260385Sscottl if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 383260385Sscottl bp->bio_ma_n = round_page(bp->bio_ma_offset + 384260385Sscottl bp->bio_length) / PAGE_SIZE; 385260385Sscottl } 386260385Sscottl if (excess > 0) 387260385Sscottl CTR3(KTR_GEOM, "g_down truncated bio " 388260385Sscottl "%p provider %s by %d", bp, 389260385Sscottl bp->bio_to->name, excess); 390260385Sscottl } 391260385Sscottl 392260385Sscottl /* Deliver zero length transfers right here. */ 393260385Sscottl if (bp->bio_length == 0) { 394260385Sscottl CTR2(KTR_GEOM, "g_down terminated 0-length " 395260385Sscottl "bp %p provider %s", bp, bp->bio_to->name); 396260385Sscottl return (0); 397260385Sscottl } 398260385Sscottl 399260385Sscottl if ((bp->bio_flags & BIO_UNMAPPED) != 0 && 400260385Sscottl (bp->bio_to->flags & G_PF_ACCEPT_UNMAPPED) == 0 && 401260385Sscottl (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) { 402260385Sscottl if ((error = g_io_transient_map_bio(bp)) >= 0) 403260385Sscottl return (error); 404260385Sscottl } 40593778Sphk break; 40693778Sphk default: 40793778Sphk break; 40892108Sphk } 409260385Sscottl return (EJUSTRETURN); 410110471Sphk} 411110471Sphk 412193981Sluigi/* 413193981Sluigi * bio classification support. 414193981Sluigi * 415193981Sluigi * g_register_classifier() and g_unregister_classifier() 416193981Sluigi * are used to add/remove a classifier from the list. 417193981Sluigi * The list is protected using the g_bio_run_down lock, 418193981Sluigi * because the classifiers are called in this path. 419193981Sluigi * 420193981Sluigi * g_io_request() passes bio's that are not already classified 421193981Sluigi * (i.e. those with bio_classifier1 == NULL) to g_run_classifiers(). 422193981Sluigi * Classifiers can store their result in the two fields 423193981Sluigi * bio_classifier1 and bio_classifier2. 424193981Sluigi * A classifier that updates one of the fields should 425193981Sluigi * return a non-zero value. 426193981Sluigi * If no classifier updates the field, g_run_classifiers() sets 427193981Sluigi * bio_classifier1 = BIO_NOTCLASSIFIED to avoid further calls. 428193981Sluigi */ 429193981Sluigi 430193981Sluigiint 431193981Sluigig_register_classifier(struct g_classifier_hook *hook) 432193981Sluigi{ 433193981Sluigi 434193981Sluigi g_bioq_lock(&g_bio_run_down); 435193981Sluigi TAILQ_INSERT_TAIL(&g_classifier_tailq, hook, link); 436193981Sluigi g_bioq_unlock(&g_bio_run_down); 437193981Sluigi 438193981Sluigi return (0); 439193981Sluigi} 440193981Sluigi 441110471Sphkvoid 442193981Sluigig_unregister_classifier(struct g_classifier_hook *hook) 443193981Sluigi{ 444193981Sluigi struct g_classifier_hook *entry; 445193981Sluigi 446193981Sluigi g_bioq_lock(&g_bio_run_down); 447193981Sluigi TAILQ_FOREACH(entry, &g_classifier_tailq, link) { 448193981Sluigi if (entry == hook) { 449193981Sluigi TAILQ_REMOVE(&g_classifier_tailq, hook, link); 450193981Sluigi break; 451193981Sluigi } 452193981Sluigi } 453193981Sluigi g_bioq_unlock(&g_bio_run_down); 454193981Sluigi} 455193981Sluigi 456193981Sluigistatic void 457193981Sluigig_run_classifiers(struct bio *bp) 458193981Sluigi{ 459193981Sluigi struct g_classifier_hook *hook; 460193981Sluigi int classified = 0; 461193981Sluigi 462193981Sluigi TAILQ_FOREACH(hook, &g_classifier_tailq, link) 463193981Sluigi classified |= hook->func(hook->arg, bp); 464193981Sluigi 465193981Sluigi if (!classified) 466193981Sluigi bp->bio_classifier1 = BIO_NOTCLASSIFIED; 467193981Sluigi} 468193981Sluigi 469193981Sluigivoid 470110471Sphkg_io_request(struct bio *bp, struct g_consumer *cp) 471110471Sphk{ 472110523Sphk struct g_provider *pp; 473260385Sscottl struct mtx *mtxp; 474260385Sscottl int direct, error, first; 475110471Sphk 476110471Sphk KASSERT(cp != NULL, ("NULL cp in g_io_request")); 477110471Sphk KASSERT(bp != NULL, ("NULL bp in g_io_request")); 478119973Sphk pp = cp->provider; 479110523Sphk KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 480156170Spjd#ifdef DIAGNOSTIC 481156170Spjd KASSERT(bp->bio_driver1 == NULL, 482156170Spjd ("bio_driver1 used by the consumer (geom %s)", cp->geom->name)); 483156170Spjd KASSERT(bp->bio_driver2 == NULL, 484156170Spjd ("bio_driver2 used by the consumer (geom %s)", cp->geom->name)); 485156170Spjd KASSERT(bp->bio_pflags == 0, 486156170Spjd ("bio_pflags used by the consumer (geom %s)", cp->geom->name)); 487156170Spjd /* 488156170Spjd * Remember consumer's private fields, so we can detect if they were 489156170Spjd * modified by the provider. 490156170Spjd */ 491156170Spjd bp->_bio_caller1 = bp->bio_caller1; 492156170Spjd bp->_bio_caller2 = bp->bio_caller2; 493156170Spjd bp->_bio_cflags = bp->bio_cflags; 494156170Spjd#endif 495110523Sphk 496166325Spjd if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_GETATTR)) { 497163832Spjd KASSERT(bp->bio_data != NULL, 498166325Spjd ("NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd)); 499163832Spjd } 500166325Spjd if (bp->bio_cmd & (BIO_DELETE|BIO_FLUSH)) { 501166325Spjd KASSERT(bp->bio_data == NULL, 502166325Spjd ("non-NULL bp->data in g_io_request(cmd=%hhu)", 503166325Spjd bp->bio_cmd)); 504166325Spjd } 505134519Sphk if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) { 506134519Sphk KASSERT(bp->bio_offset % cp->provider->sectorsize == 0, 507134519Sphk ("wrong offset %jd for sectorsize %u", 508134519Sphk bp->bio_offset, cp->provider->sectorsize)); 509134519Sphk KASSERT(bp->bio_length % cp->provider->sectorsize == 0, 510134519Sphk ("wrong length %jd for sectorsize %u", 511134519Sphk bp->bio_length, cp->provider->sectorsize)); 512134519Sphk } 513134519Sphk 514136399Sups g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 515136399Sups bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 516136399Sups 517110471Sphk bp->bio_from = cp; 518110523Sphk bp->bio_to = pp; 519110471Sphk bp->bio_error = 0; 520110471Sphk bp->bio_completed = 0; 521110471Sphk 522135876Sphk KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 523135876Sphk ("Bio already on queue bp=%p", bp)); 524260385Sscottl if ((g_collectstats & G_STATS_CONSUMERS) != 0 || 525260385Sscottl ((g_collectstats & G_STATS_PROVIDERS) != 0 && pp->stat != NULL)) 526205619Smav binuptime(&bp->bio_t0); 527205619Smav else 528205619Smav getbinuptime(&bp->bio_t0); 529148410Sphk 530260385Sscottl#ifdef GET_STACK_USAGE 531286772Skib direct = (cp->flags & G_CF_DIRECT_SEND) != 0 && 532286772Skib (pp->flags & G_PF_DIRECT_RECEIVE) != 0 && 533286772Skib !g_is_geom_thread(curthread) && 534286772Skib ((pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 || 535287569Simp (bp->bio_flags & BIO_UNMAPPED) == 0 || THREAD_CAN_SLEEP()) && 536287569Simp pace == 0; 537260385Sscottl if (direct) { 538260385Sscottl /* Block direct execution if less then half of stack left. */ 539260385Sscottl size_t st, su; 540260385Sscottl GET_STACK_USAGE(st, su); 541260385Sscottl if (su * 2 > st) 542260385Sscottl direct = 0; 543260385Sscottl } 544260385Sscottl#else 545260385Sscottl direct = 0; 546260385Sscottl#endif 547260385Sscottl 548260385Sscottl if (!TAILQ_EMPTY(&g_classifier_tailq) && !bp->bio_classifier1) { 549260385Sscottl g_bioq_lock(&g_bio_run_down); 550260385Sscottl g_run_classifiers(bp); 551260385Sscottl g_bioq_unlock(&g_bio_run_down); 552260385Sscottl } 553260385Sscottl 554148410Sphk /* 555148410Sphk * The statistics collection is lockless, as such, but we 556148410Sphk * can not update one instance of the statistics from more 557148410Sphk * than one thread at a time, so grab the lock first. 558148410Sphk */ 559260385Sscottl mtxp = mtx_pool_find(mtxpool_sleep, pp); 560260385Sscottl mtx_lock(mtxp); 561260385Sscottl if (g_collectstats & G_STATS_PROVIDERS) 562135876Sphk devstat_start_transaction(pp->stat, &bp->bio_t0); 563260385Sscottl if (g_collectstats & G_STATS_CONSUMERS) 564135876Sphk devstat_start_transaction(cp->stat, &bp->bio_t0); 565130280Sphk pp->nstart++; 566112027Sphk cp->nstart++; 567260385Sscottl mtx_unlock(mtxp); 568110471Sphk 569260385Sscottl if (direct) { 570260385Sscottl error = g_io_check(bp); 571260385Sscottl if (error >= 0) { 572260385Sscottl CTR3(KTR_GEOM, "g_io_request g_io_check on bp %p " 573260385Sscottl "provider %s returned %d", bp, bp->bio_to->name, 574260385Sscottl error); 575260385Sscottl g_io_deliver(bp, error); 576260385Sscottl return; 577260385Sscottl } 578260385Sscottl bp->bio_to->geom->start(bp); 579260385Sscottl } else { 580260385Sscottl g_bioq_lock(&g_bio_run_down); 581260385Sscottl first = TAILQ_EMPTY(&g_bio_run_down.bio_queue); 582260385Sscottl TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue); 583260385Sscottl bp->bio_flags |= BIO_ONQUEUE; 584260385Sscottl g_bio_run_down.bio_queue_length++; 585260385Sscottl g_bioq_unlock(&g_bio_run_down); 586260385Sscottl /* Pass it on down. */ 587260385Sscottl if (first) 588260385Sscottl wakeup(&g_wait_down); 589260385Sscottl } 59092108Sphk} 59192108Sphk 59292108Sphkvoid 593104195Sphkg_io_deliver(struct bio *bp, int error) 59492108Sphk{ 595260385Sscottl struct bintime now; 596110523Sphk struct g_consumer *cp; 597110523Sphk struct g_provider *pp; 598260385Sscottl struct mtx *mtxp; 599260385Sscottl int direct, first; 60092108Sphk 601119973Sphk KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 602120852Sphk pp = bp->bio_to; 603120852Sphk KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 604110523Sphk cp = bp->bio_from; 605120852Sphk if (cp == NULL) { 606120852Sphk bp->bio_error = error; 607120852Sphk bp->bio_done(bp); 608120852Sphk return; 609120852Sphk } 610110523Sphk KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 611110523Sphk KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 612195195Strasz#ifdef DIAGNOSTIC 613195195Strasz /* 614195195Strasz * Some classes - GJournal in particular - can modify bio's 615195195Strasz * private fields while the bio is in transit; G_GEOM_VOLATILE_BIO 616195195Strasz * flag means it's an expected behaviour for that particular geom. 617195195Strasz */ 618195195Strasz if ((cp->geom->flags & G_GEOM_VOLATILE_BIO) == 0) { 619195195Strasz KASSERT(bp->bio_caller1 == bp->_bio_caller1, 620195195Strasz ("bio_caller1 used by the provider %s", pp->name)); 621195195Strasz KASSERT(bp->bio_caller2 == bp->_bio_caller2, 622195195Strasz ("bio_caller2 used by the provider %s", pp->name)); 623195195Strasz KASSERT(bp->bio_cflags == bp->_bio_cflags, 624195195Strasz ("bio_cflags used by the provider %s", pp->name)); 625195195Strasz } 626195195Strasz#endif 627127863Spjd KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0")); 628127863Spjd KASSERT(bp->bio_completed <= bp->bio_length, 629127863Spjd ("bio_completed can't be greater than bio_length")); 630108297Sphk 63192108Sphk g_trace(G_T_BIO, 632108297Sphk"g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 633110523Sphk bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 634105506Sphk (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 63592108Sphk 636135876Sphk KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 637135876Sphk ("Bio already on queue bp=%p", bp)); 638135876Sphk 639134519Sphk /* 640134519Sphk * XXX: next two doesn't belong here 641134519Sphk */ 642112370Sphk bp->bio_bcount = bp->bio_length; 643130280Sphk bp->bio_resid = bp->bio_bcount - bp->bio_completed; 644135876Sphk 645260385Sscottl#ifdef GET_STACK_USAGE 646260385Sscottl direct = (pp->flags & G_PF_DIRECT_SEND) && 647260385Sscottl (cp->flags & G_CF_DIRECT_RECEIVE) && 648260385Sscottl !g_is_geom_thread(curthread); 649260385Sscottl if (direct) { 650260385Sscottl /* Block direct execution if less then half of stack left. */ 651260385Sscottl size_t st, su; 652260385Sscottl GET_STACK_USAGE(st, su); 653260385Sscottl if (su * 2 > st) 654260385Sscottl direct = 0; 655260385Sscottl } 656260385Sscottl#else 657260385Sscottl direct = 0; 658260385Sscottl#endif 659260385Sscottl 660148410Sphk /* 661148410Sphk * The statistics collection is lockless, as such, but we 662148410Sphk * can not update one instance of the statistics from more 663148410Sphk * than one thread at a time, so grab the lock first. 664148410Sphk */ 665260385Sscottl if ((g_collectstats & G_STATS_CONSUMERS) != 0 || 666260385Sscottl ((g_collectstats & G_STATS_PROVIDERS) != 0 && pp->stat != NULL)) 667260385Sscottl binuptime(&now); 668260385Sscottl mtxp = mtx_pool_find(mtxpool_sleep, cp); 669260385Sscottl mtx_lock(mtxp); 670260385Sscottl if (g_collectstats & G_STATS_PROVIDERS) 671260385Sscottl devstat_end_transaction_bio_bt(pp->stat, bp, &now); 672260385Sscottl if (g_collectstats & G_STATS_CONSUMERS) 673260385Sscottl devstat_end_transaction_bio_bt(cp->stat, bp, &now); 674112027Sphk cp->nend++; 675112027Sphk pp->nend++; 676260385Sscottl mtx_unlock(mtxp); 677260385Sscottl 678135876Sphk if (error != ENOMEM) { 679135876Sphk bp->bio_error = error; 680260385Sscottl if (direct) { 681260385Sscottl biodone(bp); 682260385Sscottl } else { 683260385Sscottl g_bioq_lock(&g_bio_run_up); 684260385Sscottl first = TAILQ_EMPTY(&g_bio_run_up.bio_queue); 685260385Sscottl TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue); 686260385Sscottl bp->bio_flags |= BIO_ONQUEUE; 687260385Sscottl g_bio_run_up.bio_queue_length++; 688260385Sscottl g_bioq_unlock(&g_bio_run_up); 689260385Sscottl if (first) 690260385Sscottl wakeup(&g_wait_up); 691260385Sscottl } 692106338Sphk return; 693106338Sphk } 694135876Sphk 695135876Sphk if (bootverbose) 696135876Sphk printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 697135876Sphk bp->bio_children = 0; 698135876Sphk bp->bio_inbed = 0; 699244716Spjd bp->bio_driver1 = NULL; 700244716Spjd bp->bio_driver2 = NULL; 701244716Spjd bp->bio_pflags = 0; 702135876Sphk g_io_request(bp, cp); 703287569Simp pace = 1; 704135876Sphk return; 70592108Sphk} 70692108Sphk 707248508SkibSYSCTL_DECL(_kern_geom); 708248508Skib 709248508Skibstatic long transient_maps; 710248508SkibSYSCTL_LONG(_kern_geom, OID_AUTO, transient_maps, CTLFLAG_RD, 711248508Skib &transient_maps, 0, 712248508Skib "Total count of the transient mapping requests"); 713248508Skibu_int transient_map_retries = 10; 714248508SkibSYSCTL_UINT(_kern_geom, OID_AUTO, transient_map_retries, CTLFLAG_RW, 715248508Skib &transient_map_retries, 0, 716248508Skib "Max count of retries used before giving up on creating transient map"); 717248508Skibint transient_map_hard_failures; 718248508SkibSYSCTL_INT(_kern_geom, OID_AUTO, transient_map_hard_failures, CTLFLAG_RD, 719248508Skib &transient_map_hard_failures, 0, 720248508Skib "Failures to establish the transient mapping due to retry attempts " 721248508Skib "exhausted"); 722248508Skibint transient_map_soft_failures; 723248508SkibSYSCTL_INT(_kern_geom, OID_AUTO, transient_map_soft_failures, CTLFLAG_RD, 724248508Skib &transient_map_soft_failures, 0, 725248508Skib "Count of retried failures to establish the transient mapping"); 726248508Skibint inflight_transient_maps; 727248508SkibSYSCTL_INT(_kern_geom, OID_AUTO, inflight_transient_maps, CTLFLAG_RD, 728248508Skib &inflight_transient_maps, 0, 729248508Skib "Current count of the active transient maps"); 730248508Skib 731248508Skibstatic int 732248508Skibg_io_transient_map_bio(struct bio *bp) 733248508Skib{ 734248508Skib vm_offset_t addr; 735248508Skib long size; 736248508Skib u_int retried; 737248508Skib 738248568Skib KASSERT(unmapped_buf_allowed, ("unmapped disabled")); 739248568Skib 740248508Skib size = round_page(bp->bio_ma_offset + bp->bio_length); 741248508Skib KASSERT(size / PAGE_SIZE == bp->bio_ma_n, ("Bio too short %p", bp)); 742248508Skib addr = 0; 743248508Skib retried = 0; 744248508Skib atomic_add_long(&transient_maps, 1); 745248508Skibretry: 746252330Sjeff if (vmem_alloc(transient_arena, size, M_BESTFIT | M_NOWAIT, &addr)) { 747248508Skib if (transient_map_retries != 0 && 748248508Skib retried >= transient_map_retries) { 749248508Skib CTR2(KTR_GEOM, "g_down cannot map bp %p provider %s", 750248508Skib bp, bp->bio_to->name); 751248508Skib atomic_add_int(&transient_map_hard_failures, 1); 752260385Sscottl return (EDEADLK/* XXXKIB */); 753248508Skib } else { 754248508Skib /* 755248508Skib * Naive attempt to quisce the I/O to get more 756248508Skib * in-flight requests completed and defragment 757252330Sjeff * the transient_arena. 758248508Skib */ 759248508Skib CTR3(KTR_GEOM, "g_down retrymap bp %p provider %s r %d", 760248508Skib bp, bp->bio_to->name, retried); 761248508Skib pause("g_d_tra", hz / 10); 762248508Skib retried++; 763248508Skib atomic_add_int(&transient_map_soft_failures, 1); 764248508Skib goto retry; 765248508Skib } 766248508Skib } 767248508Skib atomic_add_int(&inflight_transient_maps, 1); 768248508Skib pmap_qenter((vm_offset_t)addr, bp->bio_ma, OFF_TO_IDX(size)); 769248508Skib bp->bio_data = (caddr_t)addr + bp->bio_ma_offset; 770248508Skib bp->bio_flags |= BIO_TRANSIENT_MAPPING; 771248508Skib bp->bio_flags &= ~BIO_UNMAPPED; 772260385Sscottl return (EJUSTRETURN); 773248508Skib} 774248508Skib 77592108Sphkvoid 77692108Sphkg_io_schedule_down(struct thread *tp __unused) 77792108Sphk{ 77892108Sphk struct bio *bp; 779110471Sphk int error; 78092108Sphk 78192108Sphk for(;;) { 782110736Sphk g_bioq_lock(&g_bio_run_down); 78392108Sphk bp = g_bioq_first(&g_bio_run_down); 784110736Sphk if (bp == NULL) { 785136755Srwatson CTR0(KTR_GEOM, "g_down going to sleep"); 786110736Sphk msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 787196904Smav PRIBIO | PDROP, "-", 0); 788110736Sphk continue; 789110736Sphk } 790136755Srwatson CTR0(KTR_GEOM, "g_down has work to do"); 791110736Sphk g_bioq_unlock(&g_bio_run_down); 792287569Simp if (pace != 0) { 793287569Simp /* 794287569Simp * There has been at least one memory allocation 795287569Simp * failure since the last I/O completed. Pause 1ms to 796287569Simp * give the system a chance to free up memory. We only 797287569Simp * do this once because a large number of allocations 798287569Simp * can fail in the direct dispatch case and there's no 799287569Simp * relationship between the number of these failures and 800287569Simp * the length of the outage. If there's still an outage, 801287569Simp * we'll pause again and again until it's 802287569Simp * resolved. Older versions paused longer and once per 803287569Simp * allocation failure. This was OK for a single threaded 804287569Simp * g_down, but with direct dispatch would lead to max of 805287569Simp * 10 IOPs for minutes at a time when transient memory 806287569Simp * issues prevented allocation for a batch of requests 807287569Simp * from the upper layers. 808287569Simp * 809287569Simp * XXX This pacing is really lame. It needs to be solved 810287569Simp * by other methods. This is OK only because the worst 811287569Simp * case scenario is so rare. In the worst case scenario 812287569Simp * all memory is tied up waiting for I/O to complete 813287569Simp * which can never happen since we can't allocate bios 814287569Simp * for that I/O. 815287569Simp */ 816287569Simp CTR0(KTR_GEOM, "g_down pacing self"); 817287569Simp pause("g_down", min(hz/1000, 1)); 818287569Simp pace = 0; 819112830Sphk } 820260385Sscottl CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp, 821260385Sscottl bp->bio_to->name); 822110471Sphk error = g_io_check(bp); 823260385Sscottl if (error >= 0) { 824136755Srwatson CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider " 825136755Srwatson "%s returned %d", bp, bp->bio_to->name, error); 826110471Sphk g_io_deliver(bp, error); 827110471Sphk continue; 828110471Sphk } 829150177Sjhb THREAD_NO_SLEEPING(); 830136755Srwatson CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld " 831136755Srwatson "len %ld", bp, bp->bio_to->name, bp->bio_offset, 832136755Srwatson bp->bio_length); 83392108Sphk bp->bio_to->geom->start(bp); 834150177Sjhb THREAD_SLEEPING_OK(); 83592108Sphk } 83692108Sphk} 83792108Sphk 83892108Sphkvoid 839125137Sphkbio_taskqueue(struct bio *bp, bio_task_t *func, void *arg) 840125137Sphk{ 841125137Sphk bp->bio_task = func; 842125137Sphk bp->bio_task_arg = arg; 843125137Sphk /* 844125137Sphk * The taskqueue is actually just a second queue off the "up" 845125137Sphk * queue, so we use the same lock. 846125137Sphk */ 847125137Sphk g_bioq_lock(&g_bio_run_up); 848134519Sphk KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 849134519Sphk ("Bio already on queue bp=%p target taskq", bp)); 850134519Sphk bp->bio_flags |= BIO_ONQUEUE; 851125137Sphk TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue); 852125137Sphk g_bio_run_task.bio_queue_length++; 853125137Sphk wakeup(&g_wait_up); 854125137Sphk g_bioq_unlock(&g_bio_run_up); 855125137Sphk} 856125137Sphk 857125137Sphk 858125137Sphkvoid 85992108Sphkg_io_schedule_up(struct thread *tp __unused) 86092108Sphk{ 86192108Sphk struct bio *bp; 86292108Sphk for(;;) { 863110736Sphk g_bioq_lock(&g_bio_run_up); 864125137Sphk bp = g_bioq_first(&g_bio_run_task); 865125137Sphk if (bp != NULL) { 866125137Sphk g_bioq_unlock(&g_bio_run_up); 867150177Sjhb THREAD_NO_SLEEPING(); 868136755Srwatson CTR1(KTR_GEOM, "g_up processing task bp %p", bp); 869125137Sphk bp->bio_task(bp->bio_task_arg); 870150177Sjhb THREAD_SLEEPING_OK(); 871125137Sphk continue; 872125137Sphk } 87392108Sphk bp = g_bioq_first(&g_bio_run_up); 874110736Sphk if (bp != NULL) { 875110736Sphk g_bioq_unlock(&g_bio_run_up); 876150177Sjhb THREAD_NO_SLEEPING(); 877136755Srwatson CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off " 878183146Ssbruno "%jd len %ld", bp, bp->bio_to->name, 879136755Srwatson bp->bio_offset, bp->bio_length); 880110736Sphk biodone(bp); 881150177Sjhb THREAD_SLEEPING_OK(); 882110736Sphk continue; 883110736Sphk } 884136755Srwatson CTR0(KTR_GEOM, "g_up going to sleep"); 885110736Sphk msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 886196904Smav PRIBIO | PDROP, "-", 0); 88792108Sphk } 88892108Sphk} 88992108Sphk 89092108Sphkvoid * 89192108Sphkg_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 89292108Sphk{ 89392108Sphk struct bio *bp; 89492108Sphk void *ptr; 89592108Sphk int errorc; 89692108Sphk 897135873Spjd KASSERT(length > 0 && length >= cp->provider->sectorsize && 898135873Spjd length <= MAXPHYS, ("g_read_data(): invalid length %jd", 899135873Spjd (intmax_t)length)); 900120493Sphk 901134379Sphk bp = g_alloc_bio(); 902104665Sphk bp->bio_cmd = BIO_READ; 903104665Sphk bp->bio_done = NULL; 904104665Sphk bp->bio_offset = offset; 905104665Sphk bp->bio_length = length; 906111119Simp ptr = g_malloc(length, M_WAITOK); 907104665Sphk bp->bio_data = ptr; 908104665Sphk g_io_request(bp, cp); 909104665Sphk errorc = biowait(bp, "gread"); 910104665Sphk if (error != NULL) 911104665Sphk *error = errorc; 912104665Sphk g_destroy_bio(bp); 913104665Sphk if (errorc) { 914104665Sphk g_free(ptr); 915104665Sphk ptr = NULL; 916104665Sphk } 91792108Sphk return (ptr); 91892108Sphk} 919104194Sphk 920104194Sphkint 921104194Sphkg_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 922104194Sphk{ 923104194Sphk struct bio *bp; 924104194Sphk int error; 925104194Sphk 926135873Spjd KASSERT(length > 0 && length >= cp->provider->sectorsize && 927135873Spjd length <= MAXPHYS, ("g_write_data(): invalid length %jd", 928135873Spjd (intmax_t)length)); 929120493Sphk 930134379Sphk bp = g_alloc_bio(); 931104194Sphk bp->bio_cmd = BIO_WRITE; 932104194Sphk bp->bio_done = NULL; 933104194Sphk bp->bio_offset = offset; 934104194Sphk bp->bio_length = length; 935104194Sphk bp->bio_data = ptr; 936104194Sphk g_io_request(bp, cp); 937104194Sphk error = biowait(bp, "gwrite"); 938104194Sphk g_destroy_bio(bp); 939104194Sphk return (error); 940104194Sphk} 941125713Spjd 942169283Spjdint 943169283Spjdg_delete_data(struct g_consumer *cp, off_t offset, off_t length) 944169283Spjd{ 945169283Spjd struct bio *bp; 946169283Spjd int error; 947169283Spjd 948174669Sphk KASSERT(length > 0 && length >= cp->provider->sectorsize, 949174669Sphk ("g_delete_data(): invalid length %jd", (intmax_t)length)); 950169283Spjd 951169283Spjd bp = g_alloc_bio(); 952169283Spjd bp->bio_cmd = BIO_DELETE; 953169283Spjd bp->bio_done = NULL; 954169283Spjd bp->bio_offset = offset; 955169283Spjd bp->bio_length = length; 956169283Spjd bp->bio_data = NULL; 957169283Spjd g_io_request(bp, cp); 958169283Spjd error = biowait(bp, "gdelete"); 959169283Spjd g_destroy_bio(bp); 960169283Spjd return (error); 961169283Spjd} 962169283Spjd 963125713Spjdvoid 964125713Spjdg_print_bio(struct bio *bp) 965125713Spjd{ 966125713Spjd const char *pname, *cmd = NULL; 967125713Spjd 968125713Spjd if (bp->bio_to != NULL) 969125713Spjd pname = bp->bio_to->name; 970125713Spjd else 971125713Spjd pname = "[unknown]"; 972125713Spjd 973125713Spjd switch (bp->bio_cmd) { 974125713Spjd case BIO_GETATTR: 975125713Spjd cmd = "GETATTR"; 976125713Spjd printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute); 977125713Spjd return; 978163832Spjd case BIO_FLUSH: 979163832Spjd cmd = "FLUSH"; 980163832Spjd printf("%s[%s]", pname, cmd); 981163832Spjd return; 982125713Spjd case BIO_READ: 983125713Spjd cmd = "READ"; 984208992Strasz break; 985125713Spjd case BIO_WRITE: 986208992Strasz cmd = "WRITE"; 987208992Strasz break; 988125713Spjd case BIO_DELETE: 989208992Strasz cmd = "DELETE"; 990208992Strasz break; 991125713Spjd default: 992125713Spjd cmd = "UNKNOWN"; 993125713Spjd printf("%s[%s()]", pname, cmd); 994125713Spjd return; 995125713Spjd } 996208992Strasz printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd, 997208992Strasz (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 998125713Spjd} 999