zio.c revision 260763
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2013 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 */ 26 27#include <sys/zfs_context.h> 28#include <sys/fm/fs/zfs.h> 29#include <sys/spa.h> 30#include <sys/txg.h> 31#include <sys/spa_impl.h> 32#include <sys/vdev_impl.h> 33#include <sys/zio_impl.h> 34#include <sys/zio_compress.h> 35#include <sys/zio_checksum.h> 36#include <sys/dmu_objset.h> 37#include <sys/arc.h> 38#include <sys/ddt.h> 39#include <sys/trim_map.h> 40 41SYSCTL_DECL(_vfs_zfs); 42SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); 43#if defined(__amd64__) 44static int zio_use_uma = 1; 45#else 46static int zio_use_uma = 0; 47#endif 48TUNABLE_INT("vfs.zfs.zio.use_uma", &zio_use_uma); 49SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, 50 "Use uma(9) for ZIO allocations"); 51static int zio_exclude_metadata = 0; 52TUNABLE_INT("vfs.zfs.zio.exclude_metadata", &zio_exclude_metadata); 53SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, exclude_metadata, CTLFLAG_RDTUN, &zio_exclude_metadata, 0, 54 "Exclude metadata buffers from dumps as well"); 55 56zio_trim_stats_t zio_trim_stats = { 57 { "bytes", KSTAT_DATA_UINT64, 58 "Number of bytes successfully TRIMmed" }, 59 { "success", KSTAT_DATA_UINT64, 60 "Number of successful TRIM requests" }, 61 { "unsupported", KSTAT_DATA_UINT64, 62 "Number of TRIM requests that failed because TRIM is not supported" }, 63 { "failed", KSTAT_DATA_UINT64, 64 "Number of TRIM requests that failed for reasons other than not supported" }, 65}; 66 67static kstat_t *zio_trim_ksp; 68 69/* 70 * ========================================================================== 71 * I/O type descriptions 72 * ========================================================================== 73 */ 74const char *zio_type_name[ZIO_TYPES] = { 75 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 76 "zio_ioctl" 77}; 78 79/* 80 * ========================================================================== 81 * I/O kmem caches 82 * ========================================================================== 83 */ 84kmem_cache_t *zio_cache; 85kmem_cache_t *zio_link_cache; 86kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 87kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 88 89#ifdef _KERNEL 90extern vmem_t *zio_alloc_arena; 91#endif 92extern int zfs_mg_alloc_failures; 93 94/* 95 * The following actions directly effect the spa's sync-to-convergence logic. 96 * The values below define the sync pass when we start performing the action. 97 * Care should be taken when changing these values as they directly impact 98 * spa_sync() performance. Tuning these values may introduce subtle performance 99 * pathologies and should only be done in the context of performance analysis. 100 * These tunables will eventually be removed and replaced with #defines once 101 * enough analysis has been done to determine optimal values. 102 * 103 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 104 * regular blocks are not deferred. 105 */ 106int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 107TUNABLE_INT("vfs.zfs.sync_pass_deferred_free", &zfs_sync_pass_deferred_free); 108SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_deferred_free, CTLFLAG_RDTUN, 109 &zfs_sync_pass_deferred_free, 0, "defer frees starting in this pass"); 110int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 111TUNABLE_INT("vfs.zfs.sync_pass_dont_compress", &zfs_sync_pass_dont_compress); 112SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_dont_compress, CTLFLAG_RDTUN, 113 &zfs_sync_pass_dont_compress, 0, "don't compress starting in this pass"); 114int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 115TUNABLE_INT("vfs.zfs.sync_pass_rewrite", &zfs_sync_pass_rewrite); 116SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_rewrite, CTLFLAG_RDTUN, 117 &zfs_sync_pass_rewrite, 0, "rewrite new bps starting in this pass"); 118 119/* 120 * An allocating zio is one that either currently has the DVA allocate 121 * stage set or will have it later in its lifetime. 122 */ 123#define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 124 125boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 126 127#ifdef ZFS_DEBUG 128int zio_buf_debug_limit = 16384; 129#else 130int zio_buf_debug_limit = 0; 131#endif 132 133void 134zio_init(void) 135{ 136 size_t c; 137 zio_cache = kmem_cache_create("zio_cache", 138 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 139 zio_link_cache = kmem_cache_create("zio_link_cache", 140 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 141 if (!zio_use_uma) 142 goto out; 143 144 /* 145 * For small buffers, we want a cache for each multiple of 146 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 147 * for each quarter-power of 2. For large buffers, we want 148 * a cache for each multiple of PAGESIZE. 149 */ 150 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 151 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 152 size_t p2 = size; 153 size_t align = 0; 154 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 155 156 while (p2 & (p2 - 1)) 157 p2 &= p2 - 1; 158 159#ifdef illumos 160#ifndef _KERNEL 161 /* 162 * If we are using watchpoints, put each buffer on its own page, 163 * to eliminate the performance overhead of trapping to the 164 * kernel when modifying a non-watched buffer that shares the 165 * page with a watched buffer. 166 */ 167 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 168 continue; 169#endif 170#endif /* illumos */ 171 if (size <= 4 * SPA_MINBLOCKSIZE) { 172 align = SPA_MINBLOCKSIZE; 173 } else if (IS_P2ALIGNED(size, PAGESIZE)) { 174 align = PAGESIZE; 175 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 176 align = p2 >> 2; 177 } 178 179 if (align != 0) { 180 char name[36]; 181 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 182 zio_buf_cache[c] = kmem_cache_create(name, size, 183 align, NULL, NULL, NULL, NULL, NULL, cflags); 184 185 /* 186 * Since zio_data bufs do not appear in crash dumps, we 187 * pass KMC_NOTOUCH so that no allocator metadata is 188 * stored with the buffers. 189 */ 190 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 191 zio_data_buf_cache[c] = kmem_cache_create(name, size, 192 align, NULL, NULL, NULL, NULL, NULL, 193 cflags | KMC_NOTOUCH | KMC_NODEBUG); 194 } 195 } 196 197 while (--c != 0) { 198 ASSERT(zio_buf_cache[c] != NULL); 199 if (zio_buf_cache[c - 1] == NULL) 200 zio_buf_cache[c - 1] = zio_buf_cache[c]; 201 202 ASSERT(zio_data_buf_cache[c] != NULL); 203 if (zio_data_buf_cache[c - 1] == NULL) 204 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 205 } 206out: 207 208 /* 209 * The zio write taskqs have 1 thread per cpu, allow 1/2 of the taskqs 210 * to fail 3 times per txg or 8 failures, whichever is greater. 211 */ 212 if (zfs_mg_alloc_failures == 0) 213 zfs_mg_alloc_failures = MAX((3 * max_ncpus / 2), 8); 214 else if (zfs_mg_alloc_failures < 8) 215 zfs_mg_alloc_failures = 8; 216 217 zio_inject_init(); 218 219 zio_trim_ksp = kstat_create("zfs", 0, "zio_trim", "misc", 220 KSTAT_TYPE_NAMED, 221 sizeof(zio_trim_stats) / sizeof(kstat_named_t), 222 KSTAT_FLAG_VIRTUAL); 223 224 if (zio_trim_ksp != NULL) { 225 zio_trim_ksp->ks_data = &zio_trim_stats; 226 kstat_install(zio_trim_ksp); 227 } 228} 229 230void 231zio_fini(void) 232{ 233 size_t c; 234 kmem_cache_t *last_cache = NULL; 235 kmem_cache_t *last_data_cache = NULL; 236 237 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 238 if (zio_buf_cache[c] != last_cache) { 239 last_cache = zio_buf_cache[c]; 240 kmem_cache_destroy(zio_buf_cache[c]); 241 } 242 zio_buf_cache[c] = NULL; 243 244 if (zio_data_buf_cache[c] != last_data_cache) { 245 last_data_cache = zio_data_buf_cache[c]; 246 kmem_cache_destroy(zio_data_buf_cache[c]); 247 } 248 zio_data_buf_cache[c] = NULL; 249 } 250 251 kmem_cache_destroy(zio_link_cache); 252 kmem_cache_destroy(zio_cache); 253 254 zio_inject_fini(); 255 256 if (zio_trim_ksp != NULL) { 257 kstat_delete(zio_trim_ksp); 258 zio_trim_ksp = NULL; 259 } 260} 261 262/* 263 * ========================================================================== 264 * Allocate and free I/O buffers 265 * ========================================================================== 266 */ 267 268/* 269 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 270 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 271 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 272 * excess / transient data in-core during a crashdump. 273 */ 274void * 275zio_buf_alloc(size_t size) 276{ 277 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 278 int flags = zio_exclude_metadata ? KM_NODEBUG : 0; 279 280 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 281 282 if (zio_use_uma) 283 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 284 else 285 return (kmem_alloc(size, KM_SLEEP|flags)); 286} 287 288/* 289 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 290 * crashdump if the kernel panics. This exists so that we will limit the amount 291 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 292 * of kernel heap dumped to disk when the kernel panics) 293 */ 294void * 295zio_data_buf_alloc(size_t size) 296{ 297 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 298 299 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 300 301 if (zio_use_uma) 302 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 303 else 304 return (kmem_alloc(size, KM_SLEEP | KM_NODEBUG)); 305} 306 307void 308zio_buf_free(void *buf, size_t size) 309{ 310 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 311 312 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 313 314 if (zio_use_uma) 315 kmem_cache_free(zio_buf_cache[c], buf); 316 else 317 kmem_free(buf, size); 318} 319 320void 321zio_data_buf_free(void *buf, size_t size) 322{ 323 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 324 325 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 326 327 if (zio_use_uma) 328 kmem_cache_free(zio_data_buf_cache[c], buf); 329 else 330 kmem_free(buf, size); 331} 332 333/* 334 * ========================================================================== 335 * Push and pop I/O transform buffers 336 * ========================================================================== 337 */ 338static void 339zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 340 zio_transform_func_t *transform) 341{ 342 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 343 344 zt->zt_orig_data = zio->io_data; 345 zt->zt_orig_size = zio->io_size; 346 zt->zt_bufsize = bufsize; 347 zt->zt_transform = transform; 348 349 zt->zt_next = zio->io_transform_stack; 350 zio->io_transform_stack = zt; 351 352 zio->io_data = data; 353 zio->io_size = size; 354} 355 356static void 357zio_pop_transforms(zio_t *zio) 358{ 359 zio_transform_t *zt; 360 361 while ((zt = zio->io_transform_stack) != NULL) { 362 if (zt->zt_transform != NULL) 363 zt->zt_transform(zio, 364 zt->zt_orig_data, zt->zt_orig_size); 365 366 if (zt->zt_bufsize != 0) 367 zio_buf_free(zio->io_data, zt->zt_bufsize); 368 369 zio->io_data = zt->zt_orig_data; 370 zio->io_size = zt->zt_orig_size; 371 zio->io_transform_stack = zt->zt_next; 372 373 kmem_free(zt, sizeof (zio_transform_t)); 374 } 375} 376 377/* 378 * ========================================================================== 379 * I/O transform callbacks for subblocks and decompression 380 * ========================================================================== 381 */ 382static void 383zio_subblock(zio_t *zio, void *data, uint64_t size) 384{ 385 ASSERT(zio->io_size > size); 386 387 if (zio->io_type == ZIO_TYPE_READ) 388 bcopy(zio->io_data, data, size); 389} 390 391static void 392zio_decompress(zio_t *zio, void *data, uint64_t size) 393{ 394 if (zio->io_error == 0 && 395 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 396 zio->io_data, data, zio->io_size, size) != 0) 397 zio->io_error = SET_ERROR(EIO); 398} 399 400/* 401 * ========================================================================== 402 * I/O parent/child relationships and pipeline interlocks 403 * ========================================================================== 404 */ 405/* 406 * NOTE - Callers to zio_walk_parents() and zio_walk_children must 407 * continue calling these functions until they return NULL. 408 * Otherwise, the next caller will pick up the list walk in 409 * some indeterminate state. (Otherwise every caller would 410 * have to pass in a cookie to keep the state represented by 411 * io_walk_link, which gets annoying.) 412 */ 413zio_t * 414zio_walk_parents(zio_t *cio) 415{ 416 zio_link_t *zl = cio->io_walk_link; 417 list_t *pl = &cio->io_parent_list; 418 419 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl); 420 cio->io_walk_link = zl; 421 422 if (zl == NULL) 423 return (NULL); 424 425 ASSERT(zl->zl_child == cio); 426 return (zl->zl_parent); 427} 428 429zio_t * 430zio_walk_children(zio_t *pio) 431{ 432 zio_link_t *zl = pio->io_walk_link; 433 list_t *cl = &pio->io_child_list; 434 435 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl); 436 pio->io_walk_link = zl; 437 438 if (zl == NULL) 439 return (NULL); 440 441 ASSERT(zl->zl_parent == pio); 442 return (zl->zl_child); 443} 444 445zio_t * 446zio_unique_parent(zio_t *cio) 447{ 448 zio_t *pio = zio_walk_parents(cio); 449 450 VERIFY(zio_walk_parents(cio) == NULL); 451 return (pio); 452} 453 454void 455zio_add_child(zio_t *pio, zio_t *cio) 456{ 457 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 458 459 /* 460 * Logical I/Os can have logical, gang, or vdev children. 461 * Gang I/Os can have gang or vdev children. 462 * Vdev I/Os can only have vdev children. 463 * The following ASSERT captures all of these constraints. 464 */ 465 ASSERT(cio->io_child_type <= pio->io_child_type); 466 467 zl->zl_parent = pio; 468 zl->zl_child = cio; 469 470 mutex_enter(&cio->io_lock); 471 mutex_enter(&pio->io_lock); 472 473 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 474 475 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 476 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 477 478 list_insert_head(&pio->io_child_list, zl); 479 list_insert_head(&cio->io_parent_list, zl); 480 481 pio->io_child_count++; 482 cio->io_parent_count++; 483 484 mutex_exit(&pio->io_lock); 485 mutex_exit(&cio->io_lock); 486} 487 488static void 489zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 490{ 491 ASSERT(zl->zl_parent == pio); 492 ASSERT(zl->zl_child == cio); 493 494 mutex_enter(&cio->io_lock); 495 mutex_enter(&pio->io_lock); 496 497 list_remove(&pio->io_child_list, zl); 498 list_remove(&cio->io_parent_list, zl); 499 500 pio->io_child_count--; 501 cio->io_parent_count--; 502 503 mutex_exit(&pio->io_lock); 504 mutex_exit(&cio->io_lock); 505 506 kmem_cache_free(zio_link_cache, zl); 507} 508 509static boolean_t 510zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 511{ 512 uint64_t *countp = &zio->io_children[child][wait]; 513 boolean_t waiting = B_FALSE; 514 515 mutex_enter(&zio->io_lock); 516 ASSERT(zio->io_stall == NULL); 517 if (*countp != 0) { 518 zio->io_stage >>= 1; 519 zio->io_stall = countp; 520 waiting = B_TRUE; 521 } 522 mutex_exit(&zio->io_lock); 523 524 return (waiting); 525} 526 527static void 528zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 529{ 530 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 531 int *errorp = &pio->io_child_error[zio->io_child_type]; 532 533 mutex_enter(&pio->io_lock); 534 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 535 *errorp = zio_worst_error(*errorp, zio->io_error); 536 pio->io_reexecute |= zio->io_reexecute; 537 ASSERT3U(*countp, >, 0); 538 539 (*countp)--; 540 541 if (*countp == 0 && pio->io_stall == countp) { 542 pio->io_stall = NULL; 543 mutex_exit(&pio->io_lock); 544 zio_execute(pio); 545 } else { 546 mutex_exit(&pio->io_lock); 547 } 548} 549 550static void 551zio_inherit_child_errors(zio_t *zio, enum zio_child c) 552{ 553 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 554 zio->io_error = zio->io_child_error[c]; 555} 556 557/* 558 * ========================================================================== 559 * Create the various types of I/O (read, write, free, etc) 560 * ========================================================================== 561 */ 562static zio_t * 563zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 564 void *data, uint64_t size, zio_done_func_t *done, void *private, 565 zio_type_t type, zio_priority_t priority, enum zio_flag flags, 566 vdev_t *vd, uint64_t offset, const zbookmark_t *zb, 567 enum zio_stage stage, enum zio_stage pipeline) 568{ 569 zio_t *zio; 570 571 ASSERT3U(type == ZIO_TYPE_FREE || size, <=, SPA_MAXBLOCKSIZE); 572 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 573 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 574 575 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 576 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 577 ASSERT(vd || stage == ZIO_STAGE_OPEN); 578 579 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 580 bzero(zio, sizeof (zio_t)); 581 582 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 583 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 584 585 list_create(&zio->io_parent_list, sizeof (zio_link_t), 586 offsetof(zio_link_t, zl_parent_node)); 587 list_create(&zio->io_child_list, sizeof (zio_link_t), 588 offsetof(zio_link_t, zl_child_node)); 589 590 if (vd != NULL) 591 zio->io_child_type = ZIO_CHILD_VDEV; 592 else if (flags & ZIO_FLAG_GANG_CHILD) 593 zio->io_child_type = ZIO_CHILD_GANG; 594 else if (flags & ZIO_FLAG_DDT_CHILD) 595 zio->io_child_type = ZIO_CHILD_DDT; 596 else 597 zio->io_child_type = ZIO_CHILD_LOGICAL; 598 599 if (bp != NULL) { 600 zio->io_bp = (blkptr_t *)bp; 601 zio->io_bp_copy = *bp; 602 zio->io_bp_orig = *bp; 603 if (type != ZIO_TYPE_WRITE || 604 zio->io_child_type == ZIO_CHILD_DDT) 605 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 606 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 607 zio->io_logical = zio; 608 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 609 pipeline |= ZIO_GANG_STAGES; 610 } 611 612 zio->io_spa = spa; 613 zio->io_txg = txg; 614 zio->io_done = done; 615 zio->io_private = private; 616 zio->io_type = type; 617 zio->io_priority = priority; 618 zio->io_vd = vd; 619 zio->io_offset = offset; 620 zio->io_orig_data = zio->io_data = data; 621 zio->io_orig_size = zio->io_size = size; 622 zio->io_orig_flags = zio->io_flags = flags; 623 zio->io_orig_stage = zio->io_stage = stage; 624 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 625 626 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 627 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 628 629 if (zb != NULL) 630 zio->io_bookmark = *zb; 631 632 if (pio != NULL) { 633 if (zio->io_logical == NULL) 634 zio->io_logical = pio->io_logical; 635 if (zio->io_child_type == ZIO_CHILD_GANG) 636 zio->io_gang_leader = pio->io_gang_leader; 637 zio_add_child(pio, zio); 638 } 639 640 return (zio); 641} 642 643static void 644zio_destroy(zio_t *zio) 645{ 646 list_destroy(&zio->io_parent_list); 647 list_destroy(&zio->io_child_list); 648 mutex_destroy(&zio->io_lock); 649 cv_destroy(&zio->io_cv); 650 kmem_cache_free(zio_cache, zio); 651} 652 653zio_t * 654zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 655 void *private, enum zio_flag flags) 656{ 657 zio_t *zio; 658 659 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 660 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 661 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 662 663 return (zio); 664} 665 666zio_t * 667zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 668{ 669 return (zio_null(NULL, spa, NULL, done, private, flags)); 670} 671 672zio_t * 673zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 674 void *data, uint64_t size, zio_done_func_t *done, void *private, 675 zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb) 676{ 677 zio_t *zio; 678 679 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 680 data, size, done, private, 681 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 682 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 683 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 684 685 return (zio); 686} 687 688zio_t * 689zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 690 void *data, uint64_t size, const zio_prop_t *zp, 691 zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done, 692 void *private, 693 zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb) 694{ 695 zio_t *zio; 696 697 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 698 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 699 zp->zp_compress >= ZIO_COMPRESS_OFF && 700 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 701 DMU_OT_IS_VALID(zp->zp_type) && 702 zp->zp_level < 32 && 703 zp->zp_copies > 0 && 704 zp->zp_copies <= spa_max_replication(spa)); 705 706 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 707 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 708 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 709 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 710 711 zio->io_ready = ready; 712 zio->io_physdone = physdone; 713 zio->io_prop = *zp; 714 715 return (zio); 716} 717 718zio_t * 719zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 720 uint64_t size, zio_done_func_t *done, void *private, 721 zio_priority_t priority, enum zio_flag flags, zbookmark_t *zb) 722{ 723 zio_t *zio; 724 725 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 726 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 727 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 728 729 return (zio); 730} 731 732void 733zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 734{ 735 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 736 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 737 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 738 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 739 740 /* 741 * We must reset the io_prop to match the values that existed 742 * when the bp was first written by dmu_sync() keeping in mind 743 * that nopwrite and dedup are mutually exclusive. 744 */ 745 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 746 zio->io_prop.zp_nopwrite = nopwrite; 747 zio->io_prop.zp_copies = copies; 748 zio->io_bp_override = bp; 749} 750 751void 752zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 753{ 754 metaslab_check_free(spa, bp); 755 756 /* 757 * Frees that are for the currently-syncing txg, are not going to be 758 * deferred, and which will not need to do a read (i.e. not GANG or 759 * DEDUP), can be processed immediately. Otherwise, put them on the 760 * in-memory list for later processing. 761 */ 762 if (zfs_trim_enabled || BP_IS_GANG(bp) || BP_GET_DEDUP(bp) || 763 txg != spa->spa_syncing_txg || 764 spa_sync_pass(spa) >= zfs_sync_pass_deferred_free) { 765 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 766 } else { 767 VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 768 BP_GET_PSIZE(bp), 0))); 769 } 770} 771 772zio_t * 773zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 774 uint64_t size, enum zio_flag flags) 775{ 776 zio_t *zio; 777 enum zio_stage stage = ZIO_FREE_PIPELINE; 778 779 dprintf_bp(bp, "freeing in txg %llu, pass %u", 780 (longlong_t)txg, spa->spa_sync_pass); 781 782 ASSERT(!BP_IS_HOLE(bp)); 783 ASSERT(spa_syncing_txg(spa) == txg); 784 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free); 785 786 metaslab_check_free(spa, bp); 787 arc_freed(spa, bp); 788 789 if (zfs_trim_enabled) 790 stage |= ZIO_STAGE_ISSUE_ASYNC | ZIO_STAGE_VDEV_IO_START | 791 ZIO_STAGE_VDEV_IO_ASSESS; 792 /* 793 * GANG and DEDUP blocks can induce a read (for the gang block header, 794 * or the DDT), so issue them asynchronously so that this thread is 795 * not tied up. 796 */ 797 else if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) 798 stage |= ZIO_STAGE_ISSUE_ASYNC; 799 800 zio = zio_create(pio, spa, txg, bp, NULL, size, 801 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, flags, 802 NULL, 0, NULL, ZIO_STAGE_OPEN, stage); 803 804 return (zio); 805} 806 807zio_t * 808zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 809 zio_done_func_t *done, void *private, enum zio_flag flags) 810{ 811 zio_t *zio; 812 813 /* 814 * A claim is an allocation of a specific block. Claims are needed 815 * to support immediate writes in the intent log. The issue is that 816 * immediate writes contain committed data, but in a txg that was 817 * *not* committed. Upon opening the pool after an unclean shutdown, 818 * the intent log claims all blocks that contain immediate write data 819 * so that the SPA knows they're in use. 820 * 821 * All claims *must* be resolved in the first txg -- before the SPA 822 * starts allocating blocks -- so that nothing is allocated twice. 823 * If txg == 0 we just verify that the block is claimable. 824 */ 825 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 826 ASSERT(txg == spa_first_txg(spa) || txg == 0); 827 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 828 829 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 830 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 831 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 832 833 return (zio); 834} 835 836zio_t * 837zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, uint64_t offset, 838 uint64_t size, zio_done_func_t *done, void *private, 839 enum zio_flag flags) 840{ 841 zio_t *zio; 842 int c; 843 844 if (vd->vdev_children == 0) { 845 zio = zio_create(pio, spa, 0, NULL, NULL, size, done, private, 846 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, offset, NULL, 847 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 848 849 zio->io_cmd = cmd; 850 } else { 851 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 852 853 for (c = 0; c < vd->vdev_children; c++) 854 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 855 offset, size, done, private, flags)); 856 } 857 858 return (zio); 859} 860 861zio_t * 862zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 863 void *data, int checksum, zio_done_func_t *done, void *private, 864 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 865{ 866 zio_t *zio; 867 868 ASSERT(vd->vdev_children == 0); 869 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 870 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 871 ASSERT3U(offset + size, <=, vd->vdev_psize); 872 873 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 874 ZIO_TYPE_READ, priority, flags, vd, offset, NULL, 875 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 876 877 zio->io_prop.zp_checksum = checksum; 878 879 return (zio); 880} 881 882zio_t * 883zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 884 void *data, int checksum, zio_done_func_t *done, void *private, 885 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 886{ 887 zio_t *zio; 888 889 ASSERT(vd->vdev_children == 0); 890 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 891 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 892 ASSERT3U(offset + size, <=, vd->vdev_psize); 893 894 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 895 ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL, 896 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 897 898 zio->io_prop.zp_checksum = checksum; 899 900 if (zio_checksum_table[checksum].ci_eck) { 901 /* 902 * zec checksums are necessarily destructive -- they modify 903 * the end of the write buffer to hold the verifier/checksum. 904 * Therefore, we must make a local copy in case the data is 905 * being written to multiple places in parallel. 906 */ 907 void *wbuf = zio_buf_alloc(size); 908 bcopy(data, wbuf, size); 909 zio_push_transform(zio, wbuf, size, size, NULL); 910 } 911 912 return (zio); 913} 914 915/* 916 * Create a child I/O to do some work for us. 917 */ 918zio_t * 919zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 920 void *data, uint64_t size, int type, zio_priority_t priority, 921 enum zio_flag flags, zio_done_func_t *done, void *private) 922{ 923 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 924 zio_t *zio; 925 926 ASSERT(vd->vdev_parent == 927 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 928 929 if (type == ZIO_TYPE_READ && bp != NULL) { 930 /* 931 * If we have the bp, then the child should perform the 932 * checksum and the parent need not. This pushes error 933 * detection as close to the leaves as possible and 934 * eliminates redundant checksums in the interior nodes. 935 */ 936 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 937 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 938 } 939 940 if (vd->vdev_children == 0) 941 offset += VDEV_LABEL_START_SIZE; 942 943 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 944 945 /* 946 * If we've decided to do a repair, the write is not speculative -- 947 * even if the original read was. 948 */ 949 if (flags & ZIO_FLAG_IO_REPAIR) 950 flags &= ~ZIO_FLAG_SPECULATIVE; 951 952 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 953 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 954 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 955 956 zio->io_physdone = pio->io_physdone; 957 if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL) 958 zio->io_logical->io_phys_children++; 959 960 return (zio); 961} 962 963zio_t * 964zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 965 int type, zio_priority_t priority, enum zio_flag flags, 966 zio_done_func_t *done, void *private) 967{ 968 zio_t *zio; 969 970 ASSERT(vd->vdev_ops->vdev_op_leaf); 971 972 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 973 data, size, done, private, type, priority, 974 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 975 vd, offset, NULL, 976 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 977 978 return (zio); 979} 980 981void 982zio_flush(zio_t *zio, vdev_t *vd) 983{ 984 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 0, 0, 985 NULL, NULL, 986 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 987} 988 989zio_t * 990zio_trim(zio_t *zio, spa_t *spa, vdev_t *vd, uint64_t offset, uint64_t size) 991{ 992 993 ASSERT(vd->vdev_ops->vdev_op_leaf); 994 995 return zio_ioctl(zio, spa, vd, DKIOCTRIM, offset, size, 996 NULL, NULL, 997 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY); 998} 999 1000void 1001zio_shrink(zio_t *zio, uint64_t size) 1002{ 1003 ASSERT(zio->io_executor == NULL); 1004 ASSERT(zio->io_orig_size == zio->io_size); 1005 ASSERT(size <= zio->io_size); 1006 1007 /* 1008 * We don't shrink for raidz because of problems with the 1009 * reconstruction when reading back less than the block size. 1010 * Note, BP_IS_RAIDZ() assumes no compression. 1011 */ 1012 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 1013 if (!BP_IS_RAIDZ(zio->io_bp)) 1014 zio->io_orig_size = zio->io_size = size; 1015} 1016 1017/* 1018 * ========================================================================== 1019 * Prepare to read and write logical blocks 1020 * ========================================================================== 1021 */ 1022 1023static int 1024zio_read_bp_init(zio_t *zio) 1025{ 1026 blkptr_t *bp = zio->io_bp; 1027 1028 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 1029 zio->io_child_type == ZIO_CHILD_LOGICAL && 1030 !(zio->io_flags & ZIO_FLAG_RAW)) { 1031 uint64_t psize = BP_GET_PSIZE(bp); 1032 void *cbuf = zio_buf_alloc(psize); 1033 1034 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 1035 } 1036 1037 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 1038 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1039 1040 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 1041 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1042 1043 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 1044 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 1045 1046 return (ZIO_PIPELINE_CONTINUE); 1047} 1048 1049static int 1050zio_write_bp_init(zio_t *zio) 1051{ 1052 spa_t *spa = zio->io_spa; 1053 zio_prop_t *zp = &zio->io_prop; 1054 enum zio_compress compress = zp->zp_compress; 1055 blkptr_t *bp = zio->io_bp; 1056 uint64_t lsize = zio->io_size; 1057 uint64_t psize = lsize; 1058 int pass = 1; 1059 1060 /* 1061 * If our children haven't all reached the ready stage, 1062 * wait for them and then repeat this pipeline stage. 1063 */ 1064 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 1065 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 1066 return (ZIO_PIPELINE_STOP); 1067 1068 if (!IO_IS_ALLOCATING(zio)) 1069 return (ZIO_PIPELINE_CONTINUE); 1070 1071 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1072 1073 if (zio->io_bp_override) { 1074 ASSERT(bp->blk_birth != zio->io_txg); 1075 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 1076 1077 *bp = *zio->io_bp_override; 1078 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1079 1080 /* 1081 * If we've been overridden and nopwrite is set then 1082 * set the flag accordingly to indicate that a nopwrite 1083 * has already occurred. 1084 */ 1085 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1086 ASSERT(!zp->zp_dedup); 1087 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1088 return (ZIO_PIPELINE_CONTINUE); 1089 } 1090 1091 ASSERT(!zp->zp_nopwrite); 1092 1093 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1094 return (ZIO_PIPELINE_CONTINUE); 1095 1096 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup || 1097 zp->zp_dedup_verify); 1098 1099 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 1100 BP_SET_DEDUP(bp, 1); 1101 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1102 return (ZIO_PIPELINE_CONTINUE); 1103 } 1104 zio->io_bp_override = NULL; 1105 BP_ZERO(bp); 1106 } 1107 1108 if (bp->blk_birth == zio->io_txg) { 1109 /* 1110 * We're rewriting an existing block, which means we're 1111 * working on behalf of spa_sync(). For spa_sync() to 1112 * converge, it must eventually be the case that we don't 1113 * have to allocate new blocks. But compression changes 1114 * the blocksize, which forces a reallocate, and makes 1115 * convergence take longer. Therefore, after the first 1116 * few passes, stop compressing to ensure convergence. 1117 */ 1118 pass = spa_sync_pass(spa); 1119 1120 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1121 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1122 ASSERT(!BP_GET_DEDUP(bp)); 1123 1124 if (pass >= zfs_sync_pass_dont_compress) 1125 compress = ZIO_COMPRESS_OFF; 1126 1127 /* Make sure someone doesn't change their mind on overwrites */ 1128 ASSERT(MIN(zp->zp_copies + BP_IS_GANG(bp), 1129 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1130 } 1131 1132 if (compress != ZIO_COMPRESS_OFF) { 1133 metaslab_class_t *mc = spa_normal_class(spa); 1134 void *cbuf = zio_buf_alloc(lsize); 1135 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize, 1136 (size_t)metaslab_class_get_minblocksize(mc)); 1137 if (psize == 0 || psize == lsize) { 1138 compress = ZIO_COMPRESS_OFF; 1139 zio_buf_free(cbuf, lsize); 1140 } else { 1141 ASSERT(psize < lsize); 1142 zio_push_transform(zio, cbuf, psize, lsize, NULL); 1143 } 1144 } 1145 1146 /* 1147 * The final pass of spa_sync() must be all rewrites, but the first 1148 * few passes offer a trade-off: allocating blocks defers convergence, 1149 * but newly allocated blocks are sequential, so they can be written 1150 * to disk faster. Therefore, we allow the first few passes of 1151 * spa_sync() to allocate new blocks, but force rewrites after that. 1152 * There should only be a handful of blocks after pass 1 in any case. 1153 */ 1154 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == psize && 1155 pass >= zfs_sync_pass_rewrite) { 1156 ASSERT(psize != 0); 1157 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1158 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1159 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1160 } else { 1161 BP_ZERO(bp); 1162 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1163 } 1164 1165 if (psize == 0) { 1166 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1167 } else { 1168 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1169 BP_SET_LSIZE(bp, lsize); 1170 BP_SET_PSIZE(bp, psize); 1171 BP_SET_COMPRESS(bp, compress); 1172 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1173 BP_SET_TYPE(bp, zp->zp_type); 1174 BP_SET_LEVEL(bp, zp->zp_level); 1175 BP_SET_DEDUP(bp, zp->zp_dedup); 1176 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1177 if (zp->zp_dedup) { 1178 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1179 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1180 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1181 } 1182 if (zp->zp_nopwrite) { 1183 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1184 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1185 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1186 } 1187 } 1188 1189 return (ZIO_PIPELINE_CONTINUE); 1190} 1191 1192static int 1193zio_free_bp_init(zio_t *zio) 1194{ 1195 blkptr_t *bp = zio->io_bp; 1196 1197 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1198 if (BP_GET_DEDUP(bp)) 1199 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1200 } 1201 1202 return (ZIO_PIPELINE_CONTINUE); 1203} 1204 1205/* 1206 * ========================================================================== 1207 * Execute the I/O pipeline 1208 * ========================================================================== 1209 */ 1210 1211static void 1212zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1213{ 1214 spa_t *spa = zio->io_spa; 1215 zio_type_t t = zio->io_type; 1216 int flags = (cutinline ? TQ_FRONT : 0); 1217 1218 ASSERT(q == ZIO_TASKQ_ISSUE || q == ZIO_TASKQ_INTERRUPT); 1219 1220 /* 1221 * If we're a config writer or a probe, the normal issue and 1222 * interrupt threads may all be blocked waiting for the config lock. 1223 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1224 */ 1225 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1226 t = ZIO_TYPE_NULL; 1227 1228 /* 1229 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1230 */ 1231 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1232 t = ZIO_TYPE_NULL; 1233 1234 /* 1235 * If this is a high priority I/O, then use the high priority taskq if 1236 * available. 1237 */ 1238 if (zio->io_priority == ZIO_PRIORITY_NOW && 1239 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 1240 q++; 1241 1242 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1243 1244 /* 1245 * NB: We are assuming that the zio can only be dispatched 1246 * to a single taskq at a time. It would be a grievous error 1247 * to dispatch the zio to another taskq at the same time. 1248 */ 1249#if defined(illumos) || !defined(_KERNEL) 1250 ASSERT(zio->io_tqent.tqent_next == NULL); 1251#else 1252 ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); 1253#endif 1254 spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio, 1255 flags, &zio->io_tqent); 1256} 1257 1258static boolean_t 1259zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 1260{ 1261 kthread_t *executor = zio->io_executor; 1262 spa_t *spa = zio->io_spa; 1263 1264 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 1265 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1266 uint_t i; 1267 for (i = 0; i < tqs->stqs_count; i++) { 1268 if (taskq_member(tqs->stqs_taskq[i], executor)) 1269 return (B_TRUE); 1270 } 1271 } 1272 1273 return (B_FALSE); 1274} 1275 1276static int 1277zio_issue_async(zio_t *zio) 1278{ 1279 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1280 1281 return (ZIO_PIPELINE_STOP); 1282} 1283 1284void 1285zio_interrupt(zio_t *zio) 1286{ 1287 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1288} 1289 1290/* 1291 * Execute the I/O pipeline until one of the following occurs: 1292 * 1293 * (1) the I/O completes 1294 * (2) the pipeline stalls waiting for dependent child I/Os 1295 * (3) the I/O issues, so we're waiting for an I/O completion interrupt 1296 * (4) the I/O is delegated by vdev-level caching or aggregation 1297 * (5) the I/O is deferred due to vdev-level queueing 1298 * (6) the I/O is handed off to another thread. 1299 * 1300 * In all cases, the pipeline stops whenever there's no CPU work; it never 1301 * burns a thread in cv_wait(). 1302 * 1303 * There's no locking on io_stage because there's no legitimate way 1304 * for multiple threads to be attempting to process the same I/O. 1305 */ 1306static zio_pipe_stage_t *zio_pipeline[]; 1307 1308void 1309zio_execute(zio_t *zio) 1310{ 1311 zio->io_executor = curthread; 1312 1313 while (zio->io_stage < ZIO_STAGE_DONE) { 1314 enum zio_stage pipeline = zio->io_pipeline; 1315 enum zio_stage stage = zio->io_stage; 1316 int rv; 1317 1318 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1319 ASSERT(ISP2(stage)); 1320 ASSERT(zio->io_stall == NULL); 1321 1322 do { 1323 stage <<= 1; 1324 } while ((stage & pipeline) == 0); 1325 1326 ASSERT(stage <= ZIO_STAGE_DONE); 1327 1328 /* 1329 * If we are in interrupt context and this pipeline stage 1330 * will grab a config lock that is held across I/O, 1331 * or may wait for an I/O that needs an interrupt thread 1332 * to complete, issue async to avoid deadlock. 1333 * 1334 * For VDEV_IO_START, we cut in line so that the io will 1335 * be sent to disk promptly. 1336 */ 1337 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1338 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1339 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1340 zio_requeue_io_start_cut_in_line : B_FALSE; 1341 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1342 return; 1343 } 1344 1345 zio->io_stage = stage; 1346 rv = zio_pipeline[highbit(stage) - 1](zio); 1347 1348 if (rv == ZIO_PIPELINE_STOP) 1349 return; 1350 1351 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1352 } 1353} 1354 1355/* 1356 * ========================================================================== 1357 * Initiate I/O, either sync or async 1358 * ========================================================================== 1359 */ 1360int 1361zio_wait(zio_t *zio) 1362{ 1363 int error; 1364 1365 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1366 ASSERT(zio->io_executor == NULL); 1367 1368 zio->io_waiter = curthread; 1369 1370 zio_execute(zio); 1371 1372 mutex_enter(&zio->io_lock); 1373 while (zio->io_executor != NULL) 1374 cv_wait(&zio->io_cv, &zio->io_lock); 1375 mutex_exit(&zio->io_lock); 1376 1377 error = zio->io_error; 1378 zio_destroy(zio); 1379 1380 return (error); 1381} 1382 1383void 1384zio_nowait(zio_t *zio) 1385{ 1386 ASSERT(zio->io_executor == NULL); 1387 1388 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1389 zio_unique_parent(zio) == NULL) { 1390 /* 1391 * This is a logical async I/O with no parent to wait for it. 1392 * We add it to the spa_async_root_zio "Godfather" I/O which 1393 * will ensure they complete prior to unloading the pool. 1394 */ 1395 spa_t *spa = zio->io_spa; 1396 1397 zio_add_child(spa->spa_async_zio_root, zio); 1398 } 1399 1400 zio_execute(zio); 1401} 1402 1403/* 1404 * ========================================================================== 1405 * Reexecute or suspend/resume failed I/O 1406 * ========================================================================== 1407 */ 1408 1409static void 1410zio_reexecute(zio_t *pio) 1411{ 1412 zio_t *cio, *cio_next; 1413 1414 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1415 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1416 ASSERT(pio->io_gang_leader == NULL); 1417 ASSERT(pio->io_gang_tree == NULL); 1418 1419 pio->io_flags = pio->io_orig_flags; 1420 pio->io_stage = pio->io_orig_stage; 1421 pio->io_pipeline = pio->io_orig_pipeline; 1422 pio->io_reexecute = 0; 1423 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1424 pio->io_error = 0; 1425 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1426 pio->io_state[w] = 0; 1427 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1428 pio->io_child_error[c] = 0; 1429 1430 if (IO_IS_ALLOCATING(pio)) 1431 BP_ZERO(pio->io_bp); 1432 1433 /* 1434 * As we reexecute pio's children, new children could be created. 1435 * New children go to the head of pio's io_child_list, however, 1436 * so we will (correctly) not reexecute them. The key is that 1437 * the remainder of pio's io_child_list, from 'cio_next' onward, 1438 * cannot be affected by any side effects of reexecuting 'cio'. 1439 */ 1440 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) { 1441 cio_next = zio_walk_children(pio); 1442 mutex_enter(&pio->io_lock); 1443 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1444 pio->io_children[cio->io_child_type][w]++; 1445 mutex_exit(&pio->io_lock); 1446 zio_reexecute(cio); 1447 } 1448 1449 /* 1450 * Now that all children have been reexecuted, execute the parent. 1451 * We don't reexecute "The Godfather" I/O here as it's the 1452 * responsibility of the caller to wait on him. 1453 */ 1454 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) 1455 zio_execute(pio); 1456} 1457 1458void 1459zio_suspend(spa_t *spa, zio_t *zio) 1460{ 1461 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1462 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1463 "failure and the failure mode property for this pool " 1464 "is set to panic.", spa_name(spa)); 1465 1466 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1467 1468 mutex_enter(&spa->spa_suspend_lock); 1469 1470 if (spa->spa_suspend_zio_root == NULL) 1471 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1472 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1473 ZIO_FLAG_GODFATHER); 1474 1475 spa->spa_suspended = B_TRUE; 1476 1477 if (zio != NULL) { 1478 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1479 ASSERT(zio != spa->spa_suspend_zio_root); 1480 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1481 ASSERT(zio_unique_parent(zio) == NULL); 1482 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1483 zio_add_child(spa->spa_suspend_zio_root, zio); 1484 } 1485 1486 mutex_exit(&spa->spa_suspend_lock); 1487} 1488 1489int 1490zio_resume(spa_t *spa) 1491{ 1492 zio_t *pio; 1493 1494 /* 1495 * Reexecute all previously suspended i/o. 1496 */ 1497 mutex_enter(&spa->spa_suspend_lock); 1498 spa->spa_suspended = B_FALSE; 1499 cv_broadcast(&spa->spa_suspend_cv); 1500 pio = spa->spa_suspend_zio_root; 1501 spa->spa_suspend_zio_root = NULL; 1502 mutex_exit(&spa->spa_suspend_lock); 1503 1504 if (pio == NULL) 1505 return (0); 1506 1507 zio_reexecute(pio); 1508 return (zio_wait(pio)); 1509} 1510 1511void 1512zio_resume_wait(spa_t *spa) 1513{ 1514 mutex_enter(&spa->spa_suspend_lock); 1515 while (spa_suspended(spa)) 1516 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1517 mutex_exit(&spa->spa_suspend_lock); 1518} 1519 1520/* 1521 * ========================================================================== 1522 * Gang blocks. 1523 * 1524 * A gang block is a collection of small blocks that looks to the DMU 1525 * like one large block. When zio_dva_allocate() cannot find a block 1526 * of the requested size, due to either severe fragmentation or the pool 1527 * being nearly full, it calls zio_write_gang_block() to construct the 1528 * block from smaller fragments. 1529 * 1530 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1531 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1532 * an indirect block: it's an array of block pointers. It consumes 1533 * only one sector and hence is allocatable regardless of fragmentation. 1534 * The gang header's bps point to its gang members, which hold the data. 1535 * 1536 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1537 * as the verifier to ensure uniqueness of the SHA256 checksum. 1538 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1539 * not the gang header. This ensures that data block signatures (needed for 1540 * deduplication) are independent of how the block is physically stored. 1541 * 1542 * Gang blocks can be nested: a gang member may itself be a gang block. 1543 * Thus every gang block is a tree in which root and all interior nodes are 1544 * gang headers, and the leaves are normal blocks that contain user data. 1545 * The root of the gang tree is called the gang leader. 1546 * 1547 * To perform any operation (read, rewrite, free, claim) on a gang block, 1548 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1549 * in the io_gang_tree field of the original logical i/o by recursively 1550 * reading the gang leader and all gang headers below it. This yields 1551 * an in-core tree containing the contents of every gang header and the 1552 * bps for every constituent of the gang block. 1553 * 1554 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1555 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1556 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1557 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1558 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1559 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1560 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1561 * of the gang header plus zio_checksum_compute() of the data to update the 1562 * gang header's blk_cksum as described above. 1563 * 1564 * The two-phase assemble/issue model solves the problem of partial failure -- 1565 * what if you'd freed part of a gang block but then couldn't read the 1566 * gang header for another part? Assembling the entire gang tree first 1567 * ensures that all the necessary gang header I/O has succeeded before 1568 * starting the actual work of free, claim, or write. Once the gang tree 1569 * is assembled, free and claim are in-memory operations that cannot fail. 1570 * 1571 * In the event that a gang write fails, zio_dva_unallocate() walks the 1572 * gang tree to immediately free (i.e. insert back into the space map) 1573 * everything we've allocated. This ensures that we don't get ENOSPC 1574 * errors during repeated suspend/resume cycles due to a flaky device. 1575 * 1576 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1577 * the gang tree, we won't modify the block, so we can safely defer the free 1578 * (knowing that the block is still intact). If we *can* assemble the gang 1579 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1580 * each constituent bp and we can allocate a new block on the next sync pass. 1581 * 1582 * In all cases, the gang tree allows complete recovery from partial failure. 1583 * ========================================================================== 1584 */ 1585 1586static zio_t * 1587zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1588{ 1589 if (gn != NULL) 1590 return (pio); 1591 1592 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1593 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1594 &pio->io_bookmark)); 1595} 1596 1597zio_t * 1598zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1599{ 1600 zio_t *zio; 1601 1602 if (gn != NULL) { 1603 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1604 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1605 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1606 /* 1607 * As we rewrite each gang header, the pipeline will compute 1608 * a new gang block header checksum for it; but no one will 1609 * compute a new data checksum, so we do that here. The one 1610 * exception is the gang leader: the pipeline already computed 1611 * its data checksum because that stage precedes gang assembly. 1612 * (Presently, nothing actually uses interior data checksums; 1613 * this is just good hygiene.) 1614 */ 1615 if (gn != pio->io_gang_leader->io_gang_tree) { 1616 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1617 data, BP_GET_PSIZE(bp)); 1618 } 1619 /* 1620 * If we are here to damage data for testing purposes, 1621 * leave the GBH alone so that we can detect the damage. 1622 */ 1623 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1624 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1625 } else { 1626 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1627 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1628 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1629 } 1630 1631 return (zio); 1632} 1633 1634/* ARGSUSED */ 1635zio_t * 1636zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1637{ 1638 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1639 BP_IS_GANG(bp) ? SPA_GANGBLOCKSIZE : BP_GET_PSIZE(bp), 1640 ZIO_GANG_CHILD_FLAGS(pio))); 1641} 1642 1643/* ARGSUSED */ 1644zio_t * 1645zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1646{ 1647 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1648 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1649} 1650 1651static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1652 NULL, 1653 zio_read_gang, 1654 zio_rewrite_gang, 1655 zio_free_gang, 1656 zio_claim_gang, 1657 NULL 1658}; 1659 1660static void zio_gang_tree_assemble_done(zio_t *zio); 1661 1662static zio_gang_node_t * 1663zio_gang_node_alloc(zio_gang_node_t **gnpp) 1664{ 1665 zio_gang_node_t *gn; 1666 1667 ASSERT(*gnpp == NULL); 1668 1669 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1670 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1671 *gnpp = gn; 1672 1673 return (gn); 1674} 1675 1676static void 1677zio_gang_node_free(zio_gang_node_t **gnpp) 1678{ 1679 zio_gang_node_t *gn = *gnpp; 1680 1681 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1682 ASSERT(gn->gn_child[g] == NULL); 1683 1684 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1685 kmem_free(gn, sizeof (*gn)); 1686 *gnpp = NULL; 1687} 1688 1689static void 1690zio_gang_tree_free(zio_gang_node_t **gnpp) 1691{ 1692 zio_gang_node_t *gn = *gnpp; 1693 1694 if (gn == NULL) 1695 return; 1696 1697 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1698 zio_gang_tree_free(&gn->gn_child[g]); 1699 1700 zio_gang_node_free(gnpp); 1701} 1702 1703static void 1704zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1705{ 1706 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1707 1708 ASSERT(gio->io_gang_leader == gio); 1709 ASSERT(BP_IS_GANG(bp)); 1710 1711 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1712 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1713 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1714} 1715 1716static void 1717zio_gang_tree_assemble_done(zio_t *zio) 1718{ 1719 zio_t *gio = zio->io_gang_leader; 1720 zio_gang_node_t *gn = zio->io_private; 1721 blkptr_t *bp = zio->io_bp; 1722 1723 ASSERT(gio == zio_unique_parent(zio)); 1724 ASSERT(zio->io_child_count == 0); 1725 1726 if (zio->io_error) 1727 return; 1728 1729 if (BP_SHOULD_BYTESWAP(bp)) 1730 byteswap_uint64_array(zio->io_data, zio->io_size); 1731 1732 ASSERT(zio->io_data == gn->gn_gbh); 1733 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1734 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1735 1736 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1737 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1738 if (!BP_IS_GANG(gbp)) 1739 continue; 1740 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1741 } 1742} 1743 1744static void 1745zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1746{ 1747 zio_t *gio = pio->io_gang_leader; 1748 zio_t *zio; 1749 1750 ASSERT(BP_IS_GANG(bp) == !!gn); 1751 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1752 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1753 1754 /* 1755 * If you're a gang header, your data is in gn->gn_gbh. 1756 * If you're a gang member, your data is in 'data' and gn == NULL. 1757 */ 1758 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1759 1760 if (gn != NULL) { 1761 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1762 1763 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1764 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1765 if (BP_IS_HOLE(gbp)) 1766 continue; 1767 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1768 data = (char *)data + BP_GET_PSIZE(gbp); 1769 } 1770 } 1771 1772 if (gn == gio->io_gang_tree && gio->io_data != NULL) 1773 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1774 1775 if (zio != pio) 1776 zio_nowait(zio); 1777} 1778 1779static int 1780zio_gang_assemble(zio_t *zio) 1781{ 1782 blkptr_t *bp = zio->io_bp; 1783 1784 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 1785 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1786 1787 zio->io_gang_leader = zio; 1788 1789 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 1790 1791 return (ZIO_PIPELINE_CONTINUE); 1792} 1793 1794static int 1795zio_gang_issue(zio_t *zio) 1796{ 1797 blkptr_t *bp = zio->io_bp; 1798 1799 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 1800 return (ZIO_PIPELINE_STOP); 1801 1802 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 1803 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1804 1805 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 1806 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 1807 else 1808 zio_gang_tree_free(&zio->io_gang_tree); 1809 1810 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1811 1812 return (ZIO_PIPELINE_CONTINUE); 1813} 1814 1815static void 1816zio_write_gang_member_ready(zio_t *zio) 1817{ 1818 zio_t *pio = zio_unique_parent(zio); 1819 zio_t *gio = zio->io_gang_leader; 1820 dva_t *cdva = zio->io_bp->blk_dva; 1821 dva_t *pdva = pio->io_bp->blk_dva; 1822 uint64_t asize; 1823 1824 if (BP_IS_HOLE(zio->io_bp)) 1825 return; 1826 1827 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 1828 1829 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 1830 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 1831 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 1832 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 1833 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1834 1835 mutex_enter(&pio->io_lock); 1836 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 1837 ASSERT(DVA_GET_GANG(&pdva[d])); 1838 asize = DVA_GET_ASIZE(&pdva[d]); 1839 asize += DVA_GET_ASIZE(&cdva[d]); 1840 DVA_SET_ASIZE(&pdva[d], asize); 1841 } 1842 mutex_exit(&pio->io_lock); 1843} 1844 1845static int 1846zio_write_gang_block(zio_t *pio) 1847{ 1848 spa_t *spa = pio->io_spa; 1849 blkptr_t *bp = pio->io_bp; 1850 zio_t *gio = pio->io_gang_leader; 1851 zio_t *zio; 1852 zio_gang_node_t *gn, **gnpp; 1853 zio_gbh_phys_t *gbh; 1854 uint64_t txg = pio->io_txg; 1855 uint64_t resid = pio->io_size; 1856 uint64_t lsize; 1857 int copies = gio->io_prop.zp_copies; 1858 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 1859 zio_prop_t zp; 1860 int error; 1861 1862 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE, 1863 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, 1864 METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER); 1865 if (error) { 1866 pio->io_error = error; 1867 return (ZIO_PIPELINE_CONTINUE); 1868 } 1869 1870 if (pio == gio) { 1871 gnpp = &gio->io_gang_tree; 1872 } else { 1873 gnpp = pio->io_private; 1874 ASSERT(pio->io_ready == zio_write_gang_member_ready); 1875 } 1876 1877 gn = zio_gang_node_alloc(gnpp); 1878 gbh = gn->gn_gbh; 1879 bzero(gbh, SPA_GANGBLOCKSIZE); 1880 1881 /* 1882 * Create the gang header. 1883 */ 1884 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 1885 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1886 1887 /* 1888 * Create and nowait the gang children. 1889 */ 1890 for (int g = 0; resid != 0; resid -= lsize, g++) { 1891 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 1892 SPA_MINBLOCKSIZE); 1893 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 1894 1895 zp.zp_checksum = gio->io_prop.zp_checksum; 1896 zp.zp_compress = ZIO_COMPRESS_OFF; 1897 zp.zp_type = DMU_OT_NONE; 1898 zp.zp_level = 0; 1899 zp.zp_copies = gio->io_prop.zp_copies; 1900 zp.zp_dedup = B_FALSE; 1901 zp.zp_dedup_verify = B_FALSE; 1902 zp.zp_nopwrite = B_FALSE; 1903 1904 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 1905 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 1906 zio_write_gang_member_ready, NULL, NULL, &gn->gn_child[g], 1907 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1908 &pio->io_bookmark)); 1909 } 1910 1911 /* 1912 * Set pio's pipeline to just wait for zio to finish. 1913 */ 1914 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1915 1916 zio_nowait(zio); 1917 1918 return (ZIO_PIPELINE_CONTINUE); 1919} 1920 1921/* 1922 * The zio_nop_write stage in the pipeline determines if allocating 1923 * a new bp is necessary. By leveraging a cryptographically secure checksum, 1924 * such as SHA256, we can compare the checksums of the new data and the old 1925 * to determine if allocating a new block is required. The nopwrite 1926 * feature can handle writes in either syncing or open context (i.e. zil 1927 * writes) and as a result is mutually exclusive with dedup. 1928 */ 1929static int 1930zio_nop_write(zio_t *zio) 1931{ 1932 blkptr_t *bp = zio->io_bp; 1933 blkptr_t *bp_orig = &zio->io_bp_orig; 1934 zio_prop_t *zp = &zio->io_prop; 1935 1936 ASSERT(BP_GET_LEVEL(bp) == 0); 1937 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1938 ASSERT(zp->zp_nopwrite); 1939 ASSERT(!zp->zp_dedup); 1940 ASSERT(zio->io_bp_override == NULL); 1941 ASSERT(IO_IS_ALLOCATING(zio)); 1942 1943 /* 1944 * Check to see if the original bp and the new bp have matching 1945 * characteristics (i.e. same checksum, compression algorithms, etc). 1946 * If they don't then just continue with the pipeline which will 1947 * allocate a new bp. 1948 */ 1949 if (BP_IS_HOLE(bp_orig) || 1950 !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup || 1951 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 1952 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 1953 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 1954 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 1955 return (ZIO_PIPELINE_CONTINUE); 1956 1957 /* 1958 * If the checksums match then reset the pipeline so that we 1959 * avoid allocating a new bp and issuing any I/O. 1960 */ 1961 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 1962 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup); 1963 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 1964 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 1965 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 1966 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 1967 sizeof (uint64_t)) == 0); 1968 1969 *bp = *bp_orig; 1970 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1971 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1972 } 1973 1974 return (ZIO_PIPELINE_CONTINUE); 1975} 1976 1977/* 1978 * ========================================================================== 1979 * Dedup 1980 * ========================================================================== 1981 */ 1982static void 1983zio_ddt_child_read_done(zio_t *zio) 1984{ 1985 blkptr_t *bp = zio->io_bp; 1986 ddt_entry_t *dde = zio->io_private; 1987 ddt_phys_t *ddp; 1988 zio_t *pio = zio_unique_parent(zio); 1989 1990 mutex_enter(&pio->io_lock); 1991 ddp = ddt_phys_select(dde, bp); 1992 if (zio->io_error == 0) 1993 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 1994 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 1995 dde->dde_repair_data = zio->io_data; 1996 else 1997 zio_buf_free(zio->io_data, zio->io_size); 1998 mutex_exit(&pio->io_lock); 1999} 2000 2001static int 2002zio_ddt_read_start(zio_t *zio) 2003{ 2004 blkptr_t *bp = zio->io_bp; 2005 2006 ASSERT(BP_GET_DEDUP(bp)); 2007 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2008 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2009 2010 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2011 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2012 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 2013 ddt_phys_t *ddp = dde->dde_phys; 2014 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 2015 blkptr_t blk; 2016 2017 ASSERT(zio->io_vsd == NULL); 2018 zio->io_vsd = dde; 2019 2020 if (ddp_self == NULL) 2021 return (ZIO_PIPELINE_CONTINUE); 2022 2023 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2024 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 2025 continue; 2026 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 2027 &blk); 2028 zio_nowait(zio_read(zio, zio->io_spa, &blk, 2029 zio_buf_alloc(zio->io_size), zio->io_size, 2030 zio_ddt_child_read_done, dde, zio->io_priority, 2031 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 2032 &zio->io_bookmark)); 2033 } 2034 return (ZIO_PIPELINE_CONTINUE); 2035 } 2036 2037 zio_nowait(zio_read(zio, zio->io_spa, bp, 2038 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 2039 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 2040 2041 return (ZIO_PIPELINE_CONTINUE); 2042} 2043 2044static int 2045zio_ddt_read_done(zio_t *zio) 2046{ 2047 blkptr_t *bp = zio->io_bp; 2048 2049 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 2050 return (ZIO_PIPELINE_STOP); 2051 2052 ASSERT(BP_GET_DEDUP(bp)); 2053 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2054 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2055 2056 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2057 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2058 ddt_entry_t *dde = zio->io_vsd; 2059 if (ddt == NULL) { 2060 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 2061 return (ZIO_PIPELINE_CONTINUE); 2062 } 2063 if (dde == NULL) { 2064 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 2065 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2066 return (ZIO_PIPELINE_STOP); 2067 } 2068 if (dde->dde_repair_data != NULL) { 2069 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 2070 zio->io_child_error[ZIO_CHILD_DDT] = 0; 2071 } 2072 ddt_repair_done(ddt, dde); 2073 zio->io_vsd = NULL; 2074 } 2075 2076 ASSERT(zio->io_vsd == NULL); 2077 2078 return (ZIO_PIPELINE_CONTINUE); 2079} 2080 2081static boolean_t 2082zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 2083{ 2084 spa_t *spa = zio->io_spa; 2085 2086 /* 2087 * Note: we compare the original data, not the transformed data, 2088 * because when zio->io_bp is an override bp, we will not have 2089 * pushed the I/O transforms. That's an important optimization 2090 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 2091 */ 2092 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2093 zio_t *lio = dde->dde_lead_zio[p]; 2094 2095 if (lio != NULL) { 2096 return (lio->io_orig_size != zio->io_orig_size || 2097 bcmp(zio->io_orig_data, lio->io_orig_data, 2098 zio->io_orig_size) != 0); 2099 } 2100 } 2101 2102 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2103 ddt_phys_t *ddp = &dde->dde_phys[p]; 2104 2105 if (ddp->ddp_phys_birth != 0) { 2106 arc_buf_t *abuf = NULL; 2107 uint32_t aflags = ARC_WAIT; 2108 blkptr_t blk = *zio->io_bp; 2109 int error; 2110 2111 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2112 2113 ddt_exit(ddt); 2114 2115 error = arc_read(NULL, spa, &blk, 2116 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 2117 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2118 &aflags, &zio->io_bookmark); 2119 2120 if (error == 0) { 2121 if (arc_buf_size(abuf) != zio->io_orig_size || 2122 bcmp(abuf->b_data, zio->io_orig_data, 2123 zio->io_orig_size) != 0) 2124 error = SET_ERROR(EEXIST); 2125 VERIFY(arc_buf_remove_ref(abuf, &abuf)); 2126 } 2127 2128 ddt_enter(ddt); 2129 return (error != 0); 2130 } 2131 } 2132 2133 return (B_FALSE); 2134} 2135 2136static void 2137zio_ddt_child_write_ready(zio_t *zio) 2138{ 2139 int p = zio->io_prop.zp_copies; 2140 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2141 ddt_entry_t *dde = zio->io_private; 2142 ddt_phys_t *ddp = &dde->dde_phys[p]; 2143 zio_t *pio; 2144 2145 if (zio->io_error) 2146 return; 2147 2148 ddt_enter(ddt); 2149 2150 ASSERT(dde->dde_lead_zio[p] == zio); 2151 2152 ddt_phys_fill(ddp, zio->io_bp); 2153 2154 while ((pio = zio_walk_parents(zio)) != NULL) 2155 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2156 2157 ddt_exit(ddt); 2158} 2159 2160static void 2161zio_ddt_child_write_done(zio_t *zio) 2162{ 2163 int p = zio->io_prop.zp_copies; 2164 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2165 ddt_entry_t *dde = zio->io_private; 2166 ddt_phys_t *ddp = &dde->dde_phys[p]; 2167 2168 ddt_enter(ddt); 2169 2170 ASSERT(ddp->ddp_refcnt == 0); 2171 ASSERT(dde->dde_lead_zio[p] == zio); 2172 dde->dde_lead_zio[p] = NULL; 2173 2174 if (zio->io_error == 0) { 2175 while (zio_walk_parents(zio) != NULL) 2176 ddt_phys_addref(ddp); 2177 } else { 2178 ddt_phys_clear(ddp); 2179 } 2180 2181 ddt_exit(ddt); 2182} 2183 2184static void 2185zio_ddt_ditto_write_done(zio_t *zio) 2186{ 2187 int p = DDT_PHYS_DITTO; 2188 zio_prop_t *zp = &zio->io_prop; 2189 blkptr_t *bp = zio->io_bp; 2190 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2191 ddt_entry_t *dde = zio->io_private; 2192 ddt_phys_t *ddp = &dde->dde_phys[p]; 2193 ddt_key_t *ddk = &dde->dde_key; 2194 2195 ddt_enter(ddt); 2196 2197 ASSERT(ddp->ddp_refcnt == 0); 2198 ASSERT(dde->dde_lead_zio[p] == zio); 2199 dde->dde_lead_zio[p] = NULL; 2200 2201 if (zio->io_error == 0) { 2202 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2203 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2204 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2205 if (ddp->ddp_phys_birth != 0) 2206 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2207 ddt_phys_fill(ddp, bp); 2208 } 2209 2210 ddt_exit(ddt); 2211} 2212 2213static int 2214zio_ddt_write(zio_t *zio) 2215{ 2216 spa_t *spa = zio->io_spa; 2217 blkptr_t *bp = zio->io_bp; 2218 uint64_t txg = zio->io_txg; 2219 zio_prop_t *zp = &zio->io_prop; 2220 int p = zp->zp_copies; 2221 int ditto_copies; 2222 zio_t *cio = NULL; 2223 zio_t *dio = NULL; 2224 ddt_t *ddt = ddt_select(spa, bp); 2225 ddt_entry_t *dde; 2226 ddt_phys_t *ddp; 2227 2228 ASSERT(BP_GET_DEDUP(bp)); 2229 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2230 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2231 2232 ddt_enter(ddt); 2233 dde = ddt_lookup(ddt, bp, B_TRUE); 2234 ddp = &dde->dde_phys[p]; 2235 2236 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2237 /* 2238 * If we're using a weak checksum, upgrade to a strong checksum 2239 * and try again. If we're already using a strong checksum, 2240 * we can't resolve it, so just convert to an ordinary write. 2241 * (And automatically e-mail a paper to Nature?) 2242 */ 2243 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) { 2244 zp->zp_checksum = spa_dedup_checksum(spa); 2245 zio_pop_transforms(zio); 2246 zio->io_stage = ZIO_STAGE_OPEN; 2247 BP_ZERO(bp); 2248 } else { 2249 zp->zp_dedup = B_FALSE; 2250 } 2251 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2252 ddt_exit(ddt); 2253 return (ZIO_PIPELINE_CONTINUE); 2254 } 2255 2256 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2257 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2258 2259 if (ditto_copies > ddt_ditto_copies_present(dde) && 2260 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2261 zio_prop_t czp = *zp; 2262 2263 czp.zp_copies = ditto_copies; 2264 2265 /* 2266 * If we arrived here with an override bp, we won't have run 2267 * the transform stack, so we won't have the data we need to 2268 * generate a child i/o. So, toss the override bp and restart. 2269 * This is safe, because using the override bp is just an 2270 * optimization; and it's rare, so the cost doesn't matter. 2271 */ 2272 if (zio->io_bp_override) { 2273 zio_pop_transforms(zio); 2274 zio->io_stage = ZIO_STAGE_OPEN; 2275 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2276 zio->io_bp_override = NULL; 2277 BP_ZERO(bp); 2278 ddt_exit(ddt); 2279 return (ZIO_PIPELINE_CONTINUE); 2280 } 2281 2282 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2283 zio->io_orig_size, &czp, NULL, NULL, 2284 zio_ddt_ditto_write_done, dde, zio->io_priority, 2285 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2286 2287 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2288 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2289 } 2290 2291 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2292 if (ddp->ddp_phys_birth != 0) 2293 ddt_bp_fill(ddp, bp, txg); 2294 if (dde->dde_lead_zio[p] != NULL) 2295 zio_add_child(zio, dde->dde_lead_zio[p]); 2296 else 2297 ddt_phys_addref(ddp); 2298 } else if (zio->io_bp_override) { 2299 ASSERT(bp->blk_birth == txg); 2300 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2301 ddt_phys_fill(ddp, bp); 2302 ddt_phys_addref(ddp); 2303 } else { 2304 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2305 zio->io_orig_size, zp, zio_ddt_child_write_ready, NULL, 2306 zio_ddt_child_write_done, dde, zio->io_priority, 2307 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2308 2309 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2310 dde->dde_lead_zio[p] = cio; 2311 } 2312 2313 ddt_exit(ddt); 2314 2315 if (cio) 2316 zio_nowait(cio); 2317 if (dio) 2318 zio_nowait(dio); 2319 2320 return (ZIO_PIPELINE_CONTINUE); 2321} 2322 2323ddt_entry_t *freedde; /* for debugging */ 2324 2325static int 2326zio_ddt_free(zio_t *zio) 2327{ 2328 spa_t *spa = zio->io_spa; 2329 blkptr_t *bp = zio->io_bp; 2330 ddt_t *ddt = ddt_select(spa, bp); 2331 ddt_entry_t *dde; 2332 ddt_phys_t *ddp; 2333 2334 ASSERT(BP_GET_DEDUP(bp)); 2335 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2336 2337 ddt_enter(ddt); 2338 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2339 ddp = ddt_phys_select(dde, bp); 2340 ddt_phys_decref(ddp); 2341 ddt_exit(ddt); 2342 2343 return (ZIO_PIPELINE_CONTINUE); 2344} 2345 2346/* 2347 * ========================================================================== 2348 * Allocate and free blocks 2349 * ========================================================================== 2350 */ 2351static int 2352zio_dva_allocate(zio_t *zio) 2353{ 2354 spa_t *spa = zio->io_spa; 2355 metaslab_class_t *mc = spa_normal_class(spa); 2356 blkptr_t *bp = zio->io_bp; 2357 int error; 2358 int flags = 0; 2359 2360 if (zio->io_gang_leader == NULL) { 2361 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2362 zio->io_gang_leader = zio; 2363 } 2364 2365 ASSERT(BP_IS_HOLE(bp)); 2366 ASSERT0(BP_GET_NDVAS(bp)); 2367 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2368 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2369 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2370 2371 /* 2372 * The dump device does not support gang blocks so allocation on 2373 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid 2374 * the "fast" gang feature. 2375 */ 2376 flags |= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0; 2377 flags |= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ? 2378 METASLAB_GANG_CHILD : 0; 2379 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2380 zio->io_prop.zp_copies, zio->io_txg, NULL, flags); 2381 2382 if (error) { 2383 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2384 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2385 error); 2386 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2387 return (zio_write_gang_block(zio)); 2388 zio->io_error = error; 2389 } 2390 2391 return (ZIO_PIPELINE_CONTINUE); 2392} 2393 2394static int 2395zio_dva_free(zio_t *zio) 2396{ 2397 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2398 2399 return (ZIO_PIPELINE_CONTINUE); 2400} 2401 2402static int 2403zio_dva_claim(zio_t *zio) 2404{ 2405 int error; 2406 2407 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2408 if (error) 2409 zio->io_error = error; 2410 2411 return (ZIO_PIPELINE_CONTINUE); 2412} 2413 2414/* 2415 * Undo an allocation. This is used by zio_done() when an I/O fails 2416 * and we want to give back the block we just allocated. 2417 * This handles both normal blocks and gang blocks. 2418 */ 2419static void 2420zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2421{ 2422 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2423 ASSERT(zio->io_bp_override == NULL); 2424 2425 if (!BP_IS_HOLE(bp)) 2426 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2427 2428 if (gn != NULL) { 2429 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2430 zio_dva_unallocate(zio, gn->gn_child[g], 2431 &gn->gn_gbh->zg_blkptr[g]); 2432 } 2433 } 2434} 2435 2436/* 2437 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2438 */ 2439int 2440zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2441 uint64_t size, boolean_t use_slog) 2442{ 2443 int error = 1; 2444 2445 ASSERT(txg > spa_syncing_txg(spa)); 2446 2447 /* 2448 * ZIL blocks are always contiguous (i.e. not gang blocks) so we 2449 * set the METASLAB_GANG_AVOID flag so that they don't "fast gang" 2450 * when allocating them. 2451 */ 2452 if (use_slog) { 2453 error = metaslab_alloc(spa, spa_log_class(spa), size, 2454 new_bp, 1, txg, old_bp, 2455 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2456 } 2457 2458 if (error) { 2459 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2460 new_bp, 1, txg, old_bp, 2461 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2462 } 2463 2464 if (error == 0) { 2465 BP_SET_LSIZE(new_bp, size); 2466 BP_SET_PSIZE(new_bp, size); 2467 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2468 BP_SET_CHECKSUM(new_bp, 2469 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2470 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2471 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2472 BP_SET_LEVEL(new_bp, 0); 2473 BP_SET_DEDUP(new_bp, 0); 2474 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2475 } 2476 2477 return (error); 2478} 2479 2480/* 2481 * Free an intent log block. 2482 */ 2483void 2484zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2485{ 2486 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2487 ASSERT(!BP_IS_GANG(bp)); 2488 2489 zio_free(spa, txg, bp); 2490} 2491 2492/* 2493 * ========================================================================== 2494 * Read, write and delete to physical devices 2495 * ========================================================================== 2496 */ 2497static int 2498zio_vdev_io_start(zio_t *zio) 2499{ 2500 vdev_t *vd = zio->io_vd; 2501 uint64_t align; 2502 spa_t *spa = zio->io_spa; 2503 2504 ASSERT(zio->io_error == 0); 2505 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2506 2507 if (vd == NULL) { 2508 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2509 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2510 2511 /* 2512 * The mirror_ops handle multiple DVAs in a single BP. 2513 */ 2514 return (vdev_mirror_ops.vdev_op_io_start(zio)); 2515 } 2516 2517 if (vd->vdev_ops->vdev_op_leaf && zio->io_type == ZIO_TYPE_FREE) { 2518 trim_map_free(vd, zio->io_offset, zio->io_size, zio->io_txg); 2519 return (ZIO_PIPELINE_CONTINUE); 2520 } 2521 2522 /* 2523 * We keep track of time-sensitive I/Os so that the scan thread 2524 * can quickly react to certain workloads. In particular, we care 2525 * about non-scrubbing, top-level reads and writes with the following 2526 * characteristics: 2527 * - synchronous writes of user data to non-slog devices 2528 * - any reads of user data 2529 * When these conditions are met, adjust the timestamp of spa_last_io 2530 * which allows the scan thread to adjust its workload accordingly. 2531 */ 2532 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2533 vd == vd->vdev_top && !vd->vdev_islog && 2534 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2535 zio->io_txg != spa_syncing_txg(spa)) { 2536 uint64_t old = spa->spa_last_io; 2537 uint64_t new = ddi_get_lbolt64(); 2538 if (old != new) 2539 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2540 } 2541 2542 align = 1ULL << vd->vdev_top->vdev_ashift; 2543 2544 if (P2PHASE(zio->io_size, align) != 0) { 2545 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2546 char *abuf = NULL; 2547 if (zio->io_type == ZIO_TYPE_READ || 2548 zio->io_type == ZIO_TYPE_WRITE) 2549 abuf = zio_buf_alloc(asize); 2550 ASSERT(vd == vd->vdev_top); 2551 if (zio->io_type == ZIO_TYPE_WRITE) { 2552 bcopy(zio->io_data, abuf, zio->io_size); 2553 bzero(abuf + zio->io_size, asize - zio->io_size); 2554 } 2555 zio_push_transform(zio, abuf, asize, abuf ? asize : 0, 2556 zio_subblock); 2557 } 2558 2559 ASSERT(P2PHASE(zio->io_offset, align) == 0); 2560 ASSERT(P2PHASE(zio->io_size, align) == 0); 2561 VERIFY(zio->io_type == ZIO_TYPE_READ || spa_writeable(spa)); 2562 2563 /* 2564 * If this is a repair I/O, and there's no self-healing involved -- 2565 * that is, we're just resilvering what we expect to resilver -- 2566 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2567 * This prevents spurious resilvering with nested replication. 2568 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2569 * A is out of date, we'll read from C+D, then use the data to 2570 * resilver A+B -- but we don't actually want to resilver B, just A. 2571 * The top-level mirror has no way to know this, so instead we just 2572 * discard unnecessary repairs as we work our way down the vdev tree. 2573 * The same logic applies to any form of nested replication: 2574 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2575 */ 2576 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2577 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2578 zio->io_txg != 0 && /* not a delegated i/o */ 2579 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2580 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2581 zio_vdev_io_bypass(zio); 2582 return (ZIO_PIPELINE_CONTINUE); 2583 } 2584 2585 if (vd->vdev_ops->vdev_op_leaf && 2586 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2587 2588 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio) == 0) 2589 return (ZIO_PIPELINE_CONTINUE); 2590 2591 if ((zio = vdev_queue_io(zio)) == NULL) 2592 return (ZIO_PIPELINE_STOP); 2593 2594 if (!vdev_accessible(vd, zio)) { 2595 zio->io_error = SET_ERROR(ENXIO); 2596 zio_interrupt(zio); 2597 return (ZIO_PIPELINE_STOP); 2598 } 2599 } 2600 2601 /* 2602 * Note that we ignore repair writes for TRIM because they can conflict 2603 * with normal writes. This isn't an issue because, by definition, we 2604 * only repair blocks that aren't freed. 2605 */ 2606 if (vd->vdev_ops->vdev_op_leaf && zio->io_type == ZIO_TYPE_WRITE && 2607 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2608 if (!trim_map_write_start(zio)) 2609 return (ZIO_PIPELINE_STOP); 2610 } 2611 2612 return (vd->vdev_ops->vdev_op_io_start(zio)); 2613} 2614 2615static int 2616zio_vdev_io_done(zio_t *zio) 2617{ 2618 vdev_t *vd = zio->io_vd; 2619 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 2620 boolean_t unexpected_error = B_FALSE; 2621 2622 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2623 return (ZIO_PIPELINE_STOP); 2624 2625 ASSERT(zio->io_type == ZIO_TYPE_READ || 2626 zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_FREE); 2627 2628 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 2629 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2630 2631 if (zio->io_type == ZIO_TYPE_WRITE && 2632 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) 2633 trim_map_write_done(zio); 2634 2635 vdev_queue_io_done(zio); 2636 2637 if (zio->io_type == ZIO_TYPE_WRITE) 2638 vdev_cache_write(zio); 2639 2640 if (zio_injection_enabled && zio->io_error == 0) 2641 zio->io_error = zio_handle_device_injection(vd, 2642 zio, EIO); 2643 2644 if (zio_injection_enabled && zio->io_error == 0) 2645 zio->io_error = zio_handle_label_injection(zio, EIO); 2646 2647 if (zio->io_error) { 2648 if (!vdev_accessible(vd, zio)) { 2649 zio->io_error = SET_ERROR(ENXIO); 2650 } else { 2651 unexpected_error = B_TRUE; 2652 } 2653 } 2654 } 2655 2656 ops->vdev_op_io_done(zio); 2657 2658 if (unexpected_error) 2659 VERIFY(vdev_probe(vd, zio) == NULL); 2660 2661 return (ZIO_PIPELINE_CONTINUE); 2662} 2663 2664/* 2665 * For non-raidz ZIOs, we can just copy aside the bad data read from the 2666 * disk, and use that to finish the checksum ereport later. 2667 */ 2668static void 2669zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 2670 const void *good_buf) 2671{ 2672 /* no processing needed */ 2673 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 2674} 2675 2676/*ARGSUSED*/ 2677void 2678zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 2679{ 2680 void *buf = zio_buf_alloc(zio->io_size); 2681 2682 bcopy(zio->io_data, buf, zio->io_size); 2683 2684 zcr->zcr_cbinfo = zio->io_size; 2685 zcr->zcr_cbdata = buf; 2686 zcr->zcr_finish = zio_vsd_default_cksum_finish; 2687 zcr->zcr_free = zio_buf_free; 2688} 2689 2690static int 2691zio_vdev_io_assess(zio_t *zio) 2692{ 2693 vdev_t *vd = zio->io_vd; 2694 2695 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2696 return (ZIO_PIPELINE_STOP); 2697 2698 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2699 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 2700 2701 if (zio->io_vsd != NULL) { 2702 zio->io_vsd_ops->vsd_free(zio); 2703 zio->io_vsd = NULL; 2704 } 2705 2706 if (zio_injection_enabled && zio->io_error == 0) 2707 zio->io_error = zio_handle_fault_injection(zio, EIO); 2708 2709 if (zio->io_type == ZIO_TYPE_IOCTL && zio->io_cmd == DKIOCTRIM) 2710 switch (zio->io_error) { 2711 case 0: 2712 ZIO_TRIM_STAT_INCR(bytes, zio->io_size); 2713 ZIO_TRIM_STAT_BUMP(success); 2714 break; 2715 case EOPNOTSUPP: 2716 ZIO_TRIM_STAT_BUMP(unsupported); 2717 break; 2718 default: 2719 ZIO_TRIM_STAT_BUMP(failed); 2720 break; 2721 } 2722 2723 /* 2724 * If the I/O failed, determine whether we should attempt to retry it. 2725 * 2726 * On retry, we cut in line in the issue queue, since we don't want 2727 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 2728 */ 2729 if (zio->io_error && vd == NULL && 2730 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 2731 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 2732 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 2733 zio->io_error = 0; 2734 zio->io_flags |= ZIO_FLAG_IO_RETRY | 2735 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 2736 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 2737 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 2738 zio_requeue_io_start_cut_in_line); 2739 return (ZIO_PIPELINE_STOP); 2740 } 2741 2742 /* 2743 * If we got an error on a leaf device, convert it to ENXIO 2744 * if the device is not accessible at all. 2745 */ 2746 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 2747 !vdev_accessible(vd, zio)) 2748 zio->io_error = SET_ERROR(ENXIO); 2749 2750 /* 2751 * If we can't write to an interior vdev (mirror or RAID-Z), 2752 * set vdev_cant_write so that we stop trying to allocate from it. 2753 */ 2754 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 2755 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 2756 vd->vdev_cant_write = B_TRUE; 2757 } 2758 2759 if (zio->io_error) 2760 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2761 2762 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 2763 zio->io_physdone != NULL) { 2764 ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED)); 2765 ASSERT(zio->io_child_type == ZIO_CHILD_VDEV); 2766 zio->io_physdone(zio->io_logical); 2767 } 2768 2769 return (ZIO_PIPELINE_CONTINUE); 2770} 2771 2772void 2773zio_vdev_io_reissue(zio_t *zio) 2774{ 2775 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2776 ASSERT(zio->io_error == 0); 2777 2778 zio->io_stage >>= 1; 2779} 2780 2781void 2782zio_vdev_io_redone(zio_t *zio) 2783{ 2784 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 2785 2786 zio->io_stage >>= 1; 2787} 2788 2789void 2790zio_vdev_io_bypass(zio_t *zio) 2791{ 2792 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2793 ASSERT(zio->io_error == 0); 2794 2795 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 2796 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 2797} 2798 2799/* 2800 * ========================================================================== 2801 * Generate and verify checksums 2802 * ========================================================================== 2803 */ 2804static int 2805zio_checksum_generate(zio_t *zio) 2806{ 2807 blkptr_t *bp = zio->io_bp; 2808 enum zio_checksum checksum; 2809 2810 if (bp == NULL) { 2811 /* 2812 * This is zio_write_phys(). 2813 * We're either generating a label checksum, or none at all. 2814 */ 2815 checksum = zio->io_prop.zp_checksum; 2816 2817 if (checksum == ZIO_CHECKSUM_OFF) 2818 return (ZIO_PIPELINE_CONTINUE); 2819 2820 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 2821 } else { 2822 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 2823 ASSERT(!IO_IS_ALLOCATING(zio)); 2824 checksum = ZIO_CHECKSUM_GANG_HEADER; 2825 } else { 2826 checksum = BP_GET_CHECKSUM(bp); 2827 } 2828 } 2829 2830 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 2831 2832 return (ZIO_PIPELINE_CONTINUE); 2833} 2834 2835static int 2836zio_checksum_verify(zio_t *zio) 2837{ 2838 zio_bad_cksum_t info; 2839 blkptr_t *bp = zio->io_bp; 2840 int error; 2841 2842 ASSERT(zio->io_vd != NULL); 2843 2844 if (bp == NULL) { 2845 /* 2846 * This is zio_read_phys(). 2847 * We're either verifying a label checksum, or nothing at all. 2848 */ 2849 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 2850 return (ZIO_PIPELINE_CONTINUE); 2851 2852 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 2853 } 2854 2855 if ((error = zio_checksum_error(zio, &info)) != 0) { 2856 zio->io_error = error; 2857 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 2858 zfs_ereport_start_checksum(zio->io_spa, 2859 zio->io_vd, zio, zio->io_offset, 2860 zio->io_size, NULL, &info); 2861 } 2862 } 2863 2864 return (ZIO_PIPELINE_CONTINUE); 2865} 2866 2867/* 2868 * Called by RAID-Z to ensure we don't compute the checksum twice. 2869 */ 2870void 2871zio_checksum_verified(zio_t *zio) 2872{ 2873 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 2874} 2875 2876/* 2877 * ========================================================================== 2878 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 2879 * An error of 0 indictes success. ENXIO indicates whole-device failure, 2880 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 2881 * indicate errors that are specific to one I/O, and most likely permanent. 2882 * Any other error is presumed to be worse because we weren't expecting it. 2883 * ========================================================================== 2884 */ 2885int 2886zio_worst_error(int e1, int e2) 2887{ 2888 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 2889 int r1, r2; 2890 2891 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 2892 if (e1 == zio_error_rank[r1]) 2893 break; 2894 2895 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 2896 if (e2 == zio_error_rank[r2]) 2897 break; 2898 2899 return (r1 > r2 ? e1 : e2); 2900} 2901 2902/* 2903 * ========================================================================== 2904 * I/O completion 2905 * ========================================================================== 2906 */ 2907static int 2908zio_ready(zio_t *zio) 2909{ 2910 blkptr_t *bp = zio->io_bp; 2911 zio_t *pio, *pio_next; 2912 2913 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 2914 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 2915 return (ZIO_PIPELINE_STOP); 2916 2917 if (zio->io_ready) { 2918 ASSERT(IO_IS_ALLOCATING(zio)); 2919 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 2920 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 2921 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 2922 2923 zio->io_ready(zio); 2924 } 2925 2926 if (bp != NULL && bp != &zio->io_bp_copy) 2927 zio->io_bp_copy = *bp; 2928 2929 if (zio->io_error) 2930 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2931 2932 mutex_enter(&zio->io_lock); 2933 zio->io_state[ZIO_WAIT_READY] = 1; 2934 pio = zio_walk_parents(zio); 2935 mutex_exit(&zio->io_lock); 2936 2937 /* 2938 * As we notify zio's parents, new parents could be added. 2939 * New parents go to the head of zio's io_parent_list, however, 2940 * so we will (correctly) not notify them. The remainder of zio's 2941 * io_parent_list, from 'pio_next' onward, cannot change because 2942 * all parents must wait for us to be done before they can be done. 2943 */ 2944 for (; pio != NULL; pio = pio_next) { 2945 pio_next = zio_walk_parents(zio); 2946 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 2947 } 2948 2949 if (zio->io_flags & ZIO_FLAG_NODATA) { 2950 if (BP_IS_GANG(bp)) { 2951 zio->io_flags &= ~ZIO_FLAG_NODATA; 2952 } else { 2953 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 2954 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2955 } 2956 } 2957 2958 if (zio_injection_enabled && 2959 zio->io_spa->spa_syncing_txg == zio->io_txg) 2960 zio_handle_ignored_writes(zio); 2961 2962 return (ZIO_PIPELINE_CONTINUE); 2963} 2964 2965static int 2966zio_done(zio_t *zio) 2967{ 2968 spa_t *spa = zio->io_spa; 2969 zio_t *lio = zio->io_logical; 2970 blkptr_t *bp = zio->io_bp; 2971 vdev_t *vd = zio->io_vd; 2972 uint64_t psize = zio->io_size; 2973 zio_t *pio, *pio_next; 2974 2975 /* 2976 * If our children haven't all completed, 2977 * wait for them and then repeat this pipeline stage. 2978 */ 2979 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 2980 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 2981 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 2982 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 2983 return (ZIO_PIPELINE_STOP); 2984 2985 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 2986 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 2987 ASSERT(zio->io_children[c][w] == 0); 2988 2989 if (bp != NULL) { 2990 ASSERT(bp->blk_pad[0] == 0); 2991 ASSERT(bp->blk_pad[1] == 0); 2992 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 2993 (bp == zio_unique_parent(zio)->io_bp)); 2994 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 2995 zio->io_bp_override == NULL && 2996 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2997 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 2998 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 2999 ASSERT(BP_COUNT_GANG(bp) == 0 || 3000 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 3001 } 3002 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 3003 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 3004 } 3005 3006 /* 3007 * If there were child vdev/gang/ddt errors, they apply to us now. 3008 */ 3009 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 3010 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 3011 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 3012 3013 /* 3014 * If the I/O on the transformed data was successful, generate any 3015 * checksum reports now while we still have the transformed data. 3016 */ 3017 if (zio->io_error == 0) { 3018 while (zio->io_cksum_report != NULL) { 3019 zio_cksum_report_t *zcr = zio->io_cksum_report; 3020 uint64_t align = zcr->zcr_align; 3021 uint64_t asize = P2ROUNDUP(psize, align); 3022 char *abuf = zio->io_data; 3023 3024 if (asize != psize) { 3025 abuf = zio_buf_alloc(asize); 3026 bcopy(zio->io_data, abuf, psize); 3027 bzero(abuf + psize, asize - psize); 3028 } 3029 3030 zio->io_cksum_report = zcr->zcr_next; 3031 zcr->zcr_next = NULL; 3032 zcr->zcr_finish(zcr, abuf); 3033 zfs_ereport_free_checksum(zcr); 3034 3035 if (asize != psize) 3036 zio_buf_free(abuf, asize); 3037 } 3038 } 3039 3040 zio_pop_transforms(zio); /* note: may set zio->io_error */ 3041 3042 vdev_stat_update(zio, psize); 3043 3044 if (zio->io_error) { 3045 /* 3046 * If this I/O is attached to a particular vdev, 3047 * generate an error message describing the I/O failure 3048 * at the block level. We ignore these errors if the 3049 * device is currently unavailable. 3050 */ 3051 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 3052 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 3053 3054 if ((zio->io_error == EIO || !(zio->io_flags & 3055 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 3056 zio == lio) { 3057 /* 3058 * For logical I/O requests, tell the SPA to log the 3059 * error and generate a logical data ereport. 3060 */ 3061 spa_log_error(spa, zio); 3062 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 3063 0, 0); 3064 } 3065 } 3066 3067 if (zio->io_error && zio == lio) { 3068 /* 3069 * Determine whether zio should be reexecuted. This will 3070 * propagate all the way to the root via zio_notify_parent(). 3071 */ 3072 ASSERT(vd == NULL && bp != NULL); 3073 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3074 3075 if (IO_IS_ALLOCATING(zio) && 3076 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 3077 if (zio->io_error != ENOSPC) 3078 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 3079 else 3080 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3081 } 3082 3083 if ((zio->io_type == ZIO_TYPE_READ || 3084 zio->io_type == ZIO_TYPE_FREE) && 3085 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 3086 zio->io_error == ENXIO && 3087 spa_load_state(spa) == SPA_LOAD_NONE && 3088 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 3089 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3090 3091 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 3092 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3093 3094 /* 3095 * Here is a possibly good place to attempt to do 3096 * either combinatorial reconstruction or error correction 3097 * based on checksums. It also might be a good place 3098 * to send out preliminary ereports before we suspend 3099 * processing. 3100 */ 3101 } 3102 3103 /* 3104 * If there were logical child errors, they apply to us now. 3105 * We defer this until now to avoid conflating logical child 3106 * errors with errors that happened to the zio itself when 3107 * updating vdev stats and reporting FMA events above. 3108 */ 3109 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 3110 3111 if ((zio->io_error || zio->io_reexecute) && 3112 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 3113 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 3114 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 3115 3116 zio_gang_tree_free(&zio->io_gang_tree); 3117 3118 /* 3119 * Godfather I/Os should never suspend. 3120 */ 3121 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 3122 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 3123 zio->io_reexecute = 0; 3124 3125 if (zio->io_reexecute) { 3126 /* 3127 * This is a logical I/O that wants to reexecute. 3128 * 3129 * Reexecute is top-down. When an i/o fails, if it's not 3130 * the root, it simply notifies its parent and sticks around. 3131 * The parent, seeing that it still has children in zio_done(), 3132 * does the same. This percolates all the way up to the root. 3133 * The root i/o will reexecute or suspend the entire tree. 3134 * 3135 * This approach ensures that zio_reexecute() honors 3136 * all the original i/o dependency relationships, e.g. 3137 * parents not executing until children are ready. 3138 */ 3139 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3140 3141 zio->io_gang_leader = NULL; 3142 3143 mutex_enter(&zio->io_lock); 3144 zio->io_state[ZIO_WAIT_DONE] = 1; 3145 mutex_exit(&zio->io_lock); 3146 3147 /* 3148 * "The Godfather" I/O monitors its children but is 3149 * not a true parent to them. It will track them through 3150 * the pipeline but severs its ties whenever they get into 3151 * trouble (e.g. suspended). This allows "The Godfather" 3152 * I/O to return status without blocking. 3153 */ 3154 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3155 zio_link_t *zl = zio->io_walk_link; 3156 pio_next = zio_walk_parents(zio); 3157 3158 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 3159 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 3160 zio_remove_child(pio, zio, zl); 3161 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3162 } 3163 } 3164 3165 if ((pio = zio_unique_parent(zio)) != NULL) { 3166 /* 3167 * We're not a root i/o, so there's nothing to do 3168 * but notify our parent. Don't propagate errors 3169 * upward since we haven't permanently failed yet. 3170 */ 3171 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 3172 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 3173 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3174 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 3175 /* 3176 * We'd fail again if we reexecuted now, so suspend 3177 * until conditions improve (e.g. device comes online). 3178 */ 3179 zio_suspend(spa, zio); 3180 } else { 3181 /* 3182 * Reexecution is potentially a huge amount of work. 3183 * Hand it off to the otherwise-unused claim taskq. 3184 */ 3185#if defined(illumos) || !defined(_KERNEL) 3186 ASSERT(zio->io_tqent.tqent_next == NULL); 3187#else 3188 ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); 3189#endif 3190 spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM, 3191 ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio, 3192 0, &zio->io_tqent); 3193 } 3194 return (ZIO_PIPELINE_STOP); 3195 } 3196 3197 ASSERT(zio->io_child_count == 0); 3198 ASSERT(zio->io_reexecute == 0); 3199 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 3200 3201 /* 3202 * Report any checksum errors, since the I/O is complete. 3203 */ 3204 while (zio->io_cksum_report != NULL) { 3205 zio_cksum_report_t *zcr = zio->io_cksum_report; 3206 zio->io_cksum_report = zcr->zcr_next; 3207 zcr->zcr_next = NULL; 3208 zcr->zcr_finish(zcr, NULL); 3209 zfs_ereport_free_checksum(zcr); 3210 } 3211 3212 /* 3213 * It is the responsibility of the done callback to ensure that this 3214 * particular zio is no longer discoverable for adoption, and as 3215 * such, cannot acquire any new parents. 3216 */ 3217 if (zio->io_done) 3218 zio->io_done(zio); 3219 3220 mutex_enter(&zio->io_lock); 3221 zio->io_state[ZIO_WAIT_DONE] = 1; 3222 mutex_exit(&zio->io_lock); 3223 3224 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3225 zio_link_t *zl = zio->io_walk_link; 3226 pio_next = zio_walk_parents(zio); 3227 zio_remove_child(pio, zio, zl); 3228 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3229 } 3230 3231 if (zio->io_waiter != NULL) { 3232 mutex_enter(&zio->io_lock); 3233 zio->io_executor = NULL; 3234 cv_broadcast(&zio->io_cv); 3235 mutex_exit(&zio->io_lock); 3236 } else { 3237 zio_destroy(zio); 3238 } 3239 3240 return (ZIO_PIPELINE_STOP); 3241} 3242 3243/* 3244 * ========================================================================== 3245 * I/O pipeline definition 3246 * ========================================================================== 3247 */ 3248static zio_pipe_stage_t *zio_pipeline[] = { 3249 NULL, 3250 zio_read_bp_init, 3251 zio_free_bp_init, 3252 zio_issue_async, 3253 zio_write_bp_init, 3254 zio_checksum_generate, 3255 zio_nop_write, 3256 zio_ddt_read_start, 3257 zio_ddt_read_done, 3258 zio_ddt_write, 3259 zio_ddt_free, 3260 zio_gang_assemble, 3261 zio_gang_issue, 3262 zio_dva_allocate, 3263 zio_dva_free, 3264 zio_dva_claim, 3265 zio_ready, 3266 zio_vdev_io_start, 3267 zio_vdev_io_done, 3268 zio_vdev_io_assess, 3269 zio_checksum_verify, 3270 zio_done 3271}; 3272 3273/* dnp is the dnode for zb1->zb_object */ 3274boolean_t 3275zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1, 3276 const zbookmark_t *zb2) 3277{ 3278 uint64_t zb1nextL0, zb2thisobj; 3279 3280 ASSERT(zb1->zb_objset == zb2->zb_objset); 3281 ASSERT(zb2->zb_level == 0); 3282 3283 /* 3284 * A bookmark in the deadlist is considered to be after 3285 * everything else. 3286 */ 3287 if (zb2->zb_object == DMU_DEADLIST_OBJECT) 3288 return (B_TRUE); 3289 3290 /* The objset_phys_t isn't before anything. */ 3291 if (dnp == NULL) 3292 return (B_FALSE); 3293 3294 zb1nextL0 = (zb1->zb_blkid + 1) << 3295 ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)); 3296 3297 zb2thisobj = zb2->zb_object ? zb2->zb_object : 3298 zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT); 3299 3300 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 3301 uint64_t nextobj = zb1nextL0 * 3302 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT; 3303 return (nextobj <= zb2thisobj); 3304 } 3305 3306 if (zb1->zb_object < zb2thisobj) 3307 return (B_TRUE); 3308 if (zb1->zb_object > zb2thisobj) 3309 return (B_FALSE); 3310 if (zb2->zb_object == DMU_META_DNODE_OBJECT) 3311 return (B_FALSE); 3312 return (zb1nextL0 <= zb2->zb_blkid); 3313} 3314