zio.c revision 268650
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2014 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 */ 26 27#include <sys/zfs_context.h> 28#include <sys/fm/fs/zfs.h> 29#include <sys/spa.h> 30#include <sys/txg.h> 31#include <sys/spa_impl.h> 32#include <sys/vdev_impl.h> 33#include <sys/zio_impl.h> 34#include <sys/zio_compress.h> 35#include <sys/zio_checksum.h> 36#include <sys/dmu_objset.h> 37#include <sys/arc.h> 38#include <sys/ddt.h> 39#include <sys/trim_map.h> 40#include <sys/blkptr.h> 41#include <sys/zfeature.h> 42 43SYSCTL_DECL(_vfs_zfs); 44SYSCTL_NODE(_vfs_zfs, OID_AUTO, zio, CTLFLAG_RW, 0, "ZFS ZIO"); 45#if defined(__amd64__) 46static int zio_use_uma = 1; 47#else 48static int zio_use_uma = 0; 49#endif 50TUNABLE_INT("vfs.zfs.zio.use_uma", &zio_use_uma); 51SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, use_uma, CTLFLAG_RDTUN, &zio_use_uma, 0, 52 "Use uma(9) for ZIO allocations"); 53static int zio_exclude_metadata = 0; 54TUNABLE_INT("vfs.zfs.zio.exclude_metadata", &zio_exclude_metadata); 55SYSCTL_INT(_vfs_zfs_zio, OID_AUTO, exclude_metadata, CTLFLAG_RDTUN, &zio_exclude_metadata, 0, 56 "Exclude metadata buffers from dumps as well"); 57 58zio_trim_stats_t zio_trim_stats = { 59 { "bytes", KSTAT_DATA_UINT64, 60 "Number of bytes successfully TRIMmed" }, 61 { "success", KSTAT_DATA_UINT64, 62 "Number of successful TRIM requests" }, 63 { "unsupported", KSTAT_DATA_UINT64, 64 "Number of TRIM requests that failed because TRIM is not supported" }, 65 { "failed", KSTAT_DATA_UINT64, 66 "Number of TRIM requests that failed for reasons other than not supported" }, 67}; 68 69static kstat_t *zio_trim_ksp; 70 71/* 72 * ========================================================================== 73 * I/O type descriptions 74 * ========================================================================== 75 */ 76const char *zio_type_name[ZIO_TYPES] = { 77 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 78 "zio_ioctl" 79}; 80 81/* 82 * ========================================================================== 83 * I/O kmem caches 84 * ========================================================================== 85 */ 86kmem_cache_t *zio_cache; 87kmem_cache_t *zio_link_cache; 88kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 89kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 90 91#ifdef _KERNEL 92extern vmem_t *zio_alloc_arena; 93#endif 94 95/* 96 * The following actions directly effect the spa's sync-to-convergence logic. 97 * The values below define the sync pass when we start performing the action. 98 * Care should be taken when changing these values as they directly impact 99 * spa_sync() performance. Tuning these values may introduce subtle performance 100 * pathologies and should only be done in the context of performance analysis. 101 * These tunables will eventually be removed and replaced with #defines once 102 * enough analysis has been done to determine optimal values. 103 * 104 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 105 * regular blocks are not deferred. 106 */ 107int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 108TUNABLE_INT("vfs.zfs.sync_pass_deferred_free", &zfs_sync_pass_deferred_free); 109SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_deferred_free, CTLFLAG_RDTUN, 110 &zfs_sync_pass_deferred_free, 0, "defer frees starting in this pass"); 111int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 112TUNABLE_INT("vfs.zfs.sync_pass_dont_compress", &zfs_sync_pass_dont_compress); 113SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_dont_compress, CTLFLAG_RDTUN, 114 &zfs_sync_pass_dont_compress, 0, "don't compress starting in this pass"); 115int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 116TUNABLE_INT("vfs.zfs.sync_pass_rewrite", &zfs_sync_pass_rewrite); 117SYSCTL_INT(_vfs_zfs, OID_AUTO, sync_pass_rewrite, CTLFLAG_RDTUN, 118 &zfs_sync_pass_rewrite, 0, "rewrite new bps starting in this pass"); 119 120/* 121 * An allocating zio is one that either currently has the DVA allocate 122 * stage set or will have it later in its lifetime. 123 */ 124#define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 125 126boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 127 128#ifdef ZFS_DEBUG 129int zio_buf_debug_limit = 16384; 130#else 131int zio_buf_debug_limit = 0; 132#endif 133 134void 135zio_init(void) 136{ 137 size_t c; 138 zio_cache = kmem_cache_create("zio_cache", 139 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 140 zio_link_cache = kmem_cache_create("zio_link_cache", 141 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 142 if (!zio_use_uma) 143 goto out; 144 145 /* 146 * For small buffers, we want a cache for each multiple of 147 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 148 * for each quarter-power of 2. For large buffers, we want 149 * a cache for each multiple of PAGESIZE. 150 */ 151 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 152 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 153 size_t p2 = size; 154 size_t align = 0; 155 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 156 157 while (p2 & (p2 - 1)) 158 p2 &= p2 - 1; 159 160#ifdef illumos 161#ifndef _KERNEL 162 /* 163 * If we are using watchpoints, put each buffer on its own page, 164 * to eliminate the performance overhead of trapping to the 165 * kernel when modifying a non-watched buffer that shares the 166 * page with a watched buffer. 167 */ 168 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 169 continue; 170#endif 171#endif /* illumos */ 172 if (size <= 4 * SPA_MINBLOCKSIZE) { 173 align = SPA_MINBLOCKSIZE; 174 } else if (IS_P2ALIGNED(size, PAGESIZE)) { 175 align = PAGESIZE; 176 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 177 align = p2 >> 2; 178 } 179 180 if (align != 0) { 181 char name[36]; 182 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 183 zio_buf_cache[c] = kmem_cache_create(name, size, 184 align, NULL, NULL, NULL, NULL, NULL, cflags); 185 186 /* 187 * Since zio_data bufs do not appear in crash dumps, we 188 * pass KMC_NOTOUCH so that no allocator metadata is 189 * stored with the buffers. 190 */ 191 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 192 zio_data_buf_cache[c] = kmem_cache_create(name, size, 193 align, NULL, NULL, NULL, NULL, NULL, 194 cflags | KMC_NOTOUCH | KMC_NODEBUG); 195 } 196 } 197 198 while (--c != 0) { 199 ASSERT(zio_buf_cache[c] != NULL); 200 if (zio_buf_cache[c - 1] == NULL) 201 zio_buf_cache[c - 1] = zio_buf_cache[c]; 202 203 ASSERT(zio_data_buf_cache[c] != NULL); 204 if (zio_data_buf_cache[c - 1] == NULL) 205 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 206 } 207out: 208 209 zio_inject_init(); 210 211 zio_trim_ksp = kstat_create("zfs", 0, "zio_trim", "misc", 212 KSTAT_TYPE_NAMED, 213 sizeof(zio_trim_stats) / sizeof(kstat_named_t), 214 KSTAT_FLAG_VIRTUAL); 215 216 if (zio_trim_ksp != NULL) { 217 zio_trim_ksp->ks_data = &zio_trim_stats; 218 kstat_install(zio_trim_ksp); 219 } 220} 221 222void 223zio_fini(void) 224{ 225 size_t c; 226 kmem_cache_t *last_cache = NULL; 227 kmem_cache_t *last_data_cache = NULL; 228 229 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 230 if (zio_buf_cache[c] != last_cache) { 231 last_cache = zio_buf_cache[c]; 232 kmem_cache_destroy(zio_buf_cache[c]); 233 } 234 zio_buf_cache[c] = NULL; 235 236 if (zio_data_buf_cache[c] != last_data_cache) { 237 last_data_cache = zio_data_buf_cache[c]; 238 kmem_cache_destroy(zio_data_buf_cache[c]); 239 } 240 zio_data_buf_cache[c] = NULL; 241 } 242 243 kmem_cache_destroy(zio_link_cache); 244 kmem_cache_destroy(zio_cache); 245 246 zio_inject_fini(); 247 248 if (zio_trim_ksp != NULL) { 249 kstat_delete(zio_trim_ksp); 250 zio_trim_ksp = NULL; 251 } 252} 253 254/* 255 * ========================================================================== 256 * Allocate and free I/O buffers 257 * ========================================================================== 258 */ 259 260/* 261 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 262 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 263 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 264 * excess / transient data in-core during a crashdump. 265 */ 266void * 267zio_buf_alloc(size_t size) 268{ 269 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 270 int flags = zio_exclude_metadata ? KM_NODEBUG : 0; 271 272 ASSERT3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 273 274 if (zio_use_uma) 275 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 276 else 277 return (kmem_alloc(size, KM_SLEEP|flags)); 278} 279 280/* 281 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 282 * crashdump if the kernel panics. This exists so that we will limit the amount 283 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 284 * of kernel heap dumped to disk when the kernel panics) 285 */ 286void * 287zio_data_buf_alloc(size_t size) 288{ 289 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 290 291 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 292 293 if (zio_use_uma) 294 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 295 else 296 return (kmem_alloc(size, KM_SLEEP | KM_NODEBUG)); 297} 298 299void 300zio_buf_free(void *buf, size_t size) 301{ 302 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 303 304 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 305 306 if (zio_use_uma) 307 kmem_cache_free(zio_buf_cache[c], buf); 308 else 309 kmem_free(buf, size); 310} 311 312void 313zio_data_buf_free(void *buf, size_t size) 314{ 315 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 316 317 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 318 319 if (zio_use_uma) 320 kmem_cache_free(zio_data_buf_cache[c], buf); 321 else 322 kmem_free(buf, size); 323} 324 325/* 326 * ========================================================================== 327 * Push and pop I/O transform buffers 328 * ========================================================================== 329 */ 330static void 331zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 332 zio_transform_func_t *transform) 333{ 334 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 335 336 zt->zt_orig_data = zio->io_data; 337 zt->zt_orig_size = zio->io_size; 338 zt->zt_bufsize = bufsize; 339 zt->zt_transform = transform; 340 341 zt->zt_next = zio->io_transform_stack; 342 zio->io_transform_stack = zt; 343 344 zio->io_data = data; 345 zio->io_size = size; 346} 347 348static void 349zio_pop_transforms(zio_t *zio) 350{ 351 zio_transform_t *zt; 352 353 while ((zt = zio->io_transform_stack) != NULL) { 354 if (zt->zt_transform != NULL) 355 zt->zt_transform(zio, 356 zt->zt_orig_data, zt->zt_orig_size); 357 358 if (zt->zt_bufsize != 0) 359 zio_buf_free(zio->io_data, zt->zt_bufsize); 360 361 zio->io_data = zt->zt_orig_data; 362 zio->io_size = zt->zt_orig_size; 363 zio->io_transform_stack = zt->zt_next; 364 365 kmem_free(zt, sizeof (zio_transform_t)); 366 } 367} 368 369/* 370 * ========================================================================== 371 * I/O transform callbacks for subblocks and decompression 372 * ========================================================================== 373 */ 374static void 375zio_subblock(zio_t *zio, void *data, uint64_t size) 376{ 377 ASSERT(zio->io_size > size); 378 379 if (zio->io_type == ZIO_TYPE_READ) 380 bcopy(zio->io_data, data, size); 381} 382 383static void 384zio_decompress(zio_t *zio, void *data, uint64_t size) 385{ 386 if (zio->io_error == 0 && 387 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 388 zio->io_data, data, zio->io_size, size) != 0) 389 zio->io_error = SET_ERROR(EIO); 390} 391 392/* 393 * ========================================================================== 394 * I/O parent/child relationships and pipeline interlocks 395 * ========================================================================== 396 */ 397/* 398 * NOTE - Callers to zio_walk_parents() and zio_walk_children must 399 * continue calling these functions until they return NULL. 400 * Otherwise, the next caller will pick up the list walk in 401 * some indeterminate state. (Otherwise every caller would 402 * have to pass in a cookie to keep the state represented by 403 * io_walk_link, which gets annoying.) 404 */ 405zio_t * 406zio_walk_parents(zio_t *cio) 407{ 408 zio_link_t *zl = cio->io_walk_link; 409 list_t *pl = &cio->io_parent_list; 410 411 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl); 412 cio->io_walk_link = zl; 413 414 if (zl == NULL) 415 return (NULL); 416 417 ASSERT(zl->zl_child == cio); 418 return (zl->zl_parent); 419} 420 421zio_t * 422zio_walk_children(zio_t *pio) 423{ 424 zio_link_t *zl = pio->io_walk_link; 425 list_t *cl = &pio->io_child_list; 426 427 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl); 428 pio->io_walk_link = zl; 429 430 if (zl == NULL) 431 return (NULL); 432 433 ASSERT(zl->zl_parent == pio); 434 return (zl->zl_child); 435} 436 437zio_t * 438zio_unique_parent(zio_t *cio) 439{ 440 zio_t *pio = zio_walk_parents(cio); 441 442 VERIFY(zio_walk_parents(cio) == NULL); 443 return (pio); 444} 445 446void 447zio_add_child(zio_t *pio, zio_t *cio) 448{ 449 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 450 451 /* 452 * Logical I/Os can have logical, gang, or vdev children. 453 * Gang I/Os can have gang or vdev children. 454 * Vdev I/Os can only have vdev children. 455 * The following ASSERT captures all of these constraints. 456 */ 457 ASSERT(cio->io_child_type <= pio->io_child_type); 458 459 zl->zl_parent = pio; 460 zl->zl_child = cio; 461 462 mutex_enter(&cio->io_lock); 463 mutex_enter(&pio->io_lock); 464 465 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 466 467 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 468 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 469 470 list_insert_head(&pio->io_child_list, zl); 471 list_insert_head(&cio->io_parent_list, zl); 472 473 pio->io_child_count++; 474 cio->io_parent_count++; 475 476 mutex_exit(&pio->io_lock); 477 mutex_exit(&cio->io_lock); 478} 479 480static void 481zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 482{ 483 ASSERT(zl->zl_parent == pio); 484 ASSERT(zl->zl_child == cio); 485 486 mutex_enter(&cio->io_lock); 487 mutex_enter(&pio->io_lock); 488 489 list_remove(&pio->io_child_list, zl); 490 list_remove(&cio->io_parent_list, zl); 491 492 pio->io_child_count--; 493 cio->io_parent_count--; 494 495 mutex_exit(&pio->io_lock); 496 mutex_exit(&cio->io_lock); 497 498 kmem_cache_free(zio_link_cache, zl); 499} 500 501static boolean_t 502zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 503{ 504 uint64_t *countp = &zio->io_children[child][wait]; 505 boolean_t waiting = B_FALSE; 506 507 mutex_enter(&zio->io_lock); 508 ASSERT(zio->io_stall == NULL); 509 if (*countp != 0) { 510 zio->io_stage >>= 1; 511 zio->io_stall = countp; 512 waiting = B_TRUE; 513 } 514 mutex_exit(&zio->io_lock); 515 516 return (waiting); 517} 518 519static void 520zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 521{ 522 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 523 int *errorp = &pio->io_child_error[zio->io_child_type]; 524 525 mutex_enter(&pio->io_lock); 526 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 527 *errorp = zio_worst_error(*errorp, zio->io_error); 528 pio->io_reexecute |= zio->io_reexecute; 529 ASSERT3U(*countp, >, 0); 530 531 (*countp)--; 532 533 if (*countp == 0 && pio->io_stall == countp) { 534 pio->io_stall = NULL; 535 mutex_exit(&pio->io_lock); 536 zio_execute(pio); 537 } else { 538 mutex_exit(&pio->io_lock); 539 } 540} 541 542static void 543zio_inherit_child_errors(zio_t *zio, enum zio_child c) 544{ 545 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 546 zio->io_error = zio->io_child_error[c]; 547} 548 549/* 550 * ========================================================================== 551 * Create the various types of I/O (read, write, free, etc) 552 * ========================================================================== 553 */ 554static zio_t * 555zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 556 void *data, uint64_t size, zio_done_func_t *done, void *private, 557 zio_type_t type, zio_priority_t priority, enum zio_flag flags, 558 vdev_t *vd, uint64_t offset, const zbookmark_t *zb, 559 enum zio_stage stage, enum zio_stage pipeline) 560{ 561 zio_t *zio; 562 563 ASSERT3U(type == ZIO_TYPE_FREE || size, <=, SPA_MAXBLOCKSIZE); 564 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 565 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 566 567 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 568 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 569 ASSERT(vd || stage == ZIO_STAGE_OPEN); 570 571 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 572 bzero(zio, sizeof (zio_t)); 573 574 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 575 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 576 577 list_create(&zio->io_parent_list, sizeof (zio_link_t), 578 offsetof(zio_link_t, zl_parent_node)); 579 list_create(&zio->io_child_list, sizeof (zio_link_t), 580 offsetof(zio_link_t, zl_child_node)); 581 582 if (vd != NULL) 583 zio->io_child_type = ZIO_CHILD_VDEV; 584 else if (flags & ZIO_FLAG_GANG_CHILD) 585 zio->io_child_type = ZIO_CHILD_GANG; 586 else if (flags & ZIO_FLAG_DDT_CHILD) 587 zio->io_child_type = ZIO_CHILD_DDT; 588 else 589 zio->io_child_type = ZIO_CHILD_LOGICAL; 590 591 if (bp != NULL) { 592 zio->io_bp = (blkptr_t *)bp; 593 zio->io_bp_copy = *bp; 594 zio->io_bp_orig = *bp; 595 if (type != ZIO_TYPE_WRITE || 596 zio->io_child_type == ZIO_CHILD_DDT) 597 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 598 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 599 zio->io_logical = zio; 600 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 601 pipeline |= ZIO_GANG_STAGES; 602 } 603 604 zio->io_spa = spa; 605 zio->io_txg = txg; 606 zio->io_done = done; 607 zio->io_private = private; 608 zio->io_type = type; 609 zio->io_priority = priority; 610 zio->io_vd = vd; 611 zio->io_offset = offset; 612 zio->io_orig_data = zio->io_data = data; 613 zio->io_orig_size = zio->io_size = size; 614 zio->io_orig_flags = zio->io_flags = flags; 615 zio->io_orig_stage = zio->io_stage = stage; 616 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 617 618 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 619 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 620 621 if (zb != NULL) 622 zio->io_bookmark = *zb; 623 624 if (pio != NULL) { 625 if (zio->io_logical == NULL) 626 zio->io_logical = pio->io_logical; 627 if (zio->io_child_type == ZIO_CHILD_GANG) 628 zio->io_gang_leader = pio->io_gang_leader; 629 zio_add_child(pio, zio); 630 } 631 632 return (zio); 633} 634 635static void 636zio_destroy(zio_t *zio) 637{ 638 list_destroy(&zio->io_parent_list); 639 list_destroy(&zio->io_child_list); 640 mutex_destroy(&zio->io_lock); 641 cv_destroy(&zio->io_cv); 642 kmem_cache_free(zio_cache, zio); 643} 644 645zio_t * 646zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 647 void *private, enum zio_flag flags) 648{ 649 zio_t *zio; 650 651 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 652 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 653 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 654 655 return (zio); 656} 657 658zio_t * 659zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 660{ 661 return (zio_null(NULL, spa, NULL, done, private, flags)); 662} 663 664zio_t * 665zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 666 void *data, uint64_t size, zio_done_func_t *done, void *private, 667 zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb) 668{ 669 zio_t *zio; 670 671 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 672 data, size, done, private, 673 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 674 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 675 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 676 677 return (zio); 678} 679 680zio_t * 681zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 682 void *data, uint64_t size, const zio_prop_t *zp, 683 zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done, 684 void *private, 685 zio_priority_t priority, enum zio_flag flags, const zbookmark_t *zb) 686{ 687 zio_t *zio; 688 689 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 690 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 691 zp->zp_compress >= ZIO_COMPRESS_OFF && 692 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 693 DMU_OT_IS_VALID(zp->zp_type) && 694 zp->zp_level < 32 && 695 zp->zp_copies > 0 && 696 zp->zp_copies <= spa_max_replication(spa)); 697 698 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 699 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 700 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 701 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 702 703 zio->io_ready = ready; 704 zio->io_physdone = physdone; 705 zio->io_prop = *zp; 706 707 /* 708 * Data can be NULL if we are going to call zio_write_override() to 709 * provide the already-allocated BP. But we may need the data to 710 * verify a dedup hit (if requested). In this case, don't try to 711 * dedup (just take the already-allocated BP verbatim). 712 */ 713 if (data == NULL && zio->io_prop.zp_dedup_verify) { 714 zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE; 715 } 716 717 return (zio); 718} 719 720zio_t * 721zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 722 uint64_t size, zio_done_func_t *done, void *private, 723 zio_priority_t priority, enum zio_flag flags, zbookmark_t *zb) 724{ 725 zio_t *zio; 726 727 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 728 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 729 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 730 731 return (zio); 732} 733 734void 735zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 736{ 737 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 738 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 739 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 740 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 741 742 /* 743 * We must reset the io_prop to match the values that existed 744 * when the bp was first written by dmu_sync() keeping in mind 745 * that nopwrite and dedup are mutually exclusive. 746 */ 747 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 748 zio->io_prop.zp_nopwrite = nopwrite; 749 zio->io_prop.zp_copies = copies; 750 zio->io_bp_override = bp; 751} 752 753void 754zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 755{ 756 757 /* 758 * The check for EMBEDDED is a performance optimization. We 759 * process the free here (by ignoring it) rather than 760 * putting it on the list and then processing it in zio_free_sync(). 761 */ 762 if (BP_IS_EMBEDDED(bp)) 763 return; 764 metaslab_check_free(spa, bp); 765 766 /* 767 * Frees that are for the currently-syncing txg, are not going to be 768 * deferred, and which will not need to do a read (i.e. not GANG or 769 * DEDUP), can be processed immediately. Otherwise, put them on the 770 * in-memory list for later processing. 771 */ 772 if (zfs_trim_enabled || BP_IS_GANG(bp) || BP_GET_DEDUP(bp) || 773 txg != spa->spa_syncing_txg || 774 spa_sync_pass(spa) >= zfs_sync_pass_deferred_free) { 775 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 776 } else { 777 VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 778 BP_GET_PSIZE(bp), 0))); 779 } 780} 781 782zio_t * 783zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 784 uint64_t size, enum zio_flag flags) 785{ 786 zio_t *zio; 787 enum zio_stage stage = ZIO_FREE_PIPELINE; 788 789 ASSERT(!BP_IS_HOLE(bp)); 790 ASSERT(spa_syncing_txg(spa) == txg); 791 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free); 792 793 if (BP_IS_EMBEDDED(bp)) 794 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 795 796 metaslab_check_free(spa, bp); 797 arc_freed(spa, bp); 798 799 if (zfs_trim_enabled) 800 stage |= ZIO_STAGE_ISSUE_ASYNC | ZIO_STAGE_VDEV_IO_START | 801 ZIO_STAGE_VDEV_IO_ASSESS; 802 /* 803 * GANG and DEDUP blocks can induce a read (for the gang block header, 804 * or the DDT), so issue them asynchronously so that this thread is 805 * not tied up. 806 */ 807 else if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) 808 stage |= ZIO_STAGE_ISSUE_ASYNC; 809 810 zio = zio_create(pio, spa, txg, bp, NULL, size, 811 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, flags, 812 NULL, 0, NULL, ZIO_STAGE_OPEN, stage); 813 814 return (zio); 815} 816 817zio_t * 818zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 819 zio_done_func_t *done, void *private, enum zio_flag flags) 820{ 821 zio_t *zio; 822 823 dprintf_bp(bp, "claiming in txg %llu", txg); 824 825 if (BP_IS_EMBEDDED(bp)) 826 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 827 828 /* 829 * A claim is an allocation of a specific block. Claims are needed 830 * to support immediate writes in the intent log. The issue is that 831 * immediate writes contain committed data, but in a txg that was 832 * *not* committed. Upon opening the pool after an unclean shutdown, 833 * the intent log claims all blocks that contain immediate write data 834 * so that the SPA knows they're in use. 835 * 836 * All claims *must* be resolved in the first txg -- before the SPA 837 * starts allocating blocks -- so that nothing is allocated twice. 838 * If txg == 0 we just verify that the block is claimable. 839 */ 840 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 841 ASSERT(txg == spa_first_txg(spa) || txg == 0); 842 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 843 844 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 845 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 846 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 847 848 return (zio); 849} 850 851zio_t * 852zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, uint64_t offset, 853 uint64_t size, zio_done_func_t *done, void *private, 854 enum zio_flag flags) 855{ 856 zio_t *zio; 857 int c; 858 859 if (vd->vdev_children == 0) { 860 zio = zio_create(pio, spa, 0, NULL, NULL, size, done, private, 861 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, offset, NULL, 862 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 863 864 zio->io_cmd = cmd; 865 } else { 866 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 867 868 for (c = 0; c < vd->vdev_children; c++) 869 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 870 offset, size, done, private, flags)); 871 } 872 873 return (zio); 874} 875 876zio_t * 877zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 878 void *data, int checksum, zio_done_func_t *done, void *private, 879 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 880{ 881 zio_t *zio; 882 883 ASSERT(vd->vdev_children == 0); 884 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 885 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 886 ASSERT3U(offset + size, <=, vd->vdev_psize); 887 888 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 889 ZIO_TYPE_READ, priority, flags, vd, offset, NULL, 890 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 891 892 zio->io_prop.zp_checksum = checksum; 893 894 return (zio); 895} 896 897zio_t * 898zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 899 void *data, int checksum, zio_done_func_t *done, void *private, 900 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 901{ 902 zio_t *zio; 903 904 ASSERT(vd->vdev_children == 0); 905 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 906 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 907 ASSERT3U(offset + size, <=, vd->vdev_psize); 908 909 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 910 ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL, 911 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 912 913 zio->io_prop.zp_checksum = checksum; 914 915 if (zio_checksum_table[checksum].ci_eck) { 916 /* 917 * zec checksums are necessarily destructive -- they modify 918 * the end of the write buffer to hold the verifier/checksum. 919 * Therefore, we must make a local copy in case the data is 920 * being written to multiple places in parallel. 921 */ 922 void *wbuf = zio_buf_alloc(size); 923 bcopy(data, wbuf, size); 924 zio_push_transform(zio, wbuf, size, size, NULL); 925 } 926 927 return (zio); 928} 929 930/* 931 * Create a child I/O to do some work for us. 932 */ 933zio_t * 934zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 935 void *data, uint64_t size, int type, zio_priority_t priority, 936 enum zio_flag flags, zio_done_func_t *done, void *private) 937{ 938 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 939 zio_t *zio; 940 941 ASSERT(vd->vdev_parent == 942 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 943 944 if (type == ZIO_TYPE_READ && bp != NULL) { 945 /* 946 * If we have the bp, then the child should perform the 947 * checksum and the parent need not. This pushes error 948 * detection as close to the leaves as possible and 949 * eliminates redundant checksums in the interior nodes. 950 */ 951 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 952 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 953 } 954 955 if (vd->vdev_children == 0) 956 offset += VDEV_LABEL_START_SIZE; 957 958 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 959 960 /* 961 * If we've decided to do a repair, the write is not speculative -- 962 * even if the original read was. 963 */ 964 if (flags & ZIO_FLAG_IO_REPAIR) 965 flags &= ~ZIO_FLAG_SPECULATIVE; 966 967 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 968 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 969 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 970 971 zio->io_physdone = pio->io_physdone; 972 if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL) 973 zio->io_logical->io_phys_children++; 974 975 return (zio); 976} 977 978zio_t * 979zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 980 int type, zio_priority_t priority, enum zio_flag flags, 981 zio_done_func_t *done, void *private) 982{ 983 zio_t *zio; 984 985 ASSERT(vd->vdev_ops->vdev_op_leaf); 986 987 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 988 data, size, done, private, type, priority, 989 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 990 vd, offset, NULL, 991 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 992 993 return (zio); 994} 995 996void 997zio_flush(zio_t *zio, vdev_t *vd) 998{ 999 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 0, 0, 1000 NULL, NULL, 1001 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 1002} 1003 1004zio_t * 1005zio_trim(zio_t *zio, spa_t *spa, vdev_t *vd, uint64_t offset, uint64_t size) 1006{ 1007 1008 ASSERT(vd->vdev_ops->vdev_op_leaf); 1009 1010 return zio_ioctl(zio, spa, vd, DKIOCTRIM, offset, size, 1011 NULL, NULL, 1012 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY); 1013} 1014 1015void 1016zio_shrink(zio_t *zio, uint64_t size) 1017{ 1018 ASSERT(zio->io_executor == NULL); 1019 ASSERT(zio->io_orig_size == zio->io_size); 1020 ASSERT(size <= zio->io_size); 1021 1022 /* 1023 * We don't shrink for raidz because of problems with the 1024 * reconstruction when reading back less than the block size. 1025 * Note, BP_IS_RAIDZ() assumes no compression. 1026 */ 1027 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 1028 if (!BP_IS_RAIDZ(zio->io_bp)) 1029 zio->io_orig_size = zio->io_size = size; 1030} 1031 1032/* 1033 * ========================================================================== 1034 * Prepare to read and write logical blocks 1035 * ========================================================================== 1036 */ 1037 1038static int 1039zio_read_bp_init(zio_t **ziop) 1040{ 1041 zio_t *zio = *ziop; 1042 blkptr_t *bp = zio->io_bp; 1043 1044 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 1045 zio->io_child_type == ZIO_CHILD_LOGICAL && 1046 !(zio->io_flags & ZIO_FLAG_RAW)) { 1047 uint64_t psize = 1048 BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp); 1049 void *cbuf = zio_buf_alloc(psize); 1050 1051 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 1052 } 1053 1054 if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) { 1055 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1056 decode_embedded_bp_compressed(bp, zio->io_data); 1057 } else { 1058 ASSERT(!BP_IS_EMBEDDED(bp)); 1059 } 1060 1061 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 1062 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1063 1064 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 1065 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1066 1067 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 1068 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 1069 1070 return (ZIO_PIPELINE_CONTINUE); 1071} 1072 1073static int 1074zio_write_bp_init(zio_t **ziop) 1075{ 1076 zio_t *zio = *ziop; 1077 spa_t *spa = zio->io_spa; 1078 zio_prop_t *zp = &zio->io_prop; 1079 enum zio_compress compress = zp->zp_compress; 1080 blkptr_t *bp = zio->io_bp; 1081 uint64_t lsize = zio->io_size; 1082 uint64_t psize = lsize; 1083 int pass = 1; 1084 1085 /* 1086 * If our children haven't all reached the ready stage, 1087 * wait for them and then repeat this pipeline stage. 1088 */ 1089 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 1090 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 1091 return (ZIO_PIPELINE_STOP); 1092 1093 if (!IO_IS_ALLOCATING(zio)) 1094 return (ZIO_PIPELINE_CONTINUE); 1095 1096 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1097 1098 if (zio->io_bp_override) { 1099 ASSERT(bp->blk_birth != zio->io_txg); 1100 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 1101 1102 *bp = *zio->io_bp_override; 1103 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1104 1105 if (BP_IS_EMBEDDED(bp)) 1106 return (ZIO_PIPELINE_CONTINUE); 1107 1108 /* 1109 * If we've been overridden and nopwrite is set then 1110 * set the flag accordingly to indicate that a nopwrite 1111 * has already occurred. 1112 */ 1113 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1114 ASSERT(!zp->zp_dedup); 1115 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1116 return (ZIO_PIPELINE_CONTINUE); 1117 } 1118 1119 ASSERT(!zp->zp_nopwrite); 1120 1121 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1122 return (ZIO_PIPELINE_CONTINUE); 1123 1124 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup || 1125 zp->zp_dedup_verify); 1126 1127 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 1128 BP_SET_DEDUP(bp, 1); 1129 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1130 return (ZIO_PIPELINE_CONTINUE); 1131 } 1132 zio->io_bp_override = NULL; 1133 BP_ZERO(bp); 1134 } 1135 1136 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) { 1137 /* 1138 * We're rewriting an existing block, which means we're 1139 * working on behalf of spa_sync(). For spa_sync() to 1140 * converge, it must eventually be the case that we don't 1141 * have to allocate new blocks. But compression changes 1142 * the blocksize, which forces a reallocate, and makes 1143 * convergence take longer. Therefore, after the first 1144 * few passes, stop compressing to ensure convergence. 1145 */ 1146 pass = spa_sync_pass(spa); 1147 1148 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1149 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1150 ASSERT(!BP_GET_DEDUP(bp)); 1151 1152 if (pass >= zfs_sync_pass_dont_compress) 1153 compress = ZIO_COMPRESS_OFF; 1154 1155 /* Make sure someone doesn't change their mind on overwrites */ 1156 ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp), 1157 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1158 } 1159 1160 if (compress != ZIO_COMPRESS_OFF) { 1161 metaslab_class_t *mc = spa_normal_class(spa); 1162 void *cbuf = zio_buf_alloc(lsize); 1163 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize, 1164 (size_t)metaslab_class_get_minblocksize(mc)); 1165 if (psize == 0 || psize == lsize) { 1166 compress = ZIO_COMPRESS_OFF; 1167 zio_buf_free(cbuf, lsize); 1168 } else if (!zp->zp_dedup && psize <= BPE_PAYLOAD_SIZE && 1169 zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && 1170 spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { 1171 encode_embedded_bp_compressed(bp, 1172 cbuf, compress, lsize, psize); 1173 BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); 1174 BP_SET_TYPE(bp, zio->io_prop.zp_type); 1175 BP_SET_LEVEL(bp, zio->io_prop.zp_level); 1176 zio_buf_free(cbuf, lsize); 1177 bp->blk_birth = zio->io_txg; 1178 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1179 ASSERT(spa_feature_is_active(spa, 1180 SPA_FEATURE_EMBEDDED_DATA)); 1181 return (ZIO_PIPELINE_CONTINUE); 1182 } else { 1183 /* 1184 * Round up compressed size to MINBLOCKSIZE and 1185 * zero the tail. 1186 */ 1187 size_t rounded = 1188 P2ROUNDUP(psize, (size_t)SPA_MINBLOCKSIZE); 1189 if (rounded > psize) { 1190 bzero((char *)cbuf + psize, rounded - psize); 1191 psize = rounded; 1192 } 1193 if (psize == lsize) { 1194 compress = ZIO_COMPRESS_OFF; 1195 zio_buf_free(cbuf, lsize); 1196 } else { 1197 zio_push_transform(zio, cbuf, 1198 psize, lsize, NULL); 1199 } 1200 } 1201 } 1202 1203 /* 1204 * The final pass of spa_sync() must be all rewrites, but the first 1205 * few passes offer a trade-off: allocating blocks defers convergence, 1206 * but newly allocated blocks are sequential, so they can be written 1207 * to disk faster. Therefore, we allow the first few passes of 1208 * spa_sync() to allocate new blocks, but force rewrites after that. 1209 * There should only be a handful of blocks after pass 1 in any case. 1210 */ 1211 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg && 1212 BP_GET_PSIZE(bp) == psize && 1213 pass >= zfs_sync_pass_rewrite) { 1214 ASSERT(psize != 0); 1215 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1216 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1217 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1218 } else { 1219 BP_ZERO(bp); 1220 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1221 } 1222 1223 if (psize == 0) { 1224 if (zio->io_bp_orig.blk_birth != 0 && 1225 spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) { 1226 BP_SET_LSIZE(bp, lsize); 1227 BP_SET_TYPE(bp, zp->zp_type); 1228 BP_SET_LEVEL(bp, zp->zp_level); 1229 BP_SET_BIRTH(bp, zio->io_txg, 0); 1230 } 1231 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1232 } else { 1233 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1234 BP_SET_LSIZE(bp, lsize); 1235 BP_SET_TYPE(bp, zp->zp_type); 1236 BP_SET_LEVEL(bp, zp->zp_level); 1237 BP_SET_PSIZE(bp, psize); 1238 BP_SET_COMPRESS(bp, compress); 1239 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1240 BP_SET_DEDUP(bp, zp->zp_dedup); 1241 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1242 if (zp->zp_dedup) { 1243 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1244 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1245 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1246 } 1247 if (zp->zp_nopwrite) { 1248 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1249 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1250 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1251 } 1252 } 1253 1254 return (ZIO_PIPELINE_CONTINUE); 1255} 1256 1257static int 1258zio_free_bp_init(zio_t **ziop) 1259{ 1260 zio_t *zio = *ziop; 1261 blkptr_t *bp = zio->io_bp; 1262 1263 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1264 if (BP_GET_DEDUP(bp)) 1265 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1266 } 1267 1268 return (ZIO_PIPELINE_CONTINUE); 1269} 1270 1271/* 1272 * ========================================================================== 1273 * Execute the I/O pipeline 1274 * ========================================================================== 1275 */ 1276 1277static void 1278zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1279{ 1280 spa_t *spa = zio->io_spa; 1281 zio_type_t t = zio->io_type; 1282 int flags = (cutinline ? TQ_FRONT : 0); 1283 1284 ASSERT(q == ZIO_TASKQ_ISSUE || q == ZIO_TASKQ_INTERRUPT); 1285 1286 /* 1287 * If we're a config writer or a probe, the normal issue and 1288 * interrupt threads may all be blocked waiting for the config lock. 1289 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1290 */ 1291 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1292 t = ZIO_TYPE_NULL; 1293 1294 /* 1295 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1296 */ 1297 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1298 t = ZIO_TYPE_NULL; 1299 1300 /* 1301 * If this is a high priority I/O, then use the high priority taskq if 1302 * available. 1303 */ 1304 if (zio->io_priority == ZIO_PRIORITY_NOW && 1305 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 1306 q++; 1307 1308 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1309 1310 /* 1311 * NB: We are assuming that the zio can only be dispatched 1312 * to a single taskq at a time. It would be a grievous error 1313 * to dispatch the zio to another taskq at the same time. 1314 */ 1315#if defined(illumos) || !defined(_KERNEL) 1316 ASSERT(zio->io_tqent.tqent_next == NULL); 1317#else 1318 ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); 1319#endif 1320 spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio, 1321 flags, &zio->io_tqent); 1322} 1323 1324static boolean_t 1325zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 1326{ 1327 kthread_t *executor = zio->io_executor; 1328 spa_t *spa = zio->io_spa; 1329 1330 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 1331 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1332 uint_t i; 1333 for (i = 0; i < tqs->stqs_count; i++) { 1334 if (taskq_member(tqs->stqs_taskq[i], executor)) 1335 return (B_TRUE); 1336 } 1337 } 1338 1339 return (B_FALSE); 1340} 1341 1342static int 1343zio_issue_async(zio_t **ziop) 1344{ 1345 zio_t *zio = *ziop; 1346 1347 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1348 1349 return (ZIO_PIPELINE_STOP); 1350} 1351 1352void 1353zio_interrupt(zio_t *zio) 1354{ 1355 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1356} 1357 1358/* 1359 * Execute the I/O pipeline until one of the following occurs: 1360 * 1361 * (1) the I/O completes 1362 * (2) the pipeline stalls waiting for dependent child I/Os 1363 * (3) the I/O issues, so we're waiting for an I/O completion interrupt 1364 * (4) the I/O is delegated by vdev-level caching or aggregation 1365 * (5) the I/O is deferred due to vdev-level queueing 1366 * (6) the I/O is handed off to another thread. 1367 * 1368 * In all cases, the pipeline stops whenever there's no CPU work; it never 1369 * burns a thread in cv_wait(). 1370 * 1371 * There's no locking on io_stage because there's no legitimate way 1372 * for multiple threads to be attempting to process the same I/O. 1373 */ 1374static zio_pipe_stage_t *zio_pipeline[]; 1375 1376void 1377zio_execute(zio_t *zio) 1378{ 1379 zio->io_executor = curthread; 1380 1381 while (zio->io_stage < ZIO_STAGE_DONE) { 1382 enum zio_stage pipeline = zio->io_pipeline; 1383 enum zio_stage stage = zio->io_stage; 1384 int rv; 1385 1386 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1387 ASSERT(ISP2(stage)); 1388 ASSERT(zio->io_stall == NULL); 1389 1390 do { 1391 stage <<= 1; 1392 } while ((stage & pipeline) == 0); 1393 1394 ASSERT(stage <= ZIO_STAGE_DONE); 1395 1396 /* 1397 * If we are in interrupt context and this pipeline stage 1398 * will grab a config lock that is held across I/O, 1399 * or may wait for an I/O that needs an interrupt thread 1400 * to complete, issue async to avoid deadlock. 1401 * 1402 * For VDEV_IO_START, we cut in line so that the io will 1403 * be sent to disk promptly. 1404 */ 1405 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1406 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1407 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1408 zio_requeue_io_start_cut_in_line : B_FALSE; 1409 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1410 return; 1411 } 1412 1413 zio->io_stage = stage; 1414 rv = zio_pipeline[highbit64(stage) - 1](&zio); 1415 1416 if (rv == ZIO_PIPELINE_STOP) 1417 return; 1418 1419 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1420 } 1421} 1422 1423/* 1424 * ========================================================================== 1425 * Initiate I/O, either sync or async 1426 * ========================================================================== 1427 */ 1428int 1429zio_wait(zio_t *zio) 1430{ 1431 int error; 1432 1433 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1434 ASSERT(zio->io_executor == NULL); 1435 1436 zio->io_waiter = curthread; 1437 1438 zio_execute(zio); 1439 1440 mutex_enter(&zio->io_lock); 1441 while (zio->io_executor != NULL) 1442 cv_wait(&zio->io_cv, &zio->io_lock); 1443 mutex_exit(&zio->io_lock); 1444 1445 error = zio->io_error; 1446 zio_destroy(zio); 1447 1448 return (error); 1449} 1450 1451void 1452zio_nowait(zio_t *zio) 1453{ 1454 ASSERT(zio->io_executor == NULL); 1455 1456 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1457 zio_unique_parent(zio) == NULL) { 1458 /* 1459 * This is a logical async I/O with no parent to wait for it. 1460 * We add it to the spa_async_root_zio "Godfather" I/O which 1461 * will ensure they complete prior to unloading the pool. 1462 */ 1463 spa_t *spa = zio->io_spa; 1464 1465 zio_add_child(spa->spa_async_zio_root, zio); 1466 } 1467 1468 zio_execute(zio); 1469} 1470 1471/* 1472 * ========================================================================== 1473 * Reexecute or suspend/resume failed I/O 1474 * ========================================================================== 1475 */ 1476 1477static void 1478zio_reexecute(zio_t *pio) 1479{ 1480 zio_t *cio, *cio_next; 1481 1482 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1483 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1484 ASSERT(pio->io_gang_leader == NULL); 1485 ASSERT(pio->io_gang_tree == NULL); 1486 1487 pio->io_flags = pio->io_orig_flags; 1488 pio->io_stage = pio->io_orig_stage; 1489 pio->io_pipeline = pio->io_orig_pipeline; 1490 pio->io_reexecute = 0; 1491 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1492 pio->io_error = 0; 1493 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1494 pio->io_state[w] = 0; 1495 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1496 pio->io_child_error[c] = 0; 1497 1498 if (IO_IS_ALLOCATING(pio)) 1499 BP_ZERO(pio->io_bp); 1500 1501 /* 1502 * As we reexecute pio's children, new children could be created. 1503 * New children go to the head of pio's io_child_list, however, 1504 * so we will (correctly) not reexecute them. The key is that 1505 * the remainder of pio's io_child_list, from 'cio_next' onward, 1506 * cannot be affected by any side effects of reexecuting 'cio'. 1507 */ 1508 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) { 1509 cio_next = zio_walk_children(pio); 1510 mutex_enter(&pio->io_lock); 1511 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1512 pio->io_children[cio->io_child_type][w]++; 1513 mutex_exit(&pio->io_lock); 1514 zio_reexecute(cio); 1515 } 1516 1517 /* 1518 * Now that all children have been reexecuted, execute the parent. 1519 * We don't reexecute "The Godfather" I/O here as it's the 1520 * responsibility of the caller to wait on him. 1521 */ 1522 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) 1523 zio_execute(pio); 1524} 1525 1526void 1527zio_suspend(spa_t *spa, zio_t *zio) 1528{ 1529 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1530 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1531 "failure and the failure mode property for this pool " 1532 "is set to panic.", spa_name(spa)); 1533 1534 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1535 1536 mutex_enter(&spa->spa_suspend_lock); 1537 1538 if (spa->spa_suspend_zio_root == NULL) 1539 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1540 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1541 ZIO_FLAG_GODFATHER); 1542 1543 spa->spa_suspended = B_TRUE; 1544 1545 if (zio != NULL) { 1546 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1547 ASSERT(zio != spa->spa_suspend_zio_root); 1548 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1549 ASSERT(zio_unique_parent(zio) == NULL); 1550 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1551 zio_add_child(spa->spa_suspend_zio_root, zio); 1552 } 1553 1554 mutex_exit(&spa->spa_suspend_lock); 1555} 1556 1557int 1558zio_resume(spa_t *spa) 1559{ 1560 zio_t *pio; 1561 1562 /* 1563 * Reexecute all previously suspended i/o. 1564 */ 1565 mutex_enter(&spa->spa_suspend_lock); 1566 spa->spa_suspended = B_FALSE; 1567 cv_broadcast(&spa->spa_suspend_cv); 1568 pio = spa->spa_suspend_zio_root; 1569 spa->spa_suspend_zio_root = NULL; 1570 mutex_exit(&spa->spa_suspend_lock); 1571 1572 if (pio == NULL) 1573 return (0); 1574 1575 zio_reexecute(pio); 1576 return (zio_wait(pio)); 1577} 1578 1579void 1580zio_resume_wait(spa_t *spa) 1581{ 1582 mutex_enter(&spa->spa_suspend_lock); 1583 while (spa_suspended(spa)) 1584 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1585 mutex_exit(&spa->spa_suspend_lock); 1586} 1587 1588/* 1589 * ========================================================================== 1590 * Gang blocks. 1591 * 1592 * A gang block is a collection of small blocks that looks to the DMU 1593 * like one large block. When zio_dva_allocate() cannot find a block 1594 * of the requested size, due to either severe fragmentation or the pool 1595 * being nearly full, it calls zio_write_gang_block() to construct the 1596 * block from smaller fragments. 1597 * 1598 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1599 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1600 * an indirect block: it's an array of block pointers. It consumes 1601 * only one sector and hence is allocatable regardless of fragmentation. 1602 * The gang header's bps point to its gang members, which hold the data. 1603 * 1604 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1605 * as the verifier to ensure uniqueness of the SHA256 checksum. 1606 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1607 * not the gang header. This ensures that data block signatures (needed for 1608 * deduplication) are independent of how the block is physically stored. 1609 * 1610 * Gang blocks can be nested: a gang member may itself be a gang block. 1611 * Thus every gang block is a tree in which root and all interior nodes are 1612 * gang headers, and the leaves are normal blocks that contain user data. 1613 * The root of the gang tree is called the gang leader. 1614 * 1615 * To perform any operation (read, rewrite, free, claim) on a gang block, 1616 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1617 * in the io_gang_tree field of the original logical i/o by recursively 1618 * reading the gang leader and all gang headers below it. This yields 1619 * an in-core tree containing the contents of every gang header and the 1620 * bps for every constituent of the gang block. 1621 * 1622 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1623 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1624 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1625 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1626 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1627 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1628 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1629 * of the gang header plus zio_checksum_compute() of the data to update the 1630 * gang header's blk_cksum as described above. 1631 * 1632 * The two-phase assemble/issue model solves the problem of partial failure -- 1633 * what if you'd freed part of a gang block but then couldn't read the 1634 * gang header for another part? Assembling the entire gang tree first 1635 * ensures that all the necessary gang header I/O has succeeded before 1636 * starting the actual work of free, claim, or write. Once the gang tree 1637 * is assembled, free and claim are in-memory operations that cannot fail. 1638 * 1639 * In the event that a gang write fails, zio_dva_unallocate() walks the 1640 * gang tree to immediately free (i.e. insert back into the space map) 1641 * everything we've allocated. This ensures that we don't get ENOSPC 1642 * errors during repeated suspend/resume cycles due to a flaky device. 1643 * 1644 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1645 * the gang tree, we won't modify the block, so we can safely defer the free 1646 * (knowing that the block is still intact). If we *can* assemble the gang 1647 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1648 * each constituent bp and we can allocate a new block on the next sync pass. 1649 * 1650 * In all cases, the gang tree allows complete recovery from partial failure. 1651 * ========================================================================== 1652 */ 1653 1654static zio_t * 1655zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1656{ 1657 if (gn != NULL) 1658 return (pio); 1659 1660 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1661 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1662 &pio->io_bookmark)); 1663} 1664 1665zio_t * 1666zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1667{ 1668 zio_t *zio; 1669 1670 if (gn != NULL) { 1671 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1672 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1673 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1674 /* 1675 * As we rewrite each gang header, the pipeline will compute 1676 * a new gang block header checksum for it; but no one will 1677 * compute a new data checksum, so we do that here. The one 1678 * exception is the gang leader: the pipeline already computed 1679 * its data checksum because that stage precedes gang assembly. 1680 * (Presently, nothing actually uses interior data checksums; 1681 * this is just good hygiene.) 1682 */ 1683 if (gn != pio->io_gang_leader->io_gang_tree) { 1684 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1685 data, BP_GET_PSIZE(bp)); 1686 } 1687 /* 1688 * If we are here to damage data for testing purposes, 1689 * leave the GBH alone so that we can detect the damage. 1690 */ 1691 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1692 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1693 } else { 1694 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1695 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1696 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1697 } 1698 1699 return (zio); 1700} 1701 1702/* ARGSUSED */ 1703zio_t * 1704zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1705{ 1706 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1707 BP_IS_GANG(bp) ? SPA_GANGBLOCKSIZE : BP_GET_PSIZE(bp), 1708 ZIO_GANG_CHILD_FLAGS(pio))); 1709} 1710 1711/* ARGSUSED */ 1712zio_t * 1713zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1714{ 1715 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1716 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1717} 1718 1719static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1720 NULL, 1721 zio_read_gang, 1722 zio_rewrite_gang, 1723 zio_free_gang, 1724 zio_claim_gang, 1725 NULL 1726}; 1727 1728static void zio_gang_tree_assemble_done(zio_t *zio); 1729 1730static zio_gang_node_t * 1731zio_gang_node_alloc(zio_gang_node_t **gnpp) 1732{ 1733 zio_gang_node_t *gn; 1734 1735 ASSERT(*gnpp == NULL); 1736 1737 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1738 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1739 *gnpp = gn; 1740 1741 return (gn); 1742} 1743 1744static void 1745zio_gang_node_free(zio_gang_node_t **gnpp) 1746{ 1747 zio_gang_node_t *gn = *gnpp; 1748 1749 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1750 ASSERT(gn->gn_child[g] == NULL); 1751 1752 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1753 kmem_free(gn, sizeof (*gn)); 1754 *gnpp = NULL; 1755} 1756 1757static void 1758zio_gang_tree_free(zio_gang_node_t **gnpp) 1759{ 1760 zio_gang_node_t *gn = *gnpp; 1761 1762 if (gn == NULL) 1763 return; 1764 1765 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1766 zio_gang_tree_free(&gn->gn_child[g]); 1767 1768 zio_gang_node_free(gnpp); 1769} 1770 1771static void 1772zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1773{ 1774 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1775 1776 ASSERT(gio->io_gang_leader == gio); 1777 ASSERT(BP_IS_GANG(bp)); 1778 1779 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1780 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1781 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1782} 1783 1784static void 1785zio_gang_tree_assemble_done(zio_t *zio) 1786{ 1787 zio_t *gio = zio->io_gang_leader; 1788 zio_gang_node_t *gn = zio->io_private; 1789 blkptr_t *bp = zio->io_bp; 1790 1791 ASSERT(gio == zio_unique_parent(zio)); 1792 ASSERT(zio->io_child_count == 0); 1793 1794 if (zio->io_error) 1795 return; 1796 1797 if (BP_SHOULD_BYTESWAP(bp)) 1798 byteswap_uint64_array(zio->io_data, zio->io_size); 1799 1800 ASSERT(zio->io_data == gn->gn_gbh); 1801 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1802 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1803 1804 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1805 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1806 if (!BP_IS_GANG(gbp)) 1807 continue; 1808 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1809 } 1810} 1811 1812static void 1813zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1814{ 1815 zio_t *gio = pio->io_gang_leader; 1816 zio_t *zio; 1817 1818 ASSERT(BP_IS_GANG(bp) == !!gn); 1819 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1820 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1821 1822 /* 1823 * If you're a gang header, your data is in gn->gn_gbh. 1824 * If you're a gang member, your data is in 'data' and gn == NULL. 1825 */ 1826 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1827 1828 if (gn != NULL) { 1829 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1830 1831 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1832 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1833 if (BP_IS_HOLE(gbp)) 1834 continue; 1835 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1836 data = (char *)data + BP_GET_PSIZE(gbp); 1837 } 1838 } 1839 1840 if (gn == gio->io_gang_tree && gio->io_data != NULL) 1841 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1842 1843 if (zio != pio) 1844 zio_nowait(zio); 1845} 1846 1847static int 1848zio_gang_assemble(zio_t **ziop) 1849{ 1850 zio_t *zio = *ziop; 1851 blkptr_t *bp = zio->io_bp; 1852 1853 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 1854 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1855 1856 zio->io_gang_leader = zio; 1857 1858 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 1859 1860 return (ZIO_PIPELINE_CONTINUE); 1861} 1862 1863static int 1864zio_gang_issue(zio_t **ziop) 1865{ 1866 zio_t *zio = *ziop; 1867 blkptr_t *bp = zio->io_bp; 1868 1869 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 1870 return (ZIO_PIPELINE_STOP); 1871 1872 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 1873 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1874 1875 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 1876 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 1877 else 1878 zio_gang_tree_free(&zio->io_gang_tree); 1879 1880 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1881 1882 return (ZIO_PIPELINE_CONTINUE); 1883} 1884 1885static void 1886zio_write_gang_member_ready(zio_t *zio) 1887{ 1888 zio_t *pio = zio_unique_parent(zio); 1889 zio_t *gio = zio->io_gang_leader; 1890 dva_t *cdva = zio->io_bp->blk_dva; 1891 dva_t *pdva = pio->io_bp->blk_dva; 1892 uint64_t asize; 1893 1894 if (BP_IS_HOLE(zio->io_bp)) 1895 return; 1896 1897 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 1898 1899 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 1900 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 1901 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 1902 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 1903 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1904 1905 mutex_enter(&pio->io_lock); 1906 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 1907 ASSERT(DVA_GET_GANG(&pdva[d])); 1908 asize = DVA_GET_ASIZE(&pdva[d]); 1909 asize += DVA_GET_ASIZE(&cdva[d]); 1910 DVA_SET_ASIZE(&pdva[d], asize); 1911 } 1912 mutex_exit(&pio->io_lock); 1913} 1914 1915static int 1916zio_write_gang_block(zio_t *pio) 1917{ 1918 spa_t *spa = pio->io_spa; 1919 blkptr_t *bp = pio->io_bp; 1920 zio_t *gio = pio->io_gang_leader; 1921 zio_t *zio; 1922 zio_gang_node_t *gn, **gnpp; 1923 zio_gbh_phys_t *gbh; 1924 uint64_t txg = pio->io_txg; 1925 uint64_t resid = pio->io_size; 1926 uint64_t lsize; 1927 int copies = gio->io_prop.zp_copies; 1928 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 1929 zio_prop_t zp; 1930 int error; 1931 1932 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE, 1933 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, 1934 METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER); 1935 if (error) { 1936 pio->io_error = error; 1937 return (ZIO_PIPELINE_CONTINUE); 1938 } 1939 1940 if (pio == gio) { 1941 gnpp = &gio->io_gang_tree; 1942 } else { 1943 gnpp = pio->io_private; 1944 ASSERT(pio->io_ready == zio_write_gang_member_ready); 1945 } 1946 1947 gn = zio_gang_node_alloc(gnpp); 1948 gbh = gn->gn_gbh; 1949 bzero(gbh, SPA_GANGBLOCKSIZE); 1950 1951 /* 1952 * Create the gang header. 1953 */ 1954 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 1955 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1956 1957 /* 1958 * Create and nowait the gang children. 1959 */ 1960 for (int g = 0; resid != 0; resid -= lsize, g++) { 1961 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 1962 SPA_MINBLOCKSIZE); 1963 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 1964 1965 zp.zp_checksum = gio->io_prop.zp_checksum; 1966 zp.zp_compress = ZIO_COMPRESS_OFF; 1967 zp.zp_type = DMU_OT_NONE; 1968 zp.zp_level = 0; 1969 zp.zp_copies = gio->io_prop.zp_copies; 1970 zp.zp_dedup = B_FALSE; 1971 zp.zp_dedup_verify = B_FALSE; 1972 zp.zp_nopwrite = B_FALSE; 1973 1974 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 1975 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 1976 zio_write_gang_member_ready, NULL, NULL, &gn->gn_child[g], 1977 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1978 &pio->io_bookmark)); 1979 } 1980 1981 /* 1982 * Set pio's pipeline to just wait for zio to finish. 1983 */ 1984 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1985 1986 zio_nowait(zio); 1987 1988 return (ZIO_PIPELINE_CONTINUE); 1989} 1990 1991/* 1992 * The zio_nop_write stage in the pipeline determines if allocating 1993 * a new bp is necessary. By leveraging a cryptographically secure checksum, 1994 * such as SHA256, we can compare the checksums of the new data and the old 1995 * to determine if allocating a new block is required. The nopwrite 1996 * feature can handle writes in either syncing or open context (i.e. zil 1997 * writes) and as a result is mutually exclusive with dedup. 1998 */ 1999static int 2000zio_nop_write(zio_t **ziop) 2001{ 2002 zio_t *zio = *ziop; 2003 blkptr_t *bp = zio->io_bp; 2004 blkptr_t *bp_orig = &zio->io_bp_orig; 2005 zio_prop_t *zp = &zio->io_prop; 2006 2007 ASSERT(BP_GET_LEVEL(bp) == 0); 2008 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 2009 ASSERT(zp->zp_nopwrite); 2010 ASSERT(!zp->zp_dedup); 2011 ASSERT(zio->io_bp_override == NULL); 2012 ASSERT(IO_IS_ALLOCATING(zio)); 2013 2014 /* 2015 * Check to see if the original bp and the new bp have matching 2016 * characteristics (i.e. same checksum, compression algorithms, etc). 2017 * If they don't then just continue with the pipeline which will 2018 * allocate a new bp. 2019 */ 2020 if (BP_IS_HOLE(bp_orig) || 2021 !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup || 2022 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 2023 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 2024 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 2025 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 2026 return (ZIO_PIPELINE_CONTINUE); 2027 2028 /* 2029 * If the checksums match then reset the pipeline so that we 2030 * avoid allocating a new bp and issuing any I/O. 2031 */ 2032 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 2033 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup); 2034 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 2035 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 2036 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 2037 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 2038 sizeof (uint64_t)) == 0); 2039 2040 *bp = *bp_orig; 2041 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2042 zio->io_flags |= ZIO_FLAG_NOPWRITE; 2043 } 2044 2045 return (ZIO_PIPELINE_CONTINUE); 2046} 2047 2048/* 2049 * ========================================================================== 2050 * Dedup 2051 * ========================================================================== 2052 */ 2053static void 2054zio_ddt_child_read_done(zio_t *zio) 2055{ 2056 blkptr_t *bp = zio->io_bp; 2057 ddt_entry_t *dde = zio->io_private; 2058 ddt_phys_t *ddp; 2059 zio_t *pio = zio_unique_parent(zio); 2060 2061 mutex_enter(&pio->io_lock); 2062 ddp = ddt_phys_select(dde, bp); 2063 if (zio->io_error == 0) 2064 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 2065 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 2066 dde->dde_repair_data = zio->io_data; 2067 else 2068 zio_buf_free(zio->io_data, zio->io_size); 2069 mutex_exit(&pio->io_lock); 2070} 2071 2072static int 2073zio_ddt_read_start(zio_t **ziop) 2074{ 2075 zio_t *zio = *ziop; 2076 blkptr_t *bp = zio->io_bp; 2077 2078 ASSERT(BP_GET_DEDUP(bp)); 2079 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2080 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2081 2082 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2083 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2084 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 2085 ddt_phys_t *ddp = dde->dde_phys; 2086 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 2087 blkptr_t blk; 2088 2089 ASSERT(zio->io_vsd == NULL); 2090 zio->io_vsd = dde; 2091 2092 if (ddp_self == NULL) 2093 return (ZIO_PIPELINE_CONTINUE); 2094 2095 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2096 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 2097 continue; 2098 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 2099 &blk); 2100 zio_nowait(zio_read(zio, zio->io_spa, &blk, 2101 zio_buf_alloc(zio->io_size), zio->io_size, 2102 zio_ddt_child_read_done, dde, zio->io_priority, 2103 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 2104 &zio->io_bookmark)); 2105 } 2106 return (ZIO_PIPELINE_CONTINUE); 2107 } 2108 2109 zio_nowait(zio_read(zio, zio->io_spa, bp, 2110 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 2111 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 2112 2113 return (ZIO_PIPELINE_CONTINUE); 2114} 2115 2116static int 2117zio_ddt_read_done(zio_t **ziop) 2118{ 2119 zio_t *zio = *ziop; 2120 blkptr_t *bp = zio->io_bp; 2121 2122 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 2123 return (ZIO_PIPELINE_STOP); 2124 2125 ASSERT(BP_GET_DEDUP(bp)); 2126 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2127 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2128 2129 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2130 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2131 ddt_entry_t *dde = zio->io_vsd; 2132 if (ddt == NULL) { 2133 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 2134 return (ZIO_PIPELINE_CONTINUE); 2135 } 2136 if (dde == NULL) { 2137 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 2138 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2139 return (ZIO_PIPELINE_STOP); 2140 } 2141 if (dde->dde_repair_data != NULL) { 2142 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 2143 zio->io_child_error[ZIO_CHILD_DDT] = 0; 2144 } 2145 ddt_repair_done(ddt, dde); 2146 zio->io_vsd = NULL; 2147 } 2148 2149 ASSERT(zio->io_vsd == NULL); 2150 2151 return (ZIO_PIPELINE_CONTINUE); 2152} 2153 2154static boolean_t 2155zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 2156{ 2157 spa_t *spa = zio->io_spa; 2158 2159 /* 2160 * Note: we compare the original data, not the transformed data, 2161 * because when zio->io_bp is an override bp, we will not have 2162 * pushed the I/O transforms. That's an important optimization 2163 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 2164 */ 2165 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2166 zio_t *lio = dde->dde_lead_zio[p]; 2167 2168 if (lio != NULL) { 2169 return (lio->io_orig_size != zio->io_orig_size || 2170 bcmp(zio->io_orig_data, lio->io_orig_data, 2171 zio->io_orig_size) != 0); 2172 } 2173 } 2174 2175 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2176 ddt_phys_t *ddp = &dde->dde_phys[p]; 2177 2178 if (ddp->ddp_phys_birth != 0) { 2179 arc_buf_t *abuf = NULL; 2180 uint32_t aflags = ARC_WAIT; 2181 blkptr_t blk = *zio->io_bp; 2182 int error; 2183 2184 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2185 2186 ddt_exit(ddt); 2187 2188 error = arc_read(NULL, spa, &blk, 2189 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 2190 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2191 &aflags, &zio->io_bookmark); 2192 2193 if (error == 0) { 2194 if (arc_buf_size(abuf) != zio->io_orig_size || 2195 bcmp(abuf->b_data, zio->io_orig_data, 2196 zio->io_orig_size) != 0) 2197 error = SET_ERROR(EEXIST); 2198 VERIFY(arc_buf_remove_ref(abuf, &abuf)); 2199 } 2200 2201 ddt_enter(ddt); 2202 return (error != 0); 2203 } 2204 } 2205 2206 return (B_FALSE); 2207} 2208 2209static void 2210zio_ddt_child_write_ready(zio_t *zio) 2211{ 2212 int p = zio->io_prop.zp_copies; 2213 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2214 ddt_entry_t *dde = zio->io_private; 2215 ddt_phys_t *ddp = &dde->dde_phys[p]; 2216 zio_t *pio; 2217 2218 if (zio->io_error) 2219 return; 2220 2221 ddt_enter(ddt); 2222 2223 ASSERT(dde->dde_lead_zio[p] == zio); 2224 2225 ddt_phys_fill(ddp, zio->io_bp); 2226 2227 while ((pio = zio_walk_parents(zio)) != NULL) 2228 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2229 2230 ddt_exit(ddt); 2231} 2232 2233static void 2234zio_ddt_child_write_done(zio_t *zio) 2235{ 2236 int p = zio->io_prop.zp_copies; 2237 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2238 ddt_entry_t *dde = zio->io_private; 2239 ddt_phys_t *ddp = &dde->dde_phys[p]; 2240 2241 ddt_enter(ddt); 2242 2243 ASSERT(ddp->ddp_refcnt == 0); 2244 ASSERT(dde->dde_lead_zio[p] == zio); 2245 dde->dde_lead_zio[p] = NULL; 2246 2247 if (zio->io_error == 0) { 2248 while (zio_walk_parents(zio) != NULL) 2249 ddt_phys_addref(ddp); 2250 } else { 2251 ddt_phys_clear(ddp); 2252 } 2253 2254 ddt_exit(ddt); 2255} 2256 2257static void 2258zio_ddt_ditto_write_done(zio_t *zio) 2259{ 2260 int p = DDT_PHYS_DITTO; 2261 zio_prop_t *zp = &zio->io_prop; 2262 blkptr_t *bp = zio->io_bp; 2263 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2264 ddt_entry_t *dde = zio->io_private; 2265 ddt_phys_t *ddp = &dde->dde_phys[p]; 2266 ddt_key_t *ddk = &dde->dde_key; 2267 2268 ddt_enter(ddt); 2269 2270 ASSERT(ddp->ddp_refcnt == 0); 2271 ASSERT(dde->dde_lead_zio[p] == zio); 2272 dde->dde_lead_zio[p] = NULL; 2273 2274 if (zio->io_error == 0) { 2275 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2276 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2277 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2278 if (ddp->ddp_phys_birth != 0) 2279 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2280 ddt_phys_fill(ddp, bp); 2281 } 2282 2283 ddt_exit(ddt); 2284} 2285 2286static int 2287zio_ddt_write(zio_t **ziop) 2288{ 2289 zio_t *zio = *ziop; 2290 spa_t *spa = zio->io_spa; 2291 blkptr_t *bp = zio->io_bp; 2292 uint64_t txg = zio->io_txg; 2293 zio_prop_t *zp = &zio->io_prop; 2294 int p = zp->zp_copies; 2295 int ditto_copies; 2296 zio_t *cio = NULL; 2297 zio_t *dio = NULL; 2298 ddt_t *ddt = ddt_select(spa, bp); 2299 ddt_entry_t *dde; 2300 ddt_phys_t *ddp; 2301 2302 ASSERT(BP_GET_DEDUP(bp)); 2303 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2304 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2305 2306 ddt_enter(ddt); 2307 dde = ddt_lookup(ddt, bp, B_TRUE); 2308 ddp = &dde->dde_phys[p]; 2309 2310 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2311 /* 2312 * If we're using a weak checksum, upgrade to a strong checksum 2313 * and try again. If we're already using a strong checksum, 2314 * we can't resolve it, so just convert to an ordinary write. 2315 * (And automatically e-mail a paper to Nature?) 2316 */ 2317 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) { 2318 zp->zp_checksum = spa_dedup_checksum(spa); 2319 zio_pop_transforms(zio); 2320 zio->io_stage = ZIO_STAGE_OPEN; 2321 BP_ZERO(bp); 2322 } else { 2323 zp->zp_dedup = B_FALSE; 2324 } 2325 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2326 ddt_exit(ddt); 2327 return (ZIO_PIPELINE_CONTINUE); 2328 } 2329 2330 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2331 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2332 2333 if (ditto_copies > ddt_ditto_copies_present(dde) && 2334 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2335 zio_prop_t czp = *zp; 2336 2337 czp.zp_copies = ditto_copies; 2338 2339 /* 2340 * If we arrived here with an override bp, we won't have run 2341 * the transform stack, so we won't have the data we need to 2342 * generate a child i/o. So, toss the override bp and restart. 2343 * This is safe, because using the override bp is just an 2344 * optimization; and it's rare, so the cost doesn't matter. 2345 */ 2346 if (zio->io_bp_override) { 2347 zio_pop_transforms(zio); 2348 zio->io_stage = ZIO_STAGE_OPEN; 2349 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2350 zio->io_bp_override = NULL; 2351 BP_ZERO(bp); 2352 ddt_exit(ddt); 2353 return (ZIO_PIPELINE_CONTINUE); 2354 } 2355 2356 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2357 zio->io_orig_size, &czp, NULL, NULL, 2358 zio_ddt_ditto_write_done, dde, zio->io_priority, 2359 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2360 2361 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2362 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2363 } 2364 2365 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2366 if (ddp->ddp_phys_birth != 0) 2367 ddt_bp_fill(ddp, bp, txg); 2368 if (dde->dde_lead_zio[p] != NULL) 2369 zio_add_child(zio, dde->dde_lead_zio[p]); 2370 else 2371 ddt_phys_addref(ddp); 2372 } else if (zio->io_bp_override) { 2373 ASSERT(bp->blk_birth == txg); 2374 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2375 ddt_phys_fill(ddp, bp); 2376 ddt_phys_addref(ddp); 2377 } else { 2378 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2379 zio->io_orig_size, zp, zio_ddt_child_write_ready, NULL, 2380 zio_ddt_child_write_done, dde, zio->io_priority, 2381 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2382 2383 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2384 dde->dde_lead_zio[p] = cio; 2385 } 2386 2387 ddt_exit(ddt); 2388 2389 if (cio) 2390 zio_nowait(cio); 2391 if (dio) 2392 zio_nowait(dio); 2393 2394 return (ZIO_PIPELINE_CONTINUE); 2395} 2396 2397ddt_entry_t *freedde; /* for debugging */ 2398 2399static int 2400zio_ddt_free(zio_t **ziop) 2401{ 2402 zio_t *zio = *ziop; 2403 spa_t *spa = zio->io_spa; 2404 blkptr_t *bp = zio->io_bp; 2405 ddt_t *ddt = ddt_select(spa, bp); 2406 ddt_entry_t *dde; 2407 ddt_phys_t *ddp; 2408 2409 ASSERT(BP_GET_DEDUP(bp)); 2410 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2411 2412 ddt_enter(ddt); 2413 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2414 ddp = ddt_phys_select(dde, bp); 2415 ddt_phys_decref(ddp); 2416 ddt_exit(ddt); 2417 2418 return (ZIO_PIPELINE_CONTINUE); 2419} 2420 2421/* 2422 * ========================================================================== 2423 * Allocate and free blocks 2424 * ========================================================================== 2425 */ 2426static int 2427zio_dva_allocate(zio_t **ziop) 2428{ 2429 zio_t *zio = *ziop; 2430 spa_t *spa = zio->io_spa; 2431 metaslab_class_t *mc = spa_normal_class(spa); 2432 blkptr_t *bp = zio->io_bp; 2433 int error; 2434 int flags = 0; 2435 2436 if (zio->io_gang_leader == NULL) { 2437 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2438 zio->io_gang_leader = zio; 2439 } 2440 2441 ASSERT(BP_IS_HOLE(bp)); 2442 ASSERT0(BP_GET_NDVAS(bp)); 2443 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2444 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2445 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2446 2447 /* 2448 * The dump device does not support gang blocks so allocation on 2449 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid 2450 * the "fast" gang feature. 2451 */ 2452 flags |= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0; 2453 flags |= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ? 2454 METASLAB_GANG_CHILD : 0; 2455 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2456 zio->io_prop.zp_copies, zio->io_txg, NULL, flags); 2457 2458 if (error) { 2459 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2460 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2461 error); 2462 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2463 return (zio_write_gang_block(zio)); 2464 zio->io_error = error; 2465 } 2466 2467 return (ZIO_PIPELINE_CONTINUE); 2468} 2469 2470static int 2471zio_dva_free(zio_t **ziop) 2472{ 2473 zio_t *zio = *ziop; 2474 2475 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2476 2477 return (ZIO_PIPELINE_CONTINUE); 2478} 2479 2480static int 2481zio_dva_claim(zio_t **ziop) 2482{ 2483 zio_t *zio = *ziop; 2484 int error; 2485 2486 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2487 if (error) 2488 zio->io_error = error; 2489 2490 return (ZIO_PIPELINE_CONTINUE); 2491} 2492 2493/* 2494 * Undo an allocation. This is used by zio_done() when an I/O fails 2495 * and we want to give back the block we just allocated. 2496 * This handles both normal blocks and gang blocks. 2497 */ 2498static void 2499zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2500{ 2501 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2502 ASSERT(zio->io_bp_override == NULL); 2503 2504 if (!BP_IS_HOLE(bp)) 2505 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2506 2507 if (gn != NULL) { 2508 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2509 zio_dva_unallocate(zio, gn->gn_child[g], 2510 &gn->gn_gbh->zg_blkptr[g]); 2511 } 2512 } 2513} 2514 2515/* 2516 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2517 */ 2518int 2519zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2520 uint64_t size, boolean_t use_slog) 2521{ 2522 int error = 1; 2523 2524 ASSERT(txg > spa_syncing_txg(spa)); 2525 2526 /* 2527 * ZIL blocks are always contiguous (i.e. not gang blocks) so we 2528 * set the METASLAB_GANG_AVOID flag so that they don't "fast gang" 2529 * when allocating them. 2530 */ 2531 if (use_slog) { 2532 error = metaslab_alloc(spa, spa_log_class(spa), size, 2533 new_bp, 1, txg, old_bp, 2534 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2535 } 2536 2537 if (error) { 2538 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2539 new_bp, 1, txg, old_bp, 2540 METASLAB_HINTBP_AVOID); 2541 } 2542 2543 if (error == 0) { 2544 BP_SET_LSIZE(new_bp, size); 2545 BP_SET_PSIZE(new_bp, size); 2546 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2547 BP_SET_CHECKSUM(new_bp, 2548 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2549 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2550 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2551 BP_SET_LEVEL(new_bp, 0); 2552 BP_SET_DEDUP(new_bp, 0); 2553 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2554 } 2555 2556 return (error); 2557} 2558 2559/* 2560 * Free an intent log block. 2561 */ 2562void 2563zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2564{ 2565 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2566 ASSERT(!BP_IS_GANG(bp)); 2567 2568 zio_free(spa, txg, bp); 2569} 2570 2571/* 2572 * ========================================================================== 2573 * Read, write and delete to physical devices 2574 * ========================================================================== 2575 */ 2576static int 2577zio_vdev_io_start(zio_t **ziop) 2578{ 2579 zio_t *zio = *ziop; 2580 vdev_t *vd = zio->io_vd; 2581 uint64_t align; 2582 spa_t *spa = zio->io_spa; 2583 2584 ASSERT(zio->io_error == 0); 2585 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2586 2587 if (vd == NULL) { 2588 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2589 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2590 2591 /* 2592 * The mirror_ops handle multiple DVAs in a single BP. 2593 */ 2594 return (vdev_mirror_ops.vdev_op_io_start(zio)); 2595 } 2596 2597 if (vd->vdev_ops->vdev_op_leaf && zio->io_type == ZIO_TYPE_FREE) { 2598 trim_map_free(vd, zio->io_offset, zio->io_size, zio->io_txg); 2599 return (ZIO_PIPELINE_CONTINUE); 2600 } 2601 2602 /* 2603 * We keep track of time-sensitive I/Os so that the scan thread 2604 * can quickly react to certain workloads. In particular, we care 2605 * about non-scrubbing, top-level reads and writes with the following 2606 * characteristics: 2607 * - synchronous writes of user data to non-slog devices 2608 * - any reads of user data 2609 * When these conditions are met, adjust the timestamp of spa_last_io 2610 * which allows the scan thread to adjust its workload accordingly. 2611 */ 2612 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2613 vd == vd->vdev_top && !vd->vdev_islog && 2614 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2615 zio->io_txg != spa_syncing_txg(spa)) { 2616 uint64_t old = spa->spa_last_io; 2617 uint64_t new = ddi_get_lbolt64(); 2618 if (old != new) 2619 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2620 } 2621 2622 align = 1ULL << vd->vdev_top->vdev_ashift; 2623 2624 if (P2PHASE(zio->io_size, align) != 0) { 2625 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2626 char *abuf = NULL; 2627 if (zio->io_type == ZIO_TYPE_READ || 2628 zio->io_type == ZIO_TYPE_WRITE) 2629 abuf = zio_buf_alloc(asize); 2630 ASSERT(vd == vd->vdev_top); 2631 if (zio->io_type == ZIO_TYPE_WRITE) { 2632 bcopy(zio->io_data, abuf, zio->io_size); 2633 bzero(abuf + zio->io_size, asize - zio->io_size); 2634 } 2635 zio_push_transform(zio, abuf, asize, abuf ? asize : 0, 2636 zio_subblock); 2637 } 2638 2639 ASSERT(P2PHASE(zio->io_offset, align) == 0); 2640 ASSERT(P2PHASE(zio->io_size, align) == 0); 2641 VERIFY(zio->io_type == ZIO_TYPE_READ || spa_writeable(spa)); 2642 2643 /* 2644 * If this is a repair I/O, and there's no self-healing involved -- 2645 * that is, we're just resilvering what we expect to resilver -- 2646 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2647 * This prevents spurious resilvering with nested replication. 2648 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2649 * A is out of date, we'll read from C+D, then use the data to 2650 * resilver A+B -- but we don't actually want to resilver B, just A. 2651 * The top-level mirror has no way to know this, so instead we just 2652 * discard unnecessary repairs as we work our way down the vdev tree. 2653 * The same logic applies to any form of nested replication: 2654 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2655 */ 2656 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2657 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2658 zio->io_txg != 0 && /* not a delegated i/o */ 2659 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2660 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2661 zio_vdev_io_bypass(zio); 2662 return (ZIO_PIPELINE_CONTINUE); 2663 } 2664 2665 if (vd->vdev_ops->vdev_op_leaf && 2666 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2667 2668 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio)) 2669 return (ZIO_PIPELINE_CONTINUE); 2670 2671 if ((zio = vdev_queue_io(zio)) == NULL) 2672 return (ZIO_PIPELINE_STOP); 2673 *ziop = zio; 2674 2675 if (!vdev_accessible(vd, zio)) { 2676 zio->io_error = SET_ERROR(ENXIO); 2677 zio_interrupt(zio); 2678 return (ZIO_PIPELINE_STOP); 2679 } 2680 } 2681 2682 /* 2683 * Note that we ignore repair writes for TRIM because they can conflict 2684 * with normal writes. This isn't an issue because, by definition, we 2685 * only repair blocks that aren't freed. 2686 */ 2687 if (vd->vdev_ops->vdev_op_leaf && zio->io_type == ZIO_TYPE_WRITE && 2688 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2689 if (!trim_map_write_start(zio)) 2690 return (ZIO_PIPELINE_STOP); 2691 } 2692 2693 return (vd->vdev_ops->vdev_op_io_start(zio)); 2694} 2695 2696static int 2697zio_vdev_io_done(zio_t **ziop) 2698{ 2699 zio_t *zio = *ziop; 2700 vdev_t *vd = zio->io_vd; 2701 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 2702 boolean_t unexpected_error = B_FALSE; 2703 2704 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2705 return (ZIO_PIPELINE_STOP); 2706 2707 ASSERT(zio->io_type == ZIO_TYPE_READ || 2708 zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_FREE); 2709 2710 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 2711 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2712 2713 if (zio->io_type == ZIO_TYPE_WRITE && 2714 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) 2715 trim_map_write_done(zio); 2716 2717 vdev_queue_io_done(zio); 2718 2719 if (zio->io_type == ZIO_TYPE_WRITE) 2720 vdev_cache_write(zio); 2721 2722 if (zio_injection_enabled && zio->io_error == 0) 2723 zio->io_error = zio_handle_device_injection(vd, 2724 zio, EIO); 2725 2726 if (zio_injection_enabled && zio->io_error == 0) 2727 zio->io_error = zio_handle_label_injection(zio, EIO); 2728 2729 if (zio->io_error) { 2730 if (!vdev_accessible(vd, zio)) { 2731 zio->io_error = SET_ERROR(ENXIO); 2732 } else { 2733 unexpected_error = B_TRUE; 2734 } 2735 } 2736 } 2737 2738 ops->vdev_op_io_done(zio); 2739 2740 if (unexpected_error) 2741 VERIFY(vdev_probe(vd, zio) == NULL); 2742 2743 return (ZIO_PIPELINE_CONTINUE); 2744} 2745 2746/* 2747 * For non-raidz ZIOs, we can just copy aside the bad data read from the 2748 * disk, and use that to finish the checksum ereport later. 2749 */ 2750static void 2751zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 2752 const void *good_buf) 2753{ 2754 /* no processing needed */ 2755 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 2756} 2757 2758/*ARGSUSED*/ 2759void 2760zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 2761{ 2762 void *buf = zio_buf_alloc(zio->io_size); 2763 2764 bcopy(zio->io_data, buf, zio->io_size); 2765 2766 zcr->zcr_cbinfo = zio->io_size; 2767 zcr->zcr_cbdata = buf; 2768 zcr->zcr_finish = zio_vsd_default_cksum_finish; 2769 zcr->zcr_free = zio_buf_free; 2770} 2771 2772static int 2773zio_vdev_io_assess(zio_t **ziop) 2774{ 2775 zio_t *zio = *ziop; 2776 vdev_t *vd = zio->io_vd; 2777 2778 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2779 return (ZIO_PIPELINE_STOP); 2780 2781 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2782 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 2783 2784 if (zio->io_vsd != NULL) { 2785 zio->io_vsd_ops->vsd_free(zio); 2786 zio->io_vsd = NULL; 2787 } 2788 2789 if (zio_injection_enabled && zio->io_error == 0) 2790 zio->io_error = zio_handle_fault_injection(zio, EIO); 2791 2792 if (zio->io_type == ZIO_TYPE_IOCTL && zio->io_cmd == DKIOCTRIM) 2793 switch (zio->io_error) { 2794 case 0: 2795 ZIO_TRIM_STAT_INCR(bytes, zio->io_size); 2796 ZIO_TRIM_STAT_BUMP(success); 2797 break; 2798 case EOPNOTSUPP: 2799 ZIO_TRIM_STAT_BUMP(unsupported); 2800 break; 2801 default: 2802 ZIO_TRIM_STAT_BUMP(failed); 2803 break; 2804 } 2805 2806 /* 2807 * If the I/O failed, determine whether we should attempt to retry it. 2808 * 2809 * On retry, we cut in line in the issue queue, since we don't want 2810 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 2811 */ 2812 if (zio->io_error && vd == NULL && 2813 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 2814 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 2815 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 2816 zio->io_error = 0; 2817 zio->io_flags |= ZIO_FLAG_IO_RETRY | 2818 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 2819 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 2820 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 2821 zio_requeue_io_start_cut_in_line); 2822 return (ZIO_PIPELINE_STOP); 2823 } 2824 2825 /* 2826 * If we got an error on a leaf device, convert it to ENXIO 2827 * if the device is not accessible at all. 2828 */ 2829 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 2830 !vdev_accessible(vd, zio)) 2831 zio->io_error = SET_ERROR(ENXIO); 2832 2833 /* 2834 * If we can't write to an interior vdev (mirror or RAID-Z), 2835 * set vdev_cant_write so that we stop trying to allocate from it. 2836 */ 2837 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 2838 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 2839 vd->vdev_cant_write = B_TRUE; 2840 } 2841 2842 if (zio->io_error) 2843 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2844 2845 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 2846 zio->io_physdone != NULL) { 2847 ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED)); 2848 ASSERT(zio->io_child_type == ZIO_CHILD_VDEV); 2849 zio->io_physdone(zio->io_logical); 2850 } 2851 2852 return (ZIO_PIPELINE_CONTINUE); 2853} 2854 2855void 2856zio_vdev_io_reissue(zio_t *zio) 2857{ 2858 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2859 ASSERT(zio->io_error == 0); 2860 2861 zio->io_stage >>= 1; 2862} 2863 2864void 2865zio_vdev_io_redone(zio_t *zio) 2866{ 2867 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 2868 2869 zio->io_stage >>= 1; 2870} 2871 2872void 2873zio_vdev_io_bypass(zio_t *zio) 2874{ 2875 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2876 ASSERT(zio->io_error == 0); 2877 2878 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 2879 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 2880} 2881 2882/* 2883 * ========================================================================== 2884 * Generate and verify checksums 2885 * ========================================================================== 2886 */ 2887static int 2888zio_checksum_generate(zio_t **ziop) 2889{ 2890 zio_t *zio = *ziop; 2891 blkptr_t *bp = zio->io_bp; 2892 enum zio_checksum checksum; 2893 2894 if (bp == NULL) { 2895 /* 2896 * This is zio_write_phys(). 2897 * We're either generating a label checksum, or none at all. 2898 */ 2899 checksum = zio->io_prop.zp_checksum; 2900 2901 if (checksum == ZIO_CHECKSUM_OFF) 2902 return (ZIO_PIPELINE_CONTINUE); 2903 2904 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 2905 } else { 2906 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 2907 ASSERT(!IO_IS_ALLOCATING(zio)); 2908 checksum = ZIO_CHECKSUM_GANG_HEADER; 2909 } else { 2910 checksum = BP_GET_CHECKSUM(bp); 2911 } 2912 } 2913 2914 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 2915 2916 return (ZIO_PIPELINE_CONTINUE); 2917} 2918 2919static int 2920zio_checksum_verify(zio_t **ziop) 2921{ 2922 zio_t *zio = *ziop; 2923 zio_bad_cksum_t info; 2924 blkptr_t *bp = zio->io_bp; 2925 int error; 2926 2927 ASSERT(zio->io_vd != NULL); 2928 2929 if (bp == NULL) { 2930 /* 2931 * This is zio_read_phys(). 2932 * We're either verifying a label checksum, or nothing at all. 2933 */ 2934 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 2935 return (ZIO_PIPELINE_CONTINUE); 2936 2937 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 2938 } 2939 2940 if ((error = zio_checksum_error(zio, &info)) != 0) { 2941 zio->io_error = error; 2942 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 2943 zfs_ereport_start_checksum(zio->io_spa, 2944 zio->io_vd, zio, zio->io_offset, 2945 zio->io_size, NULL, &info); 2946 } 2947 } 2948 2949 return (ZIO_PIPELINE_CONTINUE); 2950} 2951 2952/* 2953 * Called by RAID-Z to ensure we don't compute the checksum twice. 2954 */ 2955void 2956zio_checksum_verified(zio_t *zio) 2957{ 2958 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 2959} 2960 2961/* 2962 * ========================================================================== 2963 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 2964 * An error of 0 indicates success. ENXIO indicates whole-device failure, 2965 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 2966 * indicate errors that are specific to one I/O, and most likely permanent. 2967 * Any other error is presumed to be worse because we weren't expecting it. 2968 * ========================================================================== 2969 */ 2970int 2971zio_worst_error(int e1, int e2) 2972{ 2973 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 2974 int r1, r2; 2975 2976 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 2977 if (e1 == zio_error_rank[r1]) 2978 break; 2979 2980 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 2981 if (e2 == zio_error_rank[r2]) 2982 break; 2983 2984 return (r1 > r2 ? e1 : e2); 2985} 2986 2987/* 2988 * ========================================================================== 2989 * I/O completion 2990 * ========================================================================== 2991 */ 2992static int 2993zio_ready(zio_t **ziop) 2994{ 2995 zio_t *zio = *ziop; 2996 blkptr_t *bp = zio->io_bp; 2997 zio_t *pio, *pio_next; 2998 2999 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 3000 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 3001 return (ZIO_PIPELINE_STOP); 3002 3003 if (zio->io_ready) { 3004 ASSERT(IO_IS_ALLOCATING(zio)); 3005 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 3006 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 3007 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 3008 3009 zio->io_ready(zio); 3010 } 3011 3012 if (bp != NULL && bp != &zio->io_bp_copy) 3013 zio->io_bp_copy = *bp; 3014 3015 if (zio->io_error) 3016 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3017 3018 mutex_enter(&zio->io_lock); 3019 zio->io_state[ZIO_WAIT_READY] = 1; 3020 pio = zio_walk_parents(zio); 3021 mutex_exit(&zio->io_lock); 3022 3023 /* 3024 * As we notify zio's parents, new parents could be added. 3025 * New parents go to the head of zio's io_parent_list, however, 3026 * so we will (correctly) not notify them. The remainder of zio's 3027 * io_parent_list, from 'pio_next' onward, cannot change because 3028 * all parents must wait for us to be done before they can be done. 3029 */ 3030 for (; pio != NULL; pio = pio_next) { 3031 pio_next = zio_walk_parents(zio); 3032 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 3033 } 3034 3035 if (zio->io_flags & ZIO_FLAG_NODATA) { 3036 if (BP_IS_GANG(bp)) { 3037 zio->io_flags &= ~ZIO_FLAG_NODATA; 3038 } else { 3039 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 3040 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 3041 } 3042 } 3043 3044 if (zio_injection_enabled && 3045 zio->io_spa->spa_syncing_txg == zio->io_txg) 3046 zio_handle_ignored_writes(zio); 3047 3048 return (ZIO_PIPELINE_CONTINUE); 3049} 3050 3051static int 3052zio_done(zio_t **ziop) 3053{ 3054 zio_t *zio = *ziop; 3055 spa_t *spa = zio->io_spa; 3056 zio_t *lio = zio->io_logical; 3057 blkptr_t *bp = zio->io_bp; 3058 vdev_t *vd = zio->io_vd; 3059 uint64_t psize = zio->io_size; 3060 zio_t *pio, *pio_next; 3061 3062 /* 3063 * If our children haven't all completed, 3064 * wait for them and then repeat this pipeline stage. 3065 */ 3066 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 3067 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 3068 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 3069 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 3070 return (ZIO_PIPELINE_STOP); 3071 3072 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 3073 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 3074 ASSERT(zio->io_children[c][w] == 0); 3075 3076 if (bp != NULL && !BP_IS_EMBEDDED(bp)) { 3077 ASSERT(bp->blk_pad[0] == 0); 3078 ASSERT(bp->blk_pad[1] == 0); 3079 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 3080 (bp == zio_unique_parent(zio)->io_bp)); 3081 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 3082 zio->io_bp_override == NULL && 3083 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 3084 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 3085 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 3086 ASSERT(BP_COUNT_GANG(bp) == 0 || 3087 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 3088 } 3089 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 3090 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 3091 } 3092 3093 /* 3094 * If there were child vdev/gang/ddt errors, they apply to us now. 3095 */ 3096 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 3097 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 3098 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 3099 3100 /* 3101 * If the I/O on the transformed data was successful, generate any 3102 * checksum reports now while we still have the transformed data. 3103 */ 3104 if (zio->io_error == 0) { 3105 while (zio->io_cksum_report != NULL) { 3106 zio_cksum_report_t *zcr = zio->io_cksum_report; 3107 uint64_t align = zcr->zcr_align; 3108 uint64_t asize = P2ROUNDUP(psize, align); 3109 char *abuf = zio->io_data; 3110 3111 if (asize != psize) { 3112 abuf = zio_buf_alloc(asize); 3113 bcopy(zio->io_data, abuf, psize); 3114 bzero(abuf + psize, asize - psize); 3115 } 3116 3117 zio->io_cksum_report = zcr->zcr_next; 3118 zcr->zcr_next = NULL; 3119 zcr->zcr_finish(zcr, abuf); 3120 zfs_ereport_free_checksum(zcr); 3121 3122 if (asize != psize) 3123 zio_buf_free(abuf, asize); 3124 } 3125 } 3126 3127 zio_pop_transforms(zio); /* note: may set zio->io_error */ 3128 3129 vdev_stat_update(zio, psize); 3130 3131 if (zio->io_error) { 3132 /* 3133 * If this I/O is attached to a particular vdev, 3134 * generate an error message describing the I/O failure 3135 * at the block level. We ignore these errors if the 3136 * device is currently unavailable. 3137 */ 3138 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 3139 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 3140 3141 if ((zio->io_error == EIO || !(zio->io_flags & 3142 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 3143 zio == lio) { 3144 /* 3145 * For logical I/O requests, tell the SPA to log the 3146 * error and generate a logical data ereport. 3147 */ 3148 spa_log_error(spa, zio); 3149 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 3150 0, 0); 3151 } 3152 } 3153 3154 if (zio->io_error && zio == lio) { 3155 /* 3156 * Determine whether zio should be reexecuted. This will 3157 * propagate all the way to the root via zio_notify_parent(). 3158 */ 3159 ASSERT(vd == NULL && bp != NULL); 3160 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3161 3162 if (IO_IS_ALLOCATING(zio) && 3163 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 3164 if (zio->io_error != ENOSPC) 3165 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 3166 else 3167 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3168 } 3169 3170 if ((zio->io_type == ZIO_TYPE_READ || 3171 zio->io_type == ZIO_TYPE_FREE) && 3172 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 3173 zio->io_error == ENXIO && 3174 spa_load_state(spa) == SPA_LOAD_NONE && 3175 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 3176 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3177 3178 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 3179 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3180 3181 /* 3182 * Here is a possibly good place to attempt to do 3183 * either combinatorial reconstruction or error correction 3184 * based on checksums. It also might be a good place 3185 * to send out preliminary ereports before we suspend 3186 * processing. 3187 */ 3188 } 3189 3190 /* 3191 * If there were logical child errors, they apply to us now. 3192 * We defer this until now to avoid conflating logical child 3193 * errors with errors that happened to the zio itself when 3194 * updating vdev stats and reporting FMA events above. 3195 */ 3196 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 3197 3198 if ((zio->io_error || zio->io_reexecute) && 3199 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 3200 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 3201 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 3202 3203 zio_gang_tree_free(&zio->io_gang_tree); 3204 3205 /* 3206 * Godfather I/Os should never suspend. 3207 */ 3208 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 3209 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 3210 zio->io_reexecute = 0; 3211 3212 if (zio->io_reexecute) { 3213 /* 3214 * This is a logical I/O that wants to reexecute. 3215 * 3216 * Reexecute is top-down. When an i/o fails, if it's not 3217 * the root, it simply notifies its parent and sticks around. 3218 * The parent, seeing that it still has children in zio_done(), 3219 * does the same. This percolates all the way up to the root. 3220 * The root i/o will reexecute or suspend the entire tree. 3221 * 3222 * This approach ensures that zio_reexecute() honors 3223 * all the original i/o dependency relationships, e.g. 3224 * parents not executing until children are ready. 3225 */ 3226 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3227 3228 zio->io_gang_leader = NULL; 3229 3230 mutex_enter(&zio->io_lock); 3231 zio->io_state[ZIO_WAIT_DONE] = 1; 3232 mutex_exit(&zio->io_lock); 3233 3234 /* 3235 * "The Godfather" I/O monitors its children but is 3236 * not a true parent to them. It will track them through 3237 * the pipeline but severs its ties whenever they get into 3238 * trouble (e.g. suspended). This allows "The Godfather" 3239 * I/O to return status without blocking. 3240 */ 3241 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3242 zio_link_t *zl = zio->io_walk_link; 3243 pio_next = zio_walk_parents(zio); 3244 3245 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 3246 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 3247 zio_remove_child(pio, zio, zl); 3248 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3249 } 3250 } 3251 3252 if ((pio = zio_unique_parent(zio)) != NULL) { 3253 /* 3254 * We're not a root i/o, so there's nothing to do 3255 * but notify our parent. Don't propagate errors 3256 * upward since we haven't permanently failed yet. 3257 */ 3258 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 3259 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 3260 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3261 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 3262 /* 3263 * We'd fail again if we reexecuted now, so suspend 3264 * until conditions improve (e.g. device comes online). 3265 */ 3266 zio_suspend(spa, zio); 3267 } else { 3268 /* 3269 * Reexecution is potentially a huge amount of work. 3270 * Hand it off to the otherwise-unused claim taskq. 3271 */ 3272#if defined(illumos) || !defined(_KERNEL) 3273 ASSERT(zio->io_tqent.tqent_next == NULL); 3274#else 3275 ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); 3276#endif 3277 spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM, 3278 ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio, 3279 0, &zio->io_tqent); 3280 } 3281 return (ZIO_PIPELINE_STOP); 3282 } 3283 3284 ASSERT(zio->io_child_count == 0); 3285 ASSERT(zio->io_reexecute == 0); 3286 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 3287 3288 /* 3289 * Report any checksum errors, since the I/O is complete. 3290 */ 3291 while (zio->io_cksum_report != NULL) { 3292 zio_cksum_report_t *zcr = zio->io_cksum_report; 3293 zio->io_cksum_report = zcr->zcr_next; 3294 zcr->zcr_next = NULL; 3295 zcr->zcr_finish(zcr, NULL); 3296 zfs_ereport_free_checksum(zcr); 3297 } 3298 3299 /* 3300 * It is the responsibility of the done callback to ensure that this 3301 * particular zio is no longer discoverable for adoption, and as 3302 * such, cannot acquire any new parents. 3303 */ 3304 if (zio->io_done) 3305 zio->io_done(zio); 3306 3307 mutex_enter(&zio->io_lock); 3308 zio->io_state[ZIO_WAIT_DONE] = 1; 3309 mutex_exit(&zio->io_lock); 3310 3311 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3312 zio_link_t *zl = zio->io_walk_link; 3313 pio_next = zio_walk_parents(zio); 3314 zio_remove_child(pio, zio, zl); 3315 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3316 } 3317 3318 if (zio->io_waiter != NULL) { 3319 mutex_enter(&zio->io_lock); 3320 zio->io_executor = NULL; 3321 cv_broadcast(&zio->io_cv); 3322 mutex_exit(&zio->io_lock); 3323 } else { 3324 zio_destroy(zio); 3325 } 3326 3327 return (ZIO_PIPELINE_STOP); 3328} 3329 3330/* 3331 * ========================================================================== 3332 * I/O pipeline definition 3333 * ========================================================================== 3334 */ 3335static zio_pipe_stage_t *zio_pipeline[] = { 3336 NULL, 3337 zio_read_bp_init, 3338 zio_free_bp_init, 3339 zio_issue_async, 3340 zio_write_bp_init, 3341 zio_checksum_generate, 3342 zio_nop_write, 3343 zio_ddt_read_start, 3344 zio_ddt_read_done, 3345 zio_ddt_write, 3346 zio_ddt_free, 3347 zio_gang_assemble, 3348 zio_gang_issue, 3349 zio_dva_allocate, 3350 zio_dva_free, 3351 zio_dva_claim, 3352 zio_ready, 3353 zio_vdev_io_start, 3354 zio_vdev_io_done, 3355 zio_vdev_io_assess, 3356 zio_checksum_verify, 3357 zio_done 3358}; 3359 3360/* dnp is the dnode for zb1->zb_object */ 3361boolean_t 3362zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1, 3363 const zbookmark_t *zb2) 3364{ 3365 uint64_t zb1nextL0, zb2thisobj; 3366 3367 ASSERT(zb1->zb_objset == zb2->zb_objset); 3368 ASSERT(zb2->zb_level == 0); 3369 3370 /* The objset_phys_t isn't before anything. */ 3371 if (dnp == NULL) 3372 return (B_FALSE); 3373 3374 zb1nextL0 = (zb1->zb_blkid + 1) << 3375 ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)); 3376 3377 zb2thisobj = zb2->zb_object ? zb2->zb_object : 3378 zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT); 3379 3380 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 3381 uint64_t nextobj = zb1nextL0 * 3382 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT; 3383 return (nextobj <= zb2thisobj); 3384 } 3385 3386 if (zb1->zb_object < zb2thisobj) 3387 return (B_TRUE); 3388 if (zb1->zb_object > zb2thisobj) 3389 return (B_FALSE); 3390 if (zb2->zb_object == DMU_META_DNODE_OBJECT) 3391 return (B_FALSE); 3392 return (zb1nextL0 <= zb2->zb_blkid); 3393} 3394