vdev_mirror.c revision 307279
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26/* 27 * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 28 */ 29 30#include <sys/zfs_context.h> 31#include <sys/spa.h> 32#include <sys/vdev_impl.h> 33#include <sys/zio.h> 34#include <sys/fs/zfs.h> 35 36/* 37 * Virtual device vector for mirroring. 38 */ 39 40typedef struct mirror_child { 41 vdev_t *mc_vd; 42 uint64_t mc_offset; 43 int mc_error; 44 int mc_load; 45 uint8_t mc_tried; 46 uint8_t mc_skipped; 47 uint8_t mc_speculative; 48} mirror_child_t; 49 50typedef struct mirror_map { 51 int *mm_preferred; 52 int mm_preferred_cnt; 53 int mm_children; 54 boolean_t mm_replacing; 55 boolean_t mm_root; 56 mirror_child_t mm_child[]; 57} mirror_map_t; 58 59static int vdev_mirror_shift = 21; 60 61#ifdef _KERNEL 62SYSCTL_DECL(_vfs_zfs_vdev); 63static SYSCTL_NODE(_vfs_zfs_vdev, OID_AUTO, mirror, CTLFLAG_RD, 0, 64 "ZFS VDEV Mirror"); 65#endif 66 67/* 68 * The load configuration settings below are tuned by default for 69 * the case where all devices are of the same rotational type. 70 * 71 * If there is a mixture of rotating and non-rotating media, setting 72 * non_rotating_seek_inc to 0 may well provide better results as it 73 * will direct more reads to the non-rotating vdevs which are more 74 * likely to have a higher performance. 75 */ 76 77/* Rotating media load calculation configuration. */ 78static int rotating_inc = 0; 79#ifdef _KERNEL 80TUNABLE_INT("vfs.zfs.vdev.mirror.rotating_inc", &rotating_inc); 81SYSCTL_INT(_vfs_zfs_vdev_mirror, OID_AUTO, rotating_inc, CTLFLAG_RW, 82 &rotating_inc, 0, "Rotating media load increment for non-seeking I/O's"); 83#endif 84 85static int rotating_seek_inc = 5; 86#ifdef _KERNEL 87TUNABLE_INT("vfs.zfs.vdev.mirror.rotating_seek_inc", &rotating_seek_inc); 88SYSCTL_INT(_vfs_zfs_vdev_mirror, OID_AUTO, rotating_seek_inc, CTLFLAG_RW, 89 &rotating_seek_inc, 0, "Rotating media load increment for seeking I/O's"); 90#endif 91 92static int rotating_seek_offset = 1 * 1024 * 1024; 93#ifdef _KERNEL 94TUNABLE_INT("vfs.zfs.vdev.mirror.rotating_seek_offset", &rotating_seek_offset); 95SYSCTL_INT(_vfs_zfs_vdev_mirror, OID_AUTO, rotating_seek_offset, CTLFLAG_RW, 96 &rotating_seek_offset, 0, "Offset in bytes from the last I/O which " 97 "triggers a reduced rotating media seek increment"); 98#endif 99 100/* Non-rotating media load calculation configuration. */ 101static int non_rotating_inc = 0; 102#ifdef _KERNEL 103TUNABLE_INT("vfs.zfs.vdev.mirror.non_rotating_inc", &non_rotating_inc); 104SYSCTL_INT(_vfs_zfs_vdev_mirror, OID_AUTO, non_rotating_inc, CTLFLAG_RW, 105 &non_rotating_inc, 0, 106 "Non-rotating media load increment for non-seeking I/O's"); 107#endif 108 109static int non_rotating_seek_inc = 1; 110#ifdef _KERNEL 111TUNABLE_INT("vfs.zfs.vdev.mirror.non_rotating_seek_inc", 112 &non_rotating_seek_inc); 113SYSCTL_INT(_vfs_zfs_vdev_mirror, OID_AUTO, non_rotating_seek_inc, CTLFLAG_RW, 114 &non_rotating_seek_inc, 0, 115 "Non-rotating media load increment for seeking I/O's"); 116#endif 117 118 119static inline size_t 120vdev_mirror_map_size(int children) 121{ 122 return (offsetof(mirror_map_t, mm_child[children]) + 123 sizeof(int) * children); 124} 125 126static inline mirror_map_t * 127vdev_mirror_map_alloc(int children, boolean_t replacing, boolean_t root) 128{ 129 mirror_map_t *mm; 130 131 mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP); 132 mm->mm_children = children; 133 mm->mm_replacing = replacing; 134 mm->mm_root = root; 135 mm->mm_preferred = (int *)((uintptr_t)mm + 136 offsetof(mirror_map_t, mm_child[children])); 137 138 return mm; 139} 140 141static void 142vdev_mirror_map_free(zio_t *zio) 143{ 144 mirror_map_t *mm = zio->io_vsd; 145 146 kmem_free(mm, vdev_mirror_map_size(mm->mm_children)); 147} 148 149static const zio_vsd_ops_t vdev_mirror_vsd_ops = { 150 vdev_mirror_map_free, 151 zio_vsd_default_cksum_report 152}; 153 154static int 155vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset) 156{ 157 uint64_t lastoffset; 158 int load; 159 160 /* All DVAs have equal weight at the root. */ 161 if (mm->mm_root) 162 return (INT_MAX); 163 164 /* 165 * We don't return INT_MAX if the device is resilvering i.e. 166 * vdev_resilver_txg != 0 as when tested performance was slightly 167 * worse overall when resilvering with compared to without. 168 */ 169 170 /* Standard load based on pending queue length. */ 171 load = vdev_queue_length(vd); 172 lastoffset = vdev_queue_lastoffset(vd); 173 174 if (vd->vdev_rotation_rate == VDEV_RATE_NON_ROTATING) { 175 /* Non-rotating media. */ 176 if (lastoffset == zio_offset) 177 return (load + non_rotating_inc); 178 179 /* 180 * Apply a seek penalty even for non-rotating devices as 181 * sequential I/O'a can be aggregated into fewer operations 182 * on the device, thus avoiding unnecessary per-command 183 * overhead and boosting performance. 184 */ 185 return (load + non_rotating_seek_inc); 186 } 187 188 /* Rotating media I/O's which directly follow the last I/O. */ 189 if (lastoffset == zio_offset) 190 return (load + rotating_inc); 191 192 /* 193 * Apply half the seek increment to I/O's within seek offset 194 * of the last I/O queued to this vdev as they should incure less 195 * of a seek increment. 196 */ 197 if (ABS(lastoffset - zio_offset) < rotating_seek_offset) 198 return (load + (rotating_seek_inc / 2)); 199 200 /* Apply the full seek increment to all other I/O's. */ 201 return (load + rotating_seek_inc); 202} 203 204 205static mirror_map_t * 206vdev_mirror_map_init(zio_t *zio) 207{ 208 mirror_map_t *mm = NULL; 209 mirror_child_t *mc; 210 vdev_t *vd = zio->io_vd; 211 int c; 212 213 if (vd == NULL) { 214 dva_t *dva = zio->io_bp->blk_dva; 215 spa_t *spa = zio->io_spa; 216 217 mm = vdev_mirror_map_alloc(BP_GET_NDVAS(zio->io_bp), B_FALSE, 218 B_TRUE); 219 for (c = 0; c < mm->mm_children; c++) { 220 mc = &mm->mm_child[c]; 221 mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c])); 222 mc->mc_offset = DVA_GET_OFFSET(&dva[c]); 223 } 224 } else { 225 mm = vdev_mirror_map_alloc(vd->vdev_children, 226 (vd->vdev_ops == &vdev_replacing_ops || 227 vd->vdev_ops == &vdev_spare_ops), B_FALSE); 228 for (c = 0; c < mm->mm_children; c++) { 229 mc = &mm->mm_child[c]; 230 mc->mc_vd = vd->vdev_child[c]; 231 mc->mc_offset = zio->io_offset; 232 } 233 } 234 235 zio->io_vsd = mm; 236 zio->io_vsd_ops = &vdev_mirror_vsd_ops; 237 return (mm); 238} 239 240static int 241vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize, 242 uint64_t *logical_ashift, uint64_t *physical_ashift) 243{ 244 int numerrors = 0; 245 int lasterror = 0; 246 247 if (vd->vdev_children == 0) { 248 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; 249 return (SET_ERROR(EINVAL)); 250 } 251 252 vdev_open_children(vd); 253 254 for (int c = 0; c < vd->vdev_children; c++) { 255 vdev_t *cvd = vd->vdev_child[c]; 256 257 if (cvd->vdev_open_error) { 258 lasterror = cvd->vdev_open_error; 259 numerrors++; 260 continue; 261 } 262 263 *asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1; 264 *max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1; 265 *logical_ashift = MAX(*logical_ashift, cvd->vdev_ashift); 266 *physical_ashift = MAX(*physical_ashift, 267 cvd->vdev_physical_ashift); 268 } 269 270 if (numerrors == vd->vdev_children) { 271 vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS; 272 return (lasterror); 273 } 274 275 return (0); 276} 277 278static void 279vdev_mirror_close(vdev_t *vd) 280{ 281 for (int c = 0; c < vd->vdev_children; c++) 282 vdev_close(vd->vdev_child[c]); 283} 284 285static void 286vdev_mirror_child_done(zio_t *zio) 287{ 288 mirror_child_t *mc = zio->io_private; 289 290 mc->mc_error = zio->io_error; 291 mc->mc_tried = 1; 292 mc->mc_skipped = 0; 293} 294 295static void 296vdev_mirror_scrub_done(zio_t *zio) 297{ 298 mirror_child_t *mc = zio->io_private; 299 300 if (zio->io_error == 0) { 301 zio_t *pio; 302 zio_link_t *zl = NULL; 303 304 mutex_enter(&zio->io_lock); 305 while ((pio = zio_walk_parents(zio, &zl)) != NULL) { 306 mutex_enter(&pio->io_lock); 307 ASSERT3U(zio->io_size, >=, pio->io_size); 308 bcopy(zio->io_data, pio->io_data, pio->io_size); 309 mutex_exit(&pio->io_lock); 310 } 311 mutex_exit(&zio->io_lock); 312 } 313 314 zio_buf_free(zio->io_data, zio->io_size); 315 316 mc->mc_error = zio->io_error; 317 mc->mc_tried = 1; 318 mc->mc_skipped = 0; 319} 320 321/* 322 * Check the other, lower-index DVAs to see if they're on the same 323 * vdev as the child we picked. If they are, use them since they 324 * are likely to have been allocated from the primary metaslab in 325 * use at the time, and hence are more likely to have locality with 326 * single-copy data. 327 */ 328static int 329vdev_mirror_dva_select(zio_t *zio, int p) 330{ 331 dva_t *dva = zio->io_bp->blk_dva; 332 mirror_map_t *mm = zio->io_vsd; 333 int preferred; 334 int c; 335 336 preferred = mm->mm_preferred[p]; 337 for (p-- ; p >= 0; p--) { 338 c = mm->mm_preferred[p]; 339 if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred])) 340 preferred = c; 341 } 342 return (preferred); 343} 344 345static int 346vdev_mirror_preferred_child_randomize(zio_t *zio) 347{ 348 mirror_map_t *mm = zio->io_vsd; 349 int p; 350 351 if (mm->mm_root) { 352 p = spa_get_random(mm->mm_preferred_cnt); 353 return (vdev_mirror_dva_select(zio, p)); 354 } 355 356 /* 357 * To ensure we don't always favour the first matching vdev, 358 * which could lead to wear leveling issues on SSD's, we 359 * use the I/O offset as a pseudo random seed into the vdevs 360 * which have the lowest load. 361 */ 362 p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt; 363 return (mm->mm_preferred[p]); 364} 365 366/* 367 * Try to find a vdev whose DTL doesn't contain the block we want to read 368 * prefering vdevs based on determined load. 369 * 370 * If we can't, try the read on any vdev we haven't already tried. 371 */ 372static int 373vdev_mirror_child_select(zio_t *zio) 374{ 375 mirror_map_t *mm = zio->io_vsd; 376 uint64_t txg = zio->io_txg; 377 int c, lowest_load; 378 379 ASSERT(zio->io_bp == NULL || BP_PHYSICAL_BIRTH(zio->io_bp) == txg); 380 381 lowest_load = INT_MAX; 382 mm->mm_preferred_cnt = 0; 383 for (c = 0; c < mm->mm_children; c++) { 384 mirror_child_t *mc; 385 386 mc = &mm->mm_child[c]; 387 if (mc->mc_tried || mc->mc_skipped) 388 continue; 389 390 if (!vdev_readable(mc->mc_vd)) { 391 mc->mc_error = SET_ERROR(ENXIO); 392 mc->mc_tried = 1; /* don't even try */ 393 mc->mc_skipped = 1; 394 continue; 395 } 396 397 if (vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1)) { 398 mc->mc_error = SET_ERROR(ESTALE); 399 mc->mc_skipped = 1; 400 mc->mc_speculative = 1; 401 continue; 402 } 403 404 mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset); 405 if (mc->mc_load > lowest_load) 406 continue; 407 408 if (mc->mc_load < lowest_load) { 409 lowest_load = mc->mc_load; 410 mm->mm_preferred_cnt = 0; 411 } 412 mm->mm_preferred[mm->mm_preferred_cnt] = c; 413 mm->mm_preferred_cnt++; 414 } 415 416 if (mm->mm_preferred_cnt == 1) { 417 vdev_queue_register_lastoffset( 418 mm->mm_child[mm->mm_preferred[0]].mc_vd, zio); 419 return (mm->mm_preferred[0]); 420 } 421 422 if (mm->mm_preferred_cnt > 1) { 423 int c = vdev_mirror_preferred_child_randomize(zio); 424 425 vdev_queue_register_lastoffset(mm->mm_child[c].mc_vd, zio); 426 return (c); 427 } 428 429 /* 430 * Every device is either missing or has this txg in its DTL. 431 * Look for any child we haven't already tried before giving up. 432 */ 433 for (c = 0; c < mm->mm_children; c++) { 434 if (!mm->mm_child[c].mc_tried) { 435 vdev_queue_register_lastoffset(mm->mm_child[c].mc_vd, 436 zio); 437 return (c); 438 } 439 } 440 441 /* 442 * Every child failed. There's no place left to look. 443 */ 444 return (-1); 445} 446 447static void 448vdev_mirror_io_start(zio_t *zio) 449{ 450 mirror_map_t *mm; 451 mirror_child_t *mc; 452 int c, children; 453 454 mm = vdev_mirror_map_init(zio); 455 456 if (zio->io_type == ZIO_TYPE_READ) { 457 if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_replacing && 458 mm->mm_children > 1) { 459 /* 460 * For scrubbing reads we need to allocate a read 461 * buffer for each child and issue reads to all 462 * children. If any child succeeds, it will copy its 463 * data into zio->io_data in vdev_mirror_scrub_done. 464 */ 465 for (c = 0; c < mm->mm_children; c++) { 466 mc = &mm->mm_child[c]; 467 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 468 mc->mc_vd, mc->mc_offset, 469 zio_buf_alloc(zio->io_size), zio->io_size, 470 zio->io_type, zio->io_priority, 0, 471 vdev_mirror_scrub_done, mc)); 472 } 473 zio_execute(zio); 474 return; 475 } 476 /* 477 * For normal reads just pick one child. 478 */ 479 c = vdev_mirror_child_select(zio); 480 children = (c >= 0); 481 } else { 482 ASSERT(zio->io_type == ZIO_TYPE_WRITE || 483 zio->io_type == ZIO_TYPE_FREE); 484 485 /* 486 * Writes and frees go to all children. 487 */ 488 c = 0; 489 children = mm->mm_children; 490 } 491 492 while (children--) { 493 mc = &mm->mm_child[c]; 494 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 495 mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size, 496 zio->io_type, zio->io_priority, 0, 497 vdev_mirror_child_done, mc)); 498 c++; 499 } 500 501 zio_execute(zio); 502} 503 504static int 505vdev_mirror_worst_error(mirror_map_t *mm) 506{ 507 int error[2] = { 0, 0 }; 508 509 for (int c = 0; c < mm->mm_children; c++) { 510 mirror_child_t *mc = &mm->mm_child[c]; 511 int s = mc->mc_speculative; 512 error[s] = zio_worst_error(error[s], mc->mc_error); 513 } 514 515 return (error[0] ? error[0] : error[1]); 516} 517 518static void 519vdev_mirror_io_done(zio_t *zio) 520{ 521 mirror_map_t *mm = zio->io_vsd; 522 mirror_child_t *mc; 523 int c; 524 int good_copies = 0; 525 int unexpected_errors = 0; 526 527 for (c = 0; c < mm->mm_children; c++) { 528 mc = &mm->mm_child[c]; 529 530 if (mc->mc_error) { 531 if (!mc->mc_skipped) 532 unexpected_errors++; 533 } else if (mc->mc_tried) { 534 good_copies++; 535 } 536 } 537 538 if (zio->io_type == ZIO_TYPE_WRITE) { 539 /* 540 * XXX -- for now, treat partial writes as success. 541 * 542 * Now that we support write reallocation, it would be better 543 * to treat partial failure as real failure unless there are 544 * no non-degraded top-level vdevs left, and not update DTLs 545 * if we intend to reallocate. 546 */ 547 /* XXPOLICY */ 548 if (good_copies != mm->mm_children) { 549 /* 550 * Always require at least one good copy. 551 * 552 * For ditto blocks (io_vd == NULL), require 553 * all copies to be good. 554 * 555 * XXX -- for replacing vdevs, there's no great answer. 556 * If the old device is really dead, we may not even 557 * be able to access it -- so we only want to 558 * require good writes to the new device. But if 559 * the new device turns out to be flaky, we want 560 * to be able to detach it -- which requires all 561 * writes to the old device to have succeeded. 562 */ 563 if (good_copies == 0 || zio->io_vd == NULL) 564 zio->io_error = vdev_mirror_worst_error(mm); 565 } 566 return; 567 } else if (zio->io_type == ZIO_TYPE_FREE) { 568 return; 569 } 570 571 ASSERT(zio->io_type == ZIO_TYPE_READ); 572 573 /* 574 * If we don't have a good copy yet, keep trying other children. 575 */ 576 /* XXPOLICY */ 577 if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) { 578 ASSERT(c >= 0 && c < mm->mm_children); 579 mc = &mm->mm_child[c]; 580 zio_vdev_io_redone(zio); 581 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 582 mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size, 583 ZIO_TYPE_READ, zio->io_priority, 0, 584 vdev_mirror_child_done, mc)); 585 return; 586 } 587 588 /* XXPOLICY */ 589 if (good_copies == 0) { 590 zio->io_error = vdev_mirror_worst_error(mm); 591 ASSERT(zio->io_error != 0); 592 } 593 594 if (good_copies && spa_writeable(zio->io_spa) && 595 (unexpected_errors || 596 (zio->io_flags & ZIO_FLAG_RESILVER) || 597 ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_replacing))) { 598 /* 599 * Use the good data we have in hand to repair damaged children. 600 */ 601 for (c = 0; c < mm->mm_children; c++) { 602 /* 603 * Don't rewrite known good children. 604 * Not only is it unnecessary, it could 605 * actually be harmful: if the system lost 606 * power while rewriting the only good copy, 607 * there would be no good copies left! 608 */ 609 mc = &mm->mm_child[c]; 610 611 if (mc->mc_error == 0) { 612 if (mc->mc_tried) 613 continue; 614 if (!(zio->io_flags & ZIO_FLAG_SCRUB) && 615 !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL, 616 zio->io_txg, 1)) 617 continue; 618 mc->mc_error = SET_ERROR(ESTALE); 619 } 620 621 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 622 mc->mc_vd, mc->mc_offset, 623 zio->io_data, zio->io_size, 624 ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE, 625 ZIO_FLAG_IO_REPAIR | (unexpected_errors ? 626 ZIO_FLAG_SELF_HEAL : 0), NULL, NULL)); 627 } 628 } 629} 630 631static void 632vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded) 633{ 634 if (faulted == vd->vdev_children) 635 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN, 636 VDEV_AUX_NO_REPLICAS); 637 else if (degraded + faulted != 0) 638 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE); 639 else 640 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE); 641} 642 643vdev_ops_t vdev_mirror_ops = { 644 vdev_mirror_open, 645 vdev_mirror_close, 646 vdev_default_asize, 647 vdev_mirror_io_start, 648 vdev_mirror_io_done, 649 vdev_mirror_state_change, 650 NULL, 651 NULL, 652 VDEV_TYPE_MIRROR, /* name of this vdev type */ 653 B_FALSE /* not a leaf vdev */ 654}; 655 656vdev_ops_t vdev_replacing_ops = { 657 vdev_mirror_open, 658 vdev_mirror_close, 659 vdev_default_asize, 660 vdev_mirror_io_start, 661 vdev_mirror_io_done, 662 vdev_mirror_state_change, 663 NULL, 664 NULL, 665 VDEV_TYPE_REPLACING, /* name of this vdev type */ 666 B_FALSE /* not a leaf vdev */ 667}; 668 669vdev_ops_t vdev_spare_ops = { 670 vdev_mirror_open, 671 vdev_mirror_close, 672 vdev_default_asize, 673 vdev_mirror_io_start, 674 vdev_mirror_io_done, 675 vdev_mirror_state_change, 676 NULL, 677 NULL, 678 VDEV_TYPE_SPARE, /* name of this vdev type */ 679 B_FALSE /* not a leaf vdev */ 680}; 681