zil.c revision 310516
1156230Smux/* 2156230Smux * CDDL HEADER START 3156230Smux * 4156230Smux * The contents of this file are subject to the terms of the 5156230Smux * Common Development and Distribution License (the "License"). 6156230Smux * You may not use this file except in compliance with the License. 7156230Smux * 8156230Smux * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9156230Smux * or http://www.opensolaris.org/os/licensing. 10156230Smux * See the License for the specific language governing permissions 11156230Smux * and limitations under the License. 12156230Smux * 13156230Smux * When distributing Covered Code, include this CDDL HEADER in each 14156230Smux * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15156230Smux * If applicable, add the following below this CDDL HEADER, with the 16156230Smux * fields enclosed by brackets "[]" replaced with your own identifying 17156230Smux * information: Portions Copyright [yyyy] [name of copyright owner] 18156230Smux * 19156230Smux * CDDL HEADER END 20156230Smux */ 21156230Smux/* 22156230Smux * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23156230Smux * Copyright (c) 2011, 2016 by Delphix. All rights reserved. 24156230Smux * Copyright (c) 2014 Integros [integros.com] 25156230Smux */ 26156230Smux 27156230Smux/* Portions Copyright 2010 Robert Milkowski */ 28156230Smux 29156230Smux#include <sys/zfs_context.h> 30156230Smux#include <sys/spa.h> 31156230Smux#include <sys/dmu.h> 32156230Smux#include <sys/zap.h> 33156230Smux#include <sys/arc.h> 34156230Smux#include <sys/stat.h> 35156230Smux#include <sys/resource.h> 36156230Smux#include <sys/zil.h> 37156230Smux#include <sys/zil_impl.h> 38156230Smux#include <sys/dsl_dataset.h> 39156230Smux#include <sys/vdev_impl.h> 40156230Smux#include <sys/dmu_tx.h> 41156230Smux#include <sys/dsl_pool.h> 42250227Sjkim 43156230Smux/* 44156230Smux * The zfs intent log (ZIL) saves transaction records of system calls 45156230Smux * that change the file system in memory with enough information 46156230Smux * to be able to replay them. These are stored in memory until 47156230Smux * either the DMU transaction group (txg) commits them to the stable pool 48156230Smux * and they can be discarded, or they are flushed to the stable log 49156230Smux * (also in the pool) due to a fsync, O_DSYNC or other synchronous 50156230Smux * requirement. In the event of a panic or power fail then those log 51156230Smux * records (transactions) are replayed. 52156230Smux * 53156230Smux * There is one ZIL per file system. Its on-disk (pool) format consists 54156230Smux * of 3 parts: 55156230Smux * 56156230Smux * - ZIL header 57156230Smux * - ZIL blocks 58156701Smux * - ZIL records 59156230Smux * 60156230Smux * A log record holds a system call transaction. Log blocks can 61156230Smux * hold many log records and the blocks are chained together. 62156230Smux * Each ZIL block contains a block pointer (blkptr_t) to the next 63156230Smux * ZIL block in the chain. The ZIL header points to the first 64156230Smux * block in the chain. Note there is not a fixed place in the pool 65156230Smux * to hold blocks. They are dynamically allocated and freed as 66156230Smux * needed from the blocks available. Figure X shows the ZIL structure: 67156230Smux */ 68156230Smux 69156230Smux/* 70156230Smux * Disable intent logging replay. This global ZIL switch affects all pools. 71156230Smux */ 72156230Smuxint zil_replay_disable = 0; 73156230SmuxSYSCTL_DECL(_vfs_zfs); 74156230SmuxTUNABLE_INT("vfs.zfs.zil_replay_disable", &zil_replay_disable); 75156230SmuxSYSCTL_INT(_vfs_zfs, OID_AUTO, zil_replay_disable, CTLFLAG_RW, 76156230Smux &zil_replay_disable, 0, "Disable intent logging replay"); 77156230Smux 78156230Smux/* 79156230Smux * Tunable parameter for debugging or performance analysis. Setting 80 * zfs_nocacheflush will cause corruption on power loss if a volatile 81 * out-of-order write cache is enabled. 82 */ 83boolean_t zfs_nocacheflush = B_FALSE; 84TUNABLE_INT("vfs.zfs.cache_flush_disable", &zfs_nocacheflush); 85SYSCTL_INT(_vfs_zfs, OID_AUTO, cache_flush_disable, CTLFLAG_RDTUN, 86 &zfs_nocacheflush, 0, "Disable cache flush"); 87boolean_t zfs_trim_enabled = B_TRUE; 88SYSCTL_DECL(_vfs_zfs_trim); 89TUNABLE_INT("vfs.zfs.trim.enabled", &zfs_trim_enabled); 90SYSCTL_INT(_vfs_zfs_trim, OID_AUTO, enabled, CTLFLAG_RDTUN, &zfs_trim_enabled, 0, 91 "Enable ZFS TRIM"); 92 93static kmem_cache_t *zil_lwb_cache; 94 95#define LWB_EMPTY(lwb) ((BP_GET_LSIZE(&lwb->lwb_blk) - \ 96 sizeof (zil_chain_t)) == (lwb->lwb_sz - lwb->lwb_nused)) 97 98 99/* 100 * ziltest is by and large an ugly hack, but very useful in 101 * checking replay without tedious work. 102 * When running ziltest we want to keep all itx's and so maintain 103 * a single list in the zl_itxg[] that uses a high txg: ZILTEST_TXG 104 * We subtract TXG_CONCURRENT_STATES to allow for common code. 105 */ 106#define ZILTEST_TXG (UINT64_MAX - TXG_CONCURRENT_STATES) 107 108static int 109zil_bp_compare(const void *x1, const void *x2) 110{ 111 const dva_t *dva1 = &((zil_bp_node_t *)x1)->zn_dva; 112 const dva_t *dva2 = &((zil_bp_node_t *)x2)->zn_dva; 113 114 if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 115 return (-1); 116 if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 117 return (1); 118 119 if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 120 return (-1); 121 if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 122 return (1); 123 124 return (0); 125} 126 127static void 128zil_bp_tree_init(zilog_t *zilog) 129{ 130 avl_create(&zilog->zl_bp_tree, zil_bp_compare, 131 sizeof (zil_bp_node_t), offsetof(zil_bp_node_t, zn_node)); 132} 133 134static void 135zil_bp_tree_fini(zilog_t *zilog) 136{ 137 avl_tree_t *t = &zilog->zl_bp_tree; 138 zil_bp_node_t *zn; 139 void *cookie = NULL; 140 141 while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 142 kmem_free(zn, sizeof (zil_bp_node_t)); 143 144 avl_destroy(t); 145} 146 147int 148zil_bp_tree_add(zilog_t *zilog, const blkptr_t *bp) 149{ 150 avl_tree_t *t = &zilog->zl_bp_tree; 151 const dva_t *dva; 152 zil_bp_node_t *zn; 153 avl_index_t where; 154 155 if (BP_IS_EMBEDDED(bp)) 156 return (0); 157 158 dva = BP_IDENTITY(bp); 159 160 if (avl_find(t, dva, &where) != NULL) 161 return (SET_ERROR(EEXIST)); 162 163 zn = kmem_alloc(sizeof (zil_bp_node_t), KM_SLEEP); 164 zn->zn_dva = *dva; 165 avl_insert(t, zn, where); 166 167 return (0); 168} 169 170static zil_header_t * 171zil_header_in_syncing_context(zilog_t *zilog) 172{ 173 return ((zil_header_t *)zilog->zl_header); 174} 175 176static void 177zil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 178{ 179 zio_cksum_t *zc = &bp->blk_cksum; 180 181 zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 182 zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 183 zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 184 zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 185} 186 187/* 188 * Read a log block and make sure it's valid. 189 */ 190static int 191zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, blkptr_t *nbp, void *dst, 192 char **end) 193{ 194 enum zio_flag zio_flags = ZIO_FLAG_CANFAIL; 195 arc_flags_t aflags = ARC_FLAG_WAIT; 196 arc_buf_t *abuf = NULL; 197 zbookmark_phys_t zb; 198 int error; 199 200 if (zilog->zl_header->zh_claim_txg == 0) 201 zio_flags |= ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB; 202 203 if (!(zilog->zl_header->zh_flags & ZIL_CLAIM_LR_SEQ_VALID)) 204 zio_flags |= ZIO_FLAG_SPECULATIVE; 205 206 SET_BOOKMARK(&zb, bp->blk_cksum.zc_word[ZIL_ZC_OBJSET], 207 ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 208 209 error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func, &abuf, 210 ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb); 211 212 if (error == 0) { 213 zio_cksum_t cksum = bp->blk_cksum; 214 215 /* 216 * Validate the checksummed log block. 217 * 218 * Sequence numbers should be... sequential. The checksum 219 * verifier for the next block should be bp's checksum plus 1. 220 * 221 * Also check the log chain linkage and size used. 222 */ 223 cksum.zc_word[ZIL_ZC_SEQ]++; 224 225 if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) { 226 zil_chain_t *zilc = abuf->b_data; 227 char *lr = (char *)(zilc + 1); 228 uint64_t len = zilc->zc_nused - sizeof (zil_chain_t); 229 230 if (bcmp(&cksum, &zilc->zc_next_blk.blk_cksum, 231 sizeof (cksum)) || BP_IS_HOLE(&zilc->zc_next_blk)) { 232 error = SET_ERROR(ECKSUM); 233 } else { 234 ASSERT3U(len, <=, SPA_OLD_MAXBLOCKSIZE); 235 bcopy(lr, dst, len); 236 *end = (char *)dst + len; 237 *nbp = zilc->zc_next_blk; 238 } 239 } else { 240 char *lr = abuf->b_data; 241 uint64_t size = BP_GET_LSIZE(bp); 242 zil_chain_t *zilc = (zil_chain_t *)(lr + size) - 1; 243 244 if (bcmp(&cksum, &zilc->zc_next_blk.blk_cksum, 245 sizeof (cksum)) || BP_IS_HOLE(&zilc->zc_next_blk) || 246 (zilc->zc_nused > (size - sizeof (*zilc)))) { 247 error = SET_ERROR(ECKSUM); 248 } else { 249 ASSERT3U(zilc->zc_nused, <=, 250 SPA_OLD_MAXBLOCKSIZE); 251 bcopy(lr, dst, zilc->zc_nused); 252 *end = (char *)dst + zilc->zc_nused; 253 *nbp = zilc->zc_next_blk; 254 } 255 } 256 257 arc_buf_destroy(abuf, &abuf); 258 } 259 260 return (error); 261} 262 263/* 264 * Read a TX_WRITE log data block. 265 */ 266static int 267zil_read_log_data(zilog_t *zilog, const lr_write_t *lr, void *wbuf) 268{ 269 enum zio_flag zio_flags = ZIO_FLAG_CANFAIL; 270 const blkptr_t *bp = &lr->lr_blkptr; 271 arc_flags_t aflags = ARC_FLAG_WAIT; 272 arc_buf_t *abuf = NULL; 273 zbookmark_phys_t zb; 274 int error; 275 276 if (BP_IS_HOLE(bp)) { 277 if (wbuf != NULL) 278 bzero(wbuf, MAX(BP_GET_LSIZE(bp), lr->lr_length)); 279 return (0); 280 } 281 282 if (zilog->zl_header->zh_claim_txg == 0) 283 zio_flags |= ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB; 284 285 SET_BOOKMARK(&zb, dmu_objset_id(zilog->zl_os), lr->lr_foid, 286 ZB_ZIL_LEVEL, lr->lr_offset / BP_GET_LSIZE(bp)); 287 288 error = arc_read(NULL, zilog->zl_spa, bp, arc_getbuf_func, &abuf, 289 ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb); 290 291 if (error == 0) { 292 if (wbuf != NULL) 293 bcopy(abuf->b_data, wbuf, arc_buf_size(abuf)); 294 arc_buf_destroy(abuf, &abuf); 295 } 296 297 return (error); 298} 299 300/* 301 * Parse the intent log, and call parse_func for each valid record within. 302 */ 303int 304zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 305 zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 306{ 307 const zil_header_t *zh = zilog->zl_header; 308 boolean_t claimed = !!zh->zh_claim_txg; 309 uint64_t claim_blk_seq = claimed ? zh->zh_claim_blk_seq : UINT64_MAX; 310 uint64_t claim_lr_seq = claimed ? zh->zh_claim_lr_seq : UINT64_MAX; 311 uint64_t max_blk_seq = 0; 312 uint64_t max_lr_seq = 0; 313 uint64_t blk_count = 0; 314 uint64_t lr_count = 0; 315 blkptr_t blk, next_blk; 316 char *lrbuf, *lrp; 317 int error = 0; 318 319 /* 320 * Old logs didn't record the maximum zh_claim_lr_seq. 321 */ 322 if (!(zh->zh_flags & ZIL_CLAIM_LR_SEQ_VALID)) 323 claim_lr_seq = UINT64_MAX; 324 325 /* 326 * Starting at the block pointed to by zh_log we read the log chain. 327 * For each block in the chain we strongly check that block to 328 * ensure its validity. We stop when an invalid block is found. 329 * For each block pointer in the chain we call parse_blk_func(). 330 * For each record in each valid block we call parse_lr_func(). 331 * If the log has been claimed, stop if we encounter a sequence 332 * number greater than the highest claimed sequence number. 333 */ 334 lrbuf = zio_buf_alloc(SPA_OLD_MAXBLOCKSIZE); 335 zil_bp_tree_init(zilog); 336 337 for (blk = zh->zh_log; !BP_IS_HOLE(&blk); blk = next_blk) { 338 uint64_t blk_seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 339 int reclen; 340 char *end; 341 342 if (blk_seq > claim_blk_seq) 343 break; 344 if ((error = parse_blk_func(zilog, &blk, arg, txg)) != 0) 345 break; 346 ASSERT3U(max_blk_seq, <, blk_seq); 347 max_blk_seq = blk_seq; 348 blk_count++; 349 350 if (max_lr_seq == claim_lr_seq && max_blk_seq == claim_blk_seq) 351 break; 352 353 error = zil_read_log_block(zilog, &blk, &next_blk, lrbuf, &end); 354 if (error != 0) 355 break; 356 357 for (lrp = lrbuf; lrp < end; lrp += reclen) { 358 lr_t *lr = (lr_t *)lrp; 359 reclen = lr->lrc_reclen; 360 ASSERT3U(reclen, >=, sizeof (lr_t)); 361 if (lr->lrc_seq > claim_lr_seq) 362 goto done; 363 if ((error = parse_lr_func(zilog, lr, arg, txg)) != 0) 364 goto done; 365 ASSERT3U(max_lr_seq, <, lr->lrc_seq); 366 max_lr_seq = lr->lrc_seq; 367 lr_count++; 368 } 369 } 370done: 371 zilog->zl_parse_error = error; 372 zilog->zl_parse_blk_seq = max_blk_seq; 373 zilog->zl_parse_lr_seq = max_lr_seq; 374 zilog->zl_parse_blk_count = blk_count; 375 zilog->zl_parse_lr_count = lr_count; 376 377 ASSERT(!claimed || !(zh->zh_flags & ZIL_CLAIM_LR_SEQ_VALID) || 378 (max_blk_seq == claim_blk_seq && max_lr_seq == claim_lr_seq)); 379 380 zil_bp_tree_fini(zilog); 381 zio_buf_free(lrbuf, SPA_OLD_MAXBLOCKSIZE); 382 383 return (error); 384} 385 386static int 387zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 388{ 389 /* 390 * Claim log block if not already committed and not already claimed. 391 * If tx == NULL, just verify that the block is claimable. 392 */ 393 if (BP_IS_HOLE(bp) || bp->blk_birth < first_txg || 394 zil_bp_tree_add(zilog, bp) != 0) 395 return (0); 396 397 return (zio_wait(zio_claim(NULL, zilog->zl_spa, 398 tx == NULL ? 0 : first_txg, bp, spa_claim_notify, NULL, 399 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB))); 400} 401 402static int 403zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 404{ 405 lr_write_t *lr = (lr_write_t *)lrc; 406 int error; 407 408 if (lrc->lrc_txtype != TX_WRITE) 409 return (0); 410 411 /* 412 * If the block is not readable, don't claim it. This can happen 413 * in normal operation when a log block is written to disk before 414 * some of the dmu_sync() blocks it points to. In this case, the 415 * transaction cannot have been committed to anyone (we would have 416 * waited for all writes to be stable first), so it is semantically 417 * correct to declare this the end of the log. 418 */ 419 if (lr->lr_blkptr.blk_birth >= first_txg && 420 (error = zil_read_log_data(zilog, lr, NULL)) != 0) 421 return (error); 422 return (zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg)); 423} 424 425/* ARGSUSED */ 426static int 427zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 428{ 429 zio_free_zil(zilog->zl_spa, dmu_tx_get_txg(tx), bp); 430 431 return (0); 432} 433 434static int 435zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 436{ 437 lr_write_t *lr = (lr_write_t *)lrc; 438 blkptr_t *bp = &lr->lr_blkptr; 439 440 /* 441 * If we previously claimed it, we need to free it. 442 */ 443 if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE && 444 bp->blk_birth >= claim_txg && zil_bp_tree_add(zilog, bp) == 0 && 445 !BP_IS_HOLE(bp)) 446 zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp); 447 448 return (0); 449} 450 451static lwb_t * 452zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, uint64_t txg) 453{ 454 lwb_t *lwb; 455 456 lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 457 lwb->lwb_zilog = zilog; 458 lwb->lwb_blk = *bp; 459 lwb->lwb_buf = zio_buf_alloc(BP_GET_LSIZE(bp)); 460 lwb->lwb_max_txg = txg; 461 lwb->lwb_zio = NULL; 462 lwb->lwb_tx = NULL; 463 if (BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_ZILOG2) { 464 lwb->lwb_nused = sizeof (zil_chain_t); 465 lwb->lwb_sz = BP_GET_LSIZE(bp); 466 } else { 467 lwb->lwb_nused = 0; 468 lwb->lwb_sz = BP_GET_LSIZE(bp) - sizeof (zil_chain_t); 469 } 470 471 mutex_enter(&zilog->zl_lock); 472 list_insert_tail(&zilog->zl_lwb_list, lwb); 473 mutex_exit(&zilog->zl_lock); 474 475 return (lwb); 476} 477 478/* 479 * Called when we create in-memory log transactions so that we know 480 * to cleanup the itxs at the end of spa_sync(). 481 */ 482void 483zilog_dirty(zilog_t *zilog, uint64_t txg) 484{ 485 dsl_pool_t *dp = zilog->zl_dmu_pool; 486 dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os); 487 488 if (ds->ds_is_snapshot) 489 panic("dirtying snapshot!"); 490 491 if (txg_list_add(&dp->dp_dirty_zilogs, zilog, txg)) { 492 /* up the hold count until we can be written out */ 493 dmu_buf_add_ref(ds->ds_dbuf, zilog); 494 } 495} 496 497/* 498 * Determine if the zil is dirty in the specified txg. Callers wanting to 499 * ensure that the dirty state does not change must hold the itxg_lock for 500 * the specified txg. Holding the lock will ensure that the zil cannot be 501 * dirtied (zil_itx_assign) or cleaned (zil_clean) while we check its current 502 * state. 503 */ 504boolean_t 505zilog_is_dirty_in_txg(zilog_t *zilog, uint64_t txg) 506{ 507 dsl_pool_t *dp = zilog->zl_dmu_pool; 508 509 if (txg_list_member(&dp->dp_dirty_zilogs, zilog, txg & TXG_MASK)) 510 return (B_TRUE); 511 return (B_FALSE); 512} 513 514/* 515 * Determine if the zil is dirty. The zil is considered dirty if it has 516 * any pending itx records that have not been cleaned by zil_clean(). 517 */ 518boolean_t 519zilog_is_dirty(zilog_t *zilog) 520{ 521 dsl_pool_t *dp = zilog->zl_dmu_pool; 522 523 for (int t = 0; t < TXG_SIZE; t++) { 524 if (txg_list_member(&dp->dp_dirty_zilogs, zilog, t)) 525 return (B_TRUE); 526 } 527 return (B_FALSE); 528} 529 530/* 531 * Create an on-disk intent log. 532 */ 533static lwb_t * 534zil_create(zilog_t *zilog) 535{ 536 const zil_header_t *zh = zilog->zl_header; 537 lwb_t *lwb = NULL; 538 uint64_t txg = 0; 539 dmu_tx_t *tx = NULL; 540 blkptr_t blk; 541 int error = 0; 542 543 /* 544 * Wait for any previous destroy to complete. 545 */ 546 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 547 548 ASSERT(zh->zh_claim_txg == 0); 549 ASSERT(zh->zh_replay_seq == 0); 550 551 blk = zh->zh_log; 552 553 /* 554 * Allocate an initial log block if: 555 * - there isn't one already 556 * - the existing block is the wrong endianess 557 */ 558 if (BP_IS_HOLE(&blk) || BP_SHOULD_BYTESWAP(&blk)) { 559 tx = dmu_tx_create(zilog->zl_os); 560 VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 561 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 562 txg = dmu_tx_get_txg(tx); 563 564 if (!BP_IS_HOLE(&blk)) { 565 zio_free_zil(zilog->zl_spa, txg, &blk); 566 BP_ZERO(&blk); 567 } 568 569 error = zio_alloc_zil(zilog->zl_spa, txg, &blk, NULL, 570 ZIL_MIN_BLKSZ, zilog->zl_logbias == ZFS_LOGBIAS_LATENCY); 571 572 if (error == 0) 573 zil_init_log_chain(zilog, &blk); 574 } 575 576 /* 577 * Allocate a log write buffer (lwb) for the first log block. 578 */ 579 if (error == 0) 580 lwb = zil_alloc_lwb(zilog, &blk, txg); 581 582 /* 583 * If we just allocated the first log block, commit our transaction 584 * and wait for zil_sync() to stuff the block poiner into zh_log. 585 * (zh is part of the MOS, so we cannot modify it in open context.) 586 */ 587 if (tx != NULL) { 588 dmu_tx_commit(tx); 589 txg_wait_synced(zilog->zl_dmu_pool, txg); 590 } 591 592 ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 593 594 return (lwb); 595} 596 597/* 598 * In one tx, free all log blocks and clear the log header. 599 * If keep_first is set, then we're replaying a log with no content. 600 * We want to keep the first block, however, so that the first 601 * synchronous transaction doesn't require a txg_wait_synced() 602 * in zil_create(). We don't need to txg_wait_synced() here either 603 * when keep_first is set, because both zil_create() and zil_destroy() 604 * will wait for any in-progress destroys to complete. 605 */ 606void 607zil_destroy(zilog_t *zilog, boolean_t keep_first) 608{ 609 const zil_header_t *zh = zilog->zl_header; 610 lwb_t *lwb; 611 dmu_tx_t *tx; 612 uint64_t txg; 613 614 /* 615 * Wait for any previous destroy to complete. 616 */ 617 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 618 619 zilog->zl_old_header = *zh; /* debugging aid */ 620 621 if (BP_IS_HOLE(&zh->zh_log)) 622 return; 623 624 tx = dmu_tx_create(zilog->zl_os); 625 VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 626 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 627 txg = dmu_tx_get_txg(tx); 628 629 mutex_enter(&zilog->zl_lock); 630 631 ASSERT3U(zilog->zl_destroy_txg, <, txg); 632 zilog->zl_destroy_txg = txg; 633 zilog->zl_keep_first = keep_first; 634 635 if (!list_is_empty(&zilog->zl_lwb_list)) { 636 ASSERT(zh->zh_claim_txg == 0); 637 VERIFY(!keep_first); 638 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 639 list_remove(&zilog->zl_lwb_list, lwb); 640 if (lwb->lwb_buf != NULL) 641 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 642 zio_free_zil(zilog->zl_spa, txg, &lwb->lwb_blk); 643 kmem_cache_free(zil_lwb_cache, lwb); 644 } 645 } else if (!keep_first) { 646 zil_destroy_sync(zilog, tx); 647 } 648 mutex_exit(&zilog->zl_lock); 649 650 dmu_tx_commit(tx); 651} 652 653void 654zil_destroy_sync(zilog_t *zilog, dmu_tx_t *tx) 655{ 656 ASSERT(list_is_empty(&zilog->zl_lwb_list)); 657 (void) zil_parse(zilog, zil_free_log_block, 658 zil_free_log_record, tx, zilog->zl_header->zh_claim_txg); 659} 660 661int 662zil_claim(dsl_pool_t *dp, dsl_dataset_t *ds, void *txarg) 663{ 664 dmu_tx_t *tx = txarg; 665 uint64_t first_txg = dmu_tx_get_txg(tx); 666 zilog_t *zilog; 667 zil_header_t *zh; 668 objset_t *os; 669 int error; 670 671 error = dmu_objset_own_obj(dp, ds->ds_object, 672 DMU_OST_ANY, B_FALSE, FTAG, &os); 673 if (error != 0) { 674 /* 675 * EBUSY indicates that the objset is inconsistent, in which 676 * case it can not have a ZIL. 677 */ 678 if (error != EBUSY) { 679 cmn_err(CE_WARN, "can't open objset for %llu, error %u", 680 (unsigned long long)ds->ds_object, error); 681 } 682 return (0); 683 } 684 685 zilog = dmu_objset_zil(os); 686 zh = zil_header_in_syncing_context(zilog); 687 688 if (spa_get_log_state(zilog->zl_spa) == SPA_LOG_CLEAR) { 689 if (!BP_IS_HOLE(&zh->zh_log)) 690 zio_free_zil(zilog->zl_spa, first_txg, &zh->zh_log); 691 BP_ZERO(&zh->zh_log); 692 dsl_dataset_dirty(dmu_objset_ds(os), tx); 693 dmu_objset_disown(os, FTAG); 694 return (0); 695 } 696 697 /* 698 * Claim all log blocks if we haven't already done so, and remember 699 * the highest claimed sequence number. This ensures that if we can 700 * read only part of the log now (e.g. due to a missing device), 701 * but we can read the entire log later, we will not try to replay 702 * or destroy beyond the last block we successfully claimed. 703 */ 704 ASSERT3U(zh->zh_claim_txg, <=, first_txg); 705 if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 706 (void) zil_parse(zilog, zil_claim_log_block, 707 zil_claim_log_record, tx, first_txg); 708 zh->zh_claim_txg = first_txg; 709 zh->zh_claim_blk_seq = zilog->zl_parse_blk_seq; 710 zh->zh_claim_lr_seq = zilog->zl_parse_lr_seq; 711 if (zilog->zl_parse_lr_count || zilog->zl_parse_blk_count > 1) 712 zh->zh_flags |= ZIL_REPLAY_NEEDED; 713 zh->zh_flags |= ZIL_CLAIM_LR_SEQ_VALID; 714 dsl_dataset_dirty(dmu_objset_ds(os), tx); 715 } 716 717 ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 718 dmu_objset_disown(os, FTAG); 719 return (0); 720} 721 722/* 723 * Check the log by walking the log chain. 724 * Checksum errors are ok as they indicate the end of the chain. 725 * Any other error (no device or read failure) returns an error. 726 */ 727/* ARGSUSED */ 728int 729zil_check_log_chain(dsl_pool_t *dp, dsl_dataset_t *ds, void *tx) 730{ 731 zilog_t *zilog; 732 objset_t *os; 733 blkptr_t *bp; 734 int error; 735 736 ASSERT(tx == NULL); 737 738 error = dmu_objset_from_ds(ds, &os); 739 if (error != 0) { 740 cmn_err(CE_WARN, "can't open objset %llu, error %d", 741 (unsigned long long)ds->ds_object, error); 742 return (0); 743 } 744 745 zilog = dmu_objset_zil(os); 746 bp = (blkptr_t *)&zilog->zl_header->zh_log; 747 748 /* 749 * Check the first block and determine if it's on a log device 750 * which may have been removed or faulted prior to loading this 751 * pool. If so, there's no point in checking the rest of the log 752 * as its content should have already been synced to the pool. 753 */ 754 if (!BP_IS_HOLE(bp)) { 755 vdev_t *vd; 756 boolean_t valid = B_TRUE; 757 758 spa_config_enter(os->os_spa, SCL_STATE, FTAG, RW_READER); 759 vd = vdev_lookup_top(os->os_spa, DVA_GET_VDEV(&bp->blk_dva[0])); 760 if (vd->vdev_islog && vdev_is_dead(vd)) 761 valid = vdev_log_state_valid(vd); 762 spa_config_exit(os->os_spa, SCL_STATE, FTAG); 763 764 if (!valid) 765 return (0); 766 } 767 768 /* 769 * Because tx == NULL, zil_claim_log_block() will not actually claim 770 * any blocks, but just determine whether it is possible to do so. 771 * In addition to checking the log chain, zil_claim_log_block() 772 * will invoke zio_claim() with a done func of spa_claim_notify(), 773 * which will update spa_max_claim_txg. See spa_load() for details. 774 */ 775 error = zil_parse(zilog, zil_claim_log_block, zil_claim_log_record, tx, 776 zilog->zl_header->zh_claim_txg ? -1ULL : spa_first_txg(os->os_spa)); 777 778 return ((error == ECKSUM || error == ENOENT) ? 0 : error); 779} 780 781static int 782zil_vdev_compare(const void *x1, const void *x2) 783{ 784 const uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev; 785 const uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev; 786 787 if (v1 < v2) 788 return (-1); 789 if (v1 > v2) 790 return (1); 791 792 return (0); 793} 794 795void 796zil_add_block(zilog_t *zilog, const blkptr_t *bp) 797{ 798 avl_tree_t *t = &zilog->zl_vdev_tree; 799 avl_index_t where; 800 zil_vdev_node_t *zv, zvsearch; 801 int ndvas = BP_GET_NDVAS(bp); 802 int i; 803 804 if (zfs_nocacheflush) 805 return; 806 807 ASSERT(zilog->zl_writer); 808 809 /* 810 * Even though we're zl_writer, we still need a lock because the 811 * zl_get_data() callbacks may have dmu_sync() done callbacks 812 * that will run concurrently. 813 */ 814 mutex_enter(&zilog->zl_vdev_lock); 815 for (i = 0; i < ndvas; i++) { 816 zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 817 if (avl_find(t, &zvsearch, &where) == NULL) { 818 zv = kmem_alloc(sizeof (*zv), KM_SLEEP); 819 zv->zv_vdev = zvsearch.zv_vdev; 820 avl_insert(t, zv, where); 821 } 822 } 823 mutex_exit(&zilog->zl_vdev_lock); 824} 825 826static void 827zil_flush_vdevs(zilog_t *zilog) 828{ 829 spa_t *spa = zilog->zl_spa; 830 avl_tree_t *t = &zilog->zl_vdev_tree; 831 void *cookie = NULL; 832 zil_vdev_node_t *zv; 833 zio_t *zio; 834 835 ASSERT(zilog->zl_writer); 836 837 /* 838 * We don't need zl_vdev_lock here because we're the zl_writer, 839 * and all zl_get_data() callbacks are done. 840 */ 841 if (avl_numnodes(t) == 0) 842 return; 843 844 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 845 846 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 847 848 while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) { 849 vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev); 850 if (vd != NULL) 851 zio_flush(zio, vd); 852 kmem_free(zv, sizeof (*zv)); 853 } 854 855 /* 856 * Wait for all the flushes to complete. Not all devices actually 857 * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 858 */ 859 (void) zio_wait(zio); 860 861 spa_config_exit(spa, SCL_STATE, FTAG); 862} 863 864/* 865 * Function called when a log block write completes 866 */ 867static void 868zil_lwb_write_done(zio_t *zio) 869{ 870 lwb_t *lwb = zio->io_private; 871 zilog_t *zilog = lwb->lwb_zilog; 872 dmu_tx_t *tx = lwb->lwb_tx; 873 874 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 875 ASSERT(BP_GET_TYPE(zio->io_bp) == DMU_OT_INTENT_LOG); 876 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 877 ASSERT(BP_GET_BYTEORDER(zio->io_bp) == ZFS_HOST_BYTEORDER); 878 ASSERT(!BP_IS_GANG(zio->io_bp)); 879 ASSERT(!BP_IS_HOLE(zio->io_bp)); 880 ASSERT(BP_GET_FILL(zio->io_bp) == 0); 881 882 /* 883 * Ensure the lwb buffer pointer is cleared before releasing 884 * the txg. If we have had an allocation failure and 885 * the txg is waiting to sync then we want want zil_sync() 886 * to remove the lwb so that it's not picked up as the next new 887 * one in zil_commit_writer(). zil_sync() will only remove 888 * the lwb if lwb_buf is null. 889 */ 890 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 891 mutex_enter(&zilog->zl_lock); 892 lwb->lwb_buf = NULL; 893 lwb->lwb_tx = NULL; 894 mutex_exit(&zilog->zl_lock); 895 896 /* 897 * Now that we've written this log block, we have a stable pointer 898 * to the next block in the chain, so it's OK to let the txg in 899 * which we allocated the next block sync. 900 */ 901 dmu_tx_commit(tx); 902} 903 904/* 905 * Initialize the io for a log block. 906 */ 907static void 908zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 909{ 910 zbookmark_phys_t zb; 911 912 SET_BOOKMARK(&zb, lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET], 913 ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, 914 lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]); 915 916 if (zilog->zl_root_zio == NULL) { 917 zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 918 ZIO_FLAG_CANFAIL); 919 } 920 if (lwb->lwb_zio == NULL) { 921 lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 922 0, &lwb->lwb_blk, lwb->lwb_buf, BP_GET_LSIZE(&lwb->lwb_blk), 923 zil_lwb_write_done, lwb, ZIO_PRIORITY_SYNC_WRITE, 924 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE, &zb); 925 } 926} 927 928/* 929 * Define a limited set of intent log block sizes. 930 * 931 * These must be a multiple of 4KB. Note only the amount used (again 932 * aligned to 4KB) actually gets written. However, we can't always just 933 * allocate SPA_OLD_MAXBLOCKSIZE as the slog space could be exhausted. 934 */ 935uint64_t zil_block_buckets[] = { 936 4096, /* non TX_WRITE */ 937 8192+4096, /* data base */ 938 32*1024 + 4096, /* NFS writes */ 939 UINT64_MAX 940}; 941 942/* 943 * Use the slog as long as the logbias is 'latency' and the current commit size 944 * is less than the limit or the total list size is less than 2X the limit. 945 * Limit checking is disabled by setting zil_slog_limit to UINT64_MAX. 946 */ 947uint64_t zil_slog_limit = 1024 * 1024; 948#define USE_SLOG(zilog) (((zilog)->zl_logbias == ZFS_LOGBIAS_LATENCY) && \ 949 (((zilog)->zl_cur_used < zil_slog_limit) || \ 950 ((zilog)->zl_itx_list_sz < (zil_slog_limit << 1)))) 951 952/* 953 * Start a log block write and advance to the next log block. 954 * Calls are serialized. 955 */ 956static lwb_t * 957zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 958{ 959 lwb_t *nlwb = NULL; 960 zil_chain_t *zilc; 961 spa_t *spa = zilog->zl_spa; 962 blkptr_t *bp; 963 dmu_tx_t *tx; 964 uint64_t txg; 965 uint64_t zil_blksz, wsz; 966 int i, error; 967 968 if (BP_GET_CHECKSUM(&lwb->lwb_blk) == ZIO_CHECKSUM_ZILOG2) { 969 zilc = (zil_chain_t *)lwb->lwb_buf; 970 bp = &zilc->zc_next_blk; 971 } else { 972 zilc = (zil_chain_t *)(lwb->lwb_buf + lwb->lwb_sz); 973 bp = &zilc->zc_next_blk; 974 } 975 976 ASSERT(lwb->lwb_nused <= lwb->lwb_sz); 977 978 /* 979 * Allocate the next block and save its address in this block 980 * before writing it in order to establish the log chain. 981 * Note that if the allocation of nlwb synced before we wrote 982 * the block that points at it (lwb), we'd leak it if we crashed. 983 * Therefore, we don't do dmu_tx_commit() until zil_lwb_write_done(). 984 * We dirty the dataset to ensure that zil_sync() will be called 985 * to clean up in the event of allocation failure or I/O failure. 986 */ 987 tx = dmu_tx_create(zilog->zl_os); 988 VERIFY(dmu_tx_assign(tx, TXG_WAIT) == 0); 989 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 990 txg = dmu_tx_get_txg(tx); 991 992 lwb->lwb_tx = tx; 993 994 /* 995 * Log blocks are pre-allocated. Here we select the size of the next 996 * block, based on size used in the last block. 997 * - first find the smallest bucket that will fit the block from a 998 * limited set of block sizes. This is because it's faster to write 999 * blocks allocated from the same metaslab as they are adjacent or 1000 * close. 1001 * - next find the maximum from the new suggested size and an array of 1002 * previous sizes. This lessens a picket fence effect of wrongly 1003 * guesssing the size if we have a stream of say 2k, 64k, 2k, 64k 1004 * requests. 1005 * 1006 * Note we only write what is used, but we can't just allocate 1007 * the maximum block size because we can exhaust the available 1008 * pool log space. 1009 */ 1010 zil_blksz = zilog->zl_cur_used + sizeof (zil_chain_t); 1011 for (i = 0; zil_blksz > zil_block_buckets[i]; i++) 1012 continue; 1013 zil_blksz = zil_block_buckets[i]; 1014 if (zil_blksz == UINT64_MAX) 1015 zil_blksz = SPA_OLD_MAXBLOCKSIZE; 1016 zilog->zl_prev_blks[zilog->zl_prev_rotor] = zil_blksz; 1017 for (i = 0; i < ZIL_PREV_BLKS; i++) 1018 zil_blksz = MAX(zil_blksz, zilog->zl_prev_blks[i]); 1019 zilog->zl_prev_rotor = (zilog->zl_prev_rotor + 1) & (ZIL_PREV_BLKS - 1); 1020 1021 BP_ZERO(bp); 1022 /* pass the old blkptr in order to spread log blocks across devs */ 1023 error = zio_alloc_zil(spa, txg, bp, &lwb->lwb_blk, zil_blksz, 1024 USE_SLOG(zilog)); 1025 if (error == 0) { 1026 ASSERT3U(bp->blk_birth, ==, txg); 1027 bp->blk_cksum = lwb->lwb_blk.blk_cksum; 1028 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 1029 1030 /* 1031 * Allocate a new log write buffer (lwb). 1032 */ 1033 nlwb = zil_alloc_lwb(zilog, bp, txg); 1034 1035 /* Record the block for later vdev flushing */ 1036 zil_add_block(zilog, &lwb->lwb_blk); 1037 } 1038 1039 if (BP_GET_CHECKSUM(&lwb->lwb_blk) == ZIO_CHECKSUM_ZILOG2) { 1040 /* For Slim ZIL only write what is used. */ 1041 wsz = P2ROUNDUP_TYPED(lwb->lwb_nused, ZIL_MIN_BLKSZ, uint64_t); 1042 ASSERT3U(wsz, <=, lwb->lwb_sz); 1043 zio_shrink(lwb->lwb_zio, wsz); 1044 1045 } else { 1046 wsz = lwb->lwb_sz; 1047 } 1048 1049 zilc->zc_pad = 0; 1050 zilc->zc_nused = lwb->lwb_nused; 1051 zilc->zc_eck.zec_cksum = lwb->lwb_blk.blk_cksum; 1052 1053 /* 1054 * clear unused data for security 1055 */ 1056 bzero(lwb->lwb_buf + lwb->lwb_nused, wsz - lwb->lwb_nused); 1057 1058 zio_nowait(lwb->lwb_zio); /* Kick off the write for the old log block */ 1059 1060 /* 1061 * If there was an allocation failure then nlwb will be null which 1062 * forces a txg_wait_synced(). 1063 */ 1064 return (nlwb); 1065} 1066 1067static lwb_t * 1068zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 1069{ 1070 lr_t *lrc = &itx->itx_lr; /* common log record */ 1071 lr_write_t *lrw = (lr_write_t *)lrc; 1072 char *lr_buf; 1073 uint64_t txg = lrc->lrc_txg; 1074 uint64_t reclen = lrc->lrc_reclen; 1075 uint64_t dlen = 0; 1076 1077 if (lwb == NULL) 1078 return (NULL); 1079 1080 ASSERT(lwb->lwb_buf != NULL); 1081 1082 if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 1083 dlen = P2ROUNDUP_TYPED( 1084 lrw->lr_length, sizeof (uint64_t), uint64_t); 1085 1086 zilog->zl_cur_used += (reclen + dlen); 1087 1088 zil_lwb_write_init(zilog, lwb); 1089 1090 /* 1091 * If this record won't fit in the current log block, start a new one. 1092 */ 1093 if (lwb->lwb_nused + reclen + dlen > lwb->lwb_sz) { 1094 lwb = zil_lwb_write_start(zilog, lwb); 1095 if (lwb == NULL) 1096 return (NULL); 1097 zil_lwb_write_init(zilog, lwb); 1098 ASSERT(LWB_EMPTY(lwb)); 1099 if (lwb->lwb_nused + reclen + dlen > lwb->lwb_sz) { 1100 txg_wait_synced(zilog->zl_dmu_pool, txg); 1101 return (lwb); 1102 } 1103 } 1104 1105 lr_buf = lwb->lwb_buf + lwb->lwb_nused; 1106 bcopy(lrc, lr_buf, reclen); 1107 lrc = (lr_t *)lr_buf; 1108 lrw = (lr_write_t *)lrc; 1109 1110 /* 1111 * If it's a write, fetch the data or get its blkptr as appropriate. 1112 */ 1113 if (lrc->lrc_txtype == TX_WRITE) { 1114 if (txg > spa_freeze_txg(zilog->zl_spa)) 1115 txg_wait_synced(zilog->zl_dmu_pool, txg); 1116 if (itx->itx_wr_state != WR_COPIED) { 1117 char *dbuf; 1118 int error; 1119 1120 if (dlen) { 1121 ASSERT(itx->itx_wr_state == WR_NEED_COPY); 1122 dbuf = lr_buf + reclen; 1123 lrw->lr_common.lrc_reclen += dlen; 1124 } else { 1125 ASSERT(itx->itx_wr_state == WR_INDIRECT); 1126 dbuf = NULL; 1127 } 1128 error = zilog->zl_get_data( 1129 itx->itx_private, lrw, dbuf, lwb->lwb_zio); 1130 if (error == EIO) { 1131 txg_wait_synced(zilog->zl_dmu_pool, txg); 1132 return (lwb); 1133 } 1134 if (error != 0) { 1135 ASSERT(error == ENOENT || error == EEXIST || 1136 error == EALREADY); 1137 return (lwb); 1138 } 1139 } 1140 } 1141 1142 /* 1143 * We're actually making an entry, so update lrc_seq to be the 1144 * log record sequence number. Note that this is generally not 1145 * equal to the itx sequence number because not all transactions 1146 * are synchronous, and sometimes spa_sync() gets there first. 1147 */ 1148 lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 1149 lwb->lwb_nused += reclen + dlen; 1150 lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 1151 ASSERT3U(lwb->lwb_nused, <=, lwb->lwb_sz); 1152 ASSERT0(P2PHASE(lwb->lwb_nused, sizeof (uint64_t))); 1153 1154 return (lwb); 1155} 1156 1157itx_t * 1158zil_itx_create(uint64_t txtype, size_t lrsize) 1159{ 1160 itx_t *itx; 1161 1162 lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 1163 1164 itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 1165 itx->itx_lr.lrc_txtype = txtype; 1166 itx->itx_lr.lrc_reclen = lrsize; 1167 itx->itx_sod = lrsize; /* if write & WR_NEED_COPY will be increased */ 1168 itx->itx_lr.lrc_seq = 0; /* defensive */ 1169 itx->itx_sync = B_TRUE; /* default is synchronous */ 1170 1171 return (itx); 1172} 1173 1174void 1175zil_itx_destroy(itx_t *itx) 1176{ 1177 kmem_free(itx, offsetof(itx_t, itx_lr) + itx->itx_lr.lrc_reclen); 1178} 1179 1180/* 1181 * Free up the sync and async itxs. The itxs_t has already been detached 1182 * so no locks are needed. 1183 */ 1184static void 1185zil_itxg_clean(itxs_t *itxs) 1186{ 1187 itx_t *itx; 1188 list_t *list; 1189 avl_tree_t *t; 1190 void *cookie; 1191 itx_async_node_t *ian; 1192 1193 list = &itxs->i_sync_list; 1194 while ((itx = list_head(list)) != NULL) { 1195 list_remove(list, itx); 1196 kmem_free(itx, offsetof(itx_t, itx_lr) + 1197 itx->itx_lr.lrc_reclen); 1198 } 1199 1200 cookie = NULL; 1201 t = &itxs->i_async_tree; 1202 while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) { 1203 list = &ian->ia_list; 1204 while ((itx = list_head(list)) != NULL) { 1205 list_remove(list, itx); 1206 kmem_free(itx, offsetof(itx_t, itx_lr) + 1207 itx->itx_lr.lrc_reclen); 1208 } 1209 list_destroy(list); 1210 kmem_free(ian, sizeof (itx_async_node_t)); 1211 } 1212 avl_destroy(t); 1213 1214 kmem_free(itxs, sizeof (itxs_t)); 1215} 1216 1217static int 1218zil_aitx_compare(const void *x1, const void *x2) 1219{ 1220 const uint64_t o1 = ((itx_async_node_t *)x1)->ia_foid; 1221 const uint64_t o2 = ((itx_async_node_t *)x2)->ia_foid; 1222 1223 if (o1 < o2) 1224 return (-1); 1225 if (o1 > o2) 1226 return (1); 1227 1228 return (0); 1229} 1230 1231/* 1232 * Remove all async itx with the given oid. 1233 */ 1234static void 1235zil_remove_async(zilog_t *zilog, uint64_t oid) 1236{ 1237 uint64_t otxg, txg; 1238 itx_async_node_t *ian; 1239 avl_tree_t *t; 1240 avl_index_t where; 1241 list_t clean_list; 1242 itx_t *itx; 1243 1244 ASSERT(oid != 0); 1245 list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 1246 1247 if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 1248 otxg = ZILTEST_TXG; 1249 else 1250 otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 1251 1252 for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 1253 itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 1254 1255 mutex_enter(&itxg->itxg_lock); 1256 if (itxg->itxg_txg != txg) { 1257 mutex_exit(&itxg->itxg_lock); 1258 continue; 1259 } 1260 1261 /* 1262 * Locate the object node and append its list. 1263 */ 1264 t = &itxg->itxg_itxs->i_async_tree; 1265 ian = avl_find(t, &oid, &where); 1266 if (ian != NULL) 1267 list_move_tail(&clean_list, &ian->ia_list); 1268 mutex_exit(&itxg->itxg_lock); 1269 } 1270 while ((itx = list_head(&clean_list)) != NULL) { 1271 list_remove(&clean_list, itx); 1272 kmem_free(itx, offsetof(itx_t, itx_lr) + 1273 itx->itx_lr.lrc_reclen); 1274 } 1275 list_destroy(&clean_list); 1276} 1277 1278void 1279zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 1280{ 1281 uint64_t txg; 1282 itxg_t *itxg; 1283 itxs_t *itxs, *clean = NULL; 1284 1285 /* 1286 * Object ids can be re-instantiated in the next txg so 1287 * remove any async transactions to avoid future leaks. 1288 * This can happen if a fsync occurs on the re-instantiated 1289 * object for a WR_INDIRECT or WR_NEED_COPY write, which gets 1290 * the new file data and flushes a write record for the old object. 1291 */ 1292 if ((itx->itx_lr.lrc_txtype & ~TX_CI) == TX_REMOVE) 1293 zil_remove_async(zilog, itx->itx_oid); 1294 1295 /* 1296 * Ensure the data of a renamed file is committed before the rename. 1297 */ 1298 if ((itx->itx_lr.lrc_txtype & ~TX_CI) == TX_RENAME) 1299 zil_async_to_sync(zilog, itx->itx_oid); 1300 1301 if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) 1302 txg = ZILTEST_TXG; 1303 else 1304 txg = dmu_tx_get_txg(tx); 1305 1306 itxg = &zilog->zl_itxg[txg & TXG_MASK]; 1307 mutex_enter(&itxg->itxg_lock); 1308 itxs = itxg->itxg_itxs; 1309 if (itxg->itxg_txg != txg) { 1310 if (itxs != NULL) { 1311 /* 1312 * The zil_clean callback hasn't got around to cleaning 1313 * this itxg. Save the itxs for release below. 1314 * This should be rare. 1315 */ 1316 atomic_add_64(&zilog->zl_itx_list_sz, -itxg->itxg_sod); 1317 itxg->itxg_sod = 0; 1318 clean = itxg->itxg_itxs; 1319 } 1320 ASSERT(itxg->itxg_sod == 0); 1321 itxg->itxg_txg = txg; 1322 itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_SLEEP); 1323 1324 list_create(&itxs->i_sync_list, sizeof (itx_t), 1325 offsetof(itx_t, itx_node)); 1326 avl_create(&itxs->i_async_tree, zil_aitx_compare, 1327 sizeof (itx_async_node_t), 1328 offsetof(itx_async_node_t, ia_node)); 1329 } 1330 if (itx->itx_sync) { 1331 list_insert_tail(&itxs->i_sync_list, itx); 1332 atomic_add_64(&zilog->zl_itx_list_sz, itx->itx_sod); 1333 itxg->itxg_sod += itx->itx_sod; 1334 } else { 1335 avl_tree_t *t = &itxs->i_async_tree; 1336 uint64_t foid = ((lr_ooo_t *)&itx->itx_lr)->lr_foid; 1337 itx_async_node_t *ian; 1338 avl_index_t where; 1339 1340 ian = avl_find(t, &foid, &where); 1341 if (ian == NULL) { 1342 ian = kmem_alloc(sizeof (itx_async_node_t), KM_SLEEP); 1343 list_create(&ian->ia_list, sizeof (itx_t), 1344 offsetof(itx_t, itx_node)); 1345 ian->ia_foid = foid; 1346 avl_insert(t, ian, where); 1347 } 1348 list_insert_tail(&ian->ia_list, itx); 1349 } 1350 1351 itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 1352 zilog_dirty(zilog, txg); 1353 mutex_exit(&itxg->itxg_lock); 1354 1355 /* Release the old itxs now we've dropped the lock */ 1356 if (clean != NULL) 1357 zil_itxg_clean(clean); 1358} 1359 1360/* 1361 * If there are any in-memory intent log transactions which have now been 1362 * synced then start up a taskq to free them. We should only do this after we 1363 * have written out the uberblocks (i.e. txg has been comitted) so that 1364 * don't inadvertently clean out in-memory log records that would be required 1365 * by zil_commit(). 1366 */ 1367void 1368zil_clean(zilog_t *zilog, uint64_t synced_txg) 1369{ 1370 itxg_t *itxg = &zilog->zl_itxg[synced_txg & TXG_MASK]; 1371 itxs_t *clean_me; 1372 1373 mutex_enter(&itxg->itxg_lock); 1374 if (itxg->itxg_itxs == NULL || itxg->itxg_txg == ZILTEST_TXG) { 1375 mutex_exit(&itxg->itxg_lock); 1376 return; 1377 } 1378 ASSERT3U(itxg->itxg_txg, <=, synced_txg); 1379 ASSERT(itxg->itxg_txg != 0); 1380 ASSERT(zilog->zl_clean_taskq != NULL); 1381 atomic_add_64(&zilog->zl_itx_list_sz, -itxg->itxg_sod); 1382 itxg->itxg_sod = 0; 1383 clean_me = itxg->itxg_itxs; 1384 itxg->itxg_itxs = NULL; 1385 itxg->itxg_txg = 0; 1386 mutex_exit(&itxg->itxg_lock); 1387 /* 1388 * Preferably start a task queue to free up the old itxs but 1389 * if taskq_dispatch can't allocate resources to do that then 1390 * free it in-line. This should be rare. Note, using TQ_SLEEP 1391 * created a bad performance problem. 1392 */ 1393 if (taskq_dispatch(zilog->zl_clean_taskq, 1394 (void (*)(void *))zil_itxg_clean, clean_me, TQ_NOSLEEP) == 0) 1395 zil_itxg_clean(clean_me); 1396} 1397 1398/* 1399 * Get the list of itxs to commit into zl_itx_commit_list. 1400 */ 1401static void 1402zil_get_commit_list(zilog_t *zilog) 1403{ 1404 uint64_t otxg, txg; 1405 list_t *commit_list = &zilog->zl_itx_commit_list; 1406 uint64_t push_sod = 0; 1407 1408 if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 1409 otxg = ZILTEST_TXG; 1410 else 1411 otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 1412 1413 /* 1414 * This is inherently racy, since there is nothing to prevent 1415 * the last synced txg from changing. That's okay since we'll 1416 * only commit things in the future. 1417 */ 1418 for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 1419 itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 1420 1421 mutex_enter(&itxg->itxg_lock); 1422 if (itxg->itxg_txg != txg) { 1423 mutex_exit(&itxg->itxg_lock); 1424 continue; 1425 } 1426 1427 /* 1428 * If we're adding itx records to the zl_itx_commit_list, 1429 * then the zil better be dirty in this "txg". We can assert 1430 * that here since we're holding the itxg_lock which will 1431 * prevent spa_sync from cleaning it. Once we add the itxs 1432 * to the zl_itx_commit_list we must commit it to disk even 1433 * if it's unnecessary (i.e. the txg was synced). 1434 */ 1435 ASSERT(zilog_is_dirty_in_txg(zilog, txg) || 1436 spa_freeze_txg(zilog->zl_spa) != UINT64_MAX); 1437 list_move_tail(commit_list, &itxg->itxg_itxs->i_sync_list); 1438 push_sod += itxg->itxg_sod; 1439 itxg->itxg_sod = 0; 1440 1441 mutex_exit(&itxg->itxg_lock); 1442 } 1443 atomic_add_64(&zilog->zl_itx_list_sz, -push_sod); 1444} 1445 1446/* 1447 * Move the async itxs for a specified object to commit into sync lists. 1448 */ 1449void 1450zil_async_to_sync(zilog_t *zilog, uint64_t foid) 1451{ 1452 uint64_t otxg, txg; 1453 itx_async_node_t *ian; 1454 avl_tree_t *t; 1455 avl_index_t where; 1456 1457 if (spa_freeze_txg(zilog->zl_spa) != UINT64_MAX) /* ziltest support */ 1458 otxg = ZILTEST_TXG; 1459 else 1460 otxg = spa_last_synced_txg(zilog->zl_spa) + 1; 1461 1462 /* 1463 * This is inherently racy, since there is nothing to prevent 1464 * the last synced txg from changing. 1465 */ 1466 for (txg = otxg; txg < (otxg + TXG_CONCURRENT_STATES); txg++) { 1467 itxg_t *itxg = &zilog->zl_itxg[txg & TXG_MASK]; 1468 1469 mutex_enter(&itxg->itxg_lock); 1470 if (itxg->itxg_txg != txg) { 1471 mutex_exit(&itxg->itxg_lock); 1472 continue; 1473 } 1474 1475 /* 1476 * If a foid is specified then find that node and append its 1477 * list. Otherwise walk the tree appending all the lists 1478 * to the sync list. We add to the end rather than the 1479 * beginning to ensure the create has happened. 1480 */ 1481 t = &itxg->itxg_itxs->i_async_tree; 1482 if (foid != 0) { 1483 ian = avl_find(t, &foid, &where); 1484 if (ian != NULL) { 1485 list_move_tail(&itxg->itxg_itxs->i_sync_list, 1486 &ian->ia_list); 1487 } 1488 } else { 1489 void *cookie = NULL; 1490 1491 while ((ian = avl_destroy_nodes(t, &cookie)) != NULL) { 1492 list_move_tail(&itxg->itxg_itxs->i_sync_list, 1493 &ian->ia_list); 1494 list_destroy(&ian->ia_list); 1495 kmem_free(ian, sizeof (itx_async_node_t)); 1496 } 1497 } 1498 mutex_exit(&itxg->itxg_lock); 1499 } 1500} 1501 1502static void 1503zil_commit_writer(zilog_t *zilog) 1504{ 1505 uint64_t txg; 1506 itx_t *itx; 1507 lwb_t *lwb; 1508 spa_t *spa = zilog->zl_spa; 1509 int error = 0; 1510 1511 ASSERT(zilog->zl_root_zio == NULL); 1512 1513 mutex_exit(&zilog->zl_lock); 1514 1515 zil_get_commit_list(zilog); 1516 1517 /* 1518 * Return if there's nothing to commit before we dirty the fs by 1519 * calling zil_create(). 1520 */ 1521 if (list_head(&zilog->zl_itx_commit_list) == NULL) { 1522 mutex_enter(&zilog->zl_lock); 1523 return; 1524 } 1525 1526 if (zilog->zl_suspend) { 1527 lwb = NULL; 1528 } else { 1529 lwb = list_tail(&zilog->zl_lwb_list); 1530 if (lwb == NULL) 1531 lwb = zil_create(zilog); 1532 } 1533 1534 DTRACE_PROBE1(zil__cw1, zilog_t *, zilog); 1535 while (itx = list_head(&zilog->zl_itx_commit_list)) { 1536 txg = itx->itx_lr.lrc_txg; 1537 ASSERT3U(txg, !=, 0); 1538 1539 /* 1540 * This is inherently racy and may result in us writing 1541 * out a log block for a txg that was just synced. This is 1542 * ok since we'll end cleaning up that log block the next 1543 * time we call zil_sync(). 1544 */ 1545 if (txg > spa_last_synced_txg(spa) || txg > spa_freeze_txg(spa)) 1546 lwb = zil_lwb_commit(zilog, itx, lwb); 1547 list_remove(&zilog->zl_itx_commit_list, itx); 1548 kmem_free(itx, offsetof(itx_t, itx_lr) 1549 + itx->itx_lr.lrc_reclen); 1550 } 1551 DTRACE_PROBE1(zil__cw2, zilog_t *, zilog); 1552 1553 /* write the last block out */ 1554 if (lwb != NULL && lwb->lwb_zio != NULL) 1555 lwb = zil_lwb_write_start(zilog, lwb); 1556 1557 zilog->zl_cur_used = 0; 1558 1559 /* 1560 * Wait if necessary for the log blocks to be on stable storage. 1561 */ 1562 if (zilog->zl_root_zio) { 1563 error = zio_wait(zilog->zl_root_zio); 1564 zilog->zl_root_zio = NULL; 1565 zil_flush_vdevs(zilog); 1566 } 1567 1568 if (error || lwb == NULL) 1569 txg_wait_synced(zilog->zl_dmu_pool, 0); 1570 1571 mutex_enter(&zilog->zl_lock); 1572 1573 /* 1574 * Remember the highest committed log sequence number for ztest. 1575 * We only update this value when all the log writes succeeded, 1576 * because ztest wants to ASSERT that it got the whole log chain. 1577 */ 1578 if (error == 0 && lwb != NULL) 1579 zilog->zl_commit_lr_seq = zilog->zl_lr_seq; 1580} 1581 1582/* 1583 * Commit zfs transactions to stable storage. 1584 * If foid is 0 push out all transactions, otherwise push only those 1585 * for that object or might reference that object. 1586 * 1587 * itxs are committed in batches. In a heavily stressed zil there will be 1588 * a commit writer thread who is writing out a bunch of itxs to the log 1589 * for a set of committing threads (cthreads) in the same batch as the writer. 1590 * Those cthreads are all waiting on the same cv for that batch. 1591 * 1592 * There will also be a different and growing batch of threads that are 1593 * waiting to commit (qthreads). When the committing batch completes 1594 * a transition occurs such that the cthreads exit and the qthreads become 1595 * cthreads. One of the new cthreads becomes the writer thread for the 1596 * batch. Any new threads arriving become new qthreads. 1597 * 1598 * Only 2 condition variables are needed and there's no transition 1599 * between the two cvs needed. They just flip-flop between qthreads 1600 * and cthreads. 1601 * 1602 * Using this scheme we can efficiently wakeup up only those threads 1603 * that have been committed. 1604 */ 1605void 1606zil_commit(zilog_t *zilog, uint64_t foid) 1607{ 1608 uint64_t mybatch; 1609 1610 if (zilog->zl_sync == ZFS_SYNC_DISABLED) 1611 return; 1612 1613 /* move the async itxs for the foid to the sync queues */ 1614 zil_async_to_sync(zilog, foid); 1615 1616 mutex_enter(&zilog->zl_lock); 1617 mybatch = zilog->zl_next_batch; 1618 while (zilog->zl_writer) { 1619 cv_wait(&zilog->zl_cv_batch[mybatch & 1], &zilog->zl_lock); 1620 if (mybatch <= zilog->zl_com_batch) { 1621 mutex_exit(&zilog->zl_lock); 1622 return; 1623 } 1624 } 1625 1626 zilog->zl_next_batch++; 1627 zilog->zl_writer = B_TRUE; 1628 zil_commit_writer(zilog); 1629 zilog->zl_com_batch = mybatch; 1630 zilog->zl_writer = B_FALSE; 1631 mutex_exit(&zilog->zl_lock); 1632 1633 /* wake up one thread to become the next writer */ 1634 cv_signal(&zilog->zl_cv_batch[(mybatch+1) & 1]); 1635 1636 /* wake up all threads waiting for this batch to be committed */ 1637 cv_broadcast(&zilog->zl_cv_batch[mybatch & 1]); 1638} 1639 1640/* 1641 * Called in syncing context to free committed log blocks and update log header. 1642 */ 1643void 1644zil_sync(zilog_t *zilog, dmu_tx_t *tx) 1645{ 1646 zil_header_t *zh = zil_header_in_syncing_context(zilog); 1647 uint64_t txg = dmu_tx_get_txg(tx); 1648 spa_t *spa = zilog->zl_spa; 1649 uint64_t *replayed_seq = &zilog->zl_replayed_seq[txg & TXG_MASK]; 1650 lwb_t *lwb; 1651 1652 /* 1653 * We don't zero out zl_destroy_txg, so make sure we don't try 1654 * to destroy it twice. 1655 */ 1656 if (spa_sync_pass(spa) != 1) 1657 return; 1658 1659 mutex_enter(&zilog->zl_lock); 1660 1661 ASSERT(zilog->zl_stop_sync == 0); 1662 1663 if (*replayed_seq != 0) { 1664 ASSERT(zh->zh_replay_seq < *replayed_seq); 1665 zh->zh_replay_seq = *replayed_seq; 1666 *replayed_seq = 0; 1667 } 1668 1669 if (zilog->zl_destroy_txg == txg) { 1670 blkptr_t blk = zh->zh_log; 1671 1672 ASSERT(list_head(&zilog->zl_lwb_list) == NULL); 1673 1674 bzero(zh, sizeof (zil_header_t)); 1675 bzero(zilog->zl_replayed_seq, sizeof (zilog->zl_replayed_seq)); 1676 1677 if (zilog->zl_keep_first) { 1678 /* 1679 * If this block was part of log chain that couldn't 1680 * be claimed because a device was missing during 1681 * zil_claim(), but that device later returns, 1682 * then this block could erroneously appear valid. 1683 * To guard against this, assign a new GUID to the new 1684 * log chain so it doesn't matter what blk points to. 1685 */ 1686 zil_init_log_chain(zilog, &blk); 1687 zh->zh_log = blk; 1688 } 1689 } 1690 1691 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1692 zh->zh_log = lwb->lwb_blk; 1693 if (lwb->lwb_buf != NULL || lwb->lwb_max_txg > txg) 1694 break; 1695 list_remove(&zilog->zl_lwb_list, lwb); 1696 zio_free_zil(spa, txg, &lwb->lwb_blk); 1697 kmem_cache_free(zil_lwb_cache, lwb); 1698 1699 /* 1700 * If we don't have anything left in the lwb list then 1701 * we've had an allocation failure and we need to zero 1702 * out the zil_header blkptr so that we don't end 1703 * up freeing the same block twice. 1704 */ 1705 if (list_head(&zilog->zl_lwb_list) == NULL) 1706 BP_ZERO(&zh->zh_log); 1707 } 1708 mutex_exit(&zilog->zl_lock); 1709} 1710 1711void 1712zil_init(void) 1713{ 1714 zil_lwb_cache = kmem_cache_create("zil_lwb_cache", 1715 sizeof (struct lwb), 0, NULL, NULL, NULL, NULL, NULL, 0); 1716} 1717 1718void 1719zil_fini(void) 1720{ 1721 kmem_cache_destroy(zil_lwb_cache); 1722} 1723 1724void 1725zil_set_sync(zilog_t *zilog, uint64_t sync) 1726{ 1727 zilog->zl_sync = sync; 1728} 1729 1730void 1731zil_set_logbias(zilog_t *zilog, uint64_t logbias) 1732{ 1733 zilog->zl_logbias = logbias; 1734} 1735 1736zilog_t * 1737zil_alloc(objset_t *os, zil_header_t *zh_phys) 1738{ 1739 zilog_t *zilog; 1740 1741 zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); 1742 1743 zilog->zl_header = zh_phys; 1744 zilog->zl_os = os; 1745 zilog->zl_spa = dmu_objset_spa(os); 1746 zilog->zl_dmu_pool = dmu_objset_pool(os); 1747 zilog->zl_destroy_txg = TXG_INITIAL - 1; 1748 zilog->zl_logbias = dmu_objset_logbias(os); 1749 zilog->zl_sync = dmu_objset_syncprop(os); 1750 zilog->zl_next_batch = 1; 1751 1752 mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL); 1753 1754 for (int i = 0; i < TXG_SIZE; i++) { 1755 mutex_init(&zilog->zl_itxg[i].itxg_lock, NULL, 1756 MUTEX_DEFAULT, NULL); 1757 } 1758 1759 list_create(&zilog->zl_lwb_list, sizeof (lwb_t), 1760 offsetof(lwb_t, lwb_node)); 1761 1762 list_create(&zilog->zl_itx_commit_list, sizeof (itx_t), 1763 offsetof(itx_t, itx_node)); 1764 1765 mutex_init(&zilog->zl_vdev_lock, NULL, MUTEX_DEFAULT, NULL); 1766 1767 avl_create(&zilog->zl_vdev_tree, zil_vdev_compare, 1768 sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node)); 1769 1770 cv_init(&zilog->zl_cv_writer, NULL, CV_DEFAULT, NULL); 1771 cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL); 1772 cv_init(&zilog->zl_cv_batch[0], NULL, CV_DEFAULT, NULL); 1773 cv_init(&zilog->zl_cv_batch[1], NULL, CV_DEFAULT, NULL); 1774 1775 return (zilog); 1776} 1777 1778void 1779zil_free(zilog_t *zilog) 1780{ 1781 zilog->zl_stop_sync = 1; 1782 1783 ASSERT0(zilog->zl_suspend); 1784 ASSERT0(zilog->zl_suspending); 1785 1786 ASSERT(list_is_empty(&zilog->zl_lwb_list)); 1787 list_destroy(&zilog->zl_lwb_list); 1788 1789 avl_destroy(&zilog->zl_vdev_tree); 1790 mutex_destroy(&zilog->zl_vdev_lock); 1791 1792 ASSERT(list_is_empty(&zilog->zl_itx_commit_list)); 1793 list_destroy(&zilog->zl_itx_commit_list); 1794 1795 for (int i = 0; i < TXG_SIZE; i++) { 1796 /* 1797 * It's possible for an itx to be generated that doesn't dirty 1798 * a txg (e.g. ztest TX_TRUNCATE). So there's no zil_clean() 1799 * callback to remove the entry. We remove those here. 1800 * 1801 * Also free up the ziltest itxs. 1802 */ 1803 if (zilog->zl_itxg[i].itxg_itxs) 1804 zil_itxg_clean(zilog->zl_itxg[i].itxg_itxs); 1805 mutex_destroy(&zilog->zl_itxg[i].itxg_lock); 1806 } 1807 1808 mutex_destroy(&zilog->zl_lock); 1809 1810 cv_destroy(&zilog->zl_cv_writer); 1811 cv_destroy(&zilog->zl_cv_suspend); 1812 cv_destroy(&zilog->zl_cv_batch[0]); 1813 cv_destroy(&zilog->zl_cv_batch[1]); 1814 1815 kmem_free(zilog, sizeof (zilog_t)); 1816} 1817 1818/* 1819 * Open an intent log. 1820 */ 1821zilog_t * 1822zil_open(objset_t *os, zil_get_data_t *get_data) 1823{ 1824 zilog_t *zilog = dmu_objset_zil(os); 1825 1826 ASSERT(zilog->zl_clean_taskq == NULL); 1827 ASSERT(zilog->zl_get_data == NULL); 1828 ASSERT(list_is_empty(&zilog->zl_lwb_list)); 1829 1830 zilog->zl_get_data = get_data; 1831 zilog->zl_clean_taskq = taskq_create("zil_clean", 1, minclsyspri, 1832 2, 2, TASKQ_PREPOPULATE); 1833 1834 return (zilog); 1835} 1836 1837/* 1838 * Close an intent log. 1839 */ 1840void 1841zil_close(zilog_t *zilog) 1842{ 1843 lwb_t *lwb; 1844 uint64_t txg = 0; 1845 1846 zil_commit(zilog, 0); /* commit all itx */ 1847 1848 /* 1849 * The lwb_max_txg for the stubby lwb will reflect the last activity 1850 * for the zil. After a txg_wait_synced() on the txg we know all the 1851 * callbacks have occurred that may clean the zil. Only then can we 1852 * destroy the zl_clean_taskq. 1853 */ 1854 mutex_enter(&zilog->zl_lock); 1855 lwb = list_tail(&zilog->zl_lwb_list); 1856 if (lwb != NULL) 1857 txg = lwb->lwb_max_txg; 1858 mutex_exit(&zilog->zl_lock); 1859 if (txg) 1860 txg_wait_synced(zilog->zl_dmu_pool, txg); 1861 1862 if (zilog_is_dirty(zilog)) 1863 zfs_dbgmsg("zil (%p) is dirty, txg %llu", zilog, txg); 1864 VERIFY(!zilog_is_dirty(zilog)); 1865 1866 taskq_destroy(zilog->zl_clean_taskq); 1867 zilog->zl_clean_taskq = NULL; 1868 zilog->zl_get_data = NULL; 1869 1870 /* 1871 * We should have only one LWB left on the list; remove it now. 1872 */ 1873 mutex_enter(&zilog->zl_lock); 1874 lwb = list_head(&zilog->zl_lwb_list); 1875 if (lwb != NULL) { 1876 ASSERT(lwb == list_tail(&zilog->zl_lwb_list)); 1877 list_remove(&zilog->zl_lwb_list, lwb); 1878 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 1879 kmem_cache_free(zil_lwb_cache, lwb); 1880 } 1881 mutex_exit(&zilog->zl_lock); 1882} 1883 1884static char *suspend_tag = "zil suspending"; 1885 1886/* 1887 * Suspend an intent log. While in suspended mode, we still honor 1888 * synchronous semantics, but we rely on txg_wait_synced() to do it. 1889 * On old version pools, we suspend the log briefly when taking a 1890 * snapshot so that it will have an empty intent log. 1891 * 1892 * Long holds are not really intended to be used the way we do here -- 1893 * held for such a short time. A concurrent caller of dsl_dataset_long_held() 1894 * could fail. Therefore we take pains to only put a long hold if it is 1895 * actually necessary. Fortunately, it will only be necessary if the 1896 * objset is currently mounted (or the ZVOL equivalent). In that case it 1897 * will already have a long hold, so we are not really making things any worse. 1898 * 1899 * Ideally, we would locate the existing long-holder (i.e. the zfsvfs_t or 1900 * zvol_state_t), and use their mechanism to prevent their hold from being 1901 * dropped (e.g. VFS_HOLD()). However, that would be even more pain for 1902 * very little gain. 1903 * 1904 * if cookiep == NULL, this does both the suspend & resume. 1905 * Otherwise, it returns with the dataset "long held", and the cookie 1906 * should be passed into zil_resume(). 1907 */ 1908int 1909zil_suspend(const char *osname, void **cookiep) 1910{ 1911 objset_t *os; 1912 zilog_t *zilog; 1913 const zil_header_t *zh; 1914 int error; 1915 1916 error = dmu_objset_hold(osname, suspend_tag, &os); 1917 if (error != 0) 1918 return (error); 1919 zilog = dmu_objset_zil(os); 1920 1921 mutex_enter(&zilog->zl_lock); 1922 zh = zilog->zl_header; 1923 1924 if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */ 1925 mutex_exit(&zilog->zl_lock); 1926 dmu_objset_rele(os, suspend_tag); 1927 return (SET_ERROR(EBUSY)); 1928 } 1929 1930 /* 1931 * Don't put a long hold in the cases where we can avoid it. This 1932 * is when there is no cookie so we are doing a suspend & resume 1933 * (i.e. called from zil_vdev_offline()), and there's nothing to do 1934 * for the suspend because it's already suspended, or there's no ZIL. 1935 */ 1936 if (cookiep == NULL && !zilog->zl_suspending && 1937 (zilog->zl_suspend > 0 || BP_IS_HOLE(&zh->zh_log))) { 1938 mutex_exit(&zilog->zl_lock); 1939 dmu_objset_rele(os, suspend_tag); 1940 return (0); 1941 } 1942 1943 dsl_dataset_long_hold(dmu_objset_ds(os), suspend_tag); 1944 dsl_pool_rele(dmu_objset_pool(os), suspend_tag); 1945 1946 zilog->zl_suspend++; 1947 1948 if (zilog->zl_suspend > 1) { 1949 /* 1950 * Someone else is already suspending it. 1951 * Just wait for them to finish. 1952 */ 1953 1954 while (zilog->zl_suspending) 1955 cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); 1956 mutex_exit(&zilog->zl_lock); 1957 1958 if (cookiep == NULL) 1959 zil_resume(os); 1960 else 1961 *cookiep = os; 1962 return (0); 1963 } 1964 1965 /* 1966 * If there is no pointer to an on-disk block, this ZIL must not 1967 * be active (e.g. filesystem not mounted), so there's nothing 1968 * to clean up. 1969 */ 1970 if (BP_IS_HOLE(&zh->zh_log)) { 1971 ASSERT(cookiep != NULL); /* fast path already handled */ 1972 1973 *cookiep = os; 1974 mutex_exit(&zilog->zl_lock); 1975 return (0); 1976 } 1977 1978 zilog->zl_suspending = B_TRUE; 1979 mutex_exit(&zilog->zl_lock); 1980 1981 zil_commit(zilog, 0); 1982 1983 zil_destroy(zilog, B_FALSE); 1984 1985 mutex_enter(&zilog->zl_lock); 1986 zilog->zl_suspending = B_FALSE; 1987 cv_broadcast(&zilog->zl_cv_suspend); 1988 mutex_exit(&zilog->zl_lock); 1989 1990 if (cookiep == NULL) 1991 zil_resume(os); 1992 else 1993 *cookiep = os; 1994 return (0); 1995} 1996 1997void 1998zil_resume(void *cookie) 1999{ 2000 objset_t *os = cookie; 2001 zilog_t *zilog = dmu_objset_zil(os); 2002 2003 mutex_enter(&zilog->zl_lock); 2004 ASSERT(zilog->zl_suspend != 0); 2005 zilog->zl_suspend--; 2006 mutex_exit(&zilog->zl_lock); 2007 dsl_dataset_long_rele(dmu_objset_ds(os), suspend_tag); 2008 dsl_dataset_rele(dmu_objset_ds(os), suspend_tag); 2009} 2010 2011typedef struct zil_replay_arg { 2012 zil_replay_func_t **zr_replay; 2013 void *zr_arg; 2014 boolean_t zr_byteswap; 2015 char *zr_lr; 2016} zil_replay_arg_t; 2017 2018static int 2019zil_replay_error(zilog_t *zilog, lr_t *lr, int error) 2020{ 2021 char name[ZFS_MAX_DATASET_NAME_LEN]; 2022 2023 zilog->zl_replaying_seq--; /* didn't actually replay this one */ 2024 2025 dmu_objset_name(zilog->zl_os, name); 2026 2027 cmn_err(CE_WARN, "ZFS replay transaction error %d, " 2028 "dataset %s, seq 0x%llx, txtype %llu %s\n", error, name, 2029 (u_longlong_t)lr->lrc_seq, 2030 (u_longlong_t)(lr->lrc_txtype & ~TX_CI), 2031 (lr->lrc_txtype & TX_CI) ? "CI" : ""); 2032 2033 return (error); 2034} 2035 2036static int 2037zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) 2038{ 2039 zil_replay_arg_t *zr = zra; 2040 const zil_header_t *zh = zilog->zl_header; 2041 uint64_t reclen = lr->lrc_reclen; 2042 uint64_t txtype = lr->lrc_txtype; 2043 int error = 0; 2044 2045 zilog->zl_replaying_seq = lr->lrc_seq; 2046 2047 if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */ 2048 return (0); 2049 2050 if (lr->lrc_txg < claim_txg) /* already committed */ 2051 return (0); 2052 2053 /* Strip case-insensitive bit, still present in log record */ 2054 txtype &= ~TX_CI; 2055 2056 if (txtype == 0 || txtype >= TX_MAX_TYPE) 2057 return (zil_replay_error(zilog, lr, EINVAL)); 2058 2059 /* 2060 * If this record type can be logged out of order, the object 2061 * (lr_foid) may no longer exist. That's legitimate, not an error. 2062 */ 2063 if (TX_OOO(txtype)) { 2064 error = dmu_object_info(zilog->zl_os, 2065 ((lr_ooo_t *)lr)->lr_foid, NULL); 2066 if (error == ENOENT || error == EEXIST) 2067 return (0); 2068 } 2069 2070 /* 2071 * Make a copy of the data so we can revise and extend it. 2072 */ 2073 bcopy(lr, zr->zr_lr, reclen); 2074 2075 /* 2076 * If this is a TX_WRITE with a blkptr, suck in the data. 2077 */ 2078 if (txtype == TX_WRITE && reclen == sizeof (lr_write_t)) { 2079 error = zil_read_log_data(zilog, (lr_write_t *)lr, 2080 zr->zr_lr + reclen); 2081 if (error != 0) 2082 return (zil_replay_error(zilog, lr, error)); 2083 } 2084 2085 /* 2086 * The log block containing this lr may have been byteswapped 2087 * so that we can easily examine common fields like lrc_txtype. 2088 * However, the log is a mix of different record types, and only the 2089 * replay vectors know how to byteswap their records. Therefore, if 2090 * the lr was byteswapped, undo it before invoking the replay vector. 2091 */ 2092 if (zr->zr_byteswap) 2093 byteswap_uint64_array(zr->zr_lr, reclen); 2094 2095 /* 2096 * We must now do two things atomically: replay this log record, 2097 * and update the log header sequence number to reflect the fact that 2098 * we did so. At the end of each replay function the sequence number 2099 * is updated if we are in replay mode. 2100 */ 2101 error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, zr->zr_byteswap); 2102 if (error != 0) { 2103 /* 2104 * The DMU's dnode layer doesn't see removes until the txg 2105 * commits, so a subsequent claim can spuriously fail with 2106 * EEXIST. So if we receive any error we try syncing out 2107 * any removes then retry the transaction. Note that we 2108 * specify B_FALSE for byteswap now, so we don't do it twice. 2109 */ 2110 txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0); 2111 error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lr, B_FALSE); 2112 if (error != 0) 2113 return (zil_replay_error(zilog, lr, error)); 2114 } 2115 return (0); 2116} 2117 2118/* ARGSUSED */ 2119static int 2120zil_incr_blks(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 2121{ 2122 zilog->zl_replay_blks++; 2123 2124 return (0); 2125} 2126 2127/* 2128 * If this dataset has a non-empty intent log, replay it and destroy it. 2129 */ 2130void 2131zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) 2132{ 2133 zilog_t *zilog = dmu_objset_zil(os); 2134 const zil_header_t *zh = zilog->zl_header; 2135 zil_replay_arg_t zr; 2136 2137 if ((zh->zh_flags & ZIL_REPLAY_NEEDED) == 0) { 2138 zil_destroy(zilog, B_TRUE); 2139 return; 2140 } 2141 2142 zr.zr_replay = replay_func; 2143 zr.zr_arg = arg; 2144 zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); 2145 zr.zr_lr = kmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); 2146 2147 /* 2148 * Wait for in-progress removes to sync before starting replay. 2149 */ 2150 txg_wait_synced(zilog->zl_dmu_pool, 0); 2151 2152 zilog->zl_replay = B_TRUE; 2153 zilog->zl_replay_time = ddi_get_lbolt(); 2154 ASSERT(zilog->zl_replay_blks == 0); 2155 (void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr, 2156 zh->zh_claim_txg); 2157 kmem_free(zr.zr_lr, 2 * SPA_MAXBLOCKSIZE); 2158 2159 zil_destroy(zilog, B_FALSE); 2160 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 2161 zilog->zl_replay = B_FALSE; 2162} 2163 2164boolean_t 2165zil_replaying(zilog_t *zilog, dmu_tx_t *tx) 2166{ 2167 if (zilog->zl_sync == ZFS_SYNC_DISABLED) 2168 return (B_TRUE); 2169 2170 if (zilog->zl_replay) { 2171 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 2172 zilog->zl_replayed_seq[dmu_tx_get_txg(tx) & TXG_MASK] = 2173 zilog->zl_replaying_seq; 2174 return (B_TRUE); 2175 } 2176 2177 return (B_FALSE); 2178} 2179 2180/* ARGSUSED */ 2181int 2182zil_vdev_offline(const char *osname, void *arg) 2183{ 2184 int error; 2185 2186 error = zil_suspend(osname, NULL); 2187 if (error != 0) 2188 return (SET_ERROR(EEXIST)); 2189 return (0); 2190} 2191