1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (c) 2000-2006 Silicon Graphics, Inc. 4 * Copyright (c) 2016-2018 Christoph Hellwig. 5 * All Rights Reserved. 6 */ 7#include "xfs.h" 8#include "xfs_fs.h" 9#include "xfs_shared.h" 10#include "xfs_format.h" 11#include "xfs_log_format.h" 12#include "xfs_trans_resv.h" 13#include "xfs_mount.h" 14#include "xfs_inode.h" 15#include "xfs_btree.h" 16#include "xfs_bmap_btree.h" 17#include "xfs_bmap.h" 18#include "xfs_bmap_util.h" 19#include "xfs_errortag.h" 20#include "xfs_error.h" 21#include "xfs_trans.h" 22#include "xfs_trans_space.h" 23#include "xfs_inode_item.h" 24#include "xfs_iomap.h" 25#include "xfs_trace.h" 26#include "xfs_quota.h" 27#include "xfs_dquot_item.h" 28#include "xfs_dquot.h" 29#include "xfs_reflink.h" 30#include "xfs_health.h" 31#include "xfs_rtbitmap.h" 32 33#define XFS_ALLOC_ALIGN(mp, off) \ 34 (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) 35 36static int 37xfs_alert_fsblock_zero( 38 xfs_inode_t *ip, 39 xfs_bmbt_irec_t *imap) 40{ 41 xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO, 42 "Access to block zero in inode %llu " 43 "start_block: %llx start_off: %llx " 44 "blkcnt: %llx extent-state: %x", 45 (unsigned long long)ip->i_ino, 46 (unsigned long long)imap->br_startblock, 47 (unsigned long long)imap->br_startoff, 48 (unsigned long long)imap->br_blockcount, 49 imap->br_state); 50 xfs_bmap_mark_sick(ip, XFS_DATA_FORK); 51 return -EFSCORRUPTED; 52} 53 54u64 55xfs_iomap_inode_sequence( 56 struct xfs_inode *ip, 57 u16 iomap_flags) 58{ 59 u64 cookie = 0; 60 61 if (iomap_flags & IOMAP_F_XATTR) 62 return READ_ONCE(ip->i_af.if_seq); 63 if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp) 64 cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32; 65 return cookie | READ_ONCE(ip->i_df.if_seq); 66} 67 68/* 69 * Check that the iomap passed to us is still valid for the given offset and 70 * length. 71 */ 72static bool 73xfs_iomap_valid( 74 struct inode *inode, 75 const struct iomap *iomap) 76{ 77 struct xfs_inode *ip = XFS_I(inode); 78 79 if (iomap->validity_cookie != 80 xfs_iomap_inode_sequence(ip, iomap->flags)) { 81 trace_xfs_iomap_invalid(ip, iomap); 82 return false; 83 } 84 85 XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS); 86 return true; 87} 88 89static const struct iomap_folio_ops xfs_iomap_folio_ops = { 90 .iomap_valid = xfs_iomap_valid, 91}; 92 93int 94xfs_bmbt_to_iomap( 95 struct xfs_inode *ip, 96 struct iomap *iomap, 97 struct xfs_bmbt_irec *imap, 98 unsigned int mapping_flags, 99 u16 iomap_flags, 100 u64 sequence_cookie) 101{ 102 struct xfs_mount *mp = ip->i_mount; 103 struct xfs_buftarg *target = xfs_inode_buftarg(ip); 104 105 if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { 106 xfs_bmap_mark_sick(ip, XFS_DATA_FORK); 107 return xfs_alert_fsblock_zero(ip, imap); 108 } 109 110 if (imap->br_startblock == HOLESTARTBLOCK) { 111 iomap->addr = IOMAP_NULL_ADDR; 112 iomap->type = IOMAP_HOLE; 113 } else if (imap->br_startblock == DELAYSTARTBLOCK || 114 isnullstartblock(imap->br_startblock)) { 115 iomap->addr = IOMAP_NULL_ADDR; 116 iomap->type = IOMAP_DELALLOC; 117 } else { 118 iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock)); 119 if (mapping_flags & IOMAP_DAX) 120 iomap->addr += target->bt_dax_part_off; 121 122 if (imap->br_state == XFS_EXT_UNWRITTEN) 123 iomap->type = IOMAP_UNWRITTEN; 124 else 125 iomap->type = IOMAP_MAPPED; 126 127 } 128 iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); 129 iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); 130 if (mapping_flags & IOMAP_DAX) 131 iomap->dax_dev = target->bt_daxdev; 132 else 133 iomap->bdev = target->bt_bdev; 134 iomap->flags = iomap_flags; 135 136 if (xfs_ipincount(ip) && 137 (ip->i_itemp->ili_fsync_fields & ~XFS_ILOG_TIMESTAMP)) 138 iomap->flags |= IOMAP_F_DIRTY; 139 140 iomap->validity_cookie = sequence_cookie; 141 iomap->folio_ops = &xfs_iomap_folio_ops; 142 return 0; 143} 144 145static void 146xfs_hole_to_iomap( 147 struct xfs_inode *ip, 148 struct iomap *iomap, 149 xfs_fileoff_t offset_fsb, 150 xfs_fileoff_t end_fsb) 151{ 152 struct xfs_buftarg *target = xfs_inode_buftarg(ip); 153 154 iomap->addr = IOMAP_NULL_ADDR; 155 iomap->type = IOMAP_HOLE; 156 iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb); 157 iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb); 158 iomap->bdev = target->bt_bdev; 159 iomap->dax_dev = target->bt_daxdev; 160} 161 162static inline xfs_fileoff_t 163xfs_iomap_end_fsb( 164 struct xfs_mount *mp, 165 loff_t offset, 166 loff_t count) 167{ 168 ASSERT(offset <= mp->m_super->s_maxbytes); 169 return min(XFS_B_TO_FSB(mp, offset + count), 170 XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes)); 171} 172 173static xfs_extlen_t 174xfs_eof_alignment( 175 struct xfs_inode *ip) 176{ 177 struct xfs_mount *mp = ip->i_mount; 178 xfs_extlen_t align = 0; 179 180 if (!XFS_IS_REALTIME_INODE(ip)) { 181 /* 182 * Round up the allocation request to a stripe unit 183 * (m_dalign) boundary if the file size is >= stripe unit 184 * size, and we are allocating past the allocation eof. 185 * 186 * If mounted with the "-o swalloc" option the alignment is 187 * increased from the strip unit size to the stripe width. 188 */ 189 if (mp->m_swidth && xfs_has_swalloc(mp)) 190 align = mp->m_swidth; 191 else if (mp->m_dalign) 192 align = mp->m_dalign; 193 194 if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align)) 195 align = 0; 196 } 197 198 return align; 199} 200 201/* 202 * Check if last_fsb is outside the last extent, and if so grow it to the next 203 * stripe unit boundary. 204 */ 205xfs_fileoff_t 206xfs_iomap_eof_align_last_fsb( 207 struct xfs_inode *ip, 208 xfs_fileoff_t end_fsb) 209{ 210 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); 211 xfs_extlen_t extsz = xfs_get_extsz_hint(ip); 212 xfs_extlen_t align = xfs_eof_alignment(ip); 213 struct xfs_bmbt_irec irec; 214 struct xfs_iext_cursor icur; 215 216 ASSERT(!xfs_need_iread_extents(ifp)); 217 218 /* 219 * Always round up the allocation request to the extent hint boundary. 220 */ 221 if (extsz) { 222 if (align) 223 align = roundup_64(align, extsz); 224 else 225 align = extsz; 226 } 227 228 if (align) { 229 xfs_fileoff_t aligned_end_fsb = roundup_64(end_fsb, align); 230 231 xfs_iext_last(ifp, &icur); 232 if (!xfs_iext_get_extent(ifp, &icur, &irec) || 233 aligned_end_fsb >= irec.br_startoff + irec.br_blockcount) 234 return aligned_end_fsb; 235 } 236 237 return end_fsb; 238} 239 240int 241xfs_iomap_write_direct( 242 struct xfs_inode *ip, 243 xfs_fileoff_t offset_fsb, 244 xfs_fileoff_t count_fsb, 245 unsigned int flags, 246 struct xfs_bmbt_irec *imap, 247 u64 *seq) 248{ 249 struct xfs_mount *mp = ip->i_mount; 250 struct xfs_trans *tp; 251 xfs_filblks_t resaligned; 252 int nimaps; 253 unsigned int dblocks, rblocks; 254 bool force = false; 255 int error; 256 int bmapi_flags = XFS_BMAPI_PREALLOC; 257 int nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT; 258 259 ASSERT(count_fsb > 0); 260 261 resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb, 262 xfs_get_extsz_hint(ip)); 263 if (unlikely(XFS_IS_REALTIME_INODE(ip))) { 264 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); 265 rblocks = resaligned; 266 } else { 267 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); 268 rblocks = 0; 269 } 270 271 error = xfs_qm_dqattach(ip); 272 if (error) 273 return error; 274 275 /* 276 * For DAX, we do not allocate unwritten extents, but instead we zero 277 * the block before we commit the transaction. Ideally we'd like to do 278 * this outside the transaction context, but if we commit and then crash 279 * we may not have zeroed the blocks and this will be exposed on 280 * recovery of the allocation. Hence we must zero before commit. 281 * 282 * Further, if we are mapping unwritten extents here, we need to zero 283 * and convert them to written so that we don't need an unwritten extent 284 * callback for DAX. This also means that we need to be able to dip into 285 * the reserve block pool for bmbt block allocation if there is no space 286 * left but we need to do unwritten extent conversion. 287 */ 288 if (flags & IOMAP_DAX) { 289 bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; 290 if (imap->br_state == XFS_EXT_UNWRITTEN) { 291 force = true; 292 nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT; 293 dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; 294 } 295 } 296 297 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks, 298 rblocks, force, &tp); 299 if (error) 300 return error; 301 302 error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); 303 if (error) 304 goto out_trans_cancel; 305 306 /* 307 * From this point onwards we overwrite the imap pointer that the 308 * caller gave to us. 309 */ 310 nimaps = 1; 311 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0, 312 imap, &nimaps); 313 if (error) 314 goto out_trans_cancel; 315 316 /* 317 * Complete the transaction 318 */ 319 error = xfs_trans_commit(tp); 320 if (error) 321 goto out_unlock; 322 323 if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { 324 xfs_bmap_mark_sick(ip, XFS_DATA_FORK); 325 error = xfs_alert_fsblock_zero(ip, imap); 326 } 327 328out_unlock: 329 *seq = xfs_iomap_inode_sequence(ip, 0); 330 xfs_iunlock(ip, XFS_ILOCK_EXCL); 331 return error; 332 333out_trans_cancel: 334 xfs_trans_cancel(tp); 335 goto out_unlock; 336} 337 338STATIC bool 339xfs_quota_need_throttle( 340 struct xfs_inode *ip, 341 xfs_dqtype_t type, 342 xfs_fsblock_t alloc_blocks) 343{ 344 struct xfs_dquot *dq = xfs_inode_dquot(ip, type); 345 346 if (!dq || !xfs_this_quota_on(ip->i_mount, type)) 347 return false; 348 349 /* no hi watermark, no throttle */ 350 if (!dq->q_prealloc_hi_wmark) 351 return false; 352 353 /* under the lo watermark, no throttle */ 354 if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark) 355 return false; 356 357 return true; 358} 359 360STATIC void 361xfs_quota_calc_throttle( 362 struct xfs_inode *ip, 363 xfs_dqtype_t type, 364 xfs_fsblock_t *qblocks, 365 int *qshift, 366 int64_t *qfreesp) 367{ 368 struct xfs_dquot *dq = xfs_inode_dquot(ip, type); 369 int64_t freesp; 370 int shift = 0; 371 372 /* no dq, or over hi wmark, squash the prealloc completely */ 373 if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) { 374 *qblocks = 0; 375 *qfreesp = 0; 376 return; 377 } 378 379 freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved; 380 if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) { 381 shift = 2; 382 if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT]) 383 shift += 2; 384 if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT]) 385 shift += 2; 386 } 387 388 if (freesp < *qfreesp) 389 *qfreesp = freesp; 390 391 /* only overwrite the throttle values if we are more aggressive */ 392 if ((freesp >> shift) < (*qblocks >> *qshift)) { 393 *qblocks = freesp; 394 *qshift = shift; 395 } 396} 397 398static int64_t 399xfs_iomap_freesp( 400 struct percpu_counter *counter, 401 uint64_t low_space[XFS_LOWSP_MAX], 402 int *shift) 403{ 404 int64_t freesp; 405 406 freesp = percpu_counter_read_positive(counter); 407 if (freesp < low_space[XFS_LOWSP_5_PCNT]) { 408 *shift = 2; 409 if (freesp < low_space[XFS_LOWSP_4_PCNT]) 410 (*shift)++; 411 if (freesp < low_space[XFS_LOWSP_3_PCNT]) 412 (*shift)++; 413 if (freesp < low_space[XFS_LOWSP_2_PCNT]) 414 (*shift)++; 415 if (freesp < low_space[XFS_LOWSP_1_PCNT]) 416 (*shift)++; 417 } 418 return freesp; 419} 420 421/* 422 * If we don't have a user specified preallocation size, dynamically increase 423 * the preallocation size as the size of the file grows. Cap the maximum size 424 * at a single extent or less if the filesystem is near full. The closer the 425 * filesystem is to being full, the smaller the maximum preallocation. 426 */ 427STATIC xfs_fsblock_t 428xfs_iomap_prealloc_size( 429 struct xfs_inode *ip, 430 int whichfork, 431 loff_t offset, 432 loff_t count, 433 struct xfs_iext_cursor *icur) 434{ 435 struct xfs_iext_cursor ncur = *icur; 436 struct xfs_bmbt_irec prev, got; 437 struct xfs_mount *mp = ip->i_mount; 438 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); 439 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 440 int64_t freesp; 441 xfs_fsblock_t qblocks; 442 xfs_fsblock_t alloc_blocks = 0; 443 xfs_extlen_t plen; 444 int shift = 0; 445 int qshift = 0; 446 447 /* 448 * As an exception we don't do any preallocation at all if the file is 449 * smaller than the minimum preallocation and we are using the default 450 * dynamic preallocation scheme, as it is likely this is the only write 451 * to the file that is going to be done. 452 */ 453 if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks)) 454 return 0; 455 456 /* 457 * Use the minimum preallocation size for small files or if we are 458 * writing right after a hole. 459 */ 460 if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || 461 !xfs_iext_prev_extent(ifp, &ncur, &prev) || 462 prev.br_startoff + prev.br_blockcount < offset_fsb) 463 return mp->m_allocsize_blocks; 464 465 /* 466 * Take the size of the preceding data extents as the basis for the 467 * preallocation size. Note that we don't care if the previous extents 468 * are written or not. 469 */ 470 plen = prev.br_blockcount; 471 while (xfs_iext_prev_extent(ifp, &ncur, &got)) { 472 if (plen > XFS_MAX_BMBT_EXTLEN / 2 || 473 isnullstartblock(got.br_startblock) || 474 got.br_startoff + got.br_blockcount != prev.br_startoff || 475 got.br_startblock + got.br_blockcount != prev.br_startblock) 476 break; 477 plen += got.br_blockcount; 478 prev = got; 479 } 480 481 /* 482 * If the size of the extents is greater than half the maximum extent 483 * length, then use the current offset as the basis. This ensures that 484 * for large files the preallocation size always extends to 485 * XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe 486 * unit/width alignment of real extents. 487 */ 488 alloc_blocks = plen * 2; 489 if (alloc_blocks > XFS_MAX_BMBT_EXTLEN) 490 alloc_blocks = XFS_B_TO_FSB(mp, offset); 491 qblocks = alloc_blocks; 492 493 /* 494 * XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc 495 * down to the nearest power of two value after throttling. To prevent 496 * the round down from unconditionally reducing the maximum supported 497 * prealloc size, we round up first, apply appropriate throttling, round 498 * down and cap the value to XFS_BMBT_MAX_EXTLEN. 499 */ 500 alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN), 501 alloc_blocks); 502 503 if (unlikely(XFS_IS_REALTIME_INODE(ip))) 504 freesp = xfs_rtx_to_rtb(mp, 505 xfs_iomap_freesp(&mp->m_frextents, 506 mp->m_low_rtexts, &shift)); 507 else 508 freesp = xfs_iomap_freesp(&mp->m_fdblocks, mp->m_low_space, 509 &shift); 510 511 /* 512 * Check each quota to cap the prealloc size, provide a shift value to 513 * throttle with and adjust amount of available space. 514 */ 515 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks)) 516 xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift, 517 &freesp); 518 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks)) 519 xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift, 520 &freesp); 521 if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks)) 522 xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift, 523 &freesp); 524 525 /* 526 * The final prealloc size is set to the minimum of free space available 527 * in each of the quotas and the overall filesystem. 528 * 529 * The shift throttle value is set to the maximum value as determined by 530 * the global low free space values and per-quota low free space values. 531 */ 532 alloc_blocks = min(alloc_blocks, qblocks); 533 shift = max(shift, qshift); 534 535 if (shift) 536 alloc_blocks >>= shift; 537 /* 538 * rounddown_pow_of_two() returns an undefined result if we pass in 539 * alloc_blocks = 0. 540 */ 541 if (alloc_blocks) 542 alloc_blocks = rounddown_pow_of_two(alloc_blocks); 543 if (alloc_blocks > XFS_MAX_BMBT_EXTLEN) 544 alloc_blocks = XFS_MAX_BMBT_EXTLEN; 545 546 /* 547 * If we are still trying to allocate more space than is 548 * available, squash the prealloc hard. This can happen if we 549 * have a large file on a small filesystem and the above 550 * lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN. 551 */ 552 while (alloc_blocks && alloc_blocks >= freesp) 553 alloc_blocks >>= 4; 554 if (alloc_blocks < mp->m_allocsize_blocks) 555 alloc_blocks = mp->m_allocsize_blocks; 556 trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift, 557 mp->m_allocsize_blocks); 558 return alloc_blocks; 559} 560 561int 562xfs_iomap_write_unwritten( 563 xfs_inode_t *ip, 564 xfs_off_t offset, 565 xfs_off_t count, 566 bool update_isize) 567{ 568 xfs_mount_t *mp = ip->i_mount; 569 xfs_fileoff_t offset_fsb; 570 xfs_filblks_t count_fsb; 571 xfs_filblks_t numblks_fsb; 572 int nimaps; 573 xfs_trans_t *tp; 574 xfs_bmbt_irec_t imap; 575 struct inode *inode = VFS_I(ip); 576 xfs_fsize_t i_size; 577 uint resblks; 578 int error; 579 580 trace_xfs_unwritten_convert(ip, offset, count); 581 582 offset_fsb = XFS_B_TO_FSBT(mp, offset); 583 count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count); 584 count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb); 585 586 /* 587 * Reserve enough blocks in this transaction for two complete extent 588 * btree splits. We may be converting the middle part of an unwritten 589 * extent and in this case we will insert two new extents in the btree 590 * each of which could cause a full split. 591 * 592 * This reservation amount will be used in the first call to 593 * xfs_bmbt_split() to select an AG with enough space to satisfy the 594 * rest of the operation. 595 */ 596 resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; 597 598 /* Attach dquots so that bmbt splits are accounted correctly. */ 599 error = xfs_qm_dqattach(ip); 600 if (error) 601 return error; 602 603 do { 604 /* 605 * Set up a transaction to convert the range of extents 606 * from unwritten to real. Do allocations in a loop until 607 * we have covered the range passed in. 608 * 609 * Note that we can't risk to recursing back into the filesystem 610 * here as we might be asked to write out the same inode that we 611 * complete here and might deadlock on the iolock. 612 */ 613 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 614 0, true, &tp); 615 if (error) 616 return error; 617 618 error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, 619 XFS_IEXT_WRITE_UNWRITTEN_CNT); 620 if (error) 621 goto error_on_bmapi_transaction; 622 623 /* 624 * Modify the unwritten extent state of the buffer. 625 */ 626 nimaps = 1; 627 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, 628 XFS_BMAPI_CONVERT, resblks, &imap, 629 &nimaps); 630 if (error) 631 goto error_on_bmapi_transaction; 632 633 /* 634 * Log the updated inode size as we go. We have to be careful 635 * to only log it up to the actual write offset if it is 636 * halfway into a block. 637 */ 638 i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb); 639 if (i_size > offset + count) 640 i_size = offset + count; 641 if (update_isize && i_size > i_size_read(inode)) 642 i_size_write(inode, i_size); 643 i_size = xfs_new_eof(ip, i_size); 644 if (i_size) { 645 ip->i_disk_size = i_size; 646 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 647 } 648 649 error = xfs_trans_commit(tp); 650 xfs_iunlock(ip, XFS_ILOCK_EXCL); 651 if (error) 652 return error; 653 654 if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) { 655 xfs_bmap_mark_sick(ip, XFS_DATA_FORK); 656 return xfs_alert_fsblock_zero(ip, &imap); 657 } 658 659 if ((numblks_fsb = imap.br_blockcount) == 0) { 660 /* 661 * The numblks_fsb value should always get 662 * smaller, otherwise the loop is stuck. 663 */ 664 ASSERT(imap.br_blockcount); 665 break; 666 } 667 offset_fsb += numblks_fsb; 668 count_fsb -= numblks_fsb; 669 } while (count_fsb > 0); 670 671 return 0; 672 673error_on_bmapi_transaction: 674 xfs_trans_cancel(tp); 675 xfs_iunlock(ip, XFS_ILOCK_EXCL); 676 return error; 677} 678 679static inline bool 680imap_needs_alloc( 681 struct inode *inode, 682 unsigned flags, 683 struct xfs_bmbt_irec *imap, 684 int nimaps) 685{ 686 /* don't allocate blocks when just zeroing */ 687 if (flags & IOMAP_ZERO) 688 return false; 689 if (!nimaps || 690 imap->br_startblock == HOLESTARTBLOCK || 691 imap->br_startblock == DELAYSTARTBLOCK) 692 return true; 693 /* we convert unwritten extents before copying the data for DAX */ 694 if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN) 695 return true; 696 return false; 697} 698 699static inline bool 700imap_needs_cow( 701 struct xfs_inode *ip, 702 unsigned int flags, 703 struct xfs_bmbt_irec *imap, 704 int nimaps) 705{ 706 if (!xfs_is_cow_inode(ip)) 707 return false; 708 709 /* when zeroing we don't have to COW holes or unwritten extents */ 710 if (flags & IOMAP_ZERO) { 711 if (!nimaps || 712 imap->br_startblock == HOLESTARTBLOCK || 713 imap->br_state == XFS_EXT_UNWRITTEN) 714 return false; 715 } 716 717 return true; 718} 719 720static int 721xfs_ilock_for_iomap( 722 struct xfs_inode *ip, 723 unsigned flags, 724 unsigned *lockmode) 725{ 726 unsigned int mode = *lockmode; 727 bool is_write = flags & (IOMAP_WRITE | IOMAP_ZERO); 728 729 /* 730 * COW writes may allocate delalloc space or convert unwritten COW 731 * extents, so we need to make sure to take the lock exclusively here. 732 */ 733 if (xfs_is_cow_inode(ip) && is_write) 734 mode = XFS_ILOCK_EXCL; 735 736 /* 737 * Extents not yet cached requires exclusive access, don't block. This 738 * is an opencoded xfs_ilock_data_map_shared() call but with 739 * non-blocking behaviour. 740 */ 741 if (xfs_need_iread_extents(&ip->i_df)) { 742 if (flags & IOMAP_NOWAIT) 743 return -EAGAIN; 744 mode = XFS_ILOCK_EXCL; 745 } 746 747relock: 748 if (flags & IOMAP_NOWAIT) { 749 if (!xfs_ilock_nowait(ip, mode)) 750 return -EAGAIN; 751 } else { 752 xfs_ilock(ip, mode); 753 } 754 755 /* 756 * The reflink iflag could have changed since the earlier unlocked 757 * check, so if we got ILOCK_SHARED for a write and but we're now a 758 * reflink inode we have to switch to ILOCK_EXCL and relock. 759 */ 760 if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) { 761 xfs_iunlock(ip, mode); 762 mode = XFS_ILOCK_EXCL; 763 goto relock; 764 } 765 766 *lockmode = mode; 767 return 0; 768} 769 770/* 771 * Check that the imap we are going to return to the caller spans the entire 772 * range that the caller requested for the IO. 773 */ 774static bool 775imap_spans_range( 776 struct xfs_bmbt_irec *imap, 777 xfs_fileoff_t offset_fsb, 778 xfs_fileoff_t end_fsb) 779{ 780 if (imap->br_startoff > offset_fsb) 781 return false; 782 if (imap->br_startoff + imap->br_blockcount < end_fsb) 783 return false; 784 return true; 785} 786 787static int 788xfs_direct_write_iomap_begin( 789 struct inode *inode, 790 loff_t offset, 791 loff_t length, 792 unsigned flags, 793 struct iomap *iomap, 794 struct iomap *srcmap) 795{ 796 struct xfs_inode *ip = XFS_I(inode); 797 struct xfs_mount *mp = ip->i_mount; 798 struct xfs_bmbt_irec imap, cmap; 799 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 800 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length); 801 int nimaps = 1, error = 0; 802 bool shared = false; 803 u16 iomap_flags = 0; 804 unsigned int lockmode = XFS_ILOCK_SHARED; 805 u64 seq; 806 807 ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO)); 808 809 if (xfs_is_shutdown(mp)) 810 return -EIO; 811 812 /* 813 * Writes that span EOF might trigger an IO size update on completion, 814 * so consider them to be dirty for the purposes of O_DSYNC even if 815 * there is no other metadata changes pending or have been made here. 816 */ 817 if (offset + length > i_size_read(inode)) 818 iomap_flags |= IOMAP_F_DIRTY; 819 820 error = xfs_ilock_for_iomap(ip, flags, &lockmode); 821 if (error) 822 return error; 823 824 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, 825 &nimaps, 0); 826 if (error) 827 goto out_unlock; 828 829 if (imap_needs_cow(ip, flags, &imap, nimaps)) { 830 error = -EAGAIN; 831 if (flags & IOMAP_NOWAIT) 832 goto out_unlock; 833 834 /* may drop and re-acquire the ilock */ 835 error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared, 836 &lockmode, 837 (flags & IOMAP_DIRECT) || IS_DAX(inode)); 838 if (error) 839 goto out_unlock; 840 if (shared) 841 goto out_found_cow; 842 end_fsb = imap.br_startoff + imap.br_blockcount; 843 length = XFS_FSB_TO_B(mp, end_fsb) - offset; 844 } 845 846 if (imap_needs_alloc(inode, flags, &imap, nimaps)) 847 goto allocate_blocks; 848 849 /* 850 * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with 851 * a single map so that we avoid partial IO failures due to the rest of 852 * the I/O range not covered by this map triggering an EAGAIN condition 853 * when it is subsequently mapped and aborting the I/O. 854 */ 855 if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) { 856 error = -EAGAIN; 857 if (!imap_spans_range(&imap, offset_fsb, end_fsb)) 858 goto out_unlock; 859 } 860 861 /* 862 * For overwrite only I/O, we cannot convert unwritten extents without 863 * requiring sub-block zeroing. This can only be done under an 864 * exclusive IOLOCK, hence return -EAGAIN if this is not a written 865 * extent to tell the caller to try again. 866 */ 867 if (flags & IOMAP_OVERWRITE_ONLY) { 868 error = -EAGAIN; 869 if (imap.br_state != XFS_EXT_NORM && 870 ((offset | length) & mp->m_blockmask)) 871 goto out_unlock; 872 } 873 874 seq = xfs_iomap_inode_sequence(ip, iomap_flags); 875 xfs_iunlock(ip, lockmode); 876 trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap); 877 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq); 878 879allocate_blocks: 880 error = -EAGAIN; 881 if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) 882 goto out_unlock; 883 884 /* 885 * We cap the maximum length we map to a sane size to keep the chunks 886 * of work done where somewhat symmetric with the work writeback does. 887 * This is a completely arbitrary number pulled out of thin air as a 888 * best guess for initial testing. 889 * 890 * Note that the values needs to be less than 32-bits wide until the 891 * lower level functions are updated. 892 */ 893 length = min_t(loff_t, length, 1024 * PAGE_SIZE); 894 end_fsb = xfs_iomap_end_fsb(mp, offset, length); 895 896 if (offset + length > XFS_ISIZE(ip)) 897 end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb); 898 else if (nimaps && imap.br_startblock == HOLESTARTBLOCK) 899 end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount); 900 xfs_iunlock(ip, lockmode); 901 902 error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb, 903 flags, &imap, &seq); 904 if (error) 905 return error; 906 907 trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap); 908 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 909 iomap_flags | IOMAP_F_NEW, seq); 910 911out_found_cow: 912 length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount); 913 trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap); 914 if (imap.br_startblock != HOLESTARTBLOCK) { 915 seq = xfs_iomap_inode_sequence(ip, 0); 916 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq); 917 if (error) 918 goto out_unlock; 919 } 920 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); 921 xfs_iunlock(ip, lockmode); 922 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq); 923 924out_unlock: 925 if (lockmode) 926 xfs_iunlock(ip, lockmode); 927 return error; 928} 929 930const struct iomap_ops xfs_direct_write_iomap_ops = { 931 .iomap_begin = xfs_direct_write_iomap_begin, 932}; 933 934static int 935xfs_dax_write_iomap_end( 936 struct inode *inode, 937 loff_t pos, 938 loff_t length, 939 ssize_t written, 940 unsigned flags, 941 struct iomap *iomap) 942{ 943 struct xfs_inode *ip = XFS_I(inode); 944 945 if (!xfs_is_cow_inode(ip)) 946 return 0; 947 948 if (!written) { 949 xfs_reflink_cancel_cow_range(ip, pos, length, true); 950 return 0; 951 } 952 953 return xfs_reflink_end_cow(ip, pos, written); 954} 955 956const struct iomap_ops xfs_dax_write_iomap_ops = { 957 .iomap_begin = xfs_direct_write_iomap_begin, 958 .iomap_end = xfs_dax_write_iomap_end, 959}; 960 961static int 962xfs_buffered_write_iomap_begin( 963 struct inode *inode, 964 loff_t offset, 965 loff_t count, 966 unsigned flags, 967 struct iomap *iomap, 968 struct iomap *srcmap) 969{ 970 struct xfs_inode *ip = XFS_I(inode); 971 struct xfs_mount *mp = ip->i_mount; 972 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 973 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count); 974 struct xfs_bmbt_irec imap, cmap; 975 struct xfs_iext_cursor icur, ccur; 976 xfs_fsblock_t prealloc_blocks = 0; 977 bool eof = false, cow_eof = false, shared = false; 978 int allocfork = XFS_DATA_FORK; 979 int error = 0; 980 unsigned int lockmode = XFS_ILOCK_EXCL; 981 u64 seq; 982 983 if (xfs_is_shutdown(mp)) 984 return -EIO; 985 986 /* we can't use delayed allocations when using extent size hints */ 987 if (xfs_get_extsz_hint(ip)) 988 return xfs_direct_write_iomap_begin(inode, offset, count, 989 flags, iomap, srcmap); 990 991 error = xfs_qm_dqattach(ip); 992 if (error) 993 return error; 994 995 error = xfs_ilock_for_iomap(ip, flags, &lockmode); 996 if (error) 997 return error; 998 999 if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) || 1000 XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { 1001 xfs_bmap_mark_sick(ip, XFS_DATA_FORK); 1002 error = -EFSCORRUPTED; 1003 goto out_unlock; 1004 } 1005 1006 XFS_STATS_INC(mp, xs_blk_mapw); 1007 1008 error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); 1009 if (error) 1010 goto out_unlock; 1011 1012 /* 1013 * Search the data fork first to look up our source mapping. We 1014 * always need the data fork map, as we have to return it to the 1015 * iomap code so that the higher level write code can read data in to 1016 * perform read-modify-write cycles for unaligned writes. 1017 */ 1018 eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap); 1019 if (eof) 1020 imap.br_startoff = end_fsb; /* fake hole until the end */ 1021 1022 /* We never need to allocate blocks for zeroing or unsharing a hole. */ 1023 if ((flags & (IOMAP_UNSHARE | IOMAP_ZERO)) && 1024 imap.br_startoff > offset_fsb) { 1025 xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff); 1026 goto out_unlock; 1027 } 1028 1029 /* 1030 * For zeroing, trim a delalloc extent that extends beyond the EOF 1031 * block. If it starts beyond the EOF block, convert it to an 1032 * unwritten extent. 1033 */ 1034 if ((flags & IOMAP_ZERO) && imap.br_startoff <= offset_fsb && 1035 isnullstartblock(imap.br_startblock)) { 1036 xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip)); 1037 1038 if (offset_fsb >= eof_fsb) 1039 goto convert_delay; 1040 if (end_fsb > eof_fsb) { 1041 end_fsb = eof_fsb; 1042 xfs_trim_extent(&imap, offset_fsb, 1043 end_fsb - offset_fsb); 1044 } 1045 } 1046 1047 /* 1048 * Search the COW fork extent list even if we did not find a data fork 1049 * extent. This serves two purposes: first this implements the 1050 * speculative preallocation using cowextsize, so that we also unshare 1051 * block adjacent to shared blocks instead of just the shared blocks 1052 * themselves. Second the lookup in the extent list is generally faster 1053 * than going out to the shared extent tree. 1054 */ 1055 if (xfs_is_cow_inode(ip)) { 1056 if (!ip->i_cowfp) { 1057 ASSERT(!xfs_is_reflink_inode(ip)); 1058 xfs_ifork_init_cow(ip); 1059 } 1060 cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, 1061 &ccur, &cmap); 1062 if (!cow_eof && cmap.br_startoff <= offset_fsb) { 1063 trace_xfs_reflink_cow_found(ip, &cmap); 1064 goto found_cow; 1065 } 1066 } 1067 1068 if (imap.br_startoff <= offset_fsb) { 1069 /* 1070 * For reflink files we may need a delalloc reservation when 1071 * overwriting shared extents. This includes zeroing of 1072 * existing extents that contain data. 1073 */ 1074 if (!xfs_is_cow_inode(ip) || 1075 ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) { 1076 trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK, 1077 &imap); 1078 goto found_imap; 1079 } 1080 1081 xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); 1082 1083 /* Trim the mapping to the nearest shared extent boundary. */ 1084 error = xfs_bmap_trim_cow(ip, &imap, &shared); 1085 if (error) 1086 goto out_unlock; 1087 1088 /* Not shared? Just report the (potentially capped) extent. */ 1089 if (!shared) { 1090 trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK, 1091 &imap); 1092 goto found_imap; 1093 } 1094 1095 /* 1096 * Fork all the shared blocks from our write offset until the 1097 * end of the extent. 1098 */ 1099 allocfork = XFS_COW_FORK; 1100 end_fsb = imap.br_startoff + imap.br_blockcount; 1101 } else { 1102 /* 1103 * We cap the maximum length we map here to MAX_WRITEBACK_PAGES 1104 * pages to keep the chunks of work done where somewhat 1105 * symmetric with the work writeback does. This is a completely 1106 * arbitrary number pulled out of thin air. 1107 * 1108 * Note that the values needs to be less than 32-bits wide until 1109 * the lower level functions are updated. 1110 */ 1111 count = min_t(loff_t, count, 1024 * PAGE_SIZE); 1112 end_fsb = xfs_iomap_end_fsb(mp, offset, count); 1113 1114 if (xfs_is_always_cow_inode(ip)) 1115 allocfork = XFS_COW_FORK; 1116 } 1117 1118 if (eof && offset + count > XFS_ISIZE(ip)) { 1119 /* 1120 * Determine the initial size of the preallocation. 1121 * We clean up any extra preallocation when the file is closed. 1122 */ 1123 if (xfs_has_allocsize(mp)) 1124 prealloc_blocks = mp->m_allocsize_blocks; 1125 else if (allocfork == XFS_DATA_FORK) 1126 prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, 1127 offset, count, &icur); 1128 else 1129 prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, 1130 offset, count, &ccur); 1131 if (prealloc_blocks) { 1132 xfs_extlen_t align; 1133 xfs_off_t end_offset; 1134 xfs_fileoff_t p_end_fsb; 1135 1136 end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1); 1137 p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) + 1138 prealloc_blocks; 1139 1140 align = xfs_eof_alignment(ip); 1141 if (align) 1142 p_end_fsb = roundup_64(p_end_fsb, align); 1143 1144 p_end_fsb = min(p_end_fsb, 1145 XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes)); 1146 ASSERT(p_end_fsb > offset_fsb); 1147 prealloc_blocks = p_end_fsb - end_fsb; 1148 } 1149 } 1150 1151retry: 1152 error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, 1153 end_fsb - offset_fsb, prealloc_blocks, 1154 allocfork == XFS_DATA_FORK ? &imap : &cmap, 1155 allocfork == XFS_DATA_FORK ? &icur : &ccur, 1156 allocfork == XFS_DATA_FORK ? eof : cow_eof); 1157 switch (error) { 1158 case 0: 1159 break; 1160 case -ENOSPC: 1161 case -EDQUOT: 1162 /* retry without any preallocation */ 1163 trace_xfs_delalloc_enospc(ip, offset, count); 1164 if (prealloc_blocks) { 1165 prealloc_blocks = 0; 1166 goto retry; 1167 } 1168 fallthrough; 1169 default: 1170 goto out_unlock; 1171 } 1172 1173 if (allocfork == XFS_COW_FORK) { 1174 trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap); 1175 goto found_cow; 1176 } 1177 1178 /* 1179 * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch 1180 * them out if the write happens to fail. 1181 */ 1182 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW); 1183 xfs_iunlock(ip, lockmode); 1184 trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap); 1185 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq); 1186 1187found_imap: 1188 seq = xfs_iomap_inode_sequence(ip, 0); 1189 xfs_iunlock(ip, lockmode); 1190 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); 1191 1192convert_delay: 1193 xfs_iunlock(ip, lockmode); 1194 truncate_pagecache(inode, offset); 1195 error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset, 1196 iomap, NULL); 1197 if (error) 1198 return error; 1199 1200 trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap); 1201 return 0; 1202 1203found_cow: 1204 seq = xfs_iomap_inode_sequence(ip, 0); 1205 if (imap.br_startoff <= offset_fsb) { 1206 error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq); 1207 if (error) 1208 goto out_unlock; 1209 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); 1210 xfs_iunlock(ip, lockmode); 1211 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 1212 IOMAP_F_SHARED, seq); 1213 } 1214 1215 xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb); 1216 xfs_iunlock(ip, lockmode); 1217 return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq); 1218 1219out_unlock: 1220 xfs_iunlock(ip, lockmode); 1221 return error; 1222} 1223 1224static int 1225xfs_buffered_write_delalloc_punch( 1226 struct inode *inode, 1227 loff_t offset, 1228 loff_t length) 1229{ 1230 xfs_bmap_punch_delalloc_range(XFS_I(inode), offset, offset + length); 1231 return 0; 1232} 1233 1234static int 1235xfs_buffered_write_iomap_end( 1236 struct inode *inode, 1237 loff_t offset, 1238 loff_t length, 1239 ssize_t written, 1240 unsigned flags, 1241 struct iomap *iomap) 1242{ 1243 1244 struct xfs_mount *mp = XFS_M(inode->i_sb); 1245 int error; 1246 1247 error = iomap_file_buffered_write_punch_delalloc(inode, iomap, offset, 1248 length, written, &xfs_buffered_write_delalloc_punch); 1249 if (error && !xfs_is_shutdown(mp)) { 1250 xfs_alert(mp, "%s: unable to clean up ino 0x%llx", 1251 __func__, XFS_I(inode)->i_ino); 1252 return error; 1253 } 1254 return 0; 1255} 1256 1257const struct iomap_ops xfs_buffered_write_iomap_ops = { 1258 .iomap_begin = xfs_buffered_write_iomap_begin, 1259 .iomap_end = xfs_buffered_write_iomap_end, 1260}; 1261 1262/* 1263 * iomap_page_mkwrite() will never fail in a way that requires delalloc extents 1264 * that it allocated to be revoked. Hence we do not need an .iomap_end method 1265 * for this operation. 1266 */ 1267const struct iomap_ops xfs_page_mkwrite_iomap_ops = { 1268 .iomap_begin = xfs_buffered_write_iomap_begin, 1269}; 1270 1271static int 1272xfs_read_iomap_begin( 1273 struct inode *inode, 1274 loff_t offset, 1275 loff_t length, 1276 unsigned flags, 1277 struct iomap *iomap, 1278 struct iomap *srcmap) 1279{ 1280 struct xfs_inode *ip = XFS_I(inode); 1281 struct xfs_mount *mp = ip->i_mount; 1282 struct xfs_bmbt_irec imap; 1283 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 1284 xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length); 1285 int nimaps = 1, error = 0; 1286 bool shared = false; 1287 unsigned int lockmode = XFS_ILOCK_SHARED; 1288 u64 seq; 1289 1290 ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO))); 1291 1292 if (xfs_is_shutdown(mp)) 1293 return -EIO; 1294 1295 error = xfs_ilock_for_iomap(ip, flags, &lockmode); 1296 if (error) 1297 return error; 1298 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, 1299 &nimaps, 0); 1300 if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode))) 1301 error = xfs_reflink_trim_around_shared(ip, &imap, &shared); 1302 seq = xfs_iomap_inode_sequence(ip, shared ? IOMAP_F_SHARED : 0); 1303 xfs_iunlock(ip, lockmode); 1304 1305 if (error) 1306 return error; 1307 trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap); 1308 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 1309 shared ? IOMAP_F_SHARED : 0, seq); 1310} 1311 1312const struct iomap_ops xfs_read_iomap_ops = { 1313 .iomap_begin = xfs_read_iomap_begin, 1314}; 1315 1316static int 1317xfs_seek_iomap_begin( 1318 struct inode *inode, 1319 loff_t offset, 1320 loff_t length, 1321 unsigned flags, 1322 struct iomap *iomap, 1323 struct iomap *srcmap) 1324{ 1325 struct xfs_inode *ip = XFS_I(inode); 1326 struct xfs_mount *mp = ip->i_mount; 1327 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 1328 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length); 1329 xfs_fileoff_t cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF; 1330 struct xfs_iext_cursor icur; 1331 struct xfs_bmbt_irec imap, cmap; 1332 int error = 0; 1333 unsigned lockmode; 1334 u64 seq; 1335 1336 if (xfs_is_shutdown(mp)) 1337 return -EIO; 1338 1339 lockmode = xfs_ilock_data_map_shared(ip); 1340 error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); 1341 if (error) 1342 goto out_unlock; 1343 1344 if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) { 1345 /* 1346 * If we found a data extent we are done. 1347 */ 1348 if (imap.br_startoff <= offset_fsb) 1349 goto done; 1350 data_fsb = imap.br_startoff; 1351 } else { 1352 /* 1353 * Fake a hole until the end of the file. 1354 */ 1355 data_fsb = xfs_iomap_end_fsb(mp, offset, length); 1356 } 1357 1358 /* 1359 * If a COW fork extent covers the hole, report it - capped to the next 1360 * data fork extent: 1361 */ 1362 if (xfs_inode_has_cow_data(ip) && 1363 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap)) 1364 cow_fsb = cmap.br_startoff; 1365 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) { 1366 if (data_fsb < cow_fsb + cmap.br_blockcount) 1367 end_fsb = min(end_fsb, data_fsb); 1368 xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb); 1369 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); 1370 error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 1371 IOMAP_F_SHARED, seq); 1372 /* 1373 * This is a COW extent, so we must probe the page cache 1374 * because there could be dirty page cache being backed 1375 * by this extent. 1376 */ 1377 iomap->type = IOMAP_UNWRITTEN; 1378 goto out_unlock; 1379 } 1380 1381 /* 1382 * Else report a hole, capped to the next found data or COW extent. 1383 */ 1384 if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb) 1385 imap.br_blockcount = cow_fsb - offset_fsb; 1386 else 1387 imap.br_blockcount = data_fsb - offset_fsb; 1388 imap.br_startoff = offset_fsb; 1389 imap.br_startblock = HOLESTARTBLOCK; 1390 imap.br_state = XFS_EXT_NORM; 1391done: 1392 seq = xfs_iomap_inode_sequence(ip, 0); 1393 xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); 1394 error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); 1395out_unlock: 1396 xfs_iunlock(ip, lockmode); 1397 return error; 1398} 1399 1400const struct iomap_ops xfs_seek_iomap_ops = { 1401 .iomap_begin = xfs_seek_iomap_begin, 1402}; 1403 1404static int 1405xfs_xattr_iomap_begin( 1406 struct inode *inode, 1407 loff_t offset, 1408 loff_t length, 1409 unsigned flags, 1410 struct iomap *iomap, 1411 struct iomap *srcmap) 1412{ 1413 struct xfs_inode *ip = XFS_I(inode); 1414 struct xfs_mount *mp = ip->i_mount; 1415 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); 1416 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length); 1417 struct xfs_bmbt_irec imap; 1418 int nimaps = 1, error = 0; 1419 unsigned lockmode; 1420 int seq; 1421 1422 if (xfs_is_shutdown(mp)) 1423 return -EIO; 1424 1425 lockmode = xfs_ilock_attr_map_shared(ip); 1426 1427 /* if there are no attribute fork or extents, return ENOENT */ 1428 if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) { 1429 error = -ENOENT; 1430 goto out_unlock; 1431 } 1432 1433 ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL); 1434 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, 1435 &nimaps, XFS_BMAPI_ATTRFORK); 1436out_unlock: 1437 1438 seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR); 1439 xfs_iunlock(ip, lockmode); 1440 1441 if (error) 1442 return error; 1443 ASSERT(nimaps); 1444 return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq); 1445} 1446 1447const struct iomap_ops xfs_xattr_iomap_ops = { 1448 .iomap_begin = xfs_xattr_iomap_begin, 1449}; 1450 1451int 1452xfs_zero_range( 1453 struct xfs_inode *ip, 1454 loff_t pos, 1455 loff_t len, 1456 bool *did_zero) 1457{ 1458 struct inode *inode = VFS_I(ip); 1459 1460 if (IS_DAX(inode)) 1461 return dax_zero_range(inode, pos, len, did_zero, 1462 &xfs_dax_write_iomap_ops); 1463 return iomap_zero_range(inode, pos, len, did_zero, 1464 &xfs_buffered_write_iomap_ops); 1465} 1466 1467int 1468xfs_truncate_page( 1469 struct xfs_inode *ip, 1470 loff_t pos, 1471 bool *did_zero) 1472{ 1473 struct inode *inode = VFS_I(ip); 1474 1475 if (IS_DAX(inode)) 1476 return dax_truncate_page(inode, pos, did_zero, 1477 &xfs_dax_write_iomap_ops); 1478 return iomap_truncate_page(inode, pos, did_zero, 1479 &xfs_buffered_write_iomap_ops); 1480} 1481