summaryrefslogtreecommitdiffstats
path: root/fs/xfs/xfs_aops.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--fs/xfs/xfs_aops.c584
1 files changed, 584 insertions, 0 deletions
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
new file mode 100644
index 000000000..5d1a995b1
--- /dev/null
+++ b/fs/xfs/xfs_aops.c
@@ -0,0 +1,584 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2000-2005 Silicon Graphics, Inc.
+ * Copyright (c) 2016-2018 Christoph Hellwig.
+ * All Rights Reserved.
+ */
+#include "xfs.h"
+#include "xfs_shared.h"
+#include "xfs_format.h"
+#include "xfs_log_format.h"
+#include "xfs_trans_resv.h"
+#include "xfs_mount.h"
+#include "xfs_inode.h"
+#include "xfs_trans.h"
+#include "xfs_iomap.h"
+#include "xfs_trace.h"
+#include "xfs_bmap.h"
+#include "xfs_bmap_util.h"
+#include "xfs_reflink.h"
+
+struct xfs_writepage_ctx {
+ struct iomap_writepage_ctx ctx;
+ unsigned int data_seq;
+ unsigned int cow_seq;
+};
+
+static inline struct xfs_writepage_ctx *
+XFS_WPC(struct iomap_writepage_ctx *ctx)
+{
+ return container_of(ctx, struct xfs_writepage_ctx, ctx);
+}
+
+/*
+ * Fast and loose check if this write could update the on-disk inode size.
+ */
+static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
+{
+ return ioend->io_offset + ioend->io_size >
+ XFS_I(ioend->io_inode)->i_disk_size;
+}
+
+/*
+ * Update on-disk file size now that data has been written to disk.
+ */
+int
+xfs_setfilesize(
+ struct xfs_inode *ip,
+ xfs_off_t offset,
+ size_t size)
+{
+ struct xfs_mount *mp = ip->i_mount;
+ struct xfs_trans *tp;
+ xfs_fsize_t isize;
+ int error;
+
+ error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
+ if (error)
+ return error;
+
+ xfs_ilock(ip, XFS_ILOCK_EXCL);
+ isize = xfs_new_eof(ip, offset + size);
+ if (!isize) {
+ xfs_iunlock(ip, XFS_ILOCK_EXCL);
+ xfs_trans_cancel(tp);
+ return 0;
+ }
+
+ trace_xfs_setfilesize(ip, offset, size);
+
+ ip->i_disk_size = isize;
+ xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
+ xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
+
+ return xfs_trans_commit(tp);
+}
+
+/*
+ * IO write completion.
+ */
+STATIC void
+xfs_end_ioend(
+ struct iomap_ioend *ioend)
+{
+ struct xfs_inode *ip = XFS_I(ioend->io_inode);
+ struct xfs_mount *mp = ip->i_mount;
+ xfs_off_t offset = ioend->io_offset;
+ size_t size = ioend->io_size;
+ unsigned int nofs_flag;
+ int error;
+
+ /*
+ * We can allocate memory here while doing writeback on behalf of
+ * memory reclaim. To avoid memory allocation deadlocks set the
+ * task-wide nofs context for the following operations.
+ */
+ nofs_flag = memalloc_nofs_save();
+
+ /*
+ * Just clean up the in-memory structures if the fs has been shut down.
+ */
+ if (xfs_is_shutdown(mp)) {
+ error = -EIO;
+ goto done;
+ }
+
+ /*
+ * Clean up all COW blocks and underlying data fork delalloc blocks on
+ * I/O error. The delalloc punch is required because this ioend was
+ * mapped to blocks in the COW fork and the associated pages are no
+ * longer dirty. If we don't remove delalloc blocks here, they become
+ * stale and can corrupt free space accounting on unmount.
+ */
+ error = blk_status_to_errno(ioend->io_bio->bi_status);
+ if (unlikely(error)) {
+ if (ioend->io_flags & IOMAP_F_SHARED) {
+ xfs_reflink_cancel_cow_range(ip, offset, size, true);
+ xfs_bmap_punch_delalloc_range(ip,
+ XFS_B_TO_FSBT(mp, offset),
+ XFS_B_TO_FSB(mp, size));
+ }
+ goto done;
+ }
+
+ /*
+ * Success: commit the COW or unwritten blocks if needed.
+ */
+ if (ioend->io_flags & IOMAP_F_SHARED)
+ error = xfs_reflink_end_cow(ip, offset, size);
+ else if (ioend->io_type == IOMAP_UNWRITTEN)
+ error = xfs_iomap_write_unwritten(ip, offset, size, false);
+
+ if (!error && xfs_ioend_is_append(ioend))
+ error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
+done:
+ iomap_finish_ioends(ioend, error);
+ memalloc_nofs_restore(nofs_flag);
+}
+
+/*
+ * Finish all pending IO completions that require transactional modifications.
+ *
+ * We try to merge physical and logically contiguous ioends before completion to
+ * minimise the number of transactions we need to perform during IO completion.
+ * Both unwritten extent conversion and COW remapping need to iterate and modify
+ * one physical extent at a time, so we gain nothing by merging physically
+ * discontiguous extents here.
+ *
+ * The ioend chain length that we can be processing here is largely unbound in
+ * length and we may have to perform significant amounts of work on each ioend
+ * to complete it. Hence we have to be careful about holding the CPU for too
+ * long in this loop.
+ */
+void
+xfs_end_io(
+ struct work_struct *work)
+{
+ struct xfs_inode *ip =
+ container_of(work, struct xfs_inode, i_ioend_work);
+ struct iomap_ioend *ioend;
+ struct list_head tmp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ip->i_ioend_lock, flags);
+ list_replace_init(&ip->i_ioend_list, &tmp);
+ spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
+
+ iomap_sort_ioends(&tmp);
+ while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
+ io_list))) {
+ list_del_init(&ioend->io_list);
+ iomap_ioend_try_merge(ioend, &tmp);
+ xfs_end_ioend(ioend);
+ cond_resched();
+ }
+}
+
+STATIC void
+xfs_end_bio(
+ struct bio *bio)
+{
+ struct iomap_ioend *ioend = bio->bi_private;
+ struct xfs_inode *ip = XFS_I(ioend->io_inode);
+ unsigned long flags;
+
+ spin_lock_irqsave(&ip->i_ioend_lock, flags);
+ if (list_empty(&ip->i_ioend_list))
+ WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
+ &ip->i_ioend_work));
+ list_add_tail(&ioend->io_list, &ip->i_ioend_list);
+ spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
+}
+
+/*
+ * Fast revalidation of the cached writeback mapping. Return true if the current
+ * mapping is valid, false otherwise.
+ */
+static bool
+xfs_imap_valid(
+ struct iomap_writepage_ctx *wpc,
+ struct xfs_inode *ip,
+ loff_t offset)
+{
+ if (offset < wpc->iomap.offset ||
+ offset >= wpc->iomap.offset + wpc->iomap.length)
+ return false;
+ /*
+ * If this is a COW mapping, it is sufficient to check that the mapping
+ * covers the offset. Be careful to check this first because the caller
+ * can revalidate a COW mapping without updating the data seqno.
+ */
+ if (wpc->iomap.flags & IOMAP_F_SHARED)
+ return true;
+
+ /*
+ * This is not a COW mapping. Check the sequence number of the data fork
+ * because concurrent changes could have invalidated the extent. Check
+ * the COW fork because concurrent changes since the last time we
+ * checked (and found nothing at this offset) could have added
+ * overlapping blocks.
+ */
+ if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
+ return false;
+ if (xfs_inode_has_cow_data(ip) &&
+ XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
+ return false;
+ return true;
+}
+
+/*
+ * Pass in a dellalloc extent and convert it to real extents, return the real
+ * extent that maps offset_fsb in wpc->iomap.
+ *
+ * The current page is held locked so nothing could have removed the block
+ * backing offset_fsb, although it could have moved from the COW to the data
+ * fork by another thread.
+ */
+static int
+xfs_convert_blocks(
+ struct iomap_writepage_ctx *wpc,
+ struct xfs_inode *ip,
+ int whichfork,
+ loff_t offset)
+{
+ int error;
+ unsigned *seq;
+
+ if (whichfork == XFS_COW_FORK)
+ seq = &XFS_WPC(wpc)->cow_seq;
+ else
+ seq = &XFS_WPC(wpc)->data_seq;
+
+ /*
+ * Attempt to allocate whatever delalloc extent currently backs offset
+ * and put the result into wpc->iomap. Allocate in a loop because it
+ * may take several attempts to allocate real blocks for a contiguous
+ * delalloc extent if free space is sufficiently fragmented.
+ */
+ do {
+ error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
+ &wpc->iomap, seq);
+ if (error)
+ return error;
+ } while (wpc->iomap.offset + wpc->iomap.length <= offset);
+
+ return 0;
+}
+
+static int
+xfs_map_blocks(
+ struct iomap_writepage_ctx *wpc,
+ struct inode *inode,
+ loff_t offset)
+{
+ struct xfs_inode *ip = XFS_I(inode);
+ struct xfs_mount *mp = ip->i_mount;
+ ssize_t count = i_blocksize(inode);
+ xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
+ xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
+ xfs_fileoff_t cow_fsb;
+ int whichfork;
+ struct xfs_bmbt_irec imap;
+ struct xfs_iext_cursor icur;
+ int retries = 0;
+ int error = 0;
+
+ if (xfs_is_shutdown(mp))
+ return -EIO;
+
+ /*
+ * COW fork blocks can overlap data fork blocks even if the blocks
+ * aren't shared. COW I/O always takes precedent, so we must always
+ * check for overlap on reflink inodes unless the mapping is already a
+ * COW one, or the COW fork hasn't changed from the last time we looked
+ * at it.
+ *
+ * It's safe to check the COW fork if_seq here without the ILOCK because
+ * we've indirectly protected against concurrent updates: writeback has
+ * the page locked, which prevents concurrent invalidations by reflink
+ * and directio and prevents concurrent buffered writes to the same
+ * page. Changes to if_seq always happen under i_lock, which protects
+ * against concurrent updates and provides a memory barrier on the way
+ * out that ensures that we always see the current value.
+ */
+ if (xfs_imap_valid(wpc, ip, offset))
+ return 0;
+
+ /*
+ * If we don't have a valid map, now it's time to get a new one for this
+ * offset. This will convert delayed allocations (including COW ones)
+ * into real extents. If we return without a valid map, it means we
+ * landed in a hole and we skip the block.
+ */
+retry:
+ cow_fsb = NULLFILEOFF;
+ whichfork = XFS_DATA_FORK;
+ xfs_ilock(ip, XFS_ILOCK_SHARED);
+ ASSERT(!xfs_need_iread_extents(&ip->i_df));
+
+ /*
+ * Check if this is offset is covered by a COW extents, and if yes use
+ * it directly instead of looking up anything in the data fork.
+ */
+ if (xfs_inode_has_cow_data(ip) &&
+ xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
+ cow_fsb = imap.br_startoff;
+ if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
+ XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+
+ whichfork = XFS_COW_FORK;
+ goto allocate_blocks;
+ }
+
+ /*
+ * No COW extent overlap. Revalidate now that we may have updated
+ * ->cow_seq. If the data mapping is still valid, we're done.
+ */
+ if (xfs_imap_valid(wpc, ip, offset)) {
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+ return 0;
+ }
+
+ /*
+ * If we don't have a valid map, now it's time to get a new one for this
+ * offset. This will convert delayed allocations (including COW ones)
+ * into real extents.
+ */
+ if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
+ imap.br_startoff = end_fsb; /* fake a hole past EOF */
+ XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
+ xfs_iunlock(ip, XFS_ILOCK_SHARED);
+
+ /* landed in a hole or beyond EOF? */
+ if (imap.br_startoff > offset_fsb) {
+ imap.br_blockcount = imap.br_startoff - offset_fsb;
+ imap.br_startoff = offset_fsb;
+ imap.br_startblock = HOLESTARTBLOCK;
+ imap.br_state = XFS_EXT_NORM;
+ }
+
+ /*
+ * Truncate to the next COW extent if there is one. This is the only
+ * opportunity to do this because we can skip COW fork lookups for the
+ * subsequent blocks in the mapping; however, the requirement to treat
+ * the COW range separately remains.
+ */
+ if (cow_fsb != NULLFILEOFF &&
+ cow_fsb < imap.br_startoff + imap.br_blockcount)
+ imap.br_blockcount = cow_fsb - imap.br_startoff;
+
+ /* got a delalloc extent? */
+ if (imap.br_startblock != HOLESTARTBLOCK &&
+ isnullstartblock(imap.br_startblock))
+ goto allocate_blocks;
+
+ xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0);
+ trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
+ return 0;
+allocate_blocks:
+ error = xfs_convert_blocks(wpc, ip, whichfork, offset);
+ if (error) {
+ /*
+ * If we failed to find the extent in the COW fork we might have
+ * raced with a COW to data fork conversion or truncate.
+ * Restart the lookup to catch the extent in the data fork for
+ * the former case, but prevent additional retries to avoid
+ * looping forever for the latter case.
+ */
+ if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
+ goto retry;
+ ASSERT(error != -EAGAIN);
+ return error;
+ }
+
+ /*
+ * Due to merging the return real extent might be larger than the
+ * original delalloc one. Trim the return extent to the next COW
+ * boundary again to force a re-lookup.
+ */
+ if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
+ loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
+
+ if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
+ wpc->iomap.length = cow_offset - wpc->iomap.offset;
+ }
+
+ ASSERT(wpc->iomap.offset <= offset);
+ ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
+ trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
+ return 0;
+}
+
+static int
+xfs_prepare_ioend(
+ struct iomap_ioend *ioend,
+ int status)
+{
+ unsigned int nofs_flag;
+
+ /*
+ * We can allocate memory here while doing writeback on behalf of
+ * memory reclaim. To avoid memory allocation deadlocks set the
+ * task-wide nofs context for the following operations.
+ */
+ nofs_flag = memalloc_nofs_save();
+
+ /* Convert CoW extents to regular */
+ if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
+ status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
+ ioend->io_offset, ioend->io_size);
+ }
+
+ memalloc_nofs_restore(nofs_flag);
+
+ /* send ioends that might require a transaction to the completion wq */
+ if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
+ (ioend->io_flags & IOMAP_F_SHARED))
+ ioend->io_bio->bi_end_io = xfs_end_bio;
+ return status;
+}
+
+/*
+ * If the page has delalloc blocks on it, we need to punch them out before we
+ * invalidate the page. If we don't, we leave a stale delalloc mapping on the
+ * inode that can trip up a later direct I/O read operation on the same region.
+ *
+ * We prevent this by truncating away the delalloc regions on the page. Because
+ * they are delalloc, we can do this without needing a transaction. Indeed - if
+ * we get ENOSPC errors, we have to be able to do this truncation without a
+ * transaction as there is no space left for block reservation (typically why we
+ * see a ENOSPC in writeback).
+ */
+static void
+xfs_discard_folio(
+ struct folio *folio,
+ loff_t pos)
+{
+ struct inode *inode = folio->mapping->host;
+ struct xfs_inode *ip = XFS_I(inode);
+ struct xfs_mount *mp = ip->i_mount;
+ size_t offset = offset_in_folio(folio, pos);
+ xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, pos);
+ xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, offset);
+ int error;
+
+ if (xfs_is_shutdown(mp))
+ return;
+
+ xfs_alert_ratelimited(mp,
+ "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.",
+ folio, ip->i_ino, pos);
+
+ error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
+ i_blocks_per_folio(inode, folio) - pageoff_fsb);
+ if (error && !xfs_is_shutdown(mp))
+ xfs_alert(mp, "page discard unable to remove delalloc mapping.");
+}
+
+static const struct iomap_writeback_ops xfs_writeback_ops = {
+ .map_blocks = xfs_map_blocks,
+ .prepare_ioend = xfs_prepare_ioend,
+ .discard_folio = xfs_discard_folio,
+};
+
+STATIC int
+xfs_vm_writepages(
+ struct address_space *mapping,
+ struct writeback_control *wbc)
+{
+ struct xfs_writepage_ctx wpc = { };
+
+ /*
+ * Writing back data in a transaction context can result in recursive
+ * transactions. This is bad, so issue a warning and get out of here.
+ */
+ if (WARN_ON_ONCE(current->journal_info))
+ return 0;
+
+ xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
+ return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
+}
+
+STATIC int
+xfs_dax_writepages(
+ struct address_space *mapping,
+ struct writeback_control *wbc)
+{
+ struct xfs_inode *ip = XFS_I(mapping->host);
+
+ xfs_iflags_clear(ip, XFS_ITRUNCATED);
+ return dax_writeback_mapping_range(mapping,
+ xfs_inode_buftarg(ip)->bt_daxdev, wbc);
+}
+
+STATIC sector_t
+xfs_vm_bmap(
+ struct address_space *mapping,
+ sector_t block)
+{
+ struct xfs_inode *ip = XFS_I(mapping->host);
+
+ trace_xfs_vm_bmap(ip);
+
+ /*
+ * The swap code (ab-)uses ->bmap to get a block mapping and then
+ * bypasses the file system for actual I/O. We really can't allow
+ * that on reflinks inodes, so we have to skip out here. And yes,
+ * 0 is the magic code for a bmap error.
+ *
+ * Since we don't pass back blockdev info, we can't return bmap
+ * information for rt files either.
+ */
+ if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
+ return 0;
+ return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
+}
+
+STATIC int
+xfs_vm_read_folio(
+ struct file *unused,
+ struct folio *folio)
+{
+ return iomap_read_folio(folio, &xfs_read_iomap_ops);
+}
+
+STATIC void
+xfs_vm_readahead(
+ struct readahead_control *rac)
+{
+ iomap_readahead(rac, &xfs_read_iomap_ops);
+}
+
+static int
+xfs_iomap_swapfile_activate(
+ struct swap_info_struct *sis,
+ struct file *swap_file,
+ sector_t *span)
+{
+ sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
+ return iomap_swapfile_activate(sis, swap_file, span,
+ &xfs_read_iomap_ops);
+}
+
+const struct address_space_operations xfs_address_space_operations = {
+ .read_folio = xfs_vm_read_folio,
+ .readahead = xfs_vm_readahead,
+ .writepages = xfs_vm_writepages,
+ .dirty_folio = filemap_dirty_folio,
+ .release_folio = iomap_release_folio,
+ .invalidate_folio = iomap_invalidate_folio,
+ .bmap = xfs_vm_bmap,
+ .direct_IO = noop_direct_IO,
+ .migrate_folio = filemap_migrate_folio,
+ .is_partially_uptodate = iomap_is_partially_uptodate,
+ .error_remove_page = generic_error_remove_page,
+ .swap_activate = xfs_iomap_swapfile_activate,
+};
+
+const struct address_space_operations xfs_dax_aops = {
+ .writepages = xfs_dax_writepages,
+ .direct_IO = noop_direct_IO,
+ .dirty_folio = noop_dirty_folio,
+ .swap_activate = xfs_iomap_swapfile_activate,
+};