summaryrefslogtreecommitdiffstats
path: root/fs/iomap
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
commit2c3c1048746a4622d8c89a29670120dc8fab93c4 (patch)
tree848558de17fb3008cdf4d861b01ac7781903ce39 /fs/iomap
parentInitial commit. (diff)
downloadlinux-2c3c1048746a4622d8c89a29670120dc8fab93c4.tar.xz
linux-2c3c1048746a4622d8c89a29670120dc8fab93c4.zip
Adding upstream version 6.1.76.upstream/6.1.76
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'fs/iomap')
-rw-r--r--fs/iomap/Makefile17
-rw-r--r--fs/iomap/buffered-io.c1548
-rw-r--r--fs/iomap/direct-io.c696
-rw-r--r--fs/iomap/fiemap.c124
-rw-r--r--fs/iomap/iter.c80
-rw-r--r--fs/iomap/seek.c104
-rw-r--r--fs/iomap/swapfile.c195
-rw-r--r--fs/iomap/trace.c12
-rw-r--r--fs/iomap/trace.h191
9 files changed, 2967 insertions, 0 deletions
diff --git a/fs/iomap/Makefile b/fs/iomap/Makefile
new file mode 100644
index 000000000..fc070184b
--- /dev/null
+++ b/fs/iomap/Makefile
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Copyright (c) 2019 Oracle.
+# All Rights Reserved.
+#
+
+ccflags-y += -I $(srctree)/$(src) # needed for trace events
+
+obj-$(CONFIG_FS_IOMAP) += iomap.o
+
+iomap-y += trace.o \
+ iter.o
+iomap-$(CONFIG_BLOCK) += buffered-io.o \
+ direct-io.o \
+ fiemap.o \
+ seek.o
+iomap-$(CONFIG_SWAP) += swapfile.o
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
new file mode 100644
index 000000000..a0a4d8de8
--- /dev/null
+++ b/fs/iomap/buffered-io.c
@@ -0,0 +1,1548 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010 Red Hat, Inc.
+ * Copyright (C) 2016-2019 Christoph Hellwig.
+ */
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include <linux/pagemap.h>
+#include <linux/uio.h>
+#include <linux/buffer_head.h>
+#include <linux/dax.h>
+#include <linux/writeback.h>
+#include <linux/list_sort.h>
+#include <linux/swap.h>
+#include <linux/bio.h>
+#include <linux/sched/signal.h>
+#include <linux/migrate.h>
+#include "trace.h"
+
+#include "../internal.h"
+
+#define IOEND_BATCH_SIZE 4096
+
+/*
+ * Structure allocated for each folio when block size < folio size
+ * to track sub-folio uptodate status and I/O completions.
+ */
+struct iomap_page {
+ atomic_t read_bytes_pending;
+ atomic_t write_bytes_pending;
+ spinlock_t uptodate_lock;
+ unsigned long uptodate[];
+};
+
+static inline struct iomap_page *to_iomap_page(struct folio *folio)
+{
+ if (folio_test_private(folio))
+ return folio_get_private(folio);
+ return NULL;
+}
+
+static struct bio_set iomap_ioend_bioset;
+
+static struct iomap_page *
+iomap_page_create(struct inode *inode, struct folio *folio, unsigned int flags)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+ unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
+ gfp_t gfp;
+
+ if (iop || nr_blocks <= 1)
+ return iop;
+
+ if (flags & IOMAP_NOWAIT)
+ gfp = GFP_NOWAIT;
+ else
+ gfp = GFP_NOFS | __GFP_NOFAIL;
+
+ iop = kzalloc(struct_size(iop, uptodate, BITS_TO_LONGS(nr_blocks)),
+ gfp);
+ if (iop) {
+ spin_lock_init(&iop->uptodate_lock);
+ if (folio_test_uptodate(folio))
+ bitmap_fill(iop->uptodate, nr_blocks);
+ folio_attach_private(folio, iop);
+ }
+ return iop;
+}
+
+static void iomap_page_release(struct folio *folio)
+{
+ struct iomap_page *iop = folio_detach_private(folio);
+ struct inode *inode = folio->mapping->host;
+ unsigned int nr_blocks = i_blocks_per_folio(inode, folio);
+
+ if (!iop)
+ return;
+ WARN_ON_ONCE(atomic_read(&iop->read_bytes_pending));
+ WARN_ON_ONCE(atomic_read(&iop->write_bytes_pending));
+ WARN_ON_ONCE(bitmap_full(iop->uptodate, nr_blocks) !=
+ folio_test_uptodate(folio));
+ kfree(iop);
+}
+
+/*
+ * Calculate the range inside the folio that we actually need to read.
+ */
+static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
+ loff_t *pos, loff_t length, size_t *offp, size_t *lenp)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+ loff_t orig_pos = *pos;
+ loff_t isize = i_size_read(inode);
+ unsigned block_bits = inode->i_blkbits;
+ unsigned block_size = (1 << block_bits);
+ size_t poff = offset_in_folio(folio, *pos);
+ size_t plen = min_t(loff_t, folio_size(folio) - poff, length);
+ unsigned first = poff >> block_bits;
+ unsigned last = (poff + plen - 1) >> block_bits;
+
+ /*
+ * If the block size is smaller than the page size, we need to check the
+ * per-block uptodate status and adjust the offset and length if needed
+ * to avoid reading in already uptodate ranges.
+ */
+ if (iop) {
+ unsigned int i;
+
+ /* move forward for each leading block marked uptodate */
+ for (i = first; i <= last; i++) {
+ if (!test_bit(i, iop->uptodate))
+ break;
+ *pos += block_size;
+ poff += block_size;
+ plen -= block_size;
+ first++;
+ }
+
+ /* truncate len if we find any trailing uptodate block(s) */
+ for ( ; i <= last; i++) {
+ if (test_bit(i, iop->uptodate)) {
+ plen -= (last - i + 1) * block_size;
+ last = i - 1;
+ break;
+ }
+ }
+ }
+
+ /*
+ * If the extent spans the block that contains the i_size, we need to
+ * handle both halves separately so that we properly zero data in the
+ * page cache for blocks that are entirely outside of i_size.
+ */
+ if (orig_pos <= isize && orig_pos + length > isize) {
+ unsigned end = offset_in_folio(folio, isize - 1) >> block_bits;
+
+ if (first <= end && last > end)
+ plen -= (last - end) * block_size;
+ }
+
+ *offp = poff;
+ *lenp = plen;
+}
+
+static void iomap_iop_set_range_uptodate(struct folio *folio,
+ struct iomap_page *iop, size_t off, size_t len)
+{
+ struct inode *inode = folio->mapping->host;
+ unsigned first = off >> inode->i_blkbits;
+ unsigned last = (off + len - 1) >> inode->i_blkbits;
+ unsigned long flags;
+
+ spin_lock_irqsave(&iop->uptodate_lock, flags);
+ bitmap_set(iop->uptodate, first, last - first + 1);
+ if (bitmap_full(iop->uptodate, i_blocks_per_folio(inode, folio)))
+ folio_mark_uptodate(folio);
+ spin_unlock_irqrestore(&iop->uptodate_lock, flags);
+}
+
+static void iomap_set_range_uptodate(struct folio *folio,
+ struct iomap_page *iop, size_t off, size_t len)
+{
+ if (iop)
+ iomap_iop_set_range_uptodate(folio, iop, off, len);
+ else
+ folio_mark_uptodate(folio);
+}
+
+static void iomap_finish_folio_read(struct folio *folio, size_t offset,
+ size_t len, int error)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+
+ if (unlikely(error)) {
+ folio_clear_uptodate(folio);
+ folio_set_error(folio);
+ } else {
+ iomap_set_range_uptodate(folio, iop, offset, len);
+ }
+
+ if (!iop || atomic_sub_and_test(len, &iop->read_bytes_pending))
+ folio_unlock(folio);
+}
+
+static void iomap_read_end_io(struct bio *bio)
+{
+ int error = blk_status_to_errno(bio->bi_status);
+ struct folio_iter fi;
+
+ bio_for_each_folio_all(fi, bio)
+ iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
+ bio_put(bio);
+}
+
+struct iomap_readpage_ctx {
+ struct folio *cur_folio;
+ bool cur_folio_in_bio;
+ struct bio *bio;
+ struct readahead_control *rac;
+};
+
+/**
+ * iomap_read_inline_data - copy inline data into the page cache
+ * @iter: iteration structure
+ * @folio: folio to copy to
+ *
+ * Copy the inline data in @iter into @folio and zero out the rest of the folio.
+ * Only a single IOMAP_INLINE extent is allowed at the end of each file.
+ * Returns zero for success to complete the read, or the usual negative errno.
+ */
+static int iomap_read_inline_data(const struct iomap_iter *iter,
+ struct folio *folio)
+{
+ struct iomap_page *iop;
+ const struct iomap *iomap = iomap_iter_srcmap(iter);
+ size_t size = i_size_read(iter->inode) - iomap->offset;
+ size_t poff = offset_in_page(iomap->offset);
+ size_t offset = offset_in_folio(folio, iomap->offset);
+ void *addr;
+
+ if (folio_test_uptodate(folio))
+ return 0;
+
+ if (WARN_ON_ONCE(size > PAGE_SIZE - poff))
+ return -EIO;
+ if (WARN_ON_ONCE(size > PAGE_SIZE -
+ offset_in_page(iomap->inline_data)))
+ return -EIO;
+ if (WARN_ON_ONCE(size > iomap->length))
+ return -EIO;
+ if (offset > 0)
+ iop = iomap_page_create(iter->inode, folio, iter->flags);
+ else
+ iop = to_iomap_page(folio);
+
+ addr = kmap_local_folio(folio, offset);
+ memcpy(addr, iomap->inline_data, size);
+ memset(addr + size, 0, PAGE_SIZE - poff - size);
+ kunmap_local(addr);
+ iomap_set_range_uptodate(folio, iop, offset, PAGE_SIZE - poff);
+ return 0;
+}
+
+static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
+ loff_t pos)
+{
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+
+ return srcmap->type != IOMAP_MAPPED ||
+ (srcmap->flags & IOMAP_F_NEW) ||
+ pos >= i_size_read(iter->inode);
+}
+
+static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
+ struct iomap_readpage_ctx *ctx, loff_t offset)
+{
+ const struct iomap *iomap = &iter->iomap;
+ loff_t pos = iter->pos + offset;
+ loff_t length = iomap_length(iter) - offset;
+ struct folio *folio = ctx->cur_folio;
+ struct iomap_page *iop;
+ loff_t orig_pos = pos;
+ size_t poff, plen;
+ sector_t sector;
+
+ if (iomap->type == IOMAP_INLINE)
+ return iomap_read_inline_data(iter, folio);
+
+ /* zero post-eof blocks as the page may be mapped */
+ iop = iomap_page_create(iter->inode, folio, iter->flags);
+ iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, &plen);
+ if (plen == 0)
+ goto done;
+
+ if (iomap_block_needs_zeroing(iter, pos)) {
+ folio_zero_range(folio, poff, plen);
+ iomap_set_range_uptodate(folio, iop, poff, plen);
+ goto done;
+ }
+
+ ctx->cur_folio_in_bio = true;
+ if (iop)
+ atomic_add(plen, &iop->read_bytes_pending);
+
+ sector = iomap_sector(iomap, pos);
+ if (!ctx->bio ||
+ bio_end_sector(ctx->bio) != sector ||
+ !bio_add_folio(ctx->bio, folio, plen, poff)) {
+ gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
+ gfp_t orig_gfp = gfp;
+ unsigned int nr_vecs = DIV_ROUND_UP(length, PAGE_SIZE);
+
+ if (ctx->bio)
+ submit_bio(ctx->bio);
+
+ if (ctx->rac) /* same as readahead_gfp_mask */
+ gfp |= __GFP_NORETRY | __GFP_NOWARN;
+ ctx->bio = bio_alloc(iomap->bdev, bio_max_segs(nr_vecs),
+ REQ_OP_READ, gfp);
+ /*
+ * If the bio_alloc fails, try it again for a single page to
+ * avoid having to deal with partial page reads. This emulates
+ * what do_mpage_read_folio does.
+ */
+ if (!ctx->bio) {
+ ctx->bio = bio_alloc(iomap->bdev, 1, REQ_OP_READ,
+ orig_gfp);
+ }
+ if (ctx->rac)
+ ctx->bio->bi_opf |= REQ_RAHEAD;
+ ctx->bio->bi_iter.bi_sector = sector;
+ ctx->bio->bi_end_io = iomap_read_end_io;
+ bio_add_folio(ctx->bio, folio, plen, poff);
+ }
+
+done:
+ /*
+ * Move the caller beyond our range so that it keeps making progress.
+ * For that, we have to include any leading non-uptodate ranges, but
+ * we can skip trailing ones as they will be handled in the next
+ * iteration.
+ */
+ return pos - orig_pos + plen;
+}
+
+int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = folio->mapping->host,
+ .pos = folio_pos(folio),
+ .len = folio_size(folio),
+ };
+ struct iomap_readpage_ctx ctx = {
+ .cur_folio = folio,
+ };
+ int ret;
+
+ trace_iomap_readpage(iter.inode, 1);
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_readpage_iter(&iter, &ctx, 0);
+
+ if (ret < 0)
+ folio_set_error(folio);
+
+ if (ctx.bio) {
+ submit_bio(ctx.bio);
+ WARN_ON_ONCE(!ctx.cur_folio_in_bio);
+ } else {
+ WARN_ON_ONCE(ctx.cur_folio_in_bio);
+ folio_unlock(folio);
+ }
+
+ /*
+ * Just like mpage_readahead and block_read_full_folio, we always
+ * return 0 and just set the folio error flag on errors. This
+ * should be cleaned up throughout the stack eventually.
+ */
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iomap_read_folio);
+
+static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
+ struct iomap_readpage_ctx *ctx)
+{
+ loff_t length = iomap_length(iter);
+ loff_t done, ret;
+
+ for (done = 0; done < length; done += ret) {
+ if (ctx->cur_folio &&
+ offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) {
+ if (!ctx->cur_folio_in_bio)
+ folio_unlock(ctx->cur_folio);
+ ctx->cur_folio = NULL;
+ }
+ if (!ctx->cur_folio) {
+ ctx->cur_folio = readahead_folio(ctx->rac);
+ ctx->cur_folio_in_bio = false;
+ }
+ ret = iomap_readpage_iter(iter, ctx, done);
+ if (ret <= 0)
+ return ret;
+ }
+
+ return done;
+}
+
+/**
+ * iomap_readahead - Attempt to read pages from a file.
+ * @rac: Describes the pages to be read.
+ * @ops: The operations vector for the filesystem.
+ *
+ * This function is for filesystems to call to implement their readahead
+ * address_space operation.
+ *
+ * Context: The @ops callbacks may submit I/O (eg to read the addresses of
+ * blocks from disc), and may wait for it. The caller may be trying to
+ * access a different page, and so sleeping excessively should be avoided.
+ * It may allocate memory, but should avoid costly allocations. This
+ * function is called with memalloc_nofs set, so allocations will not cause
+ * the filesystem to be reentered.
+ */
+void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = rac->mapping->host,
+ .pos = readahead_pos(rac),
+ .len = readahead_length(rac),
+ };
+ struct iomap_readpage_ctx ctx = {
+ .rac = rac,
+ };
+
+ trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
+
+ while (iomap_iter(&iter, ops) > 0)
+ iter.processed = iomap_readahead_iter(&iter, &ctx);
+
+ if (ctx.bio)
+ submit_bio(ctx.bio);
+ if (ctx.cur_folio) {
+ if (!ctx.cur_folio_in_bio)
+ folio_unlock(ctx.cur_folio);
+ }
+}
+EXPORT_SYMBOL_GPL(iomap_readahead);
+
+/*
+ * iomap_is_partially_uptodate checks whether blocks within a folio are
+ * uptodate or not.
+ *
+ * Returns true if all blocks which correspond to the specified part
+ * of the folio are uptodate.
+ */
+bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+ struct inode *inode = folio->mapping->host;
+ unsigned first, last, i;
+
+ if (!iop)
+ return false;
+
+ /* Caller's range may extend past the end of this folio */
+ count = min(folio_size(folio) - from, count);
+
+ /* First and last blocks in range within folio */
+ first = from >> inode->i_blkbits;
+ last = (from + count - 1) >> inode->i_blkbits;
+
+ for (i = first; i <= last; i++)
+ if (!test_bit(i, iop->uptodate))
+ return false;
+ return true;
+}
+EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
+
+bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags)
+{
+ trace_iomap_release_folio(folio->mapping->host, folio_pos(folio),
+ folio_size(folio));
+
+ /*
+ * mm accommodates an old ext3 case where clean folios might
+ * not have had the dirty bit cleared. Thus, it can send actual
+ * dirty folios to ->release_folio() via shrink_active_list();
+ * skip those here.
+ */
+ if (folio_test_dirty(folio) || folio_test_writeback(folio))
+ return false;
+ iomap_page_release(folio);
+ return true;
+}
+EXPORT_SYMBOL_GPL(iomap_release_folio);
+
+void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len)
+{
+ trace_iomap_invalidate_folio(folio->mapping->host,
+ folio_pos(folio) + offset, len);
+
+ /*
+ * If we're invalidating the entire folio, clear the dirty state
+ * from it and release it to avoid unnecessary buildup of the LRU.
+ */
+ if (offset == 0 && len == folio_size(folio)) {
+ WARN_ON_ONCE(folio_test_writeback(folio));
+ folio_cancel_dirty(folio);
+ iomap_page_release(folio);
+ }
+}
+EXPORT_SYMBOL_GPL(iomap_invalidate_folio);
+
+static void
+iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
+{
+ loff_t i_size = i_size_read(inode);
+
+ /*
+ * Only truncate newly allocated pages beyoned EOF, even if the
+ * write started inside the existing inode size.
+ */
+ if (pos + len > i_size)
+ truncate_pagecache_range(inode, max(pos, i_size),
+ pos + len - 1);
+}
+
+static int iomap_read_folio_sync(loff_t block_start, struct folio *folio,
+ size_t poff, size_t plen, const struct iomap *iomap)
+{
+ struct bio_vec bvec;
+ struct bio bio;
+
+ bio_init(&bio, iomap->bdev, &bvec, 1, REQ_OP_READ);
+ bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
+ bio_add_folio(&bio, folio, plen, poff);
+ return submit_bio_wait(&bio);
+}
+
+static int __iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
+ size_t len, struct folio *folio)
+{
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+ struct iomap_page *iop;
+ loff_t block_size = i_blocksize(iter->inode);
+ loff_t block_start = round_down(pos, block_size);
+ loff_t block_end = round_up(pos + len, block_size);
+ unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio);
+ size_t from = offset_in_folio(folio, pos), to = from + len;
+ size_t poff, plen;
+
+ if (folio_test_uptodate(folio))
+ return 0;
+ folio_clear_error(folio);
+
+ iop = iomap_page_create(iter->inode, folio, iter->flags);
+ if ((iter->flags & IOMAP_NOWAIT) && !iop && nr_blocks > 1)
+ return -EAGAIN;
+
+ do {
+ iomap_adjust_read_range(iter->inode, folio, &block_start,
+ block_end - block_start, &poff, &plen);
+ if (plen == 0)
+ break;
+
+ if (!(iter->flags & IOMAP_UNSHARE) &&
+ (from <= poff || from >= poff + plen) &&
+ (to <= poff || to >= poff + plen))
+ continue;
+
+ if (iomap_block_needs_zeroing(iter, block_start)) {
+ if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE))
+ return -EIO;
+ folio_zero_segments(folio, poff, from, to, poff + plen);
+ } else {
+ int status;
+
+ if (iter->flags & IOMAP_NOWAIT)
+ return -EAGAIN;
+
+ status = iomap_read_folio_sync(block_start, folio,
+ poff, plen, srcmap);
+ if (status)
+ return status;
+ }
+ iomap_set_range_uptodate(folio, iop, poff, plen);
+ } while ((block_start += plen) < block_end);
+
+ return 0;
+}
+
+static int iomap_write_begin_inline(const struct iomap_iter *iter,
+ struct folio *folio)
+{
+ /* needs more work for the tailpacking case; disable for now */
+ if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0))
+ return -EIO;
+ return iomap_read_inline_data(iter, folio);
+}
+
+static int iomap_write_begin(const struct iomap_iter *iter, loff_t pos,
+ size_t len, struct folio **foliop)
+{
+ const struct iomap_page_ops *page_ops = iter->iomap.page_ops;
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+ struct folio *folio;
+ unsigned fgp = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE | FGP_NOFS;
+ int status = 0;
+
+ if (iter->flags & IOMAP_NOWAIT)
+ fgp |= FGP_NOWAIT;
+
+ BUG_ON(pos + len > iter->iomap.offset + iter->iomap.length);
+ if (srcmap != &iter->iomap)
+ BUG_ON(pos + len > srcmap->offset + srcmap->length);
+
+ if (fatal_signal_pending(current))
+ return -EINTR;
+
+ if (!mapping_large_folio_support(iter->inode->i_mapping))
+ len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
+
+ if (page_ops && page_ops->page_prepare) {
+ status = page_ops->page_prepare(iter->inode, pos, len);
+ if (status)
+ return status;
+ }
+
+ folio = __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
+ fgp, mapping_gfp_mask(iter->inode->i_mapping));
+ if (!folio) {
+ status = (iter->flags & IOMAP_NOWAIT) ? -EAGAIN : -ENOMEM;
+ goto out_no_page;
+ }
+ if (pos + len > folio_pos(folio) + folio_size(folio))
+ len = folio_pos(folio) + folio_size(folio) - pos;
+
+ if (srcmap->type == IOMAP_INLINE)
+ status = iomap_write_begin_inline(iter, folio);
+ else if (srcmap->flags & IOMAP_F_BUFFER_HEAD)
+ status = __block_write_begin_int(folio, pos, len, NULL, srcmap);
+ else
+ status = __iomap_write_begin(iter, pos, len, folio);
+
+ if (unlikely(status))
+ goto out_unlock;
+
+ *foliop = folio;
+ return 0;
+
+out_unlock:
+ folio_unlock(folio);
+ folio_put(folio);
+ iomap_write_failed(iter->inode, pos, len);
+
+out_no_page:
+ if (page_ops && page_ops->page_done)
+ page_ops->page_done(iter->inode, pos, 0, NULL);
+ return status;
+}
+
+static size_t __iomap_write_end(struct inode *inode, loff_t pos, size_t len,
+ size_t copied, struct folio *folio)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+ flush_dcache_folio(folio);
+
+ /*
+ * The blocks that were entirely written will now be uptodate, so we
+ * don't have to worry about a read_folio reading them and overwriting a
+ * partial write. However, if we've encountered a short write and only
+ * partially written into a block, it will not be marked uptodate, so a
+ * read_folio might come in and destroy our partial write.
+ *
+ * Do the simplest thing and just treat any short write to a
+ * non-uptodate page as a zero-length write, and force the caller to
+ * redo the whole thing.
+ */
+ if (unlikely(copied < len && !folio_test_uptodate(folio)))
+ return 0;
+ iomap_set_range_uptodate(folio, iop, offset_in_folio(folio, pos), len);
+ filemap_dirty_folio(inode->i_mapping, folio);
+ return copied;
+}
+
+static size_t iomap_write_end_inline(const struct iomap_iter *iter,
+ struct folio *folio, loff_t pos, size_t copied)
+{
+ const struct iomap *iomap = &iter->iomap;
+ void *addr;
+
+ WARN_ON_ONCE(!folio_test_uptodate(folio));
+ BUG_ON(!iomap_inline_data_valid(iomap));
+
+ flush_dcache_folio(folio);
+ addr = kmap_local_folio(folio, pos);
+ memcpy(iomap_inline_data(iomap, pos), addr, copied);
+ kunmap_local(addr);
+
+ mark_inode_dirty(iter->inode);
+ return copied;
+}
+
+/* Returns the number of bytes copied. May be 0. Cannot be an errno. */
+static size_t iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len,
+ size_t copied, struct folio *folio)
+{
+ const struct iomap_page_ops *page_ops = iter->iomap.page_ops;
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+ loff_t old_size = iter->inode->i_size;
+ size_t ret;
+
+ if (srcmap->type == IOMAP_INLINE) {
+ ret = iomap_write_end_inline(iter, folio, pos, copied);
+ } else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) {
+ ret = block_write_end(NULL, iter->inode->i_mapping, pos, len,
+ copied, &folio->page, NULL);
+ } else {
+ ret = __iomap_write_end(iter->inode, pos, len, copied, folio);
+ }
+
+ /*
+ * Update the in-memory inode size after copying the data into the page
+ * cache. It's up to the file system to write the updated size to disk,
+ * preferably after I/O completion so that no stale data is exposed.
+ */
+ if (pos + ret > old_size) {
+ i_size_write(iter->inode, pos + ret);
+ iter->iomap.flags |= IOMAP_F_SIZE_CHANGED;
+ }
+ folio_unlock(folio);
+
+ if (old_size < pos)
+ pagecache_isize_extended(iter->inode, old_size, pos);
+ if (page_ops && page_ops->page_done)
+ page_ops->page_done(iter->inode, pos, ret, &folio->page);
+ folio_put(folio);
+
+ if (ret < len)
+ iomap_write_failed(iter->inode, pos + ret, len - ret);
+ return ret;
+}
+
+static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
+{
+ loff_t length = iomap_length(iter);
+ loff_t pos = iter->pos;
+ ssize_t written = 0;
+ long status = 0;
+ struct address_space *mapping = iter->inode->i_mapping;
+ unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0;
+
+ do {
+ struct folio *folio;
+ struct page *page;
+ unsigned long offset; /* Offset into pagecache page */
+ unsigned long bytes; /* Bytes to write to page */
+ size_t copied; /* Bytes copied from user */
+
+ offset = offset_in_page(pos);
+ bytes = min_t(unsigned long, PAGE_SIZE - offset,
+ iov_iter_count(i));
+again:
+ status = balance_dirty_pages_ratelimited_flags(mapping,
+ bdp_flags);
+ if (unlikely(status))
+ break;
+
+ if (bytes > length)
+ bytes = length;
+
+ /*
+ * Bring in the user page that we'll copy from _first_.
+ * Otherwise there's a nasty deadlock on copying from the
+ * same page as we're writing to, without it being marked
+ * up-to-date.
+ *
+ * For async buffered writes the assumption is that the user
+ * page has already been faulted in. This can be optimized by
+ * faulting the user page.
+ */
+ if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) {
+ status = -EFAULT;
+ break;
+ }
+
+ status = iomap_write_begin(iter, pos, bytes, &folio);
+ if (unlikely(status))
+ break;
+
+ page = folio_file_page(folio, pos >> PAGE_SHIFT);
+ if (mapping_writably_mapped(mapping))
+ flush_dcache_page(page);
+
+ copied = copy_page_from_iter_atomic(page, offset, bytes, i);
+
+ status = iomap_write_end(iter, pos, bytes, copied, folio);
+
+ if (unlikely(copied != status))
+ iov_iter_revert(i, copied - status);
+
+ cond_resched();
+ if (unlikely(status == 0)) {
+ /*
+ * A short copy made iomap_write_end() reject the
+ * thing entirely. Might be memory poisoning
+ * halfway through, might be a race with munmap,
+ * might be severe memory pressure.
+ */
+ if (copied)
+ bytes = copied;
+ goto again;
+ }
+ pos += status;
+ written += status;
+ length -= status;
+ } while (iov_iter_count(i) && length);
+
+ if (status == -EAGAIN) {
+ iov_iter_revert(i, written);
+ return -EAGAIN;
+ }
+ return written ? written : status;
+}
+
+ssize_t
+iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
+ const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = iocb->ki_filp->f_mapping->host,
+ .pos = iocb->ki_pos,
+ .len = iov_iter_count(i),
+ .flags = IOMAP_WRITE,
+ };
+ int ret;
+
+ if (iocb->ki_flags & IOCB_NOWAIT)
+ iter.flags |= IOMAP_NOWAIT;
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_write_iter(&iter, i);
+ if (iter.pos == iocb->ki_pos)
+ return ret;
+ return iter.pos - iocb->ki_pos;
+}
+EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
+
+static loff_t iomap_unshare_iter(struct iomap_iter *iter)
+{
+ struct iomap *iomap = &iter->iomap;
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+ loff_t pos = iter->pos;
+ loff_t length = iomap_length(iter);
+ long status = 0;
+ loff_t written = 0;
+
+ /* don't bother with blocks that are not shared to start with */
+ if (!(iomap->flags & IOMAP_F_SHARED))
+ return length;
+ /* don't bother with holes or unwritten extents */
+ if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
+ return length;
+
+ do {
+ unsigned long offset = offset_in_page(pos);
+ unsigned long bytes = min_t(loff_t, PAGE_SIZE - offset, length);
+ struct folio *folio;
+
+ status = iomap_write_begin(iter, pos, bytes, &folio);
+ if (unlikely(status))
+ return status;
+
+ status = iomap_write_end(iter, pos, bytes, bytes, folio);
+ if (WARN_ON_ONCE(status == 0))
+ return -EIO;
+
+ cond_resched();
+
+ pos += status;
+ written += status;
+ length -= status;
+
+ balance_dirty_pages_ratelimited(iter->inode->i_mapping);
+ } while (length);
+
+ return written;
+}
+
+int
+iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
+ const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = pos,
+ .len = len,
+ .flags = IOMAP_WRITE | IOMAP_UNSHARE,
+ };
+ int ret;
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_unshare_iter(&iter);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(iomap_file_unshare);
+
+static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
+{
+ const struct iomap *srcmap = iomap_iter_srcmap(iter);
+ loff_t pos = iter->pos;
+ loff_t length = iomap_length(iter);
+ loff_t written = 0;
+
+ /* already zeroed? we're done. */
+ if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
+ return length;
+
+ do {
+ struct folio *folio;
+ int status;
+ size_t offset;
+ size_t bytes = min_t(u64, SIZE_MAX, length);
+
+ status = iomap_write_begin(iter, pos, bytes, &folio);
+ if (status)
+ return status;
+
+ offset = offset_in_folio(folio, pos);
+ if (bytes > folio_size(folio) - offset)
+ bytes = folio_size(folio) - offset;
+
+ folio_zero_range(folio, offset, bytes);
+ folio_mark_accessed(folio);
+
+ bytes = iomap_write_end(iter, pos, bytes, bytes, folio);
+ if (WARN_ON_ONCE(bytes == 0))
+ return -EIO;
+
+ pos += bytes;
+ length -= bytes;
+ written += bytes;
+ } while (length > 0);
+
+ if (did_zero)
+ *did_zero = true;
+ return written;
+}
+
+int
+iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
+ const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = pos,
+ .len = len,
+ .flags = IOMAP_ZERO,
+ };
+ int ret;
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_zero_iter(&iter, did_zero);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(iomap_zero_range);
+
+int
+iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
+ const struct iomap_ops *ops)
+{
+ unsigned int blocksize = i_blocksize(inode);
+ unsigned int off = pos & (blocksize - 1);
+
+ /* Block boundary? Nothing to do */
+ if (!off)
+ return 0;
+ return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
+}
+EXPORT_SYMBOL_GPL(iomap_truncate_page);
+
+static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
+ struct folio *folio)
+{
+ loff_t length = iomap_length(iter);
+ int ret;
+
+ if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) {
+ ret = __block_write_begin_int(folio, iter->pos, length, NULL,
+ &iter->iomap);
+ if (ret)
+ return ret;
+ block_commit_write(&folio->page, 0, length);
+ } else {
+ WARN_ON_ONCE(!folio_test_uptodate(folio));
+ folio_mark_dirty(folio);
+ }
+
+ return length;
+}
+
+vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = file_inode(vmf->vma->vm_file),
+ .flags = IOMAP_WRITE | IOMAP_FAULT,
+ };
+ struct folio *folio = page_folio(vmf->page);
+ ssize_t ret;
+
+ folio_lock(folio);
+ ret = folio_mkwrite_check_truncate(folio, iter.inode);
+ if (ret < 0)
+ goto out_unlock;
+ iter.pos = folio_pos(folio);
+ iter.len = ret;
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_folio_mkwrite_iter(&iter, folio);
+
+ if (ret < 0)
+ goto out_unlock;
+ folio_wait_stable(folio);
+ return VM_FAULT_LOCKED;
+out_unlock:
+ folio_unlock(folio);
+ return block_page_mkwrite_return(ret);
+}
+EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
+
+static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
+ size_t len, int error)
+{
+ struct iomap_page *iop = to_iomap_page(folio);
+
+ if (error) {
+ folio_set_error(folio);
+ mapping_set_error(inode->i_mapping, error);
+ }
+
+ WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !iop);
+ WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) <= 0);
+
+ if (!iop || atomic_sub_and_test(len, &iop->write_bytes_pending))
+ folio_end_writeback(folio);
+}
+
+/*
+ * We're now finished for good with this ioend structure. Update the page
+ * state, release holds on bios, and finally free up memory. Do not use the
+ * ioend after this.
+ */
+static u32
+iomap_finish_ioend(struct iomap_ioend *ioend, int error)
+{
+ struct inode *inode = ioend->io_inode;
+ struct bio *bio = &ioend->io_inline_bio;
+ struct bio *last = ioend->io_bio, *next;
+ u64 start = bio->bi_iter.bi_sector;
+ loff_t offset = ioend->io_offset;
+ bool quiet = bio_flagged(bio, BIO_QUIET);
+ u32 folio_count = 0;
+
+ for (bio = &ioend->io_inline_bio; bio; bio = next) {
+ struct folio_iter fi;
+
+ /*
+ * For the last bio, bi_private points to the ioend, so we
+ * need to explicitly end the iteration here.
+ */
+ if (bio == last)
+ next = NULL;
+ else
+ next = bio->bi_private;
+
+ /* walk all folios in bio, ending page IO on them */
+ bio_for_each_folio_all(fi, bio) {
+ iomap_finish_folio_write(inode, fi.folio, fi.length,
+ error);
+ folio_count++;
+ }
+ bio_put(bio);
+ }
+ /* The ioend has been freed by bio_put() */
+
+ if (unlikely(error && !quiet)) {
+ printk_ratelimited(KERN_ERR
+"%s: writeback error on inode %lu, offset %lld, sector %llu",
+ inode->i_sb->s_id, inode->i_ino, offset, start);
+ }
+ return folio_count;
+}
+
+/*
+ * Ioend completion routine for merged bios. This can only be called from task
+ * contexts as merged ioends can be of unbound length. Hence we have to break up
+ * the writeback completions into manageable chunks to avoid long scheduler
+ * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
+ * good batch processing throughput without creating adverse scheduler latency
+ * conditions.
+ */
+void
+iomap_finish_ioends(struct iomap_ioend *ioend, int error)
+{
+ struct list_head tmp;
+ u32 completions;
+
+ might_sleep();
+
+ list_replace_init(&ioend->io_list, &tmp);
+ completions = iomap_finish_ioend(ioend, error);
+
+ while (!list_empty(&tmp)) {
+ if (completions > IOEND_BATCH_SIZE * 8) {
+ cond_resched();
+ completions = 0;
+ }
+ ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
+ list_del_init(&ioend->io_list);
+ completions += iomap_finish_ioend(ioend, error);
+ }
+}
+EXPORT_SYMBOL_GPL(iomap_finish_ioends);
+
+/*
+ * We can merge two adjacent ioends if they have the same set of work to do.
+ */
+static bool
+iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
+{
+ if (ioend->io_bio->bi_status != next->io_bio->bi_status)
+ return false;
+ if ((ioend->io_flags & IOMAP_F_SHARED) ^
+ (next->io_flags & IOMAP_F_SHARED))
+ return false;
+ if ((ioend->io_type == IOMAP_UNWRITTEN) ^
+ (next->io_type == IOMAP_UNWRITTEN))
+ return false;
+ if (ioend->io_offset + ioend->io_size != next->io_offset)
+ return false;
+ /*
+ * Do not merge physically discontiguous ioends. The filesystem
+ * completion functions will have to iterate the physical
+ * discontiguities even if we merge the ioends at a logical level, so
+ * we don't gain anything by merging physical discontiguities here.
+ *
+ * We cannot use bio->bi_iter.bi_sector here as it is modified during
+ * submission so does not point to the start sector of the bio at
+ * completion.
+ */
+ if (ioend->io_sector + (ioend->io_size >> 9) != next->io_sector)
+ return false;
+ return true;
+}
+
+void
+iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends)
+{
+ struct iomap_ioend *next;
+
+ INIT_LIST_HEAD(&ioend->io_list);
+
+ while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
+ io_list))) {
+ if (!iomap_ioend_can_merge(ioend, next))
+ break;
+ list_move_tail(&next->io_list, &ioend->io_list);
+ ioend->io_size += next->io_size;
+ }
+}
+EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
+
+static int
+iomap_ioend_compare(void *priv, const struct list_head *a,
+ const struct list_head *b)
+{
+ struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
+ struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
+
+ if (ia->io_offset < ib->io_offset)
+ return -1;
+ if (ia->io_offset > ib->io_offset)
+ return 1;
+ return 0;
+}
+
+void
+iomap_sort_ioends(struct list_head *ioend_list)
+{
+ list_sort(NULL, ioend_list, iomap_ioend_compare);
+}
+EXPORT_SYMBOL_GPL(iomap_sort_ioends);
+
+static void iomap_writepage_end_bio(struct bio *bio)
+{
+ struct iomap_ioend *ioend = bio->bi_private;
+
+ iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status));
+}
+
+/*
+ * Submit the final bio for an ioend.
+ *
+ * If @error is non-zero, it means that we have a situation where some part of
+ * the submission process has failed after we've marked pages for writeback
+ * and unlocked them. In this situation, we need to fail the bio instead of
+ * submitting it. This typically only happens on a filesystem shutdown.
+ */
+static int
+iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend,
+ int error)
+{
+ ioend->io_bio->bi_private = ioend;
+ ioend->io_bio->bi_end_io = iomap_writepage_end_bio;
+
+ if (wpc->ops->prepare_ioend)
+ error = wpc->ops->prepare_ioend(ioend, error);
+ if (error) {
+ /*
+ * If we're failing the IO now, just mark the ioend with an
+ * error and finish it. This will run IO completion immediately
+ * as there is only one reference to the ioend at this point in
+ * time.
+ */
+ ioend->io_bio->bi_status = errno_to_blk_status(error);
+ bio_endio(ioend->io_bio);
+ return error;
+ }
+
+ submit_bio(ioend->io_bio);
+ return 0;
+}
+
+static struct iomap_ioend *
+iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc,
+ loff_t offset, sector_t sector, struct writeback_control *wbc)
+{
+ struct iomap_ioend *ioend;
+ struct bio *bio;
+
+ bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
+ REQ_OP_WRITE | wbc_to_write_flags(wbc),
+ GFP_NOFS, &iomap_ioend_bioset);
+ bio->bi_iter.bi_sector = sector;
+ wbc_init_bio(wbc, bio);
+
+ ioend = container_of(bio, struct iomap_ioend, io_inline_bio);
+ INIT_LIST_HEAD(&ioend->io_list);
+ ioend->io_type = wpc->iomap.type;
+ ioend->io_flags = wpc->iomap.flags;
+ ioend->io_inode = inode;
+ ioend->io_size = 0;
+ ioend->io_folios = 0;
+ ioend->io_offset = offset;
+ ioend->io_bio = bio;
+ ioend->io_sector = sector;
+ return ioend;
+}
+
+/*
+ * Allocate a new bio, and chain the old bio to the new one.
+ *
+ * Note that we have to perform the chaining in this unintuitive order
+ * so that the bi_private linkage is set up in the right direction for the
+ * traversal in iomap_finish_ioend().
+ */
+static struct bio *
+iomap_chain_bio(struct bio *prev)
+{
+ struct bio *new;
+
+ new = bio_alloc(prev->bi_bdev, BIO_MAX_VECS, prev->bi_opf, GFP_NOFS);
+ bio_clone_blkg_association(new, prev);
+ new->bi_iter.bi_sector = bio_end_sector(prev);
+
+ bio_chain(prev, new);
+ bio_get(prev); /* for iomap_finish_ioend */
+ submit_bio(prev);
+ return new;
+}
+
+static bool
+iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset,
+ sector_t sector)
+{
+ if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
+ (wpc->ioend->io_flags & IOMAP_F_SHARED))
+ return false;
+ if (wpc->iomap.type != wpc->ioend->io_type)
+ return false;
+ if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
+ return false;
+ if (sector != bio_end_sector(wpc->ioend->io_bio))
+ return false;
+ /*
+ * Limit ioend bio chain lengths to minimise IO completion latency. This
+ * also prevents long tight loops ending page writeback on all the
+ * folios in the ioend.
+ */
+ if (wpc->ioend->io_folios >= IOEND_BATCH_SIZE)
+ return false;
+ return true;
+}
+
+/*
+ * Test to see if we have an existing ioend structure that we could append to
+ * first; otherwise finish off the current ioend and start another.
+ */
+static void
+iomap_add_to_ioend(struct inode *inode, loff_t pos, struct folio *folio,
+ struct iomap_page *iop, struct iomap_writepage_ctx *wpc,
+ struct writeback_control *wbc, struct list_head *iolist)
+{
+ sector_t sector = iomap_sector(&wpc->iomap, pos);
+ unsigned len = i_blocksize(inode);
+ size_t poff = offset_in_folio(folio, pos);
+
+ if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos, sector)) {
+ if (wpc->ioend)
+ list_add(&wpc->ioend->io_list, iolist);
+ wpc->ioend = iomap_alloc_ioend(inode, wpc, pos, sector, wbc);
+ }
+
+ if (!bio_add_folio(wpc->ioend->io_bio, folio, len, poff)) {
+ wpc->ioend->io_bio = iomap_chain_bio(wpc->ioend->io_bio);
+ bio_add_folio(wpc->ioend->io_bio, folio, len, poff);
+ }
+
+ if (iop)
+ atomic_add(len, &iop->write_bytes_pending);
+ wpc->ioend->io_size += len;
+ wbc_account_cgroup_owner(wbc, &folio->page, len);
+}
+
+/*
+ * We implement an immediate ioend submission policy here to avoid needing to
+ * chain multiple ioends and hence nest mempool allocations which can violate
+ * the forward progress guarantees we need to provide. The current ioend we're
+ * adding blocks to is cached in the writepage context, and if the new block
+ * doesn't append to the cached ioend, it will create a new ioend and cache that
+ * instead.
+ *
+ * If a new ioend is created and cached, the old ioend is returned and queued
+ * locally for submission once the entire page is processed or an error has been
+ * detected. While ioends are submitted immediately after they are completed,
+ * batching optimisations are provided by higher level block plugging.
+ *
+ * At the end of a writeback pass, there will be a cached ioend remaining on the
+ * writepage context that the caller will need to submit.
+ */
+static int
+iomap_writepage_map(struct iomap_writepage_ctx *wpc,
+ struct writeback_control *wbc, struct inode *inode,
+ struct folio *folio, u64 end_pos)
+{
+ struct iomap_page *iop = iomap_page_create(inode, folio, 0);
+ struct iomap_ioend *ioend, *next;
+ unsigned len = i_blocksize(inode);
+ unsigned nblocks = i_blocks_per_folio(inode, folio);
+ u64 pos = folio_pos(folio);
+ int error = 0, count = 0, i;
+ LIST_HEAD(submit_list);
+
+ WARN_ON_ONCE(iop && atomic_read(&iop->write_bytes_pending) != 0);
+
+ /*
+ * Walk through the folio to find areas to write back. If we
+ * run off the end of the current map or find the current map
+ * invalid, grab a new one.
+ */
+ for (i = 0; i < nblocks && pos < end_pos; i++, pos += len) {
+ if (iop && !test_bit(i, iop->uptodate))
+ continue;
+
+ error = wpc->ops->map_blocks(wpc, inode, pos);
+ if (error)
+ break;
+ trace_iomap_writepage_map(inode, &wpc->iomap);
+ if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE))
+ continue;
+ if (wpc->iomap.type == IOMAP_HOLE)
+ continue;
+ iomap_add_to_ioend(inode, pos, folio, iop, wpc, wbc,
+ &submit_list);
+ count++;
+ }
+ if (count)
+ wpc->ioend->io_folios++;
+
+ WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
+ WARN_ON_ONCE(!folio_test_locked(folio));
+ WARN_ON_ONCE(folio_test_writeback(folio));
+ WARN_ON_ONCE(folio_test_dirty(folio));
+
+ /*
+ * We cannot cancel the ioend directly here on error. We may have
+ * already set other pages under writeback and hence we have to run I/O
+ * completion to mark the error state of the pages under writeback
+ * appropriately.
+ */
+ if (unlikely(error)) {
+ /*
+ * Let the filesystem know what portion of the current page
+ * failed to map. If the page hasn't been added to ioend, it
+ * won't be affected by I/O completion and we must unlock it
+ * now.
+ */
+ if (wpc->ops->discard_folio)
+ wpc->ops->discard_folio(folio, pos);
+ if (!count) {
+ folio_unlock(folio);
+ goto done;
+ }
+ }
+
+ folio_start_writeback(folio);
+ folio_unlock(folio);
+
+ /*
+ * Preserve the original error if there was one; catch
+ * submission errors here and propagate into subsequent ioend
+ * submissions.
+ */
+ list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
+ int error2;
+
+ list_del_init(&ioend->io_list);
+ error2 = iomap_submit_ioend(wpc, ioend, error);
+ if (error2 && !error)
+ error = error2;
+ }
+
+ /*
+ * We can end up here with no error and nothing to write only if we race
+ * with a partial page truncate on a sub-page block sized filesystem.
+ */
+ if (!count)
+ folio_end_writeback(folio);
+done:
+ mapping_set_error(inode->i_mapping, error);
+ return error;
+}
+
+/*
+ * Write out a dirty page.
+ *
+ * For delalloc space on the page, we need to allocate space and flush it.
+ * For unwritten space on the page, we need to start the conversion to
+ * regular allocated space.
+ */
+static int
+iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data)
+{
+ struct folio *folio = page_folio(page);
+ struct iomap_writepage_ctx *wpc = data;
+ struct inode *inode = folio->mapping->host;
+ u64 end_pos, isize;
+
+ trace_iomap_writepage(inode, folio_pos(folio), folio_size(folio));
+
+ /*
+ * Refuse to write the folio out if we're called from reclaim context.
+ *
+ * This avoids stack overflows when called from deeply used stacks in
+ * random callers for direct reclaim or memcg reclaim. We explicitly
+ * allow reclaim from kswapd as the stack usage there is relatively low.
+ *
+ * This should never happen except in the case of a VM regression so
+ * warn about it.
+ */
+ if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
+ PF_MEMALLOC))
+ goto redirty;
+
+ /*
+ * Is this folio beyond the end of the file?
+ *
+ * The folio index is less than the end_index, adjust the end_pos
+ * to the highest offset that this folio should represent.
+ * -----------------------------------------------------
+ * | file mapping | <EOF> |
+ * -----------------------------------------------------
+ * | Page ... | Page N-2 | Page N-1 | Page N | |
+ * ^--------------------------------^----------|--------
+ * | desired writeback range | see else |
+ * ---------------------------------^------------------|
+ */
+ isize = i_size_read(inode);
+ end_pos = folio_pos(folio) + folio_size(folio);
+ if (end_pos > isize) {
+ /*
+ * Check whether the page to write out is beyond or straddles
+ * i_size or not.
+ * -------------------------------------------------------
+ * | file mapping | <EOF> |
+ * -------------------------------------------------------
+ * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
+ * ^--------------------------------^-----------|---------
+ * | | Straddles |
+ * ---------------------------------^-----------|--------|
+ */
+ size_t poff = offset_in_folio(folio, isize);
+ pgoff_t end_index = isize >> PAGE_SHIFT;
+
+ /*
+ * Skip the page if it's fully outside i_size, e.g.
+ * due to a truncate operation that's in progress. We've
+ * cleaned this page and truncate will finish things off for
+ * us.
+ *
+ * Note that the end_index is unsigned long. If the given
+ * offset is greater than 16TB on a 32-bit system then if we
+ * checked if the page is fully outside i_size with
+ * "if (page->index >= end_index + 1)", "end_index + 1" would
+ * overflow and evaluate to 0. Hence this page would be
+ * redirtied and written out repeatedly, which would result in
+ * an infinite loop; the user program performing this operation
+ * would hang. Instead, we can detect this situation by
+ * checking if the page is totally beyond i_size or if its
+ * offset is just equal to the EOF.
+ */
+ if (folio->index > end_index ||
+ (folio->index == end_index && poff == 0))
+ goto unlock;
+
+ /*
+ * The page straddles i_size. It must be zeroed out on each
+ * and every writepage invocation because it may be mmapped.
+ * "A file is mapped in multiples of the page size. For a file
+ * that is not a multiple of the page size, the remaining
+ * memory is zeroed when mapped, and writes to that region are
+ * not written out to the file."
+ */
+ folio_zero_segment(folio, poff, folio_size(folio));
+ end_pos = isize;
+ }
+
+ return iomap_writepage_map(wpc, wbc, inode, folio, end_pos);
+
+redirty:
+ folio_redirty_for_writepage(wbc, folio);
+unlock:
+ folio_unlock(folio);
+ return 0;
+}
+
+int
+iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
+ struct iomap_writepage_ctx *wpc,
+ const struct iomap_writeback_ops *ops)
+{
+ int ret;
+
+ wpc->ops = ops;
+ ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc);
+ if (!wpc->ioend)
+ return ret;
+ return iomap_submit_ioend(wpc, wpc->ioend, ret);
+}
+EXPORT_SYMBOL_GPL(iomap_writepages);
+
+static int __init iomap_init(void)
+{
+ return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
+ offsetof(struct iomap_ioend, io_inline_bio),
+ BIOSET_NEED_BVECS);
+}
+fs_initcall(iomap_init);
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
new file mode 100644
index 000000000..105c4a1d2
--- /dev/null
+++ b/fs/iomap/direct-io.c
@@ -0,0 +1,696 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010 Red Hat, Inc.
+ * Copyright (c) 2016-2021 Christoph Hellwig.
+ */
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/fscrypt.h>
+#include <linux/pagemap.h>
+#include <linux/iomap.h>
+#include <linux/backing-dev.h>
+#include <linux/uio.h>
+#include <linux/task_io_accounting_ops.h>
+#include "trace.h"
+
+#include "../internal.h"
+
+/*
+ * Private flags for iomap_dio, must not overlap with the public ones in
+ * iomap.h:
+ */
+#define IOMAP_DIO_WRITE_FUA (1 << 28)
+#define IOMAP_DIO_NEED_SYNC (1 << 29)
+#define IOMAP_DIO_WRITE (1 << 30)
+#define IOMAP_DIO_DIRTY (1 << 31)
+
+struct iomap_dio {
+ struct kiocb *iocb;
+ const struct iomap_dio_ops *dops;
+ loff_t i_size;
+ loff_t size;
+ atomic_t ref;
+ unsigned flags;
+ int error;
+ size_t done_before;
+ bool wait_for_completion;
+
+ union {
+ /* used during submission and for synchronous completion: */
+ struct {
+ struct iov_iter *iter;
+ struct task_struct *waiter;
+ struct bio *poll_bio;
+ } submit;
+
+ /* used for aio completion: */
+ struct {
+ struct work_struct work;
+ } aio;
+ };
+};
+
+static struct bio *iomap_dio_alloc_bio(const struct iomap_iter *iter,
+ struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)
+{
+ if (dio->dops && dio->dops->bio_set)
+ return bio_alloc_bioset(iter->iomap.bdev, nr_vecs, opf,
+ GFP_KERNEL, dio->dops->bio_set);
+ return bio_alloc(iter->iomap.bdev, nr_vecs, opf, GFP_KERNEL);
+}
+
+static void iomap_dio_submit_bio(const struct iomap_iter *iter,
+ struct iomap_dio *dio, struct bio *bio, loff_t pos)
+{
+ atomic_inc(&dio->ref);
+
+ /* Sync dio can't be polled reliably */
+ if ((dio->iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(dio->iocb)) {
+ bio_set_polled(bio, dio->iocb);
+ dio->submit.poll_bio = bio;
+ }
+
+ if (dio->dops && dio->dops->submit_io)
+ dio->dops->submit_io(iter, bio, pos);
+ else
+ submit_bio(bio);
+}
+
+ssize_t iomap_dio_complete(struct iomap_dio *dio)
+{
+ const struct iomap_dio_ops *dops = dio->dops;
+ struct kiocb *iocb = dio->iocb;
+ struct inode *inode = file_inode(iocb->ki_filp);
+ loff_t offset = iocb->ki_pos;
+ ssize_t ret = dio->error;
+
+ if (dops && dops->end_io)
+ ret = dops->end_io(iocb, dio->size, ret, dio->flags);
+
+ if (likely(!ret)) {
+ ret = dio->size;
+ /* check for short read */
+ if (offset + ret > dio->i_size &&
+ !(dio->flags & IOMAP_DIO_WRITE))
+ ret = dio->i_size - offset;
+ }
+
+ /*
+ * Try again to invalidate clean pages which might have been cached by
+ * non-direct readahead, or faulted in by get_user_pages() if the source
+ * of the write was an mmap'ed region of the file we're writing. Either
+ * one is a pretty crazy thing to do, so we don't support it 100%. If
+ * this invalidation fails, tough, the write still worked...
+ *
+ * And this page cache invalidation has to be after ->end_io(), as some
+ * filesystems convert unwritten extents to real allocations in
+ * ->end_io() when necessary, otherwise a racing buffer read would cache
+ * zeros from unwritten extents.
+ */
+ if (!dio->error && dio->size &&
+ (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
+ int err;
+ err = invalidate_inode_pages2_range(inode->i_mapping,
+ offset >> PAGE_SHIFT,
+ (offset + dio->size - 1) >> PAGE_SHIFT);
+ if (err)
+ dio_warn_stale_pagecache(iocb->ki_filp);
+ }
+
+ inode_dio_end(file_inode(iocb->ki_filp));
+ if (ret > 0) {
+ iocb->ki_pos += ret;
+
+ /*
+ * If this is a DSYNC write, make sure we push it to stable
+ * storage now that we've written data.
+ */
+ if (dio->flags & IOMAP_DIO_NEED_SYNC)
+ ret = generic_write_sync(iocb, ret);
+ if (ret > 0)
+ ret += dio->done_before;
+ }
+ kfree(dio);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(iomap_dio_complete);
+
+static void iomap_dio_complete_work(struct work_struct *work)
+{
+ struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
+ struct kiocb *iocb = dio->iocb;
+
+ iocb->ki_complete(iocb, iomap_dio_complete(dio));
+}
+
+/*
+ * Set an error in the dio if none is set yet. We have to use cmpxchg
+ * as the submission context and the completion context(s) can race to
+ * update the error.
+ */
+static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
+{
+ cmpxchg(&dio->error, 0, ret);
+}
+
+void iomap_dio_bio_end_io(struct bio *bio)
+{
+ struct iomap_dio *dio = bio->bi_private;
+ bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
+
+ if (bio->bi_status)
+ iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
+
+ if (atomic_dec_and_test(&dio->ref)) {
+ if (dio->wait_for_completion) {
+ struct task_struct *waiter = dio->submit.waiter;
+ WRITE_ONCE(dio->submit.waiter, NULL);
+ blk_wake_io_task(waiter);
+ } else if (dio->flags & IOMAP_DIO_WRITE) {
+ struct inode *inode = file_inode(dio->iocb->ki_filp);
+
+ WRITE_ONCE(dio->iocb->private, NULL);
+ INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
+ queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
+ } else {
+ WRITE_ONCE(dio->iocb->private, NULL);
+ iomap_dio_complete_work(&dio->aio.work);
+ }
+ }
+
+ if (should_dirty) {
+ bio_check_pages_dirty(bio);
+ } else {
+ bio_release_pages(bio, false);
+ bio_put(bio);
+ }
+}
+EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io);
+
+static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
+ loff_t pos, unsigned len)
+{
+ struct inode *inode = file_inode(dio->iocb->ki_filp);
+ struct page *page = ZERO_PAGE(0);
+ struct bio *bio;
+
+ bio = iomap_dio_alloc_bio(iter, dio, 1, REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);
+ fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
+ GFP_KERNEL);
+ bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
+ bio->bi_private = dio;
+ bio->bi_end_io = iomap_dio_bio_end_io;
+
+ get_page(page);
+ __bio_add_page(bio, page, len, 0);
+ iomap_dio_submit_bio(iter, dio, bio, pos);
+}
+
+/*
+ * Figure out the bio's operation flags from the dio request, the
+ * mapping, and whether or not we want FUA. Note that we can end up
+ * clearing the WRITE_FUA flag in the dio request.
+ */
+static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,
+ const struct iomap *iomap, bool use_fua)
+{
+ blk_opf_t opflags = REQ_SYNC | REQ_IDLE;
+
+ if (!(dio->flags & IOMAP_DIO_WRITE)) {
+ WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND);
+ return REQ_OP_READ;
+ }
+
+ if (iomap->flags & IOMAP_F_ZONE_APPEND)
+ opflags |= REQ_OP_ZONE_APPEND;
+ else
+ opflags |= REQ_OP_WRITE;
+
+ if (use_fua)
+ opflags |= REQ_FUA;
+ else
+ dio->flags &= ~IOMAP_DIO_WRITE_FUA;
+
+ return opflags;
+}
+
+static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter,
+ struct iomap_dio *dio)
+{
+ const struct iomap *iomap = &iter->iomap;
+ struct inode *inode = iter->inode;
+ unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
+ unsigned int fs_block_size = i_blocksize(inode), pad;
+ loff_t length = iomap_length(iter);
+ loff_t pos = iter->pos;
+ blk_opf_t bio_opf;
+ struct bio *bio;
+ bool need_zeroout = false;
+ bool use_fua = false;
+ int nr_pages, ret = 0;
+ size_t copied = 0;
+ size_t orig_count;
+
+ if ((pos | length) & ((1 << blkbits) - 1) ||
+ !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter))
+ return -EINVAL;
+
+ if (iomap->type == IOMAP_UNWRITTEN) {
+ dio->flags |= IOMAP_DIO_UNWRITTEN;
+ need_zeroout = true;
+ }
+
+ if (iomap->flags & IOMAP_F_SHARED)
+ dio->flags |= IOMAP_DIO_COW;
+
+ if (iomap->flags & IOMAP_F_NEW) {
+ need_zeroout = true;
+ } else if (iomap->type == IOMAP_MAPPED) {
+ /*
+ * Use a FUA write if we need datasync semantics, this is a pure
+ * data IO that doesn't require any metadata updates (including
+ * after IO completion such as unwritten extent conversion) and
+ * the underlying device supports FUA. This allows us to avoid
+ * cache flushes on IO completion.
+ */
+ if (!(iomap->flags & (IOMAP_F_SHARED|IOMAP_F_DIRTY)) &&
+ (dio->flags & IOMAP_DIO_WRITE_FUA) && bdev_fua(iomap->bdev))
+ use_fua = true;
+ }
+
+ /*
+ * Save the original count and trim the iter to just the extent we
+ * are operating on right now. The iter will be re-expanded once
+ * we are done.
+ */
+ orig_count = iov_iter_count(dio->submit.iter);
+ iov_iter_truncate(dio->submit.iter, length);
+
+ if (!iov_iter_count(dio->submit.iter))
+ goto out;
+
+ /*
+ * We can only poll for single bio I/Os.
+ */
+ if (need_zeroout ||
+ ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode)))
+ dio->iocb->ki_flags &= ~IOCB_HIPRI;
+
+ if (need_zeroout) {
+ /* zero out from the start of the block to the write offset */
+ pad = pos & (fs_block_size - 1);
+ if (pad)
+ iomap_dio_zero(iter, dio, pos - pad, pad);
+ }
+
+ /*
+ * Set the operation flags early so that bio_iov_iter_get_pages
+ * can set up the page vector appropriately for a ZONE_APPEND
+ * operation.
+ */
+ bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
+
+ nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
+ do {
+ size_t n;
+ if (dio->error) {
+ iov_iter_revert(dio->submit.iter, copied);
+ copied = ret = 0;
+ goto out;
+ }
+
+ bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf);
+ fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
+ GFP_KERNEL);
+ bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
+ bio->bi_ioprio = dio->iocb->ki_ioprio;
+ bio->bi_private = dio;
+ bio->bi_end_io = iomap_dio_bio_end_io;
+
+ ret = bio_iov_iter_get_pages(bio, dio->submit.iter);
+ if (unlikely(ret)) {
+ /*
+ * We have to stop part way through an IO. We must fall
+ * through to the sub-block tail zeroing here, otherwise
+ * this short IO may expose stale data in the tail of
+ * the block we haven't written data to.
+ */
+ bio_put(bio);
+ goto zero_tail;
+ }
+
+ n = bio->bi_iter.bi_size;
+ if (dio->flags & IOMAP_DIO_WRITE) {
+ task_io_account_write(n);
+ } else {
+ if (dio->flags & IOMAP_DIO_DIRTY)
+ bio_set_pages_dirty(bio);
+ }
+
+ dio->size += n;
+ copied += n;
+
+ nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
+ BIO_MAX_VECS);
+ /*
+ * We can only poll for single bio I/Os.
+ */
+ if (nr_pages)
+ dio->iocb->ki_flags &= ~IOCB_HIPRI;
+ iomap_dio_submit_bio(iter, dio, bio, pos);
+ pos += n;
+ } while (nr_pages);
+
+ /*
+ * We need to zeroout the tail of a sub-block write if the extent type
+ * requires zeroing or the write extends beyond EOF. If we don't zero
+ * the block tail in the latter case, we can expose stale data via mmap
+ * reads of the EOF block.
+ */
+zero_tail:
+ if (need_zeroout ||
+ ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
+ /* zero out from the end of the write to the end of the block */
+ pad = pos & (fs_block_size - 1);
+ if (pad)
+ iomap_dio_zero(iter, dio, pos, fs_block_size - pad);
+ }
+out:
+ /* Undo iter limitation to current extent */
+ iov_iter_reexpand(dio->submit.iter, orig_count - copied);
+ if (copied)
+ return copied;
+ return ret;
+}
+
+static loff_t iomap_dio_hole_iter(const struct iomap_iter *iter,
+ struct iomap_dio *dio)
+{
+ loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);
+
+ dio->size += length;
+ if (!length)
+ return -EFAULT;
+ return length;
+}
+
+static loff_t iomap_dio_inline_iter(const struct iomap_iter *iomi,
+ struct iomap_dio *dio)
+{
+ const struct iomap *iomap = &iomi->iomap;
+ struct iov_iter *iter = dio->submit.iter;
+ void *inline_data = iomap_inline_data(iomap, iomi->pos);
+ loff_t length = iomap_length(iomi);
+ loff_t pos = iomi->pos;
+ size_t copied;
+
+ if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
+ return -EIO;
+
+ if (dio->flags & IOMAP_DIO_WRITE) {
+ loff_t size = iomi->inode->i_size;
+
+ if (pos > size)
+ memset(iomap_inline_data(iomap, size), 0, pos - size);
+ copied = copy_from_iter(inline_data, length, iter);
+ if (copied) {
+ if (pos + copied > size)
+ i_size_write(iomi->inode, pos + copied);
+ mark_inode_dirty(iomi->inode);
+ }
+ } else {
+ copied = copy_to_iter(inline_data, length, iter);
+ }
+ dio->size += copied;
+ if (!copied)
+ return -EFAULT;
+ return copied;
+}
+
+static loff_t iomap_dio_iter(const struct iomap_iter *iter,
+ struct iomap_dio *dio)
+{
+ switch (iter->iomap.type) {
+ case IOMAP_HOLE:
+ if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
+ return -EIO;
+ return iomap_dio_hole_iter(iter, dio);
+ case IOMAP_UNWRITTEN:
+ if (!(dio->flags & IOMAP_DIO_WRITE))
+ return iomap_dio_hole_iter(iter, dio);
+ return iomap_dio_bio_iter(iter, dio);
+ case IOMAP_MAPPED:
+ return iomap_dio_bio_iter(iter, dio);
+ case IOMAP_INLINE:
+ return iomap_dio_inline_iter(iter, dio);
+ case IOMAP_DELALLOC:
+ /*
+ * DIO is not serialised against mmap() access at all, and so
+ * if the page_mkwrite occurs between the writeback and the
+ * iomap_iter() call in the DIO path, then it will see the
+ * DELALLOC block that the page-mkwrite allocated.
+ */
+ pr_warn_ratelimited("Direct I/O collision with buffered writes! File: %pD4 Comm: %.20s\n",
+ dio->iocb->ki_filp, current->comm);
+ return -EIO;
+ default:
+ WARN_ON_ONCE(1);
+ return -EIO;
+ }
+}
+
+/*
+ * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
+ * is being issued as AIO or not. This allows us to optimise pure data writes
+ * to use REQ_FUA rather than requiring generic_write_sync() to issue a
+ * REQ_FLUSH post write. This is slightly tricky because a single request here
+ * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
+ * may be pure data writes. In that case, we still need to do a full data sync
+ * completion.
+ *
+ * When page faults are disabled and @dio_flags includes IOMAP_DIO_PARTIAL,
+ * __iomap_dio_rw can return a partial result if it encounters a non-resident
+ * page in @iter after preparing a transfer. In that case, the non-resident
+ * pages can be faulted in and the request resumed with @done_before set to the
+ * number of bytes previously transferred. The request will then complete with
+ * the correct total number of bytes transferred; this is essential for
+ * completing partial requests asynchronously.
+ *
+ * Returns -ENOTBLK In case of a page invalidation invalidation failure for
+ * writes. The callers needs to fall back to buffered I/O in this case.
+ */
+struct iomap_dio *
+__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+ unsigned int dio_flags, void *private, size_t done_before)
+{
+ struct address_space *mapping = iocb->ki_filp->f_mapping;
+ struct inode *inode = file_inode(iocb->ki_filp);
+ struct iomap_iter iomi = {
+ .inode = inode,
+ .pos = iocb->ki_pos,
+ .len = iov_iter_count(iter),
+ .flags = IOMAP_DIRECT,
+ .private = private,
+ };
+ loff_t end = iomi.pos + iomi.len - 1, ret = 0;
+ bool wait_for_completion =
+ is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);
+ struct blk_plug plug;
+ struct iomap_dio *dio;
+
+ if (!iomi.len)
+ return NULL;
+
+ dio = kmalloc(sizeof(*dio), GFP_KERNEL);
+ if (!dio)
+ return ERR_PTR(-ENOMEM);
+
+ dio->iocb = iocb;
+ atomic_set(&dio->ref, 1);
+ dio->size = 0;
+ dio->i_size = i_size_read(inode);
+ dio->dops = dops;
+ dio->error = 0;
+ dio->flags = 0;
+ dio->done_before = done_before;
+
+ dio->submit.iter = iter;
+ dio->submit.waiter = current;
+ dio->submit.poll_bio = NULL;
+
+ if (iov_iter_rw(iter) == READ) {
+ if (iomi.pos >= dio->i_size)
+ goto out_free_dio;
+
+ if (iocb->ki_flags & IOCB_NOWAIT) {
+ if (filemap_range_needs_writeback(mapping, iomi.pos,
+ end)) {
+ ret = -EAGAIN;
+ goto out_free_dio;
+ }
+ iomi.flags |= IOMAP_NOWAIT;
+ }
+
+ if (user_backed_iter(iter))
+ dio->flags |= IOMAP_DIO_DIRTY;
+ } else {
+ iomi.flags |= IOMAP_WRITE;
+ dio->flags |= IOMAP_DIO_WRITE;
+
+ if (iocb->ki_flags & IOCB_NOWAIT) {
+ if (filemap_range_has_page(mapping, iomi.pos, end)) {
+ ret = -EAGAIN;
+ goto out_free_dio;
+ }
+ iomi.flags |= IOMAP_NOWAIT;
+ }
+
+ /* for data sync or sync, we need sync completion processing */
+ if (iocb_is_dsync(iocb) && !(dio_flags & IOMAP_DIO_NOSYNC)) {
+ dio->flags |= IOMAP_DIO_NEED_SYNC;
+
+ /*
+ * For datasync only writes, we optimistically try
+ * using FUA for this IO. Any non-FUA write that
+ * occurs will clear this flag, hence we know before
+ * completion whether a cache flush is necessary.
+ */
+ if (!(iocb->ki_flags & IOCB_SYNC))
+ dio->flags |= IOMAP_DIO_WRITE_FUA;
+ }
+ }
+
+ if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) {
+ ret = -EAGAIN;
+ if (iomi.pos >= dio->i_size ||
+ iomi.pos + iomi.len > dio->i_size)
+ goto out_free_dio;
+ iomi.flags |= IOMAP_OVERWRITE_ONLY;
+ }
+
+ ret = filemap_write_and_wait_range(mapping, iomi.pos, end);
+ if (ret)
+ goto out_free_dio;
+
+ if (iov_iter_rw(iter) == WRITE) {
+ /*
+ * Try to invalidate cache pages for the range we are writing.
+ * If this invalidation fails, let the caller fall back to
+ * buffered I/O.
+ */
+ if (invalidate_inode_pages2_range(mapping,
+ iomi.pos >> PAGE_SHIFT, end >> PAGE_SHIFT)) {
+ trace_iomap_dio_invalidate_fail(inode, iomi.pos,
+ iomi.len);
+ ret = -ENOTBLK;
+ goto out_free_dio;
+ }
+
+ if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) {
+ ret = sb_init_dio_done_wq(inode->i_sb);
+ if (ret < 0)
+ goto out_free_dio;
+ }
+ }
+
+ inode_dio_begin(inode);
+
+ blk_start_plug(&plug);
+ while ((ret = iomap_iter(&iomi, ops)) > 0) {
+ iomi.processed = iomap_dio_iter(&iomi, dio);
+
+ /*
+ * We can only poll for single bio I/Os.
+ */
+ iocb->ki_flags &= ~IOCB_HIPRI;
+ }
+
+ blk_finish_plug(&plug);
+
+ /*
+ * We only report that we've read data up to i_size.
+ * Revert iter to a state corresponding to that as some callers (such
+ * as the splice code) rely on it.
+ */
+ if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)
+ iov_iter_revert(iter, iomi.pos - dio->i_size);
+
+ if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {
+ if (!(iocb->ki_flags & IOCB_NOWAIT))
+ wait_for_completion = true;
+ ret = 0;
+ }
+
+ /* magic error code to fall back to buffered I/O */
+ if (ret == -ENOTBLK) {
+ wait_for_completion = true;
+ ret = 0;
+ }
+ if (ret < 0)
+ iomap_dio_set_error(dio, ret);
+
+ /*
+ * If all the writes we issued were FUA, we don't need to flush the
+ * cache on IO completion. Clear the sync flag for this case.
+ */
+ if (dio->flags & IOMAP_DIO_WRITE_FUA)
+ dio->flags &= ~IOMAP_DIO_NEED_SYNC;
+
+ WRITE_ONCE(iocb->private, dio->submit.poll_bio);
+
+ /*
+ * We are about to drop our additional submission reference, which
+ * might be the last reference to the dio. There are three different
+ * ways we can progress here:
+ *
+ * (a) If this is the last reference we will always complete and free
+ * the dio ourselves.
+ * (b) If this is not the last reference, and we serve an asynchronous
+ * iocb, we must never touch the dio after the decrement, the
+ * I/O completion handler will complete and free it.
+ * (c) If this is not the last reference, but we serve a synchronous
+ * iocb, the I/O completion handler will wake us up on the drop
+ * of the final reference, and we will complete and free it here
+ * after we got woken by the I/O completion handler.
+ */
+ dio->wait_for_completion = wait_for_completion;
+ if (!atomic_dec_and_test(&dio->ref)) {
+ if (!wait_for_completion)
+ return ERR_PTR(-EIOCBQUEUED);
+
+ for (;;) {
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ if (!READ_ONCE(dio->submit.waiter))
+ break;
+
+ blk_io_schedule();
+ }
+ __set_current_state(TASK_RUNNING);
+ }
+
+ return dio;
+
+out_free_dio:
+ kfree(dio);
+ if (ret)
+ return ERR_PTR(ret);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(__iomap_dio_rw);
+
+ssize_t
+iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
+ unsigned int dio_flags, void *private, size_t done_before)
+{
+ struct iomap_dio *dio;
+
+ dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
+ done_before);
+ if (IS_ERR_OR_NULL(dio))
+ return PTR_ERR_OR_ZERO(dio);
+ return iomap_dio_complete(dio);
+}
+EXPORT_SYMBOL_GPL(iomap_dio_rw);
diff --git a/fs/iomap/fiemap.c b/fs/iomap/fiemap.c
new file mode 100644
index 000000000..610ca6f1e
--- /dev/null
+++ b/fs/iomap/fiemap.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2016-2021 Christoph Hellwig.
+ */
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include <linux/fiemap.h>
+#include <linux/pagemap.h>
+
+static int iomap_to_fiemap(struct fiemap_extent_info *fi,
+ const struct iomap *iomap, u32 flags)
+{
+ switch (iomap->type) {
+ case IOMAP_HOLE:
+ /* skip holes */
+ return 0;
+ case IOMAP_DELALLOC:
+ flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN;
+ break;
+ case IOMAP_MAPPED:
+ break;
+ case IOMAP_UNWRITTEN:
+ flags |= FIEMAP_EXTENT_UNWRITTEN;
+ break;
+ case IOMAP_INLINE:
+ flags |= FIEMAP_EXTENT_DATA_INLINE;
+ break;
+ }
+
+ if (iomap->flags & IOMAP_F_MERGED)
+ flags |= FIEMAP_EXTENT_MERGED;
+ if (iomap->flags & IOMAP_F_SHARED)
+ flags |= FIEMAP_EXTENT_SHARED;
+
+ return fiemap_fill_next_extent(fi, iomap->offset,
+ iomap->addr != IOMAP_NULL_ADDR ? iomap->addr : 0,
+ iomap->length, flags);
+}
+
+static loff_t iomap_fiemap_iter(const struct iomap_iter *iter,
+ struct fiemap_extent_info *fi, struct iomap *prev)
+{
+ int ret;
+
+ if (iter->iomap.type == IOMAP_HOLE)
+ return iomap_length(iter);
+
+ ret = iomap_to_fiemap(fi, prev, 0);
+ *prev = iter->iomap;
+ switch (ret) {
+ case 0: /* success */
+ return iomap_length(iter);
+ case 1: /* extent array full */
+ return 0;
+ default: /* error */
+ return ret;
+ }
+}
+
+int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi,
+ u64 start, u64 len, const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = start,
+ .len = len,
+ .flags = IOMAP_REPORT,
+ };
+ struct iomap prev = {
+ .type = IOMAP_HOLE,
+ };
+ int ret;
+
+ ret = fiemap_prep(inode, fi, start, &iter.len, 0);
+ if (ret)
+ return ret;
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_fiemap_iter(&iter, fi, &prev);
+
+ if (prev.type != IOMAP_HOLE) {
+ ret = iomap_to_fiemap(fi, &prev, FIEMAP_EXTENT_LAST);
+ if (ret < 0)
+ return ret;
+ }
+
+ /* inode with no (attribute) mapping will give ENOENT */
+ if (ret < 0 && ret != -ENOENT)
+ return ret;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(iomap_fiemap);
+
+/* legacy ->bmap interface. 0 is the error return (!) */
+sector_t
+iomap_bmap(struct address_space *mapping, sector_t bno,
+ const struct iomap_ops *ops)
+{
+ struct iomap_iter iter = {
+ .inode = mapping->host,
+ .pos = (loff_t)bno << mapping->host->i_blkbits,
+ .len = i_blocksize(mapping->host),
+ .flags = IOMAP_REPORT,
+ };
+ const unsigned int blkshift = mapping->host->i_blkbits - SECTOR_SHIFT;
+ int ret;
+
+ if (filemap_write_and_wait(mapping))
+ return 0;
+
+ bno = 0;
+ while ((ret = iomap_iter(&iter, ops)) > 0) {
+ if (iter.iomap.type == IOMAP_MAPPED)
+ bno = iomap_sector(&iter.iomap, iter.pos) >> blkshift;
+ /* leave iter.processed unset to abort loop */
+ }
+ if (ret)
+ return 0;
+
+ return bno;
+}
+EXPORT_SYMBOL_GPL(iomap_bmap);
diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c
new file mode 100644
index 000000000..a1c7592d2
--- /dev/null
+++ b/fs/iomap/iter.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2010 Red Hat, Inc.
+ * Copyright (c) 2016-2021 Christoph Hellwig.
+ */
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include "trace.h"
+
+static inline int iomap_iter_advance(struct iomap_iter *iter)
+{
+ /* handle the previous iteration (if any) */
+ if (iter->iomap.length) {
+ if (iter->processed <= 0)
+ return iter->processed;
+ if (WARN_ON_ONCE(iter->processed > iomap_length(iter)))
+ return -EIO;
+ iter->pos += iter->processed;
+ iter->len -= iter->processed;
+ if (!iter->len)
+ return 0;
+ }
+
+ /* clear the state for the next iteration */
+ iter->processed = 0;
+ memset(&iter->iomap, 0, sizeof(iter->iomap));
+ memset(&iter->srcmap, 0, sizeof(iter->srcmap));
+ return 1;
+}
+
+static inline void iomap_iter_done(struct iomap_iter *iter)
+{
+ WARN_ON_ONCE(iter->iomap.offset > iter->pos);
+ WARN_ON_ONCE(iter->iomap.length == 0);
+ WARN_ON_ONCE(iter->iomap.offset + iter->iomap.length <= iter->pos);
+
+ trace_iomap_iter_dstmap(iter->inode, &iter->iomap);
+ if (iter->srcmap.type != IOMAP_HOLE)
+ trace_iomap_iter_srcmap(iter->inode, &iter->srcmap);
+}
+
+/**
+ * iomap_iter - iterate over a ranges in a file
+ * @iter: iteration structue
+ * @ops: iomap ops provided by the file system
+ *
+ * Iterate over filesystem-provided space mappings for the provided file range.
+ *
+ * This function handles cleanup of resources acquired for iteration when the
+ * filesystem indicates there are no more space mappings, which means that this
+ * function must be called in a loop that continues as long it returns a
+ * positive value. If 0 or a negative value is returned, the caller must not
+ * return to the loop body. Within a loop body, there are two ways to break out
+ * of the loop body: leave @iter.processed unchanged, or set it to a negative
+ * errno.
+ */
+int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
+{
+ int ret;
+
+ if (iter->iomap.length && ops->iomap_end) {
+ ret = ops->iomap_end(iter->inode, iter->pos, iomap_length(iter),
+ iter->processed > 0 ? iter->processed : 0,
+ iter->flags, &iter->iomap);
+ if (ret < 0 && !iter->processed)
+ return ret;
+ }
+
+ trace_iomap_iter(iter, ops, _RET_IP_);
+ ret = iomap_iter_advance(iter);
+ if (ret <= 0)
+ return ret;
+
+ ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
+ &iter->iomap, &iter->srcmap);
+ if (ret < 0)
+ return ret;
+ iomap_iter_done(iter);
+ return 1;
+}
diff --git a/fs/iomap/seek.c b/fs/iomap/seek.c
new file mode 100644
index 000000000..a845c012b
--- /dev/null
+++ b/fs/iomap/seek.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Red Hat, Inc.
+ * Copyright (c) 2018-2021 Christoph Hellwig.
+ */
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include <linux/pagemap.h>
+#include <linux/pagevec.h>
+
+static loff_t iomap_seek_hole_iter(const struct iomap_iter *iter,
+ loff_t *hole_pos)
+{
+ loff_t length = iomap_length(iter);
+
+ switch (iter->iomap.type) {
+ case IOMAP_UNWRITTEN:
+ *hole_pos = mapping_seek_hole_data(iter->inode->i_mapping,
+ iter->pos, iter->pos + length, SEEK_HOLE);
+ if (*hole_pos == iter->pos + length)
+ return length;
+ return 0;
+ case IOMAP_HOLE:
+ *hole_pos = iter->pos;
+ return 0;
+ default:
+ return length;
+ }
+}
+
+loff_t
+iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops)
+{
+ loff_t size = i_size_read(inode);
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = pos,
+ .flags = IOMAP_REPORT,
+ };
+ int ret;
+
+ /* Nothing to be found before or beyond the end of the file. */
+ if (pos < 0 || pos >= size)
+ return -ENXIO;
+
+ iter.len = size - pos;
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_seek_hole_iter(&iter, &pos);
+ if (ret < 0)
+ return ret;
+ if (iter.len) /* found hole before EOF */
+ return pos;
+ return size;
+}
+EXPORT_SYMBOL_GPL(iomap_seek_hole);
+
+static loff_t iomap_seek_data_iter(const struct iomap_iter *iter,
+ loff_t *hole_pos)
+{
+ loff_t length = iomap_length(iter);
+
+ switch (iter->iomap.type) {
+ case IOMAP_HOLE:
+ return length;
+ case IOMAP_UNWRITTEN:
+ *hole_pos = mapping_seek_hole_data(iter->inode->i_mapping,
+ iter->pos, iter->pos + length, SEEK_DATA);
+ if (*hole_pos < 0)
+ return length;
+ return 0;
+ default:
+ *hole_pos = iter->pos;
+ return 0;
+ }
+}
+
+loff_t
+iomap_seek_data(struct inode *inode, loff_t pos, const struct iomap_ops *ops)
+{
+ loff_t size = i_size_read(inode);
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = pos,
+ .flags = IOMAP_REPORT,
+ };
+ int ret;
+
+ /* Nothing to be found before or beyond the end of the file. */
+ if (pos < 0 || pos >= size)
+ return -ENXIO;
+
+ iter.len = size - pos;
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_seek_data_iter(&iter, &pos);
+ if (ret < 0)
+ return ret;
+ if (iter.len) /* found data before EOF */
+ return pos;
+ /* We've reached the end of the file without finding data */
+ return -ENXIO;
+}
+EXPORT_SYMBOL_GPL(iomap_seek_data);
diff --git a/fs/iomap/swapfile.c b/fs/iomap/swapfile.c
new file mode 100644
index 000000000..5fc0ac36d
--- /dev/null
+++ b/fs/iomap/swapfile.c
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Oracle. All Rights Reserved.
+ * Author: Darrick J. Wong <darrick.wong@oracle.com>
+ */
+#include <linux/module.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/iomap.h>
+#include <linux/swap.h>
+
+/* Swapfile activation */
+
+struct iomap_swapfile_info {
+ struct iomap iomap; /* accumulated iomap */
+ struct swap_info_struct *sis;
+ uint64_t lowest_ppage; /* lowest physical addr seen (pages) */
+ uint64_t highest_ppage; /* highest physical addr seen (pages) */
+ unsigned long nr_pages; /* number of pages collected */
+ int nr_extents; /* extent count */
+ struct file *file;
+};
+
+/*
+ * Collect physical extents for this swap file. Physical extents reported to
+ * the swap code must be trimmed to align to a page boundary. The logical
+ * offset within the file is irrelevant since the swapfile code maps logical
+ * page numbers of the swap device to the physical page-aligned extents.
+ */
+static int iomap_swapfile_add_extent(struct iomap_swapfile_info *isi)
+{
+ struct iomap *iomap = &isi->iomap;
+ unsigned long nr_pages;
+ unsigned long max_pages;
+ uint64_t first_ppage;
+ uint64_t first_ppage_reported;
+ uint64_t next_ppage;
+ int error;
+
+ if (unlikely(isi->nr_pages >= isi->sis->max))
+ return 0;
+ max_pages = isi->sis->max - isi->nr_pages;
+
+ /*
+ * Round the start up and the end down so that the physical
+ * extent aligns to a page boundary.
+ */
+ first_ppage = ALIGN(iomap->addr, PAGE_SIZE) >> PAGE_SHIFT;
+ next_ppage = ALIGN_DOWN(iomap->addr + iomap->length, PAGE_SIZE) >>
+ PAGE_SHIFT;
+
+ /* Skip too-short physical extents. */
+ if (first_ppage >= next_ppage)
+ return 0;
+ nr_pages = next_ppage - first_ppage;
+ nr_pages = min(nr_pages, max_pages);
+
+ /*
+ * Calculate how much swap space we're adding; the first page contains
+ * the swap header and doesn't count. The mm still wants that first
+ * page fed to add_swap_extent, however.
+ */
+ first_ppage_reported = first_ppage;
+ if (iomap->offset == 0)
+ first_ppage_reported++;
+ if (isi->lowest_ppage > first_ppage_reported)
+ isi->lowest_ppage = first_ppage_reported;
+ if (isi->highest_ppage < (next_ppage - 1))
+ isi->highest_ppage = next_ppage - 1;
+
+ /* Add extent, set up for the next call. */
+ error = add_swap_extent(isi->sis, isi->nr_pages, nr_pages, first_ppage);
+ if (error < 0)
+ return error;
+ isi->nr_extents += error;
+ isi->nr_pages += nr_pages;
+ return 0;
+}
+
+static int iomap_swapfile_fail(struct iomap_swapfile_info *isi, const char *str)
+{
+ char *buf, *p = ERR_PTR(-ENOMEM);
+
+ buf = kmalloc(PATH_MAX, GFP_KERNEL);
+ if (buf)
+ p = file_path(isi->file, buf, PATH_MAX);
+ pr_err("swapon: file %s %s\n", IS_ERR(p) ? "<unknown>" : p, str);
+ kfree(buf);
+ return -EINVAL;
+}
+
+/*
+ * Accumulate iomaps for this swap file. We have to accumulate iomaps because
+ * swap only cares about contiguous page-aligned physical extents and makes no
+ * distinction between written and unwritten extents.
+ */
+static loff_t iomap_swapfile_iter(const struct iomap_iter *iter,
+ struct iomap *iomap, struct iomap_swapfile_info *isi)
+{
+ switch (iomap->type) {
+ case IOMAP_MAPPED:
+ case IOMAP_UNWRITTEN:
+ /* Only real or unwritten extents. */
+ break;
+ case IOMAP_INLINE:
+ /* No inline data. */
+ return iomap_swapfile_fail(isi, "is inline");
+ default:
+ return iomap_swapfile_fail(isi, "has unallocated extents");
+ }
+
+ /* No uncommitted metadata or shared blocks. */
+ if (iomap->flags & IOMAP_F_DIRTY)
+ return iomap_swapfile_fail(isi, "is not committed");
+ if (iomap->flags & IOMAP_F_SHARED)
+ return iomap_swapfile_fail(isi, "has shared extents");
+
+ /* Only one bdev per swap file. */
+ if (iomap->bdev != isi->sis->bdev)
+ return iomap_swapfile_fail(isi, "outside the main device");
+
+ if (isi->iomap.length == 0) {
+ /* No accumulated extent, so just store it. */
+ memcpy(&isi->iomap, iomap, sizeof(isi->iomap));
+ } else if (isi->iomap.addr + isi->iomap.length == iomap->addr) {
+ /* Append this to the accumulated extent. */
+ isi->iomap.length += iomap->length;
+ } else {
+ /* Otherwise, add the retained iomap and store this one. */
+ int error = iomap_swapfile_add_extent(isi);
+ if (error)
+ return error;
+ memcpy(&isi->iomap, iomap, sizeof(isi->iomap));
+ }
+ return iomap_length(iter);
+}
+
+/*
+ * Iterate a swap file's iomaps to construct physical extents that can be
+ * passed to the swapfile subsystem.
+ */
+int iomap_swapfile_activate(struct swap_info_struct *sis,
+ struct file *swap_file, sector_t *pagespan,
+ const struct iomap_ops *ops)
+{
+ struct inode *inode = swap_file->f_mapping->host;
+ struct iomap_iter iter = {
+ .inode = inode,
+ .pos = 0,
+ .len = ALIGN_DOWN(i_size_read(inode), PAGE_SIZE),
+ .flags = IOMAP_REPORT,
+ };
+ struct iomap_swapfile_info isi = {
+ .sis = sis,
+ .lowest_ppage = (sector_t)-1ULL,
+ .file = swap_file,
+ };
+ int ret;
+
+ /*
+ * Persist all file mapping metadata so that we won't have any
+ * IOMAP_F_DIRTY iomaps.
+ */
+ ret = vfs_fsync(swap_file, 1);
+ if (ret)
+ return ret;
+
+ while ((ret = iomap_iter(&iter, ops)) > 0)
+ iter.processed = iomap_swapfile_iter(&iter, &iter.iomap, &isi);
+ if (ret < 0)
+ return ret;
+
+ if (isi.iomap.length) {
+ ret = iomap_swapfile_add_extent(&isi);
+ if (ret)
+ return ret;
+ }
+
+ /*
+ * If this swapfile doesn't contain even a single page-aligned
+ * contiguous range of blocks, reject this useless swapfile to
+ * prevent confusion later on.
+ */
+ if (isi.nr_pages == 0) {
+ pr_warn("swapon: Cannot find a single usable page in file.\n");
+ return -EINVAL;
+ }
+
+ *pagespan = 1 + isi.highest_ppage - isi.lowest_ppage;
+ sis->max = isi.nr_pages;
+ sis->pages = isi.nr_pages - 1;
+ sis->highest_bit = isi.nr_pages - 1;
+ return isi.nr_extents;
+}
+EXPORT_SYMBOL_GPL(iomap_swapfile_activate);
diff --git a/fs/iomap/trace.c b/fs/iomap/trace.c
new file mode 100644
index 000000000..da217246b
--- /dev/null
+++ b/fs/iomap/trace.c
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019 Christoph Hellwig
+ */
+#include <linux/iomap.h>
+
+/*
+ * We include this last to have the helpers above available for the trace
+ * event implementations.
+ */
+#define CREATE_TRACE_POINTS
+#include "trace.h"
diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h
new file mode 100644
index 000000000..f6ea9540d
--- /dev/null
+++ b/fs/iomap/trace.h
@@ -0,0 +1,191 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2009-2021 Christoph Hellwig
+ *
+ * NOTE: none of these tracepoints shall be considered a stable kernel ABI
+ * as they can change at any time.
+ *
+ * Current conventions for printing numbers measuring specific units:
+ *
+ * offset: byte offset into a subcomponent of a file operation
+ * pos: file offset, in bytes
+ * length: length of a file operation, in bytes
+ * ino: inode number
+ *
+ * Numbers describing space allocations should be formatted in hexadecimal.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM iomap
+
+#if !defined(_IOMAP_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _IOMAP_TRACE_H
+
+#include <linux/tracepoint.h>
+
+struct inode;
+
+DECLARE_EVENT_CLASS(iomap_readpage_class,
+ TP_PROTO(struct inode *inode, int nr_pages),
+ TP_ARGS(inode, nr_pages),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(u64, ino)
+ __field(int, nr_pages)
+ ),
+ TP_fast_assign(
+ __entry->dev = inode->i_sb->s_dev;
+ __entry->ino = inode->i_ino;
+ __entry->nr_pages = nr_pages;
+ ),
+ TP_printk("dev %d:%d ino 0x%llx nr_pages %d",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->ino,
+ __entry->nr_pages)
+)
+
+#define DEFINE_READPAGE_EVENT(name) \
+DEFINE_EVENT(iomap_readpage_class, name, \
+ TP_PROTO(struct inode *inode, int nr_pages), \
+ TP_ARGS(inode, nr_pages))
+DEFINE_READPAGE_EVENT(iomap_readpage);
+DEFINE_READPAGE_EVENT(iomap_readahead);
+
+DECLARE_EVENT_CLASS(iomap_range_class,
+ TP_PROTO(struct inode *inode, loff_t off, u64 len),
+ TP_ARGS(inode, off, len),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(u64, ino)
+ __field(loff_t, size)
+ __field(loff_t, offset)
+ __field(u64, length)
+ ),
+ TP_fast_assign(
+ __entry->dev = inode->i_sb->s_dev;
+ __entry->ino = inode->i_ino;
+ __entry->size = i_size_read(inode);
+ __entry->offset = off;
+ __entry->length = len;
+ ),
+ TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx length 0x%llx",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->ino,
+ __entry->size,
+ __entry->offset,
+ __entry->length)
+)
+
+#define DEFINE_RANGE_EVENT(name) \
+DEFINE_EVENT(iomap_range_class, name, \
+ TP_PROTO(struct inode *inode, loff_t off, u64 len),\
+ TP_ARGS(inode, off, len))
+DEFINE_RANGE_EVENT(iomap_writepage);
+DEFINE_RANGE_EVENT(iomap_release_folio);
+DEFINE_RANGE_EVENT(iomap_invalidate_folio);
+DEFINE_RANGE_EVENT(iomap_dio_invalidate_fail);
+
+#define IOMAP_TYPE_STRINGS \
+ { IOMAP_HOLE, "HOLE" }, \
+ { IOMAP_DELALLOC, "DELALLOC" }, \
+ { IOMAP_MAPPED, "MAPPED" }, \
+ { IOMAP_UNWRITTEN, "UNWRITTEN" }, \
+ { IOMAP_INLINE, "INLINE" }
+
+#define IOMAP_FLAGS_STRINGS \
+ { IOMAP_WRITE, "WRITE" }, \
+ { IOMAP_ZERO, "ZERO" }, \
+ { IOMAP_REPORT, "REPORT" }, \
+ { IOMAP_FAULT, "FAULT" }, \
+ { IOMAP_DIRECT, "DIRECT" }, \
+ { IOMAP_NOWAIT, "NOWAIT" }
+
+#define IOMAP_F_FLAGS_STRINGS \
+ { IOMAP_F_NEW, "NEW" }, \
+ { IOMAP_F_DIRTY, "DIRTY" }, \
+ { IOMAP_F_SHARED, "SHARED" }, \
+ { IOMAP_F_MERGED, "MERGED" }, \
+ { IOMAP_F_BUFFER_HEAD, "BH" }, \
+ { IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }
+
+DECLARE_EVENT_CLASS(iomap_class,
+ TP_PROTO(struct inode *inode, struct iomap *iomap),
+ TP_ARGS(inode, iomap),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(u64, ino)
+ __field(u64, addr)
+ __field(loff_t, offset)
+ __field(u64, length)
+ __field(u16, type)
+ __field(u16, flags)
+ __field(dev_t, bdev)
+ ),
+ TP_fast_assign(
+ __entry->dev = inode->i_sb->s_dev;
+ __entry->ino = inode->i_ino;
+ __entry->addr = iomap->addr;
+ __entry->offset = iomap->offset;
+ __entry->length = iomap->length;
+ __entry->type = iomap->type;
+ __entry->flags = iomap->flags;
+ __entry->bdev = iomap->bdev ? iomap->bdev->bd_dev : 0;
+ ),
+ TP_printk("dev %d:%d ino 0x%llx bdev %d:%d addr 0x%llx offset 0x%llx "
+ "length 0x%llx type %s flags %s",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->ino,
+ MAJOR(__entry->bdev), MINOR(__entry->bdev),
+ __entry->addr,
+ __entry->offset,
+ __entry->length,
+ __print_symbolic(__entry->type, IOMAP_TYPE_STRINGS),
+ __print_flags(__entry->flags, "|", IOMAP_F_FLAGS_STRINGS))
+)
+
+#define DEFINE_IOMAP_EVENT(name) \
+DEFINE_EVENT(iomap_class, name, \
+ TP_PROTO(struct inode *inode, struct iomap *iomap), \
+ TP_ARGS(inode, iomap))
+DEFINE_IOMAP_EVENT(iomap_iter_dstmap);
+DEFINE_IOMAP_EVENT(iomap_iter_srcmap);
+DEFINE_IOMAP_EVENT(iomap_writepage_map);
+
+TRACE_EVENT(iomap_iter,
+ TP_PROTO(struct iomap_iter *iter, const void *ops,
+ unsigned long caller),
+ TP_ARGS(iter, ops, caller),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(u64, ino)
+ __field(loff_t, pos)
+ __field(u64, length)
+ __field(unsigned int, flags)
+ __field(const void *, ops)
+ __field(unsigned long, caller)
+ ),
+ TP_fast_assign(
+ __entry->dev = iter->inode->i_sb->s_dev;
+ __entry->ino = iter->inode->i_ino;
+ __entry->pos = iter->pos;
+ __entry->length = iomap_length(iter);
+ __entry->flags = iter->flags;
+ __entry->ops = ops;
+ __entry->caller = caller;
+ ),
+ TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx flags %s (0x%x) ops %ps caller %pS",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->ino,
+ __entry->pos,
+ __entry->length,
+ __print_flags(__entry->flags, "|", IOMAP_FLAGS_STRINGS),
+ __entry->flags,
+ __entry->ops,
+ (void *)__entry->caller)
+);
+
+#endif /* _IOMAP_TRACE_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace
+#include <trace/define_trace.h>