* [PATCH v4 2/5] exfat: increase VDL before unlocking initialized folios
2026-11-16 8:57 [PATCH v4 0/5] exfat: fix valid_size update races Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 1/5] iomap: export iomap_bio_read_folio_range_sync() Chi Zhiling
@ 2026-11-16 8:57 ` Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 3/5] exfat: zero invalid VDL tails in buffered and direct reads Chi Zhiling
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Chi Zhiling @ 2026-11-16 8:57 UTC (permalink / raw)
To: linux-xfs, linux-fsdevel, exfat, linux-kernel
Cc: Namjae Jeon, Sungjong Seo, Yuezhang Mo, Christian Brauner,
Darrick J. Wong, Chi Zhiling
From: Chi Zhiling <chizhiling@kylinos.cn>
Buffered writes and gap initialization can unlock initialized folios
before increasing valid_size. Racing read can then treat their data
as unwritten and cache zeroes instead.
Increase VDL and dirty the inode before unlocking each initialized folio.
Use put_folio for ordinary buffered writes and direct-I/O buffered
fallback, and increase VDL after initializing each gap folio.
Recheck the gap boundary after taking invalidate_lock and preserve the
previous file size and completed initialization if a later folio fails.
Increase VDL for the faulting folio only after iomap_page_mkwrite succeeds,
using the current file size.
Fixes: 82a81a7352bc ("exfat: add iomap buffered I/O support")
Fixes: 90f4264e8b01 ("exfat: dirty all new pages when extending valid_size")
Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
Co-developed-by: Namjae Jeon <linkinjeon@kernel.org>
Signed-off-by: Namjae Jeon <linkinjeon@kernel.org>
---
fs/exfat/file.c | 116 ++++++++++++++++++++++++++---------------------
fs/exfat/iomap.c | 18 ++++++++
fs/exfat/iomap.h | 1 +
3 files changed, 83 insertions(+), 52 deletions(-)
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 5b67bbf81186..13778c72c978 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -718,12 +718,14 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
* beyond the current valid_size and skips uptodate blocks.
* Mark entire folios dirty so the newly valid range is written back.
*
- * Call before advancing valid_size.
+ * Increase valid_size for each newly initialized range before unlocking
+ * its folio.
*/
static int exfat_prepare_valid_range(struct inode *inode, loff_t start,
loff_t end)
{
struct address_space *mapping = inode->i_mapping;
+ struct exfat_inode_info *ei = EXFAT_I(inode);
loff_t next, pos = start;
struct folio *folio;
pgoff_t index;
@@ -733,8 +735,6 @@ static int exfat_prepare_valid_range(struct inode *inode, loff_t start,
while (pos < end) {
index = pos >> PAGE_SHIFT;
- next = min(((loff_t)index + 1) << PAGE_SHIFT, end);
-
balance_dirty_pages_ratelimited(mapping);
folio = read_mapping_folio(mapping, index, NULL);
@@ -743,7 +743,14 @@ static int exfat_prepare_valid_range(struct inode *inode, loff_t start,
folio_lock(folio);
if (folio->mapping == mapping) {
- folio_mark_dirty(folio);
+ next = min(folio_next_pos(folio), end);
+ if (next > exfat_get_valid_size(ei)) {
+ folio_mark_dirty(folio);
+ if (exfat_advance_valid_size(ei, next))
+ mark_inode_dirty(inode);
+ }
+ exfat_advance_zeroed_size(ei,
+ round_up(next, i_blocksize(inode)));
pos = next;
}
folio_unlock(folio);
@@ -757,13 +764,12 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
{
struct exfat_inode_info *ei = EXFAT_I(inode);
loff_t old_valid_size = exfat_get_valid_size(ei);
+ loff_t old_size, gap_start;
int ret = 0;
if (old_valid_size < new_valid_size) {
inode_dio_wait(inode);
-
- /* Do not re-zero blocks already covered by zeroed_size. */
- loff_t gap_start = max(old_valid_size, exfat_get_zeroed_size(ei));
+ old_size = i_size_read(inode);
if (i_size_read(inode) < new_valid_size) {
/*
@@ -782,23 +788,31 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
* is updated.
*/
filemap_invalidate_lock(inode->i_mapping);
+ /* Fault may have initialized more of the gap while we waited. */
+ gap_start = max(exfat_get_valid_size(ei),
+ exfat_get_zeroed_size(ei));
if (gap_start < new_valid_size)
unmap_mapping_range(inode->i_mapping, gap_start,
new_valid_size - gap_start, 0);
ret = exfat_prepare_valid_range(inode, gap_start, new_valid_size);
- filemap_invalidate_unlock(inode->i_mapping);
if (ret) {
- filemap_invalidate_lock(inode->i_mapping);
- truncate_setsize(inode, old_valid_size);
- exfat_truncate(inode);
+ /* Preserve initialized data, including stores through mmap. */
+ loff_t size = max(old_size, exfat_get_valid_size(ei));
+
+ if (size < i_size_read(inode)) {
+ truncate_setsize(inode, size);
+ exfat_truncate(inode);
+ }
filemap_invalidate_unlock(inode->i_mapping);
return ret;
}
- exfat_set_valid_size(ei, new_valid_size);
+ /* The requested gap can already be covered by zeroed_size. */
+ if (exfat_advance_valid_size(ei, new_valid_size))
+ mark_inode_dirty(inode);
exfat_advance_zeroed_size(ei,
round_up(new_valid_size, i_blocksize(inode)));
- mark_inode_dirty(inode);
+ filemap_invalidate_unlock(inode->i_mapping);
}
return ret;
@@ -829,7 +843,7 @@ static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb,
inode_dio_wait(file_inode(iocb->ki_filp));
written = iomap_file_buffered_write(iocb, from, &exfat_write_iomap_ops,
- NULL, NULL);
+ &exfat_iomap_write_ops, NULL);
if (written < 0)
return written;
@@ -921,7 +935,8 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter)
*/
inode_dio_wait(inode);
ret = iomap_file_buffered_write(iocb, iter,
- &exfat_write_iomap_ops, NULL, NULL);
+ &exfat_write_iomap_ops, &exfat_iomap_write_ops,
+ NULL);
}
if (ret != -EIOCBQUEUED && end > i_size_read(inode))
exfat_write_failed(inode);
@@ -971,56 +986,53 @@ static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
{
struct inode *inode = file_inode(vmf->vma->vm_file);
+ struct address_space *mapping = inode->i_mapping;
struct exfat_inode_info *ei = EXFAT_I(inode);
+ struct folio *folio = page_folio(vmf->page);
+ loff_t fault_start = folio_pos(folio);
+ loff_t new_valid_size, gap_start;
vm_fault_t ret;
- loff_t new_valid_size, mmap_valid_size, fault_page_start;
+ int error;
- if (exfat_forced_shutdown(inode->i_sb))
- return vmf_fs_error(-EIO);
+ if (unlikely(exfat_forced_shutdown(inode->i_sb)))
+ return VM_FAULT_SIGBUS;
if (!inode_trylock(inode))
return VM_FAULT_RETRY;
- mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT;
- fault_page_start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
- new_valid_size = min(mmap_valid_size, i_size_read(inode));
-
- if (exfat_get_valid_size(ei) < new_valid_size) {
- loff_t zeroed_size = exfat_get_zeroed_size(ei);
+ sb_start_pagefault(inode->i_sb);
+ error = file_update_time(vmf->vma->vm_file);
+ if (error) {
+ ret = vmf_fs_error(error);
+ goto out_pagefault;
+ }
+ filemap_invalidate_lock_shared(mapping);
- if (zeroed_size < fault_page_start) {
- int err;
+ if (folio->mapping != mapping || fault_start >= i_size_read(inode)) {
+ ret = VM_FAULT_NOPAGE;
+ goto out_unlock;
+ }
- /*
- * Zero only the gap below the faulting page. The read
- * fault populated its folio and iomap_page_mkwrite()
- * will dirty it.
- */
- err = exfat_prepare_valid_range(inode, zeroed_size,
- fault_page_start);
- if (err < 0) {
- inode_unlock(inode);
- return vmf_fs_error(err);
- }
- }
+ /* The faulting folio is already populated; initialize only its gap. */
+ gap_start = max(exfat_get_valid_size(ei), exfat_get_zeroed_size(ei));
+ error = exfat_prepare_valid_range(inode, gap_start, fault_start);
+ if (error) {
+ ret = vmf_fs_error(error);
+ goto out_unlock;
+ }
- /*
- * Track zeroed_size by block, not page, because writeback stops
- * at i_size recording blocks wholly beyond it could skip a
- * later required zeroing.
- */
+ ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL);
+ if (ret == VM_FAULT_LOCKED) {
+ /* Buffered write may have grown EOF before we locked the folio. */
+ new_valid_size = min(folio_next_pos(folio), i_size_read(inode));
exfat_advance_zeroed_size(ei,
round_up(new_valid_size, i_blocksize(inode)));
- exfat_set_valid_size(ei, new_valid_size);
- mark_inode_dirty(inode);
+ if (exfat_advance_valid_size(ei, new_valid_size))
+ mark_inode_dirty(inode);
}
-
- sb_start_pagefault(inode->i_sb);
- file_update_time(vmf->vma->vm_file);
-
- filemap_invalidate_lock_shared(inode->i_mapping);
- ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL);
- filemap_invalidate_unlock_shared(inode->i_mapping);
+out_unlock:
+ filemap_invalidate_unlock_shared(mapping);
+out_pagefault:
sb_end_pagefault(inode->i_sb);
inode_unlock(inode);
diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c
index 2089da8439e4..0a37ed9b7330 100644
--- a/fs/exfat/iomap.c
+++ b/fs/exfat/iomap.c
@@ -252,6 +252,24 @@ const struct iomap_ops exfat_write_iomap_ops = {
.iomap_next = exfat_write_iomap_next,
};
+static void exfat_iomap_put_folio(struct inode *inode, loff_t pos,
+ unsigned int copied, struct folio *folio)
+{
+ if (copied) {
+ struct exfat_inode_info *ei = EXFAT_I(inode);
+
+ if (exfat_advance_valid_size(ei, pos + copied))
+ mark_inode_dirty(inode);
+ }
+
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+const struct iomap_write_ops exfat_iomap_write_ops = {
+ .put_folio = exfat_iomap_put_folio,
+};
+
/*
* exfat_writeback_range - Map folio during writeback
*
diff --git a/fs/exfat/iomap.h b/fs/exfat/iomap.h
index fd8a913f7794..39c93f8cd790 100644
--- a/fs/exfat/iomap.h
+++ b/fs/exfat/iomap.h
@@ -9,6 +9,7 @@
extern const struct iomap_dio_ops exfat_write_dio_ops;
extern const struct iomap_ops exfat_iomap_ops;
extern const struct iomap_ops exfat_write_iomap_ops;
+extern const struct iomap_write_ops exfat_iomap_write_ops;
extern const struct iomap_writeback_ops exfat_writeback_ops;
extern const struct iomap_read_ops exfat_iomap_bio_read_ops;
--
2.53.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH v4 3/5] exfat: zero invalid VDL tails in buffered and direct reads
2026-11-16 8:57 [PATCH v4 0/5] exfat: fix valid_size update races Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 1/5] iomap: export iomap_bio_read_folio_range_sync() Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 2/5] exfat: increase VDL before unlocking initialized folios Chi Zhiling
@ 2026-11-16 8:57 ` Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 4/5] exfat: serialize file expansion with mmap write faults Chi Zhiling
2026-11-16 8:57 ` [PATCH v4 5/5] exfat: coordinate direct writes with mmap faults Chi Zhiling
4 siblings, 0 replies; 6+ messages in thread
From: Chi Zhiling @ 2026-11-16 8:57 UTC (permalink / raw)
To: linux-xfs, linux-fsdevel, exfat, linux-kernel
Cc: Namjae Jeon, Sungjong Seo, Yuezhang Mo, Christian Brauner,
Darrick J. Wong, Chi Zhiling
From: Chi Zhiling <chizhiling@kylinos.cn>
Reading the block that contains VDL also reads bytes beyond that
boundary. Zero those bytes before the range becomes uptodate, including
when a short user copy leaves part of a write untouched.
Use the same VDL snapshot for mapping lookup and validity checks. After
locking the folio, retry a stale mapping and zero invalid bytes from
read_folio_range before iomap marks the range uptodate. Use
iomap_bio_read_folio_range_sync() for the synchronous read and preserve
the mapping snapshot until asynchronous reads complete as well.
Track initialized block tails under the folio lock and clamp that state
on truncate. Remove the earlier zeroing from iomap_begin and stop using
IOMAP_F_ZERO_TAIL.
Clip direct mappings to the exact VDL boundary. Return -ENOTBLK from
the mapping callback for an unaligned VDL tail and finish a short direct
read with filemap_read(), rather than prechecking VDL in read_iter.
Use the same iomap operations for buffered and direct reads.
Fixes: 9e5711b9cfe3 ("exfat: zero invalid tail on partial overwrites")
Fixes: 867b9c96dc83 ("exfat: add iomap direct I/O support")
Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
Co-developed-by: Namjae Jeon <linkinjeon@kernel.org>
Signed-off-by: Namjae Jeon <linkinjeon@kernel.org>
---
fs/exfat/exfat_fs.h | 2 +-
fs/exfat/file.c | 7 +-
fs/exfat/iomap.c | 188 +++++++++++++++++++++++++++-----------------
3 files changed, 120 insertions(+), 77 deletions(-)
diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h
index eee573d1697c..9c43022743ed 100644
--- a/fs/exfat/exfat_fs.h
+++ b/fs/exfat/exfat_fs.h
@@ -303,7 +303,7 @@ struct exfat_inode_info {
* concurrent update can never regress the value or tear on 32-bit.
*/
atomic64_t valid_size;
- /* block-aligned size zeroed in the page cache (>= valid_size) */
+ /* Initialized cache prefix, including block tails; may lag after DIO. */
atomic64_t zeroed_size;
/* hash by i_location */
struct hlist_node i_hash_fat;
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 13778c72c978..47e95b14d570 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -301,10 +301,10 @@ int __exfat_truncate(struct inode *inode)
ei->start_clu = EXFAT_EOF_CLUSTER;
}
- if (i_size_read(inode) < exfat_get_valid_size(ei)) {
+ if (i_size_read(inode) < exfat_get_valid_size(ei))
exfat_set_valid_size(ei, i_size_read(inode));
+ if (i_size_read(inode) < exfat_get_zeroed_size(ei))
exfat_set_zeroed_size(ei, i_size_read(inode));
- }
if (ei->type == TYPE_FILE)
ei->attr |= EXFAT_ATTR_ARCHIVE;
@@ -974,6 +974,9 @@ static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
file_accessed(iocb->ki_filp);
ret = iomap_dio_rw(iocb, iter, &exfat_iomap_ops, NULL, 0,
NULL, 0);
+ if (ret >= 0 && iov_iter_count(iter) &&
+ iocb->ki_pos < i_size_read(inode))
+ ret = filemap_read(iocb, iter, ret);
} else {
ret = generic_file_read_iter(iocb, iter);
}
diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c
index 0a37ed9b7330..7eb31a6091b3 100644
--- a/fs/exfat/iomap.c
+++ b/fs/exfat/iomap.c
@@ -7,6 +7,7 @@
#include <linux/iomap.h>
#include <linux/pagemap.h>
+#include <linux/slab.h>
#include "exfat_raw.h"
#include "exfat_fs.h"
@@ -39,17 +40,18 @@ const struct iomap_dio_ops exfat_write_dio_ops = {
};
static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
- unsigned int flags, struct iomap *iomap, bool may_alloc)
+ unsigned int flags, struct iomap *iomap, bool may_alloc,
+ loff_t valid_size)
{
struct super_block *sb = inode->i_sb;
struct exfat_sb_info *sbi = EXFAT_SB(sb);
- struct exfat_inode_info *ei = EXFAT_I(inode);
unsigned int cluster, num_clusters;
loff_t cluster_offset, cluster_length;
- loff_t valid_size = exfat_get_valid_size(ei);
int err;
bool balloc = false;
+ iomap->validity_cookie = valid_size;
+
if (!may_alloc) {
/* Completely beyond EOF. Treat as hole */
if (i_size_read(inode) <= offset) {
@@ -100,17 +102,6 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
if (may_alloc || flags & IOMAP_ZERO) {
if (balloc)
iomap->flags |= IOMAP_F_NEW;
- else if (iomap->offset + iomap->length >= valid_size) {
- /*
- * This is a write that starts at or extends beyond
- * the current valid_size. The region between the old
- * valid_size and the end of this write needs to be
- * zeroed in the page cache to prevent stale data
- * exposure (see IOMAP_F_ZERO_TAIL handling in
- * __iomap_write_begin()).
- */
- iomap->flags |= IOMAP_F_ZERO_TAIL;
- }
} else {
/*
* valid_size is tracked in byte granularity and
@@ -129,23 +120,24 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
if (offset >= valid_size) {
iomap->type = flags & IOMAP_REPORT ?
IOMAP_HOLE : IOMAP_UNWRITTEN;
+ } else if ((flags & IOMAP_DIRECT) &&
+ offset + iomap->length >= valid_size &&
+ !IS_ALIGNED(valid_size, i_blocksize(inode))) {
+ err = -ENOTBLK;
} else if (offset + iomap->length > valid_size) {
- if (flags & IOMAP_REPORT) {
- /*
- * For SEEK_HOLE/SEEK_DATA, clip the length
- * to the exact byte boundary (valid_size).
- * This ensures the caller gets the precise
- * hole position in byte units.
- */
+ /*
+ * Keep SEEK_HOLE/SEEK_DATA and direct I/O mappings
+ * at the exact valid_size boundary.
+ */
+ if (flags & (IOMAP_REPORT | IOMAP_DIRECT))
iomap->length = valid_size - iomap->offset;
- } else
+ else
iomap->length = round_up(valid_size,
i_blocksize(inode)) -
iomap->offset;
}
}
- iomap->flags |= IOMAP_F_MERGED;
out:
mutex_unlock(&sbi->s_lock);
return err;
@@ -154,33 +146,15 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
static int exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
{
- return __exfat_iomap_begin(inode, offset, length, flags, iomap, false);
+ return __exfat_iomap_begin(inode, offset, length, flags, iomap, false,
+ exfat_get_valid_size(EXFAT_I(inode)));
}
static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
{
- struct exfat_inode_info *ei = EXFAT_I(inode);
- loff_t valid_size = exfat_get_valid_size(ei);
- loff_t end;
- int err;
-
- err = __exfat_iomap_begin(inode, offset, length, flags, iomap, true);
- if (err || flags & (IOMAP_DIRECT | IOMAP_FAULT) ||
- iomap->flags & IOMAP_F_NEW)
- return err;
-
- end = iomap->offset + iomap->length;
- if (end >= valid_size ||
- round_up(end, i_blocksize(inode)) <= valid_size)
- return 0;
-
- /*
- * Zero the invalid bytes from valid_size to the end of the block
- * to prevent stale data from being exposed through the page cache.
- */
- return iomap_truncate_page(inode, valid_size, NULL, &exfat_iomap_ops,
- NULL, NULL);
+ return __exfat_iomap_begin(inode, offset, length, flags, iomap, true,
+ exfat_get_valid_size(EXFAT_I(inode)));
}
static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin);
@@ -199,7 +173,8 @@ static int exfat_swap_iomap_begin(struct inode *inode, loff_t offset,
* ranges. Do not report the VDL tail as a hole.
*/
return __exfat_iomap_begin(inode, offset, length,
- flags & ~IOMAP_REPORT, iomap, false);
+ flags & ~IOMAP_REPORT, iomap, false,
+ exfat_get_valid_size(EXFAT_I(inode)));
}
static DEFINE_IOMAP_ITER_NEXT(exfat_swap_iomap_next,
@@ -231,12 +206,6 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
if (exfat_advance_valid_size(ei, end))
dirtied = true;
- /*
- * IOMAP_F_ZERO_TAIL zeroes the remainder of the last block. Track that
- * block as zeroed so later valid_size extensions do not zero it again.
- */
- if (iomap->flags & IOMAP_F_ZERO_TAIL)
- end = round_up(end, i_blocksize(inode));
exfat_advance_zeroed_size(ei, end);
if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED)
@@ -252,6 +221,60 @@ const struct iomap_ops exfat_write_iomap_ops = {
.iomap_next = exfat_write_iomap_next,
};
+static void exfat_zero_invalid_range(struct folio *folio, loff_t pos,
+ size_t len, loff_t valid_size)
+{
+ loff_t start = max(pos, valid_size);
+
+ if (start < pos + len)
+ folio_zero_segment(folio, offset_in_folio(folio, start),
+ offset_in_folio(folio, pos) + len);
+}
+
+static struct folio *exfat_iomap_get_folio(struct iomap_iter *iter, loff_t pos,
+ unsigned int len)
+{
+ struct folio *folio = iomap_get_folio(iter, pos, len);
+
+ if (!IS_ERR(folio))
+ iter->iomap.private = folio;
+ return folio;
+}
+
+static bool exfat_iomap_valid(struct inode *inode, const struct iomap *iomap)
+{
+ struct folio *folio = iomap->private;
+ loff_t valid_size = exfat_get_valid_size(EXFAT_I(inode));
+ loff_t old_valid_size = iomap->validity_cookie;
+
+ if (valid_size == old_valid_size)
+ return true;
+ if (!folio)
+ return false;
+
+ return old_valid_size >= folio_next_pos(folio) ||
+ valid_size <= folio_pos(folio);
+}
+
+/* Called after iomap_valid(), with the destination folio still locked. */
+static int exfat_iomap_read_folio_range(const struct iomap_iter *iter,
+ struct folio *folio, loff_t pos, size_t len)
+{
+ loff_t valid_size = iter->iomap.validity_cookie;
+ int error = 0;
+
+ if (pos < valid_size)
+ error = iomap_bio_read_folio_range_sync(iter, folio, pos, len);
+
+ /*
+ * Sanitize all invalid bytes before iomap marks this range uptodate,
+ * including bytes that a short user copy may leave untouched.
+ */
+ if (!error)
+ exfat_zero_invalid_range(folio, pos, len, valid_size);
+ return error;
+}
+
static void exfat_iomap_put_folio(struct inode *inode, loff_t pos,
unsigned int copied, struct folio *folio)
{
@@ -260,6 +283,8 @@ static void exfat_iomap_put_folio(struct inode *inode, loff_t pos,
if (exfat_advance_valid_size(ei, pos + copied))
mark_inode_dirty(inode);
+ exfat_advance_zeroed_size(ei,
+ round_up(pos + copied, i_blocksize(inode)));
}
folio_unlock(folio);
@@ -267,7 +292,10 @@ static void exfat_iomap_put_folio(struct inode *inode, loff_t pos,
}
const struct iomap_write_ops exfat_iomap_write_ops = {
+ .get_folio = exfat_iomap_get_folio,
.put_folio = exfat_iomap_put_folio,
+ .iomap_valid = exfat_iomap_valid,
+ .read_folio_range = exfat_iomap_read_folio_range,
};
/*
@@ -289,7 +317,8 @@ static ssize_t exfat_writeback_range(struct iomap_writepage_ctx *wpc,
int error;
error = __exfat_iomap_begin(inode, offset, len,
- 0, &wpc->iomap, false);
+ 0, &wpc->iomap, false,
+ exfat_get_valid_size(EXFAT_I(inode)));
if (error)
return error;
}
@@ -302,41 +331,52 @@ const struct iomap_writeback_ops exfat_writeback_ops = {
.writeback_submit = iomap_ioend_writeback_submit,
};
-/**
- * exfat_iomap_read_end_io - iomap read bio completion handler for exFAT
- * @bio: bio that has completed reading
- *
- * exfat_iomap_begin() rounds up MAPPED extents to the block boundary of
- * valid_size. This ensures that any subsequent blocks are treated as
- * IOMAP_UNWRITTEN, but it also causes the "straddle block" containing
- * valid_size to be read from disk. The disk data beyond valid_size in
- * this block is stale and must be zeroed to prevent data leakage.
- */
-static void exfat_iomap_read_end_io(struct bio *bio)
+struct exfat_iomap_read_ctx {
+ loff_t valid_size;
+};
+
+static void exfat_iomap_finish_read(struct bio *bio, loff_t valid_size, int error)
{
- int error = blk_status_to_errno(bio->bi_status);
struct folio_iter iter;
bio_for_each_folio_all(iter, bio) {
struct folio *folio = iter.folio;
- struct exfat_inode_info *ei = EXFAT_I(folio->mapping->host);
- s64 valid_size;
- loff_t pos = folio_pos(folio);
-
- valid_size = exfat_get_valid_size(ei);
- if (pos + iter.offset < valid_size &&
- pos + iter.offset + iter.length > valid_size)
- folio_zero_segment(folio, offset_in_folio(folio, valid_size),
- iter.offset + iter.length);
+ exfat_zero_invalid_range(folio, folio_pos(folio) + iter.offset,
+ iter.length, valid_size);
iomap_finish_folio_read(folio, iter.offset, iter.length, error);
}
bio_put(bio);
}
+static void exfat_iomap_read_end_io(struct bio *bio)
+{
+ struct exfat_iomap_read_ctx *ctx = bio->bi_private;
+ loff_t valid_size = ctx->valid_size;
+
+ kfree(ctx);
+ exfat_iomap_finish_read(bio, valid_size,
+ blk_status_to_errno(bio->bi_status));
+}
+
static void exfat_iomap_bio_submit_read(const struct iomap_iter *iter,
struct iomap_read_folio_ctx *ctx)
{
+ struct bio *bio = ctx->read_ctx;
+ struct exfat_iomap_read_ctx *read_ctx;
+
+ /* The completion must use the boundary that produced this mapping. */
+ read_ctx = kmalloc_obj(*read_ctx, GFP_NOFS);
+ if (!read_ctx) {
+ int error;
+
+ ctx->read_ctx = NULL;
+ error = submit_bio_wait(bio);
+ exfat_iomap_finish_read(bio, iter->iomap.validity_cookie, error);
+ return;
+ }
+ read_ctx->valid_size = iter->iomap.validity_cookie;
+ bio->bi_private = read_ctx;
iomap_bio_submit_read_endio(iter, ctx, exfat_iomap_read_end_io);
}
--
2.53.0
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH v4 5/5] exfat: coordinate direct writes with mmap faults
2026-11-16 8:57 [PATCH v4 0/5] exfat: fix valid_size update races Chi Zhiling
` (3 preceding siblings ...)
2026-11-16 8:57 ` [PATCH v4 4/5] exfat: serialize file expansion with mmap write faults Chi Zhiling
@ 2026-11-16 8:57 ` Chi Zhiling
4 siblings, 0 replies; 6+ messages in thread
From: Chi Zhiling @ 2026-11-16 8:57 UTC (permalink / raw)
To: linux-xfs, linux-fsdevel, exfat, linux-kernel
Cc: Namjae Jeon, Sungjong Seo, Yuezhang Mo, Christian Brauner,
Darrick J. Wong, Chi Zhiling
From: Namjae Jeon <linkinjeon@kernel.org>
Direct writes can update valid_size before their data I/O completes. An mmap
write fault can also initialize and dirty the invalid VDL gap while a direct
write is in flight, writing zeroes over its data.
Prepare and write back the invalid range through the destination boundary
folio before direct I/O, without advancing valid_size. Update i_size and
valid_size after completed writes, and trim excess clusters after errors or
short writes. Hold invalidate_lock through DIO completion, post-write
invalidation, gap initialization, and rollback to coordinate with mmap
faults. Allocate gap clusters through the normal exFAT bitmap and FAT paths,
and account for short allocations and ENOSPC after partial progress.
Record the on-disk length in i_size and clamp valid_size to that length.
Remove inode_trylock() from page_mkwrite, since VM_FAULT_RETRY from this
callback can be ignored. Snapshot valid_size for direct reads so cache-only
VDL changes cannot make disk-invalid data appear valid.
For writes entirely within a VDL snapshot, use the caller's kiocb and
IOMAP_DIO_OVERWRITE_ONLY mappings. These non-allocating mappings allow native
AIO overwrites to run concurrently. VDL-extending writes wait for earlier DIO
and use synchronous completion. Mark overwrite mappings dirty for O_DSYNC
when earlier cached writes may have advanced VDL without persisting the
directory entry.
Return -EAGAIN for IOCB_NOWAIT writes, including buffered writes, because gap
initialization can block. Check alignment before buffered fallback. Complete
submitted BIOs after ENOSPC following partial progress, then report a short
write or I/O error.
Fixes: 867b9c96dc83 ("exfat: add iomap direct I/O support")
Fixes: 6630ea49103c ("exfat: move extend valid_size into ->page_mkwrite()")
Co-developed-by: Chi Zhiling <chizhiling@kylinos.cn>
Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
Signed-off-by: Namjae Jeon <linkinjeon@kernel.org>
---
fs/exfat/fatent.c | 6 +-
fs/exfat/file.c | 303 +++++++++++++++++++++++++++++++++++++---------
fs/exfat/inode.c | 19 +--
fs/exfat/iomap.c | 172 ++++++++++++++++++--------
fs/exfat/iomap.h | 9 +-
5 files changed, 385 insertions(+), 124 deletions(-)
diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c
index 9f4607fced7f..ebeecbe45e61 100644
--- a/fs/exfat/fatent.c
+++ b/fs/exfat/fatent.c
@@ -456,8 +456,10 @@ int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc,
}
if (num_alloc > total_cnt - sbi->used_clusters) {
- ret = -ENOSPC;
- goto unlock;
+ /* A contiguous mapping can use a shorter allocation. */
+ if (!contig || sbi->used_clusters == total_cnt)
+ goto unlock;
+ num_alloc = total_cnt - sbi->used_clusters;
}
hint_clu = p_chain->dir;
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 0d1eeeeffa76..9dfd99ad9265 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -3,7 +3,6 @@
* Copyright (C) 2012-2013 Samsung Electronics Co., Ltd.
*/
-#include <linux/slab.h>
#include <linux/compat.h>
#include <linux/cred.h>
#include <linux/buffer_head.h>
@@ -18,6 +17,7 @@
#include <linux/iomap.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
+#include <linux/uio.h>
#include "exfat_raw.h"
#include "exfat_fs.h"
@@ -32,23 +32,19 @@
* This ensures that the folio is marked RO and the post-EOF range is
* zeroed before readers or writeback can observe the updated i_size.
*/
-static void exfat_pagecache_isize_extended(struct inode *inode, loff_t from,
- loff_t to)
+static struct folio *exfat_prepare_isize_extended(struct inode *inode,
+ loff_t from, loff_t to)
{
struct folio *folio;
- if (!(from & (PAGE_SIZE - 1))) {
- i_size_write(inode, to);
- return;
- }
+ if (!(from & (PAGE_SIZE - 1)))
+ return NULL;
folio = filemap_lock_folio(inode->i_mapping, from >> PAGE_SHIFT);
/* Folio not cached? Nothing to do */
- if (IS_ERR(folio)) {
- i_size_write(inode, to);
- return;
- }
+ if (IS_ERR(folio))
+ return NULL;
/*
* See folio_clear_dirty_for_io() for details why folio_mark_dirty()
@@ -71,14 +67,28 @@ static void exfat_pagecache_isize_extended(struct inode *inode, loff_t from,
folio_zero_segment(folio, offset, end);
}
+ return folio;
+}
+
+static void exfat_unlock_isize_folio(struct folio *folio)
+{
+ if (!folio)
+ return;
+ folio_unlock(folio);
+ folio_put(folio);
+}
+
+static void exfat_pagecache_isize_extended(struct inode *inode, loff_t from,
+ loff_t to)
+{
+ struct folio *folio = exfat_prepare_isize_extended(inode, from, to);
+
/*
* Publish the new size after zeroing the tail. Uptodate folios can
* be read without taking the folio lock.
*/
i_size_write(inode, to);
-
- folio_unlock(folio);
- folio_put(folio);
+ exfat_unlock_isize_folio(folio);
}
static int __exfat_cont_expand(struct inode *inode, loff_t size)
@@ -723,17 +733,16 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
}
/*
- * exfat_prepare_valid_range - populate and dirty folios covering [start, end)
+ * exfat_prepare_range - populate and dirty folios covering [start, end)
*
* Populate missing blocks via the read path, which zero-fills data
* beyond the current valid_size and skips uptodate blocks.
- * Mark entire folios dirty so the newly valid range is written back.
+ * Mark entire folios dirty so the prepared range is written back.
*
- * Increase valid_size for each newly initialized range before unlocking
- * its folio.
+ * If requested, increase valid_size for each range before unlocking its folio.
*/
-static int exfat_prepare_valid_range(struct inode *inode, loff_t start,
- loff_t end)
+static int exfat_prepare_range(struct inode *inode, loff_t start, loff_t end,
+ bool publish)
{
struct address_space *mapping = inode->i_mapping;
struct exfat_inode_info *ei = EXFAT_I(inode);
@@ -757,11 +766,12 @@ static int exfat_prepare_valid_range(struct inode *inode, loff_t start,
next = min(folio_next_pos(folio), end);
if (next > exfat_get_valid_size(ei)) {
folio_mark_dirty(folio);
- if (exfat_advance_valid_size(ei, next))
+ if (publish && exfat_advance_valid_size(ei, next))
mark_inode_dirty(inode);
}
- exfat_advance_zeroed_size(ei,
- round_up(next, i_blocksize(inode)));
+ if (publish)
+ exfat_advance_zeroed_size(ei,
+ round_up(next, i_blocksize(inode)));
pos = next;
}
folio_unlock(folio);
@@ -805,7 +815,7 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
if (gap_start < new_valid_size)
unmap_mapping_range(inode->i_mapping, gap_start,
new_valid_size - gap_start, 0);
- ret = exfat_prepare_valid_range(inode, gap_start, new_valid_size);
+ ret = exfat_prepare_range(inode, gap_start, new_valid_size, true);
if (ret) {
/* Preserve initialized data, including stores through mmap. */
loff_t size = max(old_size, exfat_get_valid_size(ei));
@@ -829,6 +839,105 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
return ret;
}
+/*
+ * A later mmap fault can initialize the gap up to its folio. Prepare and
+ * write back through the DIO range and its boundary folio so that gap
+ * initialization in a later folio does not dirty the DIO range.
+ * i_rwsem serializes preparation, allowing later overwrites to reuse the
+ * prepared range without dirtying folios containing in-flight DIO.
+ */
+static int exfat_prepare_dio_range(struct inode *inode, loff_t end)
+{
+ struct address_space *mapping = inode->i_mapping;
+ struct exfat_inode_info *ei = EXFAT_I(inode);
+ loff_t valid, start, size;
+ int ret = 0;
+
+ filemap_invalidate_lock(mapping);
+ size = i_size_read(inode);
+ valid = exfat_get_valid_size(ei);
+ start = max(valid, exfat_get_zeroed_size(ei));
+ end = max(valid, min(end, size));
+ end = min_t(u64, round_up((u64)end, mapping_max_folio_size(mapping)),
+ size);
+ if (start < end) {
+ unmap_mapping_range(mapping, start, end - start, 0);
+ ret = exfat_prepare_range(inode, start, end, false);
+ if (!ret)
+ ret = filemap_write_and_wait_range(mapping, start, end - 1);
+ if (!ret)
+ exfat_advance_zeroed_size(ei,
+ round_up(end, i_blocksize(inode)));
+ }
+ filemap_invalidate_unlock(mapping);
+ return ret;
+}
+
+static int exfat_file_write_dio_end_io(struct kiocb *iocb, ssize_t size,
+ int error, unsigned int flags)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb);
+ struct exfat_dio *dio = container_of(iocb, struct exfat_dio, iocb);
+ struct folio *folio = NULL;
+ loff_t end = dio->start + size;
+ bool dirtied = false;
+
+ if (error || !size)
+ return error;
+
+ if (i_size_read(inode) < end)
+ folio = exfat_prepare_isize_extended(inode, i_size_read(inode),
+ end);
+
+ mutex_lock(&sbi->s_lock);
+ if (i_size_read(inode) < end) {
+ i_size_write(inode, end);
+ dirtied = true;
+ }
+
+ if (dio->start <= exfat_get_valid_size(EXFAT_I(inode)) &&
+ exfat_advance_valid_size(EXFAT_I(inode), end))
+ dirtied = true;
+ if (dirtied)
+ mark_inode_dirty(inode);
+ dio->completed = size;
+
+ mutex_unlock(&sbi->s_lock);
+ exfat_unlock_isize_folio(folio);
+ return 0;
+}
+
+static const struct iomap_dio_ops exfat_write_dio_ops = {
+ .end_io = exfat_file_write_dio_end_io,
+};
+
+/*
+ * Run after iomap's post-write invalidation, so gap folios cannot be
+ * mistaken for stale cache pages by the direct I/O completion.
+ */
+static int exfat_finish_dio_gap(struct exfat_dio *dio)
+{
+ struct inode *inode = file_inode(dio->iocb.ki_filp);
+ struct exfat_inode_info *ei = EXFAT_I(inode);
+ loff_t start = max(exfat_get_valid_size(ei), exfat_get_zeroed_size(ei));
+ int ret;
+
+ if (exfat_get_valid_size(ei) >= dio->start)
+ return 0;
+ if (start < dio->start) {
+ unmap_mapping_range(inode->i_mapping, start, dio->start - start, 0);
+ ret = iomap_zero_range(inode, start, dio->start - start, NULL,
+ &exfat_write_iomap_ops,
+ &exfat_dio_gap_write_ops, dio);
+ if (ret)
+ return ret;
+ }
+ if (exfat_advance_valid_size(ei, dio->start + dio->completed))
+ mark_inode_dirty(inode);
+ return 0;
+}
+
/*
* The clusters for a write are allocated before the data is copied. Free the
* ones that a failed or short write has left beyond the end of the file.
@@ -845,20 +954,23 @@ static void exfat_write_failed(struct inode *inode)
static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb,
struct iov_iter *from)
{
+ struct iov_iter source = *from;
+ struct kiocb buffered;
loff_t offset = iocb->ki_pos, end;
ssize_t written;
int ret;
- iocb->ki_flags &= ~IOCB_DIRECT;
+ kiocb_clone(&buffered, iocb, iocb->ki_filp);
+ buffered.ki_flags &= ~IOCB_DIRECT;
inode_dio_wait(file_inode(iocb->ki_filp));
- written = iomap_file_buffered_write(iocb, from, &exfat_write_iomap_ops,
+ written = iomap_file_buffered_write(&buffered, &source, &exfat_write_iomap_ops,
&exfat_iomap_write_ops, NULL);
- if (written < 0)
+ if (written <= 0)
return written;
- end = iocb->ki_pos + written - 1;
+ end = buffered.ki_pos - 1;
ret = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,
offset, end);
if (ret)
@@ -868,30 +980,109 @@ static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb,
offset >> PAGE_SHIFT,
end >> PAGE_SHIFT);
+ /* Report progress only after the buffered data has been flushed. */
+ iocb->ki_pos = buffered.ki_pos;
+ iov_iter_advance(from, written);
return written;
}
static ssize_t exfat_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ struct block_device *bdev = inode->i_sb->s_bdev;
+ struct exfat_dio dio = { };
+ struct iov_iter source = *from;
+ struct iomap_dio *iomap_dio;
+ loff_t old_size = i_size_read(inode);
+ loff_t valid_size = exfat_get_valid_size(EXFAT_I(inode));
+ loff_t end = iocb->ki_pos + iov_iter_count(from);
ssize_t ret;
+ int error;
- ret = iomap_dio_rw(iocb, from, &exfat_write_iomap_ops,
- &exfat_write_dio_ops, 0, NULL, 0);
- if (ret == -ENOTBLK)
- ret = 0;
- else if (ret < 0)
+ /* VDL extension must wait for outstanding overwrites. */
+ if (end > valid_size)
+ inode_dio_wait(inode);
+ ret = exfat_prepare_dio_range(inode, end);
+ if (ret)
return ret;
- if (iov_iter_count(from)) {
- ssize_t written;
+ if (end <= valid_size) {
+ /* Use existing mappings and allow asynchronous completion. */
+ ret = iomap_dio_rw(iocb, from, &exfat_iomap_ops, NULL,
+ IOMAP_DIO_OVERWRITE_ONLY, &valid_size, 0);
+ if (ret != -ENOTBLK)
+ return ret;
+ goto buffered;
+ }
+
+ kiocb_clone(&dio.iocb, iocb, iocb->ki_filp);
+ dio.start = iocb->ki_pos;
+ /* User page faults can take invalidate_lock via page_mkwrite. */
+ iomap_dio = __iomap_dio_rw(&dio.iocb, &source,
+ &exfat_write_iomap_ops,
+ &exfat_write_dio_ops,
+ 0, &dio, 0);
- written = exfat_fallback_buffered_write(iocb, from);
- if (written < 0)
- return written;
- ret += written;
+ /* Serialize size updates, post-invalidation and gap completion. */
+ filemap_invalidate_lock(inode->i_mapping);
+ if (IS_ERR_OR_NULL(iomap_dio))
+ ret = PTR_ERR_OR_ZERO(iomap_dio);
+ else
+ ret = iomap_dio_complete(iomap_dio);
+ if (dio.completed) {
+ error = exfat_finish_dio_gap(&dio);
+ if (error) {
+ exfat_err(inode->i_sb,
+ "write: fail to initialize direct I/O gap (%d)",
+ error);
+ if (ret >= 0)
+ ret = error;
+ }
+ }
+ if (ret < 0) {
+ struct exfat_inode_info *ei = EXFAT_I(inode);
+ loff_t size = max(old_size, exfat_get_valid_size(ei));
+ loff_t initialized = max(exfat_get_valid_size(ei), dio.start);
+
+ if (size < i_size_read(inode)) {
+ truncate_setsize(inode, size);
+ exfat_truncate(inode);
+ }
+ /* Failed DIO may have changed the prepared invalid tail. */
+ if (exfat_get_zeroed_size(ei) > initialized)
+ exfat_set_zeroed_size(ei, initialized);
+ }
+ if (exfat_bytes_to_cluster(EXFAT_SB(inode->i_sb),
+ exfat_ondisk_size(inode)) >
+ exfat_bytes_to_cluster_round_up(EXFAT_SB(inode->i_sb),
+ i_size_read(inode)))
+ exfat_truncate(inode);
+ filemap_invalidate_unlock(inode->i_mapping);
+
+ if (ret > 0) {
+ iocb->ki_pos = dio.iocb.ki_pos;
+ iov_iter_advance(from, ret);
+ return ret;
}
+ if (ret < 0 && ret != -ENOTBLK)
+ return ret;
- return ret;
+buffered:
+ /* Invalidation can fail before iomap checks alignment. */
+ if (((iocb->ki_pos | iov_iter_count(from)) &
+ (bdev_logical_block_size(bdev) - 1)) ||
+ (iov_iter_alignment(from) & bdev_dma_alignment(bdev)))
+ return -EINVAL;
+ if (user_backed_iter(from) && !from->nofault &&
+ fault_in_iov_iter_readable(from, 1))
+ return -EFAULT;
+ if (iocb->ki_pos > exfat_get_valid_size(EXFAT_I(inode))) {
+ ret = exfat_extend_valid_size(inode, iocb->ki_pos);
+ if (ret)
+ return ret;
+ }
+
+ return exfat_fallback_buffered_write(iocb, from);
}
static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter)
@@ -906,6 +1097,10 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter)
if (unlikely(exfat_forced_shutdown(inode->i_sb)))
return -EIO;
+ /* Gap initialization and source faults require blocking operations. */
+ if (iocb->ki_flags & IOCB_NOWAIT)
+ return -EAGAIN;
+
inode_lock(inode);
ret = generic_write_checks(iocb, iter);
@@ -925,15 +1120,16 @@ static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter)
valid_size = exfat_get_valid_size(ei);
if (pos > valid_size) {
- ret = exfat_extend_valid_size(inode, pos);
- if (ret < 0 && ret != -ENOSPC) {
- exfat_err(inode->i_sb,
- "write: fail to zero from %llu to %llu(%zd)",
- valid_size, pos, ret);
+ if (!(iocb->ki_flags & IOCB_DIRECT)) {
+ ret = exfat_extend_valid_size(inode, pos);
+ if (ret < 0 && ret != -ENOSPC) {
+ exfat_err(inode->i_sb,
+ "write: fail to zero from %llu to %llu(%zd)",
+ valid_size, pos, ret);
+ }
+ if (ret < 0)
+ goto unlock;
}
- if (ret < 0)
- goto unlock;
-
pos = valid_size;
}
@@ -982,9 +1178,12 @@ static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
inode_lock_shared(inode);
if (iocb->ki_flags & IOCB_DIRECT) {
+ /* Snapshot before writeback or destination faults can advance VDL. */
+ loff_t valid_size = exfat_get_valid_size(EXFAT_I(inode));
+
file_accessed(iocb->ki_filp);
ret = iomap_dio_rw(iocb, iter, &exfat_iomap_ops, NULL, 0,
- NULL, 0);
+ &valid_size, 0);
if (ret >= 0 && iov_iter_count(iter) &&
iocb->ki_pos < i_size_read(inode))
ret = filemap_read(iocb, iter, ret);
@@ -1011,9 +1210,6 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
if (unlikely(exfat_forced_shutdown(inode->i_sb)))
return VM_FAULT_SIGBUS;
- if (!inode_trylock(inode))
- return VM_FAULT_RETRY;
-
sb_start_pagefault(inode->i_sb);
error = file_update_time(vmf->vma->vm_file);
if (error) {
@@ -1029,7 +1225,7 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
/* The faulting folio is already populated; initialize only its gap. */
gap_start = max(exfat_get_valid_size(ei), exfat_get_zeroed_size(ei));
- error = exfat_prepare_valid_range(inode, gap_start, fault_start);
+ error = exfat_prepare_range(inode, gap_start, fault_start, true);
if (error) {
ret = vmf_fs_error(error);
goto out_unlock;
@@ -1048,7 +1244,6 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
filemap_invalidate_unlock_shared(mapping);
out_pagefault:
sb_end_pagefault(inode->i_sb);
- inode_unlock(inode);
return ret;
}
diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c
index 4bea9e9deb94..7071f603b57b 100644
--- a/fs/exfat/inode.c
+++ b/fs/exfat/inode.c
@@ -73,25 +73,16 @@ int __exfat_write_inode(struct inode *inode, int sync)
&ep->dentry.file.access_date,
NULL);
- /*
- * During a DIO write, valid_size is updated eagerly in iomap_end (so
- * that concurrent buffered reads see IOMAP_MAPPED) while i_size is
- * updated asynchronously in end_io. The FAT chain was already
- * extended to cover ceil(valid_size/cluster_size) clusters. Use the
- * maximum so the on-disk size field always covers the FAT chain,
- * preventing fsck from reporting "more clusters are allocated".
- */
- on_disk_size = max_t(unsigned long long, i_size_read(inode),
- valid_size);
+ /* Direct writes publish the completed size in their end_io handler. */
+ on_disk_size = i_size_read(inode);
if (ei->start_clu == EXFAT_EOF_CLUSTER)
on_disk_size = 0;
/*
- * valid_size on disk must reflect only confirmed data (up to i_size)
- * and must not exceed on_disk_size.
+ * Truncate can publish a smaller i_size before lowering valid_size.
+ * Never record a valid_size greater than the recorded file size.
*/
- on_disk_valid_size = min_t(unsigned long long, valid_size,
- i_size_read(inode));
+ on_disk_valid_size = min_t(unsigned long long, valid_size, on_disk_size);
if (ei->start_clu == EXFAT_EOF_CLUSTER)
on_disk_valid_size = 0;
diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c
index 7eb31a6091b3..1b3ba1b8829a 100644
--- a/fs/exfat/iomap.c
+++ b/fs/exfat/iomap.c
@@ -13,32 +13,6 @@
#include "exfat_fs.h"
#include "iomap.h"
-/*
- * exfat_file_write_dio_end_io - Direct I/O write completion handler
- *
- * Updates i_size if the write extended the file. Called from the dio layer
- * after I/O completion.
- */
-static int exfat_file_write_dio_end_io(struct kiocb *iocb, ssize_t size,
- int error, unsigned int flags)
-{
- struct inode *inode = file_inode(iocb->ki_filp);
-
- if (error)
- return error;
-
- if (size && i_size_read(inode) < iocb->ki_pos + size) {
- i_size_write(inode, iocb->ki_pos + size);
- mark_inode_dirty(inode);
- }
-
- return 0;
-}
-
-const struct iomap_dio_ops exfat_write_dio_ops = {
- .end_io = exfat_file_write_dio_end_io,
-};
-
static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned int flags, struct iomap *iomap, bool may_alloc,
loff_t valid_size)
@@ -50,6 +24,9 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
int err;
bool balloc = false;
+ if ((flags & IOMAP_OVERWRITE_ONLY) && offset + length > valid_size)
+ return -EAGAIN;
+
iomap->validity_cookie = valid_size;
if (!may_alloc) {
@@ -75,6 +52,27 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
iomap->bdev = inode->i_sb->s_bdev;
iomap->offset = offset;
+ if (may_alloc && flags & IOMAP_DIRECT) {
+ unsigned int allocated = exfat_bytes_to_cluster(sbi,
+ exfat_ondisk_size(inode));
+ unsigned int first = exfat_bytes_to_cluster(sbi, offset);
+
+ /* Allocate the gap before mapping a write beyond the last cluster. */
+ while (allocated < first) {
+ unsigned int count = first - allocated, clu;
+
+ err = exfat_map_cluster(inode, allocated, &clu, &count,
+ true, NULL);
+ if (err)
+ goto out;
+ if (WARN_ON_ONCE(!count)) {
+ err = -EIO;
+ goto out;
+ }
+ allocated += count;
+ }
+ }
+
err = exfat_map_cluster(inode, exfat_bytes_to_cluster(sbi, offset),
&cluster, &num_clusters, may_alloc, &balloc);
if (err == -ENOSPC && !(flags & IOMAP_DIRECT)) {
@@ -99,7 +97,11 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
iomap->type = IOMAP_MAPPED;
iomap->flags = IOMAP_F_MERGED;
- if (may_alloc || flags & IOMAP_ZERO) {
+ /* Cached VDL update may still need to be persisted for O_DSYNC. */
+ if (flags & IOMAP_OVERWRITE_ONLY)
+ iomap->flags |= IOMAP_F_DIRTY;
+
+ if (may_alloc || flags & (IOMAP_ZERO | IOMAP_OVERWRITE_ONLY)) {
if (balloc)
iomap->flags |= IOMAP_F_NEW;
} else {
@@ -143,22 +145,40 @@ static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length
return err;
}
-static int exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
- unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
+static int exfat_iomap_next(const struct iomap_iter *iter,
+ struct iomap *iomap, struct iomap *srcmap)
{
- return __exfat_iomap_begin(inode, offset, length, flags, iomap, false,
- exfat_get_valid_size(EXFAT_I(inode)));
+ const loff_t *dio_valid_size = iter->private;
+ loff_t valid_size;
+ int ret;
+
+ if (iomap->length) {
+ ret = iomap_iter_continue(iter, iomap, srcmap, 0);
+ if (ret <= 0)
+ return ret;
+ }
+ if (iter->flags & IOMAP_DIRECT)
+ valid_size = *dio_valid_size;
+ else
+ valid_size = exfat_get_valid_size(EXFAT_I(iter->inode));
+ ret = __exfat_iomap_begin(iter->inode, iter->pos, iter->len,
+ iter->flags, iomap, false, valid_size);
+ return ret ?: 1;
}
static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
{
- return __exfat_iomap_begin(inode, offset, length, flags, iomap, true,
- exfat_get_valid_size(EXFAT_I(inode)));
+ int ret;
+
+ ret = __exfat_iomap_begin(inode, offset, length, flags, iomap, true,
+ exfat_get_valid_size(EXFAT_I(inode)));
+ if (!ret && (flags & IOMAP_DIRECT) &&
+ offset + iomap->length > iomap->validity_cookie)
+ iomap->flags |= IOMAP_F_DIRTY;
+ return ret;
}
-static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin);
-
const struct iomap_ops exfat_iomap_ops = {
.iomap_next = exfat_iomap_next,
};
@@ -186,36 +206,40 @@ static const struct iomap_ops exfat_swap_iomap_ops = {
#endif
/*
- * exfat_write_iomap_end - Update the state after write
- *
- * Extends ->valid_size to cover the newly written range.
- * Marks the inode dirty if metadata was changed.
+ * exfat_write_iomap_end - Record buffered file size changes
*/
static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
ssize_t written, unsigned int flags, struct iomap *iomap)
{
- struct exfat_inode_info *ei = EXFAT_I(inode);
- bool dirtied = false;
- loff_t end;
-
if (!written)
return 0;
- end = pos + written;
-
- if (exfat_advance_valid_size(ei, end))
- dirtied = true;
-
- exfat_advance_zeroed_size(ei, end);
-
- if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED)
+ if (iomap->flags & IOMAP_F_SIZE_CHANGED)
mark_inode_dirty(inode);
return written;
}
-static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next,
- exfat_write_iomap_begin, exfat_write_iomap_end);
+static int exfat_write_iomap_next(const struct iomap_iter *iter,
+ struct iomap *iomap, struct iomap *srcmap)
+{
+ int ret;
+
+ ret = iomap_iter_next(iter, iomap, srcmap, exfat_write_iomap_begin,
+ exfat_write_iomap_end);
+ if (ret == -ENOSPC && (iter->flags & IOMAP_DIRECT)) {
+ const struct exfat_dio *dio = iter->private;
+
+ /*
+ * ENOTBLK makes iomap wait for the submitted bios and report
+ * a short write only if they succeeded.
+ */
+ if (iter->pos > dio->start)
+ return -ENOTBLK;
+ }
+
+ return ret;
+}
const struct iomap_ops exfat_write_iomap_ops = {
.iomap_next = exfat_write_iomap_next,
@@ -298,6 +322,48 @@ const struct iomap_write_ops exfat_iomap_write_ops = {
.read_folio_range = exfat_iomap_read_folio_range,
};
+/*
+ * Gap zeroing follows the direct write. Keep completed data intact when the
+ * gap and the write share a filesystem block.
+ */
+static int exfat_dio_gap_read_folio_range(const struct iomap_iter *iter,
+ struct folio *folio, loff_t pos, size_t len)
+{
+ const struct exfat_dio *dio = iter->private;
+ loff_t valid = iter->iomap.validity_cookie;
+ loff_t end = pos + len;
+ loff_t start, stop;
+ int error;
+
+ if (pos < valid || end > dio->start) {
+ error = iomap_bio_read_folio_range_sync(iter, folio, pos, len);
+ if (error)
+ return error;
+ } else {
+ folio_zero_range(folio, offset_in_folio(folio, pos), len);
+ return 0;
+ }
+
+ start = max(pos, valid);
+ stop = min(end, dio->start);
+ if (start < stop)
+ folio_zero_segment(folio, offset_in_folio(folio, start),
+ offset_in_folio(folio, pos) + stop - pos);
+
+ start = max(pos, dio->start + dio->completed);
+ if (start < end)
+ folio_zero_segment(folio, offset_in_folio(folio, start),
+ offset_in_folio(folio, pos) + len);
+ return 0;
+}
+
+const struct iomap_write_ops exfat_dio_gap_write_ops = {
+ .get_folio = exfat_iomap_get_folio,
+ .put_folio = exfat_iomap_put_folio,
+ .iomap_valid = exfat_iomap_valid,
+ .read_folio_range = exfat_dio_gap_read_folio_range,
+};
+
/*
* exfat_writeback_range - Map folio during writeback
*
diff --git a/fs/exfat/iomap.h b/fs/exfat/iomap.h
index 39c93f8cd790..f8f018f87230 100644
--- a/fs/exfat/iomap.h
+++ b/fs/exfat/iomap.h
@@ -6,10 +6,17 @@
#ifndef _LINUX_EXFAT_IOMAP_H
#define _LINUX_EXFAT_IOMAP_H
-extern const struct iomap_dio_ops exfat_write_dio_ops;
+/* One synchronous attempt for a direct write extending valid_size. */
+struct exfat_dio {
+ struct kiocb iocb;
+ loff_t start;
+ loff_t completed;
+};
+
extern const struct iomap_ops exfat_iomap_ops;
extern const struct iomap_ops exfat_write_iomap_ops;
extern const struct iomap_write_ops exfat_iomap_write_ops;
+extern const struct iomap_write_ops exfat_dio_gap_write_ops;
extern const struct iomap_writeback_ops exfat_writeback_ops;
extern const struct iomap_read_ops exfat_iomap_bio_read_ops;
--
2.53.0
^ permalink raw reply [flat|nested] 6+ messages in thread