[PATCH v2 0/5] mm/shmem: optimize read with reduced xarray lookups and folio batching

[PATCH v2 0/5] mm/shmem: optimize read with reduced xarray lookups and folio batching

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

This series improves shmem read performance by implementing folio
batching in the read path and reducing unnecessary xarray lookups.

Performance Results:

fio --ioengine=sync --rw=read --bs=$1 --size=1G --runtime=180 --time_based --group_reporting --name=seq_read_test --filename=testfile

| THP disabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
| ---------------------- | ------------ | ----------------- | ----------- |
| 1M + normal file       | bw=11.5GiB/s | bw=12.7GiB/s      | +10.4%      |
| 64k + normal file      | bw=11.0GiB/s | bw=12.3GiB/s      | +11.8%      |
| 4k + normal file       | bw=3826MiB/s | bw=3849MiB/s      | +0.6%       |
| 1M + fallocated file   | bw=23.8GiB/s | bw=28.6GiB/s      | +20.2%      |
| 64k + fallocated file  | bw=22.5GiB/s | bw=27.3GiB/s      | +21.3%      |
| 4k + fallocated file   | bw=4655MiB/s | bw=4680MiB/s      | +0.5%       |
| 1M + hole              | bw=24.2GiB/s | bw=28.6GiB/s      | +18.2%      |
| 64k + hole             | bw=22.6GiB/s | bw=27.6GiB/s      | +22.1%      |
| 4k + hole              | bw=4652MiB/s | bw=4489MiB/s      | -3.5%       |


| THP enabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
| --------------------- | ------------ | ----------------- | ----------- |
| 1M + normal file      | bw=13.7GiB/s | bw=13.9GiB/s      | +1.4%       |
| 64k + normal file     | bw=13.5GiB/s | bw=13.5GiB/s      | +0.0%       |
| 4k + normal file      | bw=3833MiB/s | bw=3859MiB/s      | +0.7%       |
| 1M + fallocated file  | bw=24.9GiB/s | bw=34.2GiB/s      | +37.3%      |
| 64k + fallocated file | bw=23.0GiB/s | bw=31.4GiB/s      | +36.5%      |
| 4k + fallocated file  | bw=4710MiB/s | bw=4655MiB/s      | -1.2%       |
| 1M + hole             | bw=24.3GiB/s | bw=34.5GiB/s      | +42.0%      |
| 64k + hole            | bw=23.5GiB/s | bw=31.1GiB/s      | +32.3%      |
| 4k + hole             | bw=4690MiB/s | bw=4647MiB/s      | -0.9%       |


v1:
https://lore.kernel.org/linux-mm/20260520101538.58745-1-chizhiling@163.com/#t
rfc:
https://lore.kernel.org/linux-fsdevel/20260515094702.1092355-1-chizhiling@163.com/


Chi Zhiling (5):
  mm/filemap: reduce unnecessary xarray lookups when read cached pages
  mm/filemap: reduce xarray lookups in filemap_get_folios_contig()
  mm/shmem: introduce copy_zero_to_iter() for large zeroing
  mm/shmem: remove page-copy fallback in shmem read path
  mm/shmem: optimize file read with folio batching

 mm/filemap.c |  46 +++++++++++--------
 mm/shmem.c   | 126 +++++++++++++++++++++++++++++++++++----------------
 2 files changed, 113 insertions(+), 59 deletions(-)

-- 
2.43.0

[PATCH v2 1/5] mm/filemap: reduce unnecessary xarray lookups when read cached pages

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

When reading small amounts of data from the page cache, only a single
folio is typically returned from filemap_read_get_batch(). In this case,
calling xas_advance() or xas_next() after adding the folio to the batch
is unnecessary and only introduces extra branches.

The same issue exists for large reads, where one additional xarray walk
is always performed before termination.

Quit the loop once we get the last folio in the range, so the final
redundant xarray advancement can be avoided.

The xas_next() does not update xa_index when xas->xa_node is set to
XAS_RESTART, so the put and retry path would not update xa_index,
hence the warning should therefore never trigger.

During the 4k reads test, the overhead of this function dropped from
2.91% to 2.53%.

Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
---
 mm/filemap.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/mm/filemap.c b/mm/filemap.c
index 4e636647100c..d54450e529bd 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2458,12 +2458,16 @@ static void filemap_get_read_batch(struct address_space *mapping,
 {
 	XA_STATE(xas, &mapping->i_pages, index);
 	struct folio *folio;
+	pgoff_t next;
+
+	if (unlikely(index > max))
+		return;
 
 	rcu_read_lock();
 	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
 		if (xas_retry(&xas, folio))
 			continue;
-		if (xas.xa_index > max || xa_is_value(folio))
+		if (xa_is_value(folio) || WARN_ON(xas.xa_index > max))
 			break;
 		if (xa_is_sibling(folio))
 			break;
@@ -2479,7 +2483,11 @@ static void filemap_get_read_batch(struct address_space *mapping,
 			break;
 		if (folio_test_readahead(folio))
 			break;
-		xas_advance(&xas, folio_next_index(folio) - 1);
+
+		next = folio_next_index(folio);
+		if (next > max)
+			break;
+		xas_advance(&xas, next - 1);
 		continue;
 put_folio:
 		folio_put(folio);
-- 
2.43.0

[PATCH v2 2/5] mm/filemap: reduce xarray lookups in filemap_get_folios_contig()

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

Apply the same optimization used in filemap_get_read_batch() by moving
the boundary check from the loop condition to before xas_advance(),
avoiding an unnecessary xarray lookup and reducing branches in the fast
path.

Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
---
 mm/filemap.c | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/mm/filemap.c b/mm/filemap.c
index d54450e529bd..6bc717f205a7 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2258,11 +2258,13 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 	XA_STATE(xas, &mapping->i_pages, *start);
 	unsigned long nr;
 	struct folio *folio;
+	pgoff_t next;
 
-	rcu_read_lock();
+	if (unlikely(*start > end))
+		return 0;
 
-	for (folio = xas_load(&xas); folio && xas.xa_index <= end;
-			folio = xas_next(&xas)) {
+	rcu_read_lock();
+	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
 		if (xas_retry(&xas, folio))
 			continue;
 		/*
@@ -2270,11 +2272,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 		 * No current caller is looking for DAX entries.
 		 */
 		if (xa_is_value(folio))
-			goto update_start;
+			break;
 
 		/* If we landed in the middle of a THP, continue at its end. */
 		if (xa_is_sibling(folio))
-			goto update_start;
+			break;
 
 		if (!folio_try_get(folio))
 			goto retry;
@@ -2282,30 +2284,28 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 		if (unlikely(folio != xas_reload(&xas)))
 			goto put_folio;
 
-		if (!folio_batch_add(fbatch, folio)) {
-			nr = folio_nr_pages(folio);
-			*start = folio->index + nr;
-			goto out;
-		}
-		xas_advance(&xas, folio_next_index(folio) - 1);
+		if (!folio_batch_add(fbatch, folio))
+			break;
+
+		next = folio_next_index(folio);
+		if (next > end)
+			break;
+		xas_advance(&xas, next - 1);
 		continue;
+
 put_folio:
 		folio_put(folio);
-
 retry:
 		xas_reset(&xas);
 	}
+	rcu_read_unlock();
 
-update_start:
 	nr = folio_batch_count(fbatch);
-
 	if (nr) {
 		folio = fbatch->folios[nr - 1];
 		*start = folio_next_index(folio);
 	}
-out:
-	rcu_read_unlock();
-	return folio_batch_count(fbatch);
+	return nr;
 }
 EXPORT_SYMBOL(filemap_get_folios_contig);
 
-- 
2.43.0

[PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

This is a prep patch for shmem folio batching in the read path, where
non-uptodate folios need to be handled in the main iteration loop. A
large non-uptodate folio should be treated as a hole.

Currently, holes larger than PAGE_SIZE cannot be handled because
ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
wrapper to support copying larger zero ranges to the iterator.

Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
---
 mm/shmem.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/mm/shmem.c b/mm/shmem.c
index 3b5dc21b323c..b08118fbd275 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3339,6 +3339,20 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping,
 	return copied;
 }
 
+static size_t copy_zero_to_iter(size_t bytes, struct iov_iter *i)
+{
+	struct folio *zero = largest_zero_folio();
+	unsigned long nr, ret, written = 0;
+
+	do {
+		nr = min(bytes - written, folio_size(zero));
+		ret = copy_folio_to_iter(zero, 0, nr, i);
+		written += ret;
+	} while (written < bytes && ret == nr);
+
+	return written;
+}
+
 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct file *file = iocb->ki_filp;
@@ -3433,7 +3447,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			 * clear_user() not so much, that it is noticeably
 			 * faster to copy the zero page instead of clearing.
 			 */
-			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
+			ret = copy_zero_to_iter(nr, to);
 		} else {
 			/*
 			 * But submitting the same page twice in a row to
-- 
2.43.0

[PATCH v2 4/5] mm/shmem: remove page-copy fallback in shmem read path

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

This is perp patch for folio batch support which removing the fallback
to page-copy mode from shmem reads.

The existing hwpoison fallback would require fetching the same folio
multiple times to read its each pages, which complicates the
batching model. Instead, move hwpoison handling into
copy_each_pages_to_iter(), allowing the error path to be centralized.

This simplifies the shmem fast read path and avoids special-case
handling that conflicts with folio batching support.

Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
---
 mm/shmem.c | 63 +++++++++++++++++++++++++++++-------------------------
 1 file changed, 34 insertions(+), 29 deletions(-)

diff --git a/mm/shmem.c b/mm/shmem.c
index b08118fbd275..cac355685e49 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3353,6 +3353,30 @@ static size_t copy_zero_to_iter(size_t bytes, struct iov_iter *i)
 	return written;
 }
 
+static size_t copy_pages_to_iter(struct folio *folio, size_t offset,
+		size_t bytes, struct iov_iter *i, int *error)
+{
+	struct page *page;
+	unsigned long off, nr, ret, written = 0;
+
+	do {
+		page = folio_page(folio, offset >> PAGE_SHIFT);
+		if (PageHWPoison(page)) {
+			*error = -EIO;
+			break;
+		}
+
+		off = offset_in_page(offset);
+		nr = min(PAGE_SIZE - off, bytes - written);
+
+		ret = copy_page_to_iter(page, off, nr, i);
+		offset += ret;
+		written += ret;
+	} while (written < bytes && ret == nr);
+
+	return written;
+}
+
 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct file *file = iocb->ki_filp;
@@ -3365,10 +3389,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 	for (;;) {
 		struct folio *folio = NULL;
-		struct page *page = NULL;
 		unsigned long nr, ret;
 		loff_t end_offset, i_size = i_size_read(inode);
-		bool fallback_page_copy = false;
 		size_t fsize;
 
 		if (unlikely(iocb->ki_pos >= i_size))
@@ -3376,25 +3398,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 		index = iocb->ki_pos >> PAGE_SHIFT;
 		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
+		if (folio)
+			folio_unlock(folio);
 		if (error) {
 			if (error == -EINVAL)
 				error = 0;
 			break;
 		}
-		if (folio) {
-			folio_unlock(folio);
-
-			page = folio_file_page(folio, index);
-			if (PageHWPoison(page)) {
-				folio_put(folio);
-				error = -EIO;
-				break;
-			}
-
-			if (folio_test_large(folio) &&
-			    folio_test_has_hwpoisoned(folio))
-				fallback_page_copy = true;
-		}
 
 		/*
 		 * We must evaluate after, since reads (unlike writes)
@@ -3406,12 +3416,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 				folio_put(folio);
 			break;
 		}
-		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
-		if (folio && likely(!fallback_page_copy))
-			fsize = folio_size(folio);
-		else
-			fsize = PAGE_SIZE;
+		fsize = folio ? folio_size(folio) : PAGE_SIZE;
 		offset = iocb->ki_pos & (fsize - 1);
+		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
 		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
 
 		if (folio) {
@@ -3420,12 +3427,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			 * virtual addresses, take care about potential aliasing
 			 * before reading the page on the kernel side.
 			 */
-			if (mapping_writably_mapped(mapping)) {
-				if (likely(!fallback_page_copy))
-					flush_dcache_folio(folio);
-				else
-					flush_dcache_page(page);
-			}
+			if (mapping_writably_mapped(mapping))
+				flush_dcache_folio(folio);
 
 			/*
 			 * Mark the folio accessed if we read the beginning.
@@ -3436,10 +3439,10 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			 * Ok, we have the page, and it's up-to-date, so
 			 * now we can copy it to user space...
 			 */
-			if (likely(!fallback_page_copy))
+			if (likely(!folio_contain_hwpoisoned_page(folio)))
 				ret = copy_folio_to_iter(folio, offset, nr, to);
 			else
-				ret = copy_page_to_iter(page, offset, nr, to);
+				ret = copy_pages_to_iter(folio, offset, nr, to, &error);
 			folio_put(folio);
 		} else if (user_backed_iter(to)) {
 			/*
@@ -3462,6 +3465,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 		if (!iov_iter_count(to))
 			break;
+		if (unlikely(error))
+			break;
 		if (ret < nr) {
 			error = -EFAULT;
 			break;
-- 
2.43.0

[PATCH v2 5/5] mm/shmem: optimize file read with folio batching

[Chi Zhiling]

From: Chi Zhiling <chizhiling@kylinos.cn>

Optimize shmem file read by using filemap_get_folios_contig() to batch
fetch contiguous folios from the page cache, reducing the overhead of
repeated shmem_get_folio() calls.

This patch checks the uptodate flag without holding the folio lock, so
it may observe a non-uptodate state on a locked folio that is still
being initialized. This is safe because only zero-filled data can be
copied to the user buffer in that scenario.

A non-uptodate folio in the swap cache cannot be added to the shmem page
cache. This creates a semantic conflict, as shmem zeroes the folio out,
but the swap cache would fill it by reading from the swap backing store.

Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
---
 mm/shmem.c | 57 ++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/mm/shmem.c b/mm/shmem.c
index cac355685e49..61937582f08c 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -891,6 +891,14 @@ int shmem_add_to_page_cache(struct folio *folio,
 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
 
+	/*
+	 * Don't add a non-uptodate folio that is in swap cache to page
+	 * cache, since shmem will zero it instead of reading from swap
+	 * backing.
+	 */
+	VM_BUG_ON_FOLIO(folio_test_swapcache(folio) &&
+			!folio_test_uptodate(folio), folio);
+
 	folio_ref_add(folio, nr);
 	folio->mapping = mapping;
 	folio->index = index;
@@ -3382,11 +3390,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file_inode(file);
 	struct address_space *mapping = inode->i_mapping;
-	pgoff_t index;
+	struct folio_batch fbatch;
 	unsigned long offset;
 	int error = 0;
 	ssize_t retval = 0;
 
+	folio_batch_init(&fbatch);
+
 	for (;;) {
 		struct folio *folio = NULL;
 		unsigned long nr, ret;
@@ -3395,15 +3405,33 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 		if (unlikely(iocb->ki_pos >= i_size))
 			break;
+fetch:
+		folio = folio_batch_next(&fbatch);
+		if (!folio) {
+			pgoff_t start = iocb->ki_pos >> PAGE_SHIFT;
+			pgoff_t end = (iocb->ki_pos + to->count - 1) >> PAGE_SHIFT;
+
+			if (folio_batch_count(&fbatch)) {
+				for (int i = 0; i < folio_batch_count(&fbatch); i++)
+					folio_put(fbatch.folios[i]);
+				folio_batch_reinit(&fbatch);
+			}
 
-		index = iocb->ki_pos >> PAGE_SHIFT;
-		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
-		if (folio)
-			folio_unlock(folio);
-		if (error) {
-			if (error == -EINVAL)
-				error = 0;
-			break;
+			filemap_get_folios_contig(inode->i_mapping, &start, end, &fbatch);
+			if (folio_batch_count(&fbatch))
+				goto fetch;
+
+			error = shmem_get_folio(inode, start, 0, &folio, SGP_READ);
+			if (unlikely(error)) {
+				if (error == -EINVAL)
+					error = 0;
+				break;
+			}
+			if (folio) {
+				folio_unlock(folio);
+				folio_batch_add(&fbatch, folio);
+				fbatch.i++;
+			}
 		}
 
 		/*
@@ -3411,17 +3439,15 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		 * are called without i_rwsem protection against truncate
 		 */
 		i_size = i_size_read(inode);
-		if (unlikely(iocb->ki_pos >= i_size)) {
-			if (folio)
-				folio_put(folio);
+		if (unlikely(iocb->ki_pos >= i_size))
 			break;
-		}
+
 		fsize = folio ? folio_size(folio) : PAGE_SIZE;
 		offset = iocb->ki_pos & (fsize - 1);
 		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
 		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
 
-		if (folio) {
+		if (folio && folio_test_uptodate(folio)) {
 			/*
 			 * If users can be writing to this page using arbitrary
 			 * virtual addresses, take care about potential aliasing
@@ -3443,7 +3469,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 				ret = copy_folio_to_iter(folio, offset, nr, to);
 			else
 				ret = copy_pages_to_iter(folio, offset, nr, to, &error);
-			folio_put(folio);
 		} else if (user_backed_iter(to)) {
 			/*
 			 * Copy to user tends to be so well optimized, but
@@ -3474,6 +3499,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		cond_resched();
 	}
 
+	for (int i = 0; i < folio_batch_count(&fbatch); i++)
+		folio_put(fbatch.folios[i]);
 	file_accessed(file);
 	return retval ? retval : error;
 }
-- 
2.43.0

Re: [PATCH v2 0/5] mm/shmem: optimize read with reduced xarray lookups and folio batching

[Chi Zhiling]

On 6/1/26 13:56, Chi Zhiling wrote:
> Performance Results:
> 
> fio --ioengine=sync --rw=read --bs=$1 --size=1G --runtime=180 --time_based --group_reporting --name=seq_read_test --filename=testfile
> 
> | THP disabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
> | ---------------------- | ------------ | ----------------- | ----------- |
> | 1M + normal file       | bw=11.5GiB/s | bw=12.7GiB/s      | +10.4%      |
> | 64k + normal file      | bw=11.0GiB/s | bw=12.3GiB/s      | +11.8%      |
> | 4k + normal file       | bw=3826MiB/s | bw=3849MiB/s      | +0.6%       |
> | 1M + fallocated file   | bw=23.8GiB/s | bw=28.6GiB/s      | +20.2%      |
> | 64k + fallocated file  | bw=22.5GiB/s | bw=27.3GiB/s      | +21.3%      |
> | 4k + fallocated file   | bw=4655MiB/s | bw=4680MiB/s      | +0.5%       |
> | 1M + hole              | bw=24.2GiB/s | bw=28.6GiB/s      | +18.2%      |
> | 64k + hole             | bw=22.6GiB/s | bw=27.6GiB/s      | +22.1%      |
> | 4k + hole              | bw=4652MiB/s | bw=4489MiB/s      | -3.5%       |
> 
> 
> | THP enabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
> | --------------------- | ------------ | ----------------- | ----------- |
> | 1M + normal file      | bw=13.7GiB/s | bw=13.9GiB/s      | +1.4%       |
> | 64k + normal file     | bw=13.5GiB/s | bw=13.5GiB/s      | +0.0%       |
> | 4k + normal file      | bw=3833MiB/s | bw=3859MiB/s      | +0.7%       |
> | 1M + fallocated file  | bw=24.9GiB/s | bw=34.2GiB/s      | +37.3%      |
> | 64k + fallocated file | bw=23.0GiB/s | bw=31.4GiB/s      | +36.5%      |
> | 4k + fallocated file  | bw=4710MiB/s | bw=4655MiB/s      | -1.2%       |
> | 1M + hole             | bw=24.3GiB/s | bw=34.5GiB/s      | +42.0%      |
> | 64k + hole            | bw=23.5GiB/s | bw=31.1GiB/s      | +32.3%      |
> | 4k + hole             | bw=4690MiB/s | bw=4647MiB/s      | -0.9%       |
> 
Apologies, due to my oversight, the tests involving hole were incorrect, 
the holes were not successfully created in the files during testing.

Below are the corrected results from a retest:

| THP disabled | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
| ------------ | ------------ | ----------------- | ----------- |
| 1M + hole    | bw=27.3GiB/s | bw=23.4GiB/s      | -14.3%      |
| 64k + hole   | bw=27.3GiB/s | bw=23.3GiB/s      | -14.7%      |
| 4k + hole    | bw=4825MiB/s | bw=4624MiB/s      | -4.2%       |


| THP enabled | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
| ----------- | ------------ | ----------------- | ----------- |
| 1M + hole   | bw=27.0GiB/s | bw=23.1GiB/s      | -14.4%      |
| 64k + hole  | bw=27.5GiB/s | bw=23.3GiB/s      | -15.3%      |
| 4k + hole   | bw=4777MiB/s | bw=4640MiB/s      | -2.9%       |


There is a noticeable performance drop when accessing holes, as every 
read triggers a fallback. I will address this in the next version.

Re: [PATCH v2 1/5] mm/filemap: reduce unnecessary xarray lookups when read cached pages

[Matthew Wilcox]

On Mon, Jun 01, 2026 at 01:57:00PM +0800, Chi Zhiling wrote:
> From: Chi Zhiling <chizhiling@kylinos.cn>
> 
> When reading small amounts of data from the page cache, only a single
> folio is typically returned from filemap_read_get_batch(). In this case,
> calling xas_advance() or xas_next() after adding the folio to the batch
> is unnecessary and only introduces extra branches.

I don't think xas_advance() is all that expensive.  The expensive part
is going to be calling xas_next() and realising that the folio now lies
beyond the required range.  I think you'll get just as much benefit from
doing this:

diff --git a/mm/filemap.c b/mm/filemap.c
index 4e636647100c..578b03bd5514 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2459,11 +2459,14 @@ static void filemap_get_read_batch(struct address_space *mapping,
 	XA_STATE(xas, &mapping->i_pages, index);
 	struct folio *folio;
 
+	if (index > max)
+		return;
+
 	rcu_read_lock();
 	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
 		if (xas_retry(&xas, folio))
 			continue;
-		if (xas.xa_index > max || xa_is_value(folio))
+		if (xa_is_value(folio))
 			break;
 		if (xa_is_sibling(folio))
 			break;
@@ -2480,6 +2483,8 @@ static void filemap_get_read_batch(struct address_space *mapping,
 		if (folio_test_readahead(folio))
 			break;
 		xas_advance(&xas, folio_next_index(folio) - 1);
+		if (xas.index >= max)
+			break;
 		continue;
 put_folio:
 		folio_put(folio);
> 

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Matthew Wilcox]

On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:
> Currently, holes larger than PAGE_SIZE cannot be handled because
> ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
> wrapper to support copying larger zero ranges to the iterator.

I think Hugh put this optimisation in the wrong place, and you're
perpetuating that ;-)

So perhaps we can start by moving this optimisation to lib/iov_iter.c?
And then you can redo your optimisation on top of that.

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 243662af1af7..06c54d719fcd 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -451,7 +451,23 @@ static __always_inline
 size_t zero_to_user_iter(void __user *iter_to, size_t progress,
 			 size_t len, void *priv, void *priv2)
 {
-	return clear_user(iter_to, len);
+	/*
+	 * it is noticeably faster to copy the zero page instead of
+	 * calling clear_user().  Shame.
+	 */
+	void *from = page_address(ZERO_PAGE(0));
+	size_t res = 0;
+
+	while (1) {
+		size_t n = min(len, PAGE_SIZE);
+		n = copy_to_user_iter(iter_to, progress, n, from, priv2);
+		res += n;
+		len -= n;
+		if (!len || !n)
+			break;
+	}
+
+	return res;
 }
 
 static __always_inline
diff --git a/mm/shmem.c b/mm/shmem.c
index 3b5dc21b323c..112cae9f9e4f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3427,19 +3427,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			else
 				ret = copy_page_to_iter(page, offset, nr, to);
 			folio_put(folio);
-		} else if (user_backed_iter(to)) {
-			/*
-			 * Copy to user tends to be so well optimized, but
-			 * clear_user() not so much, that it is noticeably
-			 * faster to copy the zero page instead of clearing.
-			 */
-			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
 		} else {
-			/*
-			 * But submitting the same page twice in a row to
-			 * splice() - or others? - can result in confusion:
-			 * so don't attempt that optimization on pipes etc.
-			 */
 			ret = iov_iter_zero(nr, to);
 		}
 

Re: [PATCH v2 5/5] mm/shmem: optimize file read with folio batching

[Matthew Wilcox]

On Mon, Jun 01, 2026 at 01:57:04PM +0800, Chi Zhiling wrote:
> +			if (folio_batch_count(&fbatch)) {
> +				for (int i = 0; i < folio_batch_count(&fbatch); i++)
> +					folio_put(fbatch.folios[i]);
> +				folio_batch_reinit(&fbatch);
> +			}

folios_put().

> +	for (int i = 0; i < folio_batch_count(&fbatch); i++)
> +		folio_put(fbatch.folios[i]);

Also folios_put().

Re: [PATCH v2 1/5] mm/filemap: reduce unnecessary xarray lookups when read cached pages

[Chi Zhiling]

Hi, Matthew

On 6/1/26 8:51 PM, Matthew Wilcox wrote:
> On Mon, Jun 01, 2026 at 01:57:00PM +0800, Chi Zhiling wrote:
>> From: Chi Zhiling <chizhiling@kylinos.cn>
>>
>> When reading small amounts of data from the page cache, only a single
>> folio is typically returned from filemap_read_get_batch(). In this case,
>> calling xas_advance() or xas_next() after adding the folio to the batch
>> is unnecessary and only introduces extra branches.
> 
> I don't think xas_advance() is all that expensive.  The expensive part
> is going to be calling xas_next() and realising that the folio now lies
> beyond the required range.  I think you'll get just as much benefit from
> doing this:
> 
> diff --git a/mm/filemap.c b/mm/filemap.c
> index 4e636647100c..578b03bd5514 100644
> --- a/mm/filemap.c
> +++ b/mm/filemap.c
> @@ -2459,11 +2459,14 @@ static void filemap_get_read_batch(struct address_space *mapping,
>   	XA_STATE(xas, &mapping->i_pages, index);
>   	struct folio *folio;
>   
> +	if (index > max)
> +		return;
> +
>   	rcu_read_lock();
>   	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
>   		if (xas_retry(&xas, folio))
>   			continue;
> -		if (xas.xa_index > max || xa_is_value(folio))
> +		if (xa_is_value(folio))
>   			break;
>   		if (xa_is_sibling(folio))
>   			break;
> @@ -2480,6 +2483,8 @@ static void filemap_get_read_batch(struct address_space *mapping,
>   		if (folio_test_readahead(folio))
>   			break;
>   		xas_advance(&xas, folio_next_index(folio) - 1);
> +		if (xas.index >= max)
> +			break;
>   		continue;
>   put_folio:
>   		folio_put(folio);
>>

Looks good. I'll include it in the next version and make the same change 
in patch 2/5.

Thanks!

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Chi Zhiling]

On 6/1/26 9:22 PM, Matthew Wilcox wrote:
> On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:
>> Currently, holes larger than PAGE_SIZE cannot be handled because
>> ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
>> wrapper to support copying larger zero ranges to the iterator.
> 
> I think Hugh put this optimisation in the wrong place, and you're
> perpetuating that ;-)
> 
> So perhaps we can start by moving this optimisation to lib/iov_iter.c?
> And then you can redo your optimisation on top of that.

Sure, it's good to see this optimization moved into another place :)

In that case, I'll assume iov_iter_zero() is already well optimized and 
drop copy_page_to_iter() in v3.

By the way, I assume the benefit of this optimization depends on the CPU 
architecture, right? If so, I wonder whether lib/iov_iter.c is the most 
appropriate place for it.

> diff --git a/mm/shmem.c b/mm/shmem.c
> index 3b5dc21b323c..112cae9f9e4f 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -3427,19 +3427,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   			else
>   				ret = copy_page_to_iter(page, offset, nr, to);
>   			folio_put(folio);
> -		} else if (user_backed_iter(to)) {
> -			/*
> -			 * Copy to user tends to be so well optimized, but
> -			 * clear_user() not so much, that it is noticeably
> -			 * faster to copy the zero page instead of clearing.
> -			 */
> -			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
>   		} else {
> -			/*
> -			 * But submitting the same page twice in a row to
> -			 * splice() - or others? - can result in confusion:
> -			 * so don't attempt that optimization on pipes etc.
> -			 */
>   			ret = iov_iter_zero(nr, to);
>   		}

I'll include this change to v3.

Thanks,

Re: [PATCH v2 5/5] mm/shmem: optimize file read with folio batching

[Chi Zhiling]

On 6/1/26 9:59 PM, Matthew Wilcox wrote:
> On Mon, Jun 01, 2026 at 01:57:04PM +0800, Chi Zhiling wrote:
>> +			if (folio_batch_count(&fbatch)) {
>> +				for (int i = 0; i < folio_batch_count(&fbatch); i++)
>> +					folio_put(fbatch.folios[i]);
>> +				folio_batch_reinit(&fbatch);
>> +			}
> 
> folios_put().
> 
>> +	for (int i = 0; i < folio_batch_count(&fbatch); i++)
>> +		folio_put(fbatch.folios[i]);
> 
> Also folios_put().

Thanks, I'll replace it in v3.

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Mateusz Guzik]

On Mon, Jun 01, 2026 at 02:22:04PM +0100, Matthew Wilcox wrote:
> On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:
> > Currently, holes larger than PAGE_SIZE cannot be handled because
> > ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
> > wrapper to support copying larger zero ranges to the iterator.
> 
> I think Hugh put this optimisation in the wrong place, and you're
> perpetuating that ;-)
> 
> So perhaps we can start by moving this optimisation to lib/iov_iter.c?
> And then you can redo your optimisation on top of that.
> 
> diff --git a/lib/iov_iter.c b/lib/iov_iter.c
> index 243662af1af7..06c54d719fcd 100644
> --- a/lib/iov_iter.c
> +++ b/lib/iov_iter.c
> @@ -451,7 +451,23 @@ static __always_inline
>  size_t zero_to_user_iter(void __user *iter_to, size_t progress,
>  			 size_t len, void *priv, void *priv2)
>  {
> -	return clear_user(iter_to, len);
> +	/*
> +	 * it is noticeably faster to copy the zero page instead of
> +	 * calling clear_user().  Shame.
> +	 */


This is a rather suspicious claim. If clear_user is indeed so terrible
that it is faster to copy, the routine needs to get unfucked instead of
the problem being worked around.

I can't speak for arm64 or other non-amd64 archs, maybe these are
horrendeously broken.

On amd64 some archeology shows the following:
1. 0db7058e8e23e6bb ("x86/clear_user: Make it faster")

2022 vintage, replaces thoroughly terrible 8-byte per-iteration write
with rep stos usage

2. 8c9b6a88b7e2f33c ("x86: improve on the non-rep 'clear_user' function")

inlines rep stosb at the callsite if the CPU has FSRS, otherwise
fallsback to a new routine which does 64-byte writes per loop iteration.

FSRS is reasonably popular by now and chances are decent the test jig
used by Chi has it.

For a size like 4096 bytes, the 64-byte loop will be slower than rep
movsb and even rep stosq. This needs to be patched and maybe I'll get
around to doing the needful(tm) in few days (it's not hard to write, but
some care with testing is needed).

I could not be bothered to check how the workaround showed up, but it
definitely needs to be removed as opposed to being perpetuated.

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Mateusz Guzik]

On Mon, Jun 01, 2026 at 05:02:01PM +0200, Mateusz Guzik wrote:
> On Mon, Jun 01, 2026 at 02:22:04PM +0100, Matthew Wilcox wrote:
> > On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:
> > > Currently, holes larger than PAGE_SIZE cannot be handled because
> > > ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
> > > wrapper to support copying larger zero ranges to the iterator.
> > 
> > I think Hugh put this optimisation in the wrong place, and you're
> > perpetuating that ;-)
> > 
> > So perhaps we can start by moving this optimisation to lib/iov_iter.c?
> > And then you can redo your optimisation on top of that.
> > 
> > diff --git a/lib/iov_iter.c b/lib/iov_iter.c
> > index 243662af1af7..06c54d719fcd 100644
> > --- a/lib/iov_iter.c
> > +++ b/lib/iov_iter.c
> > @@ -451,7 +451,23 @@ static __always_inline
> >  size_t zero_to_user_iter(void __user *iter_to, size_t progress,
> >  			 size_t len, void *priv, void *priv2)
> >  {
> > -	return clear_user(iter_to, len);
> > +	/*
> > +	 * it is noticeably faster to copy the zero page instead of
> > +	 * calling clear_user().  Shame.
> > +	 */
> 
> 
> This is a rather suspicious claim. If clear_user is indeed so terrible
> that it is faster to copy, the routine needs to get unfucked instead of
> the problem being worked around.
> 
> I can't speak for arm64 or other non-amd64 archs, maybe these are
> horrendeously broken.
> 
> On amd64 some archeology shows the following:
> 1. 0db7058e8e23e6bb ("x86/clear_user: Make it faster")
> 
> 2022 vintage, replaces thoroughly terrible 8-byte per-iteration write
> with rep stos usage
> 
> 2. 8c9b6a88b7e2f33c ("x86: improve on the non-rep 'clear_user' function")
> 
> inlines rep stosb at the callsite if the CPU has FSRS, otherwise
> fallsback to a new routine which does 64-byte writes per loop iteration.
> 
> FSRS is reasonably popular by now and chances are decent the test jig
> used by Chi has it.
> 
> For a size like 4096 bytes, the 64-byte loop will be slower than rep
> movsb and even rep stosq. This needs to be patched and maybe I'll get
> around to doing the needful(tm) in few days (it's not hard to write, but
> some care with testing is needed).
> 
> I could not be bothered to check how the workaround showed up, but it
> definitely needs to be removed as opposed to being perpetuated.

Maybe I was not clear enough, so here it is stated differently:

The routine was incredibly bad on amd64 prior to 2022, I presume the
claim of bad performance predates the fixup. The current implementation
for FSRS_enabled CPUs is fine and I guarantee it wont be slower than
copying. For CPUs without FSRS the fallback is slower than it needs to
be for sizes like 4096 bytes, that I'm going to fix soon(tm).

No matter what, there is no justification for issuing a copy from zero
page just to avoid zeroing in place.

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Matthew Wilcox]

On Mon, Jun 01, 2026 at 05:02:01PM +0200, Mateusz Guzik wrote:
> On Mon, Jun 01, 2026 at 02:22:04PM +0100, Matthew Wilcox wrote:
> > On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:
> > > Currently, holes larger than PAGE_SIZE cannot be handled because
> > > ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
> > > wrapper to support copying larger zero ranges to the iterator.
> > 
> > I think Hugh put this optimisation in the wrong place, and you're
> > perpetuating that ;-)
> > 
> > So perhaps we can start by moving this optimisation to lib/iov_iter.c?
> > And then you can redo your optimisation on top of that.
> 
> This is a rather suspicious claim. If clear_user is indeed so terrible
> that it is faster to copy, the routine needs to get unfucked instead of
> the problem being worked around.

Oh, I agree.  Putting it in lib/iov_iter.c means more people will see it
than if it's hidden in shmem.c.

> I can't speak for arm64 or other non-amd64 archs, maybe these are
> horrendeously broken.
> 
> On amd64 some archeology shows the following:
> 1. 0db7058e8e23e6bb ("x86/clear_user: Make it faster")
> 
> 2022 vintage, replaces thoroughly terrible 8-byte per-iteration write
> with rep stos usage

That's a good candidate for fixing this problem.  56a8c8eb1eaf is from
March 2022 and mentions the slowness of clear_user() on x86.  So a
commit from May 2022 might have fixed the problem without anyone going
back to measure and remove the workaround.

> 2. 8c9b6a88b7e2f33c ("x86: improve on the non-rep 'clear_user' function")
> 
> inlines rep stosb at the callsite if the CPU has FSRS, otherwise
> fallsback to a new routine which does 64-byte writes per loop iteration.
> 
> FSRS is reasonably popular by now and chances are decent the test jig
> used by Chi has it.
> 
> For a size like 4096 bytes, the 64-byte loop will be slower than rep
> movsb and even rep stosq. This needs to be patched and maybe I'll get
> around to doing the needful(tm) in few days (it's not hard to write, but
> some care with testing is needed).
> 
> I could not be bothered to check how the workaround showed up, but it
> definitely needs to be removed as opposed to being perpetuated.

I'd be delighted if somebody tested just this patch.  I'm not really set
up for performance testing here.

diff --git a/mm/shmem.c b/mm/shmem.c
index 3b5dc21b323c..112cae9f9e4f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3427,19 +3427,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 			else
 				ret = copy_page_to_iter(page, offset, nr, to);
 			folio_put(folio);
-		} else if (user_backed_iter(to)) {
-			/*
-			 * Copy to user tends to be so well optimized, but
-			 * clear_user() not so much, that it is noticeably
-			 * faster to copy the zero page instead of clearing.
-			 */
-			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
 		} else {
-			/*
-			 * But submitting the same page twice in a row to
-			 * splice() - or others? - can result in confusion:
-			 * so don't attempt that optimization on pipes etc.
-			 */
 			ret = iov_iter_zero(nr, to);
 		}
 

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[Mateusz Guzik]

On Mon, Jun 1, 2026 at 5:14 PM Matthew Wilcox <willy@infradead.org> wrote:
>
> On Mon, Jun 01, 2026 at 05:02:01PM +0200, Mateusz Guzik wrote:
> > On Mon, Jun 01, 2026 at 02:22:04PM +0100, Matthew Wilcox wrote:
> > On amd64 some archeology shows the following:
> > 1. 0db7058e8e23e6bb ("x86/clear_user: Make it faster")
> >
> > 2022 vintage, replaces thoroughly terrible 8-byte per-iteration write
> > with rep stos usage
>
> That's a good candidate for fixing this problem.  56a8c8eb1eaf is from
> March 2022 and mentions the slowness of clear_user() on x86.  So a
> commit from May 2022 might have fixed the problem without anyone going
> back to measure and remove the workaround.
>

Ok in that case that's basically settled I think.

> I'd be delighted if somebody tested just this patch.  I'm not really set
> up for performance testing here.
>
> diff --git a/mm/shmem.c b/mm/shmem.c
> index 3b5dc21b323c..112cae9f9e4f 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -3427,19 +3427,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>                         else
>                                 ret = copy_page_to_iter(page, offset, nr, to);
>                         folio_put(folio);
> -               } else if (user_backed_iter(to)) {
> -                       /*
> -                        * Copy to user tends to be so well optimized, but
> -                        * clear_user() not so much, that it is noticeably
> -                        * faster to copy the zero page instead of clearing.
> -                        */
> -                       ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
>                 } else {
> -                       /*
> -                        * But submitting the same page twice in a row to
> -                        * splice() - or others? - can result in confusion:
> -                        * so don't attempt that optimization on pipes etc.
> -                        */
>                         ret = iov_iter_zero(nr, to);
>                 }
>

I'm not in position to test myself at the moment. But should someone
be interested and try this on amd64, either make sure you have FSRS or
in the worst case apply this hack which forces rep stosb:

diff --git a/arch/x86/include/asm/uaccess_64.h
b/arch/x86/include/asm/uaccess_64.h
index 20de34cc9aa6..552d1b883579 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -189,8 +189,7 @@ static __always_inline __must_check unsigned long
__clear_user(void __user *addr
         */
        asm volatile(
                "1:\n\t"
-               ALTERNATIVE("rep stosb",
-                           "call rep_stos_alternative",
ALT_NOT(X86_FEATURE_FSRS))
+               "rep stosb\n"
                "2:\n"
               _ASM_EXTABLE_UA(1b, 2b)
               : "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT

It is always fine to use it from correctness pov. It would cause a
slowdown for any size on old yellers predating Sandy Bridge (that's 14
years now I think?) or for small sizes on CPUs without FSRS. Given
something like 4K I/O that's not a concern.

Re: [PATCH v2 3/5] mm/shmem: introduce copy_zero_to_iter() for large zeroing

[David Laight]

On Mon, 1 Jun 2026 17:02:01 +0200
Mateusz Guzik <mjguzik@gmail.com> wrote:

> On Mon, Jun 01, 2026 at 02:22:04PM +0100, Matthew Wilcox wrote:
> > On Mon, Jun 01, 2026 at 01:57:02PM +0800, Chi Zhiling wrote:  
> > > Currently, holes larger than PAGE_SIZE cannot be handled because
> > > ZERO_PAGE is limited to a single page. Add copy_zero_to_iter() as a
> > > wrapper to support copying larger zero ranges to the iterator.  
> > 
> > I think Hugh put this optimisation in the wrong place, and you're
> > perpetuating that ;-)
> > 
> > So perhaps we can start by moving this optimisation to lib/iov_iter.c?
> > And then you can redo your optimisation on top of that.
> > 
> > diff --git a/lib/iov_iter.c b/lib/iov_iter.c
> > index 243662af1af7..06c54d719fcd 100644
> > --- a/lib/iov_iter.c
> > +++ b/lib/iov_iter.c
> > @@ -451,7 +451,23 @@ static __always_inline
> >  size_t zero_to_user_iter(void __user *iter_to, size_t progress,
> >  			 size_t len, void *priv, void *priv2)
> >  {
> > -	return clear_user(iter_to, len);
> > +	/*
> > +	 * it is noticeably faster to copy the zero page instead of
> > +	 * calling clear_user().  Shame.
> > +	 */  
> 
> 
> This is a rather suspicious claim. If clear_user is indeed so terrible
> that it is faster to copy, the routine needs to get unfucked instead of
> the problem being worked around.
> 
> I can't speak for arm64 or other non-amd64 archs, maybe these are
> horrendeously broken.
> 
> On amd64 some archeology shows the following:
> 1. 0db7058e8e23e6bb ("x86/clear_user: Make it faster")
> 
> 2022 vintage, replaces thoroughly terrible 8-byte per-iteration write
> with rep stos usage
> 
> 2. 8c9b6a88b7e2f33c ("x86: improve on the non-rep 'clear_user' function")
> 
> inlines rep stosb at the callsite if the CPU has FSRS, otherwise
> fallsback to a new routine which does 64-byte writes per loop iteration.
> 
> FSRS is reasonably popular by now and chances are decent the test jig
> used by Chi has it.
> 
> For a size like 4096 bytes, the 64-byte loop will be slower than rep
> movsb and even rep stosq. This needs to be patched and maybe I'll get
> around to doing the needful(tm) in few days (it's not hard to write, but
> some care with testing is needed).

I think Intel cpu from Sandy bridge onwards execute 'rep stosb' just
as fast as 'rep stosq'.
(I'm sure I've done the measurements for 'rep movs' and stos ought to
be similar.)
I suspect you get the same big gain (twice as fast) from an aligned
destination (IIRC 64 bytes on later cpu).
(But I doubt it is worth the cost of aligning the destination.)
The source alignment (for rep movs) make no difference at all.

The 'elephant in the room' is older zen cpu.
Some of those are no where near as fast as you might expect.
(Look at the issues using 'rep movsb' for all copies.)

I can test a range of old Intel cpu, but not amd ones.

-- David

> 
> I could not be bothered to check how the workaround showed up, but it
> definitely needs to be removed as opposed to being perpetuated.
> 

Re: [PATCH v2 0/5] mm/shmem: optimize read with reduced xarray lookups and folio batching

[Andrew Morton]

On Mon,  1 Jun 2026 13:56:59 +0800 Chi Zhiling <chizhiling@163.com> wrote:

> From: Chi Zhiling <chizhiling@kylinos.cn>
> 
> This series improves shmem read performance by implementing folio
> batching in the read path and reducing unnecessary xarray lookups.
> 
> Performance Results:
> 
> fio --ioengine=sync --rw=read --bs=$1 --size=1G --runtime=180 --time_based --group_reporting --name=seq_read_test --filename=testfile
> 
> | THP disabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
> | ---------------------- | ------------ | ----------------- | ----------- |
> | 1M + normal file       | bw=11.5GiB/s | bw=12.7GiB/s      | +10.4%      |
> | 64k + normal file      | bw=11.0GiB/s | bw=12.3GiB/s      | +11.8%      |
> | 4k + normal file       | bw=3826MiB/s | bw=3849MiB/s      | +0.6%       |
> | 1M + fallocated file   | bw=23.8GiB/s | bw=28.6GiB/s      | +20.2%      |
> | 64k + fallocated file  | bw=22.5GiB/s | bw=27.3GiB/s      | +21.3%      |
> | 4k + fallocated file   | bw=4655MiB/s | bw=4680MiB/s      | +0.5%       |
> | 1M + hole              | bw=24.2GiB/s | bw=28.6GiB/s      | +18.2%      |
> | 64k + hole             | bw=22.6GiB/s | bw=27.6GiB/s      | +22.1%      |
> | 4k + hole              | bw=4652MiB/s | bw=4489MiB/s      | -3.5%       |
> 
> 
> | THP enabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
> | --------------------- | ------------ | ----------------- | ----------- |
> | 1M + normal file      | bw=13.7GiB/s | bw=13.9GiB/s      | +1.4%       |
> | 64k + normal file     | bw=13.5GiB/s | bw=13.5GiB/s      | +0.0%       |
> | 4k + normal file      | bw=3833MiB/s | bw=3859MiB/s      | +0.7%       |
> | 1M + fallocated file  | bw=24.9GiB/s | bw=34.2GiB/s      | +37.3%      |
> | 64k + fallocated file | bw=23.0GiB/s | bw=31.4GiB/s      | +36.5%      |
> | 4k + fallocated file  | bw=4710MiB/s | bw=4655MiB/s      | -1.2%       |
> | 1M + hole             | bw=24.3GiB/s | bw=34.5GiB/s      | +42.0%      |
> | 64k + hole            | bw=23.5GiB/s | bw=31.1GiB/s      | +32.3%      |
> | 4k + hole             | bw=4690MiB/s | bw=4647MiB/s      | -0.9%       |
> 

That looks nice.

Microbenchmarks are useful, but are you able to help us understand how
much benefit our users might see in real-world workloads?

I'll take no action at this time - it's late in the cycle and reviewers
have yet to participate.

AI review flagged a few possible issues, so please take a look:
	https://sashiko.dev/#/patchset/20260601055704.167436-1-chizhiling@163.com

Re: [PATCH v2 0/5] mm/shmem: optimize read with reduced xarray lookups and folio batching

[Chi Zhiling]

On 6/2/26 08:43, Andrew Morton wrote:
> On Mon,  1 Jun 2026 13:56:59 +0800 Chi Zhiling <chizhiling@163.com> wrote:
> 
>> From: Chi Zhiling <chizhiling@kylinos.cn>
>>
>> This series improves shmem read performance by implementing folio
>> batching in the read path and reducing unnecessary xarray lookups.
>>
>> Performance Results:
>>
>> fio --ioengine=sync --rw=read --bs=$1 --size=1G --runtime=180 --time_based --group_reporting --name=seq_read_test --filename=testfile
>>
>> | THP disabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
>> | ---------------------- | ------------ | ----------------- | ----------- |
>> | 1M + normal file       | bw=11.5GiB/s | bw=12.7GiB/s      | +10.4%      |
>> | 64k + normal file      | bw=11.0GiB/s | bw=12.3GiB/s      | +11.8%      |
>> | 4k + normal file       | bw=3826MiB/s | bw=3849MiB/s      | +0.6%       |
>> | 1M + fallocated file   | bw=23.8GiB/s | bw=28.6GiB/s      | +20.2%      |
>> | 64k + fallocated file  | bw=22.5GiB/s | bw=27.3GiB/s      | +21.3%      |
>> | 4k + fallocated file   | bw=4655MiB/s | bw=4680MiB/s      | +0.5%       |
>> | 1M + hole              | bw=24.2GiB/s | bw=28.6GiB/s      | +18.2%      |
>> | 64k + hole             | bw=22.6GiB/s | bw=27.6GiB/s      | +22.1%      |
>> | 4k + hole              | bw=4652MiB/s | bw=4489MiB/s      | -3.5%       |
>>
>>
>> | THP enabled in tmpfs  | v7.1-rc5     | v7.1-rc5 + fbatch | Improvement |
>> | --------------------- | ------------ | ----------------- | ----------- |
>> | 1M + normal file      | bw=13.7GiB/s | bw=13.9GiB/s      | +1.4%       |
>> | 64k + normal file     | bw=13.5GiB/s | bw=13.5GiB/s      | +0.0%       |
>> | 4k + normal file      | bw=3833MiB/s | bw=3859MiB/s      | +0.7%       |
>> | 1M + fallocated file  | bw=24.9GiB/s | bw=34.2GiB/s      | +37.3%      |
>> | 64k + fallocated file | bw=23.0GiB/s | bw=31.4GiB/s      | +36.5%      |
>> | 4k + fallocated file  | bw=4710MiB/s | bw=4655MiB/s      | -1.2%       |
>> | 1M + hole             | bw=24.3GiB/s | bw=34.5GiB/s      | +42.0%      |
>> | 64k + hole            | bw=23.5GiB/s | bw=31.1GiB/s      | +32.3%      |
>> | 4k + hole             | bw=4690MiB/s | bw=4647MiB/s      | -0.9%       |
>>
> 
> That looks nice.
> 
> Microbenchmarks are useful, but are you able to help us understand how
> much benefit our users might see in real-world workloads?

Hi, Andrew

I don't have real-world performance data yet. I'm working on this simply 
because the patch shows decent gains in microbenchmarks. Even with THP 
enabled, it can still reduce some unnecessary overhead.

> 
> I'll take no action at this time - it's late in the cycle and reviewers
> have yet to participate.

Yes, it's unlikely to land in 7.2, and I still need to resolve some 
performance regressions.

> 
> AI review flagged a few possible issues, so please take a look:
> 	https://sashiko.dev/#/patchset/20260601055704.167436-1-chizhiling@163.com

Okay, I will take a close look.


Thanks!

Re: [PATCH v2 4/5] mm/shmem: remove page-copy fallback in shmem read path

[Baolin Wang]

On 6/1/26 1:57 PM, Chi Zhiling wrote:
> From: Chi Zhiling <chizhiling@kylinos.cn>
> 
> This is perp patch for folio batch support which removing the fallback
> to page-copy mode from shmem reads.
> 
> The existing hwpoison fallback would require fetching the same folio
> multiple times to read its each pages, which complicates the
> batching model. Instead, move hwpoison handling into
> copy_each_pages_to_iter(), allowing the error path to be centralized.
> 
> This simplifies the shmem fast read path and avoids special-case
> handling that conflicts with folio batching support.
> 
> Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
> ---
>   mm/shmem.c | 63 +++++++++++++++++++++++++++++-------------------------
>   1 file changed, 34 insertions(+), 29 deletions(-)
> 
> diff --git a/mm/shmem.c b/mm/shmem.c
> index b08118fbd275..cac355685e49 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -3353,6 +3353,30 @@ static size_t copy_zero_to_iter(size_t bytes, struct iov_iter *i)
>   	return written;
>   }
>   
> +static size_t copy_pages_to_iter(struct folio *folio, size_t offset,
> +		size_t bytes, struct iov_iter *i, int *error)
> +{
> +	struct page *page;
> +	unsigned long off, nr, ret, written = 0;
> +
> +	do {
> +		page = folio_page(folio, offset >> PAGE_SHIFT);
> +		if (PageHWPoison(page)) {
> +			*error = -EIO;
> +			break;
> +		}
> +
> +		off = offset_in_page(offset);
> +		nr = min(PAGE_SIZE - off, bytes - written);
> +
> +		ret = copy_page_to_iter(page, off, nr, i);
> +		offset += ret;
> +		written += ret;
> +	} while (written < bytes && ret == nr);
> +
> +	return written;
> +}
> +
>   static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   {
>   	struct file *file = iocb->ki_filp;
> @@ -3365,10 +3389,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   
>   	for (;;) {
>   		struct folio *folio = NULL;
> -		struct page *page = NULL;
>   		unsigned long nr, ret;
>   		loff_t end_offset, i_size = i_size_read(inode);
> -		bool fallback_page_copy = false;
>   		size_t fsize;
>   
>   		if (unlikely(iocb->ki_pos >= i_size))
> @@ -3376,25 +3398,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   
>   		index = iocb->ki_pos >> PAGE_SHIFT;
>   		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
> +		if (folio)
> +			folio_unlock(folio);
>   		if (error) {
>   			if (error == -EINVAL)
>   				error = 0;
>   			break;
>   		}
> -		if (folio) {
> -			folio_unlock(folio);
> -
> -			page = folio_file_page(folio, index);
> -			if (PageHWPoison(page)) {
> -				folio_put(folio);
> -				error = -EIO;
> -				break;
> -			}
> -
> -			if (folio_test_large(folio) &&
> -			    folio_test_has_hwpoisoned(folio))
> -				fallback_page_copy = true;
> -		}
>   
>   		/*
>   		 * We must evaluate after, since reads (unlike writes)
> @@ -3406,12 +3416,9 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   				folio_put(folio);
>   			break;
>   		}
> -		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
> -		if (folio && likely(!fallback_page_copy))
> -			fsize = folio_size(folio);
> -		else
> -			fsize = PAGE_SIZE;
> +		fsize = folio ? folio_size(folio) : PAGE_SIZE;
>   		offset = iocb->ki_pos & (fsize - 1);
> +		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);

Do not add this unnecessary change.

>   		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
>   
>   		if (folio) {
> @@ -3420,12 +3427,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   			 * virtual addresses, take care about potential aliasing
>   			 * before reading the page on the kernel side.
>   			 */
> -			if (mapping_writably_mapped(mapping)) {
> -				if (likely(!fallback_page_copy))
> -					flush_dcache_folio(folio);
> -				else
> -					flush_dcache_page(page);
> -			}
> +			if (mapping_writably_mapped(mapping))
> +				flush_dcache_folio(folio);

I'm not sure this is safe for all architectures (it's safe for arm64 
since no actual data cache flush happens here), but for architectures 
that do perform an actual data cache flush, if the page has already been 
hwpoisoned, flushing the cache could trigger a RAS error?

I think you should move this into copy_pages_to_iter() for hwpoisoned 
folios.

Re: [PATCH v2 5/5] mm/shmem: optimize file read with folio batching

[Baolin Wang]

On 6/1/26 1:57 PM, Chi Zhiling wrote:
> From: Chi Zhiling <chizhiling@kylinos.cn>
> 
> Optimize shmem file read by using filemap_get_folios_contig() to batch
> fetch contiguous folios from the page cache, reducing the overhead of
> repeated shmem_get_folio() calls.
> 
> This patch checks the uptodate flag without holding the folio lock, so
> it may observe a non-uptodate state on a locked folio that is still
> being initialized. This is safe because only zero-filled data can be
> copied to the user buffer in that scenario.
> 
> A non-uptodate folio in the swap cache cannot be added to the shmem page
> cache. This creates a semantic conflict, as shmem zeroes the folio out,
> but the swap cache would fill it by reading from the swap backing store.
> 
> Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
> ---
>   mm/shmem.c | 57 ++++++++++++++++++++++++++++++++++++++++--------------
>   1 file changed, 42 insertions(+), 15 deletions(-)
> 
> diff --git a/mm/shmem.c b/mm/shmem.c
> index cac355685e49..61937582f08c 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -891,6 +891,14 @@ int shmem_add_to_page_cache(struct folio *folio,
>   	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
>   	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
>   
> +	/*
> +	 * Don't add a non-uptodate folio that is in swap cache to page
> +	 * cache, since shmem will zero it instead of reading from swap
> +	 * backing.
> +	 */
> +	VM_BUG_ON_FOLIO(folio_test_swapcache(folio) &&
> +			!folio_test_uptodate(folio), folio);

It's impossible for a folio to be in both the swap cache and the shmem 
page cache. We can drop this.

>   	folio_ref_add(folio, nr);
>   	folio->mapping = mapping;
>   	folio->index = index;
> @@ -3382,11 +3390,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   	struct file *file = iocb->ki_filp;
>   	struct inode *inode = file_inode(file);
>   	struct address_space *mapping = inode->i_mapping;
> -	pgoff_t index;
> +	struct folio_batch fbatch;
>   	unsigned long offset;
>   	int error = 0;
>   	ssize_t retval = 0;
>   
> +	folio_batch_init(&fbatch);
> +
>   	for (;;) {
>   		struct folio *folio = NULL;
>   		unsigned long nr, ret;
> @@ -3395,15 +3405,33 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   
>   		if (unlikely(iocb->ki_pos >= i_size))
>   			break;
> +fetch:
> +		folio = folio_batch_next(&fbatch);
> +		if (!folio) {
> +			pgoff_t start = iocb->ki_pos >> PAGE_SHIFT;
> +			pgoff_t end = (iocb->ki_pos + to->count - 1) >> PAGE_SHIFT;

You should consider the inode size when calculating the 'end'. You can 
reuse the 'end_offset':

end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);

then pass 'end_offset - 1' to filemap_get_folios_contig().

> +
> +			if (folio_batch_count(&fbatch)) {
> +				for (int i = 0; i < folio_batch_count(&fbatch); i++)
> +					folio_put(fbatch.folios[i]);
> +				folio_batch_reinit(&fbatch);
> +			}
>   
> -		index = iocb->ki_pos >> PAGE_SHIFT;
> -		error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
> -		if (folio)
> -			folio_unlock(folio);
> -		if (error) {
> -			if (error == -EINVAL)
> -				error = 0;
> -			break;
> +			filemap_get_folios_contig(inode->i_mapping, &start, end, &fbatch);
> +			if (folio_batch_count(&fbatch))
> +				goto fetch;
> +
> +			error = shmem_get_folio(inode, start, 0, &folio, SGP_READ);
> +			if (unlikely(error)) {
> +				if (error == -EINVAL)
> +					error = 0;
> +				break;
> +			}
> +			if (folio) {
> +				folio_unlock(folio);
> +				folio_batch_add(&fbatch, folio);
> +				fbatch.i++;
> +			}
>   		}
>   
>   		/*
> @@ -3411,17 +3439,15 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   		 * are called without i_rwsem protection against truncate
>   		 */
>   		i_size = i_size_read(inode);
> -		if (unlikely(iocb->ki_pos >= i_size)) {
> -			if (folio)
> -				folio_put(folio);
> +		if (unlikely(iocb->ki_pos >= i_size))
>   			break;
> -		}
> +
>   		fsize = folio ? folio_size(folio) : PAGE_SIZE;
>   		offset = iocb->ki_pos & (fsize - 1);
>   		end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
>   		nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
>   
> -		if (folio) {
> +		if (folio && folio_test_uptodate(folio)) {
>   			/*
>   			 * If users can be writing to this page using arbitrary
>   			 * virtual addresses, take care about potential aliasing
> @@ -3443,7 +3469,6 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   				ret = copy_folio_to_iter(folio, offset, nr, to);
>   			else
>   				ret = copy_pages_to_iter(folio, offset, nr, to, &error);
> -			folio_put(folio);
>   		} else if (user_backed_iter(to)) {
>   			/*
>   			 * Copy to user tends to be so well optimized, but
> @@ -3474,6 +3499,8 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
>   		cond_resched();
>   	}
>   
> +	for (int i = 0; i < folio_batch_count(&fbatch); i++)
> +		folio_put(fbatch.folios[i]);
>   	file_accessed(file);
>   	return retval ? retval : error;
>   }

Re: [PATCH v2 4/5] mm/shmem: remove page-copy fallback in shmem read path

[Chi Zhiling]

On 6/2/26 13:39, Baolin Wang wrote:
> 
> 
> On 6/1/26 1:57 PM, Chi Zhiling wrote:
>> From: Chi Zhiling <chizhiling@kylinos.cn>
>>
>> This is perp patch for folio batch support which removing the fallback
>> to page-copy mode from shmem reads.
>>
>> The existing hwpoison fallback would require fetching the same folio
>> multiple times to read its each pages, which complicates the
>> batching model. Instead, move hwpoison handling into
>> copy_each_pages_to_iter(), allowing the error path to be centralized.
>>
>> This simplifies the shmem fast read path and avoids special-case
>> handling that conflicts with folio batching support.
>>
>> Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
>> ---
>>   mm/shmem.c | 63 +++++++++++++++++++++++++++++-------------------------
>>   1 file changed, 34 insertions(+), 29 deletions(-)
>>
>> diff --git a/mm/shmem.c b/mm/shmem.c
>> index b08118fbd275..cac355685e49 100644
>> --- a/mm/shmem.c
>> +++ b/mm/shmem.c
>> @@ -3353,6 +3353,30 @@ static size_t copy_zero_to_iter(size_t bytes, 
>> struct iov_iter *i)
>>       return written;
>>   }
>> +static size_t copy_pages_to_iter(struct folio *folio, size_t offset,
>> +        size_t bytes, struct iov_iter *i, int *error)
>> +{
>> +    struct page *page;
>> +    unsigned long off, nr, ret, written = 0;
>> +
>> +    do {
>> +        page = folio_page(folio, offset >> PAGE_SHIFT);
>> +        if (PageHWPoison(page)) {
>> +            *error = -EIO;
>> +            break;
>> +        }
>> +
>> +        off = offset_in_page(offset);
>> +        nr = min(PAGE_SIZE - off, bytes - written);
>> +
>> +        ret = copy_page_to_iter(page, off, nr, i);
>> +        offset += ret;
>> +        written += ret;
>> +    } while (written < bytes && ret == nr);
>> +
>> +    return written;
>> +}
>> +
>>   static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct 
>> iov_iter *to)
>>   {
>>       struct file *file = iocb->ki_filp;
>> @@ -3365,10 +3389,8 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>       for (;;) {
>>           struct folio *folio = NULL;
>> -        struct page *page = NULL;
>>           unsigned long nr, ret;
>>           loff_t end_offset, i_size = i_size_read(inode);
>> -        bool fallback_page_copy = false;
>>           size_t fsize;
>>           if (unlikely(iocb->ki_pos >= i_size))
>> @@ -3376,25 +3398,13 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>           index = iocb->ki_pos >> PAGE_SHIFT;
>>           error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
>> +        if (folio)
>> +            folio_unlock(folio);
>>           if (error) {
>>               if (error == -EINVAL)
>>                   error = 0;
>>               break;
>>           }
>> -        if (folio) {
>> -            folio_unlock(folio);
>> -
>> -            page = folio_file_page(folio, index);
>> -            if (PageHWPoison(page)) {
>> -                folio_put(folio);
>> -                error = -EIO;
>> -                break;
>> -            }
>> -
>> -            if (folio_test_large(folio) &&
>> -                folio_test_has_hwpoisoned(folio))
>> -                fallback_page_copy = true;
>> -        }
>>           /*
>>            * We must evaluate after, since reads (unlike writes)
>> @@ -3406,12 +3416,9 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>                   folio_put(folio);
>>               break;
>>           }
>> -        end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
>> -        if (folio && likely(!fallback_page_copy))
>> -            fsize = folio_size(folio);
>> -        else
>> -            fsize = PAGE_SIZE;
>> +        fsize = folio ? folio_size(folio) : PAGE_SIZE;
>>           offset = iocb->ki_pos & (fsize - 1);
>> +        end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
> 
> Do not add this unnecessary change.

Okay,

> 
>>           nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
>>           if (folio) {
>> @@ -3420,12 +3427,8 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>                * virtual addresses, take care about potential aliasing
>>                * before reading the page on the kernel side.
>>                */
>> -            if (mapping_writably_mapped(mapping)) {
>> -                if (likely(!fallback_page_copy))
>> -                    flush_dcache_folio(folio);
>> -                else
>> -                    flush_dcache_page(page);
>> -            }
>> +            if (mapping_writably_mapped(mapping))
>> +                flush_dcache_folio(folio);
> 
> I'm not sure this is safe for all architectures (it's safe for arm64 
> since no actual data cache flush happens here), but for architectures 
> that do perform an actual data cache flush, if the page has already been 
> hwpoisoned, flushing the cache could trigger a RAS error?
> 
> I think you should move this into copy_pages_to_iter() for hwpoisoned 
> folios.

Okay, will fix it, thanks!

Re: [PATCH v2 5/5] mm/shmem: optimize file read with folio batching

[Chi Zhiling]

On 6/2/26 13:54, Baolin Wang wrote:
> On 6/1/26 1:57 PM, Chi Zhiling wrote:
>> From: Chi Zhiling <chizhiling@kylinos.cn>
>>
>> Optimize shmem file read by using filemap_get_folios_contig() to batch
>> fetch contiguous folios from the page cache, reducing the overhead of
>> repeated shmem_get_folio() calls.
>>
>> This patch checks the uptodate flag without holding the folio lock, so
>> it may observe a non-uptodate state on a locked folio that is still
>> being initialized. This is safe because only zero-filled data can be
>> copied to the user buffer in that scenario.
>>
>> A non-uptodate folio in the swap cache cannot be added to the shmem page
>> cache. This creates a semantic conflict, as shmem zeroes the folio out,
>> but the swap cache would fill it by reading from the swap backing store.
>>
>> Signed-off-by: Chi Zhiling <chizhiling@kylinos.cn>
>> ---
>>   mm/shmem.c | 57 ++++++++++++++++++++++++++++++++++++++++--------------
>>   1 file changed, 42 insertions(+), 15 deletions(-)
>>
>> diff --git a/mm/shmem.c b/mm/shmem.c
>> index cac355685e49..61937582f08c 100644
>> --- a/mm/shmem.c
>> +++ b/mm/shmem.c
>> @@ -891,6 +891,14 @@ int shmem_add_to_page_cache(struct folio *folio,
>>       VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
>>       VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
>> +    /*
>> +     * Don't add a non-uptodate folio that is in swap cache to page
>> +     * cache, since shmem will zero it instead of reading from swap
>> +     * backing.
>> +     */
>> +    VM_BUG_ON_FOLIO(folio_test_swapcache(folio) &&
>> +            !folio_test_uptodate(folio), folio);
> 
> It's impossible for a folio to be in both the swap cache and the shmem 
> page cache. We can drop this.

Okay, I just want to set a reminder for others.

> 
>>       folio_ref_add(folio, nr);
>>       folio->mapping = mapping;
>>       folio->index = index;
>> @@ -3382,11 +3390,13 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>       struct file *file = iocb->ki_filp;
>>       struct inode *inode = file_inode(file);
>>       struct address_space *mapping = inode->i_mapping;
>> -    pgoff_t index;
>> +    struct folio_batch fbatch;
>>       unsigned long offset;
>>       int error = 0;
>>       ssize_t retval = 0;
>> +    folio_batch_init(&fbatch);
>> +
>>       for (;;) {
>>           struct folio *folio = NULL;
>>           unsigned long nr, ret;
>> @@ -3395,15 +3405,33 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>           if (unlikely(iocb->ki_pos >= i_size))
>>               break;
>> +fetch:
>> +        folio = folio_batch_next(&fbatch);
>> +        if (!folio) {
>> +            pgoff_t start = iocb->ki_pos >> PAGE_SHIFT;
>> +            pgoff_t end = (iocb->ki_pos + to->count - 1) >> PAGE_SHIFT;
> 
> You should consider the inode size when calculating the 'end'. You can 
> reuse the 'end_offset':
> 
> end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
> 
> then pass 'end_offset - 1' to filemap_get_folios_contig().

Okay, I will.

And I'm considering whether to reintroduce shmem_get_read_batch as a 
replacement for filemap_get_folios_contig.

The main reason is to be able to quickly skip over holes, so that we 
don't fall back when encountering holes during reads, thereby addressing 
the performance regression when reading holes.

> 
>> +
>> +            if (folio_batch_count(&fbatch)) {
>> +                for (int i = 0; i < folio_batch_count(&fbatch); i++)
>> +                    folio_put(fbatch.folios[i]);
>> +                folio_batch_reinit(&fbatch);
>> +            }
>> -        index = iocb->ki_pos >> PAGE_SHIFT;
>> -        error = shmem_get_folio(inode, index, 0, &folio, SGP_READ);
>> -        if (folio)
>> -            folio_unlock(folio);
>> -        if (error) {
>> -            if (error == -EINVAL)
>> -                error = 0;
>> -            break;
>> +            filemap_get_folios_contig(inode->i_mapping, &start, end, 
>> &fbatch);
>> +            if (folio_batch_count(&fbatch))
>> +                goto fetch;
>> +
>> +            error = shmem_get_folio(inode, start, 0, &folio, SGP_READ);
>> +            if (unlikely(error)) {
>> +                if (error == -EINVAL)
>> +                    error = 0;
>> +                break;
>> +            }
>> +            if (folio) {
>> +                folio_unlock(folio);
>> +                folio_batch_add(&fbatch, folio);
>> +                fbatch.i++;
>> +            }
>>           }
>>           /*
>> @@ -3411,17 +3439,15 @@ static ssize_t shmem_file_read_iter(struct 
>> kiocb *iocb, struct iov_iter *to)
>>            * are called without i_rwsem protection against truncate
>>            */
>>           i_size = i_size_read(inode);
>> -        if (unlikely(iocb->ki_pos >= i_size)) {
>> -            if (folio)
>> -                folio_put(folio);
>> +        if (unlikely(iocb->ki_pos >= i_size))
>>               break;
>> -        }
>> +
>>           fsize = folio ? folio_size(folio) : PAGE_SIZE;
>>           offset = iocb->ki_pos & (fsize - 1);
>>           end_offset = min_t(loff_t, i_size, iocb->ki_pos + to->count);
>>           nr = min_t(loff_t, end_offset - iocb->ki_pos, fsize - offset);
>> -        if (folio) {
>> +        if (folio && folio_test_uptodate(folio)) {
>>               /*
>>                * If users can be writing to this page using arbitrary
>>                * virtual addresses, take care about potential aliasing
>> @@ -3443,7 +3469,6 @@ static ssize_t shmem_file_read_iter(struct kiocb 
>> *iocb, struct iov_iter *to)
>>                   ret = copy_folio_to_iter(folio, offset, nr, to);
>>               else
>>                   ret = copy_pages_to_iter(folio, offset, nr, to, 
>> &error);
>> -            folio_put(folio);
>>           } else if (user_backed_iter(to)) {
>>               /*
>>                * Copy to user tends to be so well optimized, but
>> @@ -3474,6 +3499,8 @@ static ssize_t shmem_file_read_iter(struct kiocb 
>> *iocb, struct iov_iter *to)
>>           cond_resched();
>>       }
>> +    for (int i = 0; i < folio_batch_count(&fbatch); i++)
>> +        folio_put(fbatch.folios[i]);
>>       file_accessed(file);
>>       return retval ? retval : error;
>>   }