[PATCH v2 0/2] drivers/char/mem: splice the zero page for /dev/zero and /dev/full

[PATCH v2 0/2] drivers/char/mem: splice the zero page for /dev/zero and /dev/full

[Kris Pan]

Commit 1b057bd800c3 ("drivers/char/mem: implement splice() for
/dev/zero, /dev/full") added splice support to /dev/zero and /dev/full
using copy_splice_read(). The author noted that this could be optimized
by pushing references to the global zero page instead of allocating and
zeroing fresh pages for every pipe buffer, deferring it as "an
optimization for another day".

This series implements that optimization by splicing the zero page into
the pipe for /dev/zero and /dev/full, reusing the existing zero-page
splice logic from mm/shmem.c.

v2:
- Move zero_pipe_buf_ops and splice_zeropage_into_pipe() from mm/shmem.c
  into fs/splice.c so they can be shared, instead of duplicating the
  boilerplate (Max Kellermann's suggestion).
- Rework splice_read_zero() to use the shared helper.

Kris Pan (2):
  fs: move splice_zeropage_into_pipe() and zero_pipe_buf_ops to
    fs/splice.c
  drivers/char/mem: splice the zero page for /dev/zero and /dev/full

 drivers/char/mem.c     | 29 ++++++++++++++++++--
 fs/splice.c            | 62 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/splice.h |  3 ++
 mm/shmem.c             | 45 ------------------------------
 4 files changed, 92 insertions(+), 47 deletions(-)

-- 
2.43.0

[PATCH v2 1/2] fs: move splice_zeropage_into_pipe() and zero_pipe_buf_ops to fs/splice.c

[Kris Pan]

The zero-page splice logic in mm/shmem.c (zero_pipe_buf_ops and
splice_zeropage_into_pipe()) is generally useful for any file that wants
to splice zero pages into a pipe, such as /dev/zero and /dev/full.

Move them to fs/splice.c next to the other pipe buffer operations and
declare them in include/linux/splice.h so they can be shared by other
callers without duplicating the boilerplate.

No functional change.

Signed-off-by: Kris Pan <kris.pan@intel.com>
---
 fs/splice.c            | 62 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/splice.h |  3 ++
 mm/shmem.c             | 45 ------------------------------
 3 files changed, 65 insertions(+), 45 deletions(-)

diff --git a/fs/splice.c b/fs/splice.c
index 9d8f63e2fd1ab..b6ea98cc62fd5 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -398,6 +398,68 @@ ssize_t copy_splice_read(struct file *in, loff_t *ppos,
 }
 EXPORT_SYMBOL(copy_splice_read);
 
+/*
+ * Pipe buffer operations for splicing zero pages into a pipe, used to
+ * optimize reads from sparse holes in files and from zero devices such as
+ * /dev/zero and /dev/full.
+ */
+static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
+			      struct pipe_buffer *buf)
+{
+	return true;
+}
+
+static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
+				  struct pipe_buffer *buf)
+{
+}
+
+static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+				    struct pipe_buffer *buf)
+{
+	return false;
+}
+
+const struct pipe_buf_operations zero_pipe_buf_ops = {
+	.release	= zero_pipe_buf_release,
+	.try_steal	= zero_pipe_buf_try_steal,
+	.get		= zero_pipe_buf_get,
+};
+
+/**
+ * splice_zeropage_into_pipe - splice a zero page into a pipe
+ * @pipe: pipe to splice into
+ * @fpos: file position
+ * @size: number of bytes to splice
+ *
+ * Splice at most one page of zeroes into the pipe.  The zero page is not
+ * refcounted, so @zero_pipe_buf_ops must be used to avoid taking a
+ * reference on it.
+ *
+ * Return: the number of bytes spliced.
+ */
+size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
+				 loff_t fpos, size_t size)
+{
+	size_t offset = fpos & ~PAGE_MASK;
+
+	size = min_t(size_t, size, PAGE_SIZE - offset);
+
+	if (!pipe_is_full(pipe)) {
+		struct pipe_buffer *buf = pipe_head_buf(pipe);
+
+		*buf = (struct pipe_buffer) {
+			.ops	= &zero_pipe_buf_ops,
+			.page	= ZERO_PAGE(0),
+			.offset	= offset,
+			.len	= size,
+		};
+		pipe->head++;
+	}
+
+	return size;
+}
+
 const struct pipe_buf_operations default_pipe_buf_ops = {
 	.release	= generic_pipe_buf_release,
 	.try_steal	= generic_pipe_buf_try_steal,
diff --git a/include/linux/splice.h b/include/linux/splice.h
index 9dec4861d09f6..f1d4aefa4ec2c 100644
--- a/include/linux/splice.h
+++ b/include/linux/splice.h
@@ -108,4 +108,7 @@ extern void splice_shrink_spd(struct splice_pipe_desc *);
 
 extern const struct pipe_buf_operations page_cache_pipe_buf_ops;
 extern const struct pipe_buf_operations default_pipe_buf_ops;
+extern const struct pipe_buf_operations zero_pipe_buf_ops;
+size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe, loff_t fpos,
+				 size_t size);
 #endif
diff --git a/mm/shmem.c b/mm/shmem.c
index 9001aaf3b7b94..39d8dfefbe9f5 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3427,51 +3427,6 @@ static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return ret;
 }
 
-static bool zero_pipe_buf_get(struct pipe_inode_info *pipe,
-			      struct pipe_buffer *buf)
-{
-	return true;
-}
-
-static void zero_pipe_buf_release(struct pipe_inode_info *pipe,
-				  struct pipe_buffer *buf)
-{
-}
-
-static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe,
-				    struct pipe_buffer *buf)
-{
-	return false;
-}
-
-static const struct pipe_buf_operations zero_pipe_buf_ops = {
-	.release	= zero_pipe_buf_release,
-	.try_steal	= zero_pipe_buf_try_steal,
-	.get		= zero_pipe_buf_get,
-};
-
-static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe,
-					loff_t fpos, size_t size)
-{
-	size_t offset = fpos & ~PAGE_MASK;
-
-	size = min_t(size_t, size, PAGE_SIZE - offset);
-
-	if (!pipe_is_full(pipe)) {
-		struct pipe_buffer *buf = pipe_head_buf(pipe);
-
-		*buf = (struct pipe_buffer) {
-			.ops	= &zero_pipe_buf_ops,
-			.page	= ZERO_PAGE(0),
-			.offset	= offset,
-			.len	= size,
-		};
-		pipe->head++;
-	}
-
-	return size;
-}
-
 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos,
 				      struct pipe_inode_info *pipe,
 				      size_t len, unsigned int flags)
-- 
2.43.0

[PATCH v2 2/2] drivers/char/mem: splice the zero page for /dev/zero and /dev/full

[Kris Pan]

Implement a splice_read handler for /dev/zero and /dev/full that pushes
references to the global zero page into the pipe, instead of allocating
and zeroing a fresh page for every pipe buffer.

This is much faster than copy_splice_read() for splicing zeroes, since it
avoids the page allocation and zeroing on every pipe buffer.

Use the shared splice_zeropage_into_pipe() helper from fs/splice.c.

Signed-off-by: Kris Pan <kris.pan@intel.com>
Reviewed-by: Max Kellermann <max.kellermann@ionos.com>
---
 drivers/char/mem.c | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 63253d1de5d70..6d3ef359b2d7b 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -471,6 +471,31 @@ static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter)
 	return written;
 }
 
+static ssize_t splice_read_zero(struct file *in, loff_t *ppos,
+				struct pipe_inode_info *pipe, size_t len,
+				unsigned int flags)
+{
+	size_t total = 0;
+	loff_t pos = 0;
+	size_t used, npages;
+
+	used = pipe_buf_usage(pipe);
+	if (used >= pipe->max_usage)
+		return -EAGAIN;
+	npages = pipe->max_usage - used;
+	len = min_t(size_t, len, npages * PAGE_SIZE);
+
+	while (len) {
+		size_t n = splice_zeropage_into_pipe(pipe, pos, len);
+
+		pos += n;
+		total += n;
+		len -= n;
+	}
+
+	return total;
+}
+
 static ssize_t read_zero(struct file *file, char __user *buf,
 			 size_t count, loff_t *ppos)
 {
@@ -669,7 +694,7 @@ static const struct file_operations zero_fops = {
 	.read_iter	= read_iter_zero,
 	.read		= read_zero,
 	.write_iter	= write_iter_zero,
-	.splice_read	= copy_splice_read,
+	.splice_read	= splice_read_zero,
 	.splice_write	= splice_write_zero,
 	.mmap_prepare	= mmap_zero_prepare,
 	.get_unmapped_area = get_unmapped_area_zero,
@@ -682,7 +707,7 @@ static const struct file_operations full_fops = {
 	.llseek		= full_lseek,
 	.read_iter	= read_iter_zero,
 	.write		= write_full,
-	.splice_read	= copy_splice_read,
+	.splice_read	= splice_read_zero,
 };
 
 static const struct memdev {
-- 
2.43.0