[PATCH] io_uring/register: preserve SQ array entries on resize

[guzebing <guzebing1612@gmail.com>]

Ring resizing copies pending SQEs from the old SQE array into the new
one so submissions queued before the resize can still be consumed
afterwards.

That copy currently walks the SQ head/tail range directly. This is only
correct when there is no SQ array indirection. With a regular SQ array,
each pending SQ entry contains an index into the SQE array. After resize,
ctx->sq_array is repointed at the newly allocated array, so pending
entries lose their old logical-to-physical mapping and may submit the
wrong SQE.

Remember the old and new SQ arrays while migrating pending SQ entries. For
each pending entry, copy the SQE selected by the old array into the new
destination slot and rebuild the new array entry to point at the copied
SQE. Keep invalid user-provided entries invalid so the normal submission
path still drops them after resize.

Fixes: 79cfe9e59c2a1 ("io_uring/register: add IORING_REGISTER_RESIZE_RINGS")
Signed-off-by: guzebing <guzebing1612@gmail.com>
---
 io_uring/register.c | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/io_uring/register.c b/io_uring/register.c
index dce5e2f9cf770..02bc103bcc9d5 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -503,6 +503,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	unsigned i, tail, old_head;
 	struct io_uring_params *p = &config.p;
 	struct io_rings_layout *rl = &config.layout;
+	u32 *o_sq_array, *n_sq_array = NULL;
 	int ret;
 
 	memset(&config, 0, sizeof(config));
@@ -589,6 +590,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	ctx->rings = NULL;
 	o.sq_sqes = ctx->sq_sqes;
 	ctx->sq_sqes = NULL;
+	o_sq_array = ctx->sq_array;
+	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
+		n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
 
 	/*
 	 * Now copy SQ and CQ entries, if any. If either of the destination
@@ -599,20 +603,27 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (tail - old_head > p->sq_entries)
 		goto overflow;
 	for (i = old_head; i < tail; i++) {
-		unsigned index, dst_mask, src_mask;
+		unsigned int dst, src;
 		size_t sq_size;
 
-		index = i;
+		dst = i & (p->sq_entries - 1);
+		src = i & (ctx->sq_entries - 1);
+		if (n_sq_array) {
+			src = READ_ONCE(o_sq_array[src]);
+			if (unlikely(src >= ctx->sq_entries)) {
+				WRITE_ONCE(n_sq_array[dst], UINT_MAX);
+				continue;
+			}
+			WRITE_ONCE(n_sq_array[dst], dst);
+		}
+
 		sq_size = sizeof(struct io_uring_sqe);
-		src_mask = ctx->sq_entries - 1;
-		dst_mask = p->sq_entries - 1;
 		if (ctx->flags & IORING_SETUP_SQE128) {
-			index <<= 1;
+			dst <<= 1;
+			src <<= 1;
 			sq_size <<= 1;
-			src_mask = (ctx->sq_entries << 1) - 1;
-			dst_mask = (p->sq_entries << 1) - 1;
 		}
-		memcpy(&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size);
+		memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size);
 	}
 	WRITE_ONCE(n.rings->sq.head, old_head);
 	WRITE_ONCE(n.rings->sq.tail, tail);
@@ -655,8 +666,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow));
 
 	/* all done, store old pointers and assign new ones */
-	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
+	if (n_sq_array)
+		ctx->sq_array = n_sq_array;
 
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
-- 
2.20.1