mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Lian Wang <lianux.mm@gmail.com>
To: linux-mm@kvack.org
Cc: linux-kernel@vger.kernel.org, akpm@linux-foundation.org,
	chrisl@kernel.org, nphamcs@gmail.com, baoquan.he@linux.dev,
	baohua@kernel.org, youngjun.park@lge.com,
	shikemeng@huaweicloud.com, kasong@tencent.com,
	linjh22s@gmail.com, kunwu.chan@linux.dev, kees@kernel.org,
	gustavoars@kernel.org, tglx@kernel.org,
	linux-hardening@vger.kernel.org
Subject: [RFC PATCH v3 16/18] mm/swap: advance the global queue cursor directly between priority rings
Date: Wed, 28 Oct 2026 00:12:56 +0800	[thread overview]
Message-ID: <03c9f51439f1b457d7f9af40d6bbdf846f2776ed.1793106426.git.lianux.mm@gmail.com> (raw)
In-Reply-To: <cover.1793106426.git.lianux.mm@gmail.com>

The allocation walk previously passed a total retry index to the selector.
Each retry translated it back to a priority ring by scanning from the
beginning of swap_queue. Keep the ring index and the number of attempted
peers in the task-local cursor instead.

Snapshot the starting device only on the first visit to each ring and
advance each peer exactly once, including masked entries. Preserve the
large-folio same-ring boundary and the stale reader-offset correction.
The selector's complete walk is linear in the devices and priority groups;
device cluster scanning and availability updates have their own costs.

Keep the topology read lock and CPU migration protection across the walk.
The global selector uses only existing priorities and adds no per-memcg
device-selection state or ABI. Existing swap charging and limits remain
in folio_alloc_swap(), whose failure path classifies allocation errors.

The walk no longer returns an ignored internal error to that caller.

Signed-off-by: Lian Wang <lianux.mm@gmail.com>
---
 mm/swapfile.c | 49 +++++++++++++++++++++++--------------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/mm/swapfile.c b/mm/swapfile.c
index 5b6492b1bdc6..bbd114f043ab 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -194,10 +194,10 @@ struct swap_queue_reader {
 };
 
 struct swap_queue_cursor {
-	bool valid;
 	bool ring_only;
 	unsigned int ring_idx;
 	unsigned int offset;
+	unsigned int nr_iter;
 };
 
 static __percpu struct swap_queue_reader *swap_queue_readers;
@@ -307,28 +307,32 @@ static void swap_queue_install_readers(struct swap_queue_reader __percpu *reader
 	swap_queue_reset_readers();
 }
 
-static struct swap_info_struct *swap_queue_get_device(long nr_alloc, int nr_iter,
-						      struct swap_queue_cursor *cursor)
+static struct swap_info_struct *
+swap_queue_get_device(long nr_alloc, struct swap_queue_cursor *cursor)
 {
 	bool rotate = false;
 	struct swap_info_struct *si;
 	struct swap_ring_iterator *ri;
 	struct swap_prio_ring *ring;
-	unsigned int dev_idx, queue_idx;
+	unsigned int dev_idx, queue_idx, nr_iter;
 
-	if (!swap_queue_len)
+	queue_idx = cursor->ring_idx;
+	if (queue_idx >= swap_queue_len)
 		return ERR_PTR(-ENOENT);
 
-	queue_idx = 0;
-	while (nr_iter >= swap_queue[queue_idx]->size) {
-		nr_iter -= swap_queue[queue_idx]->size;
-		if (++queue_idx >= swap_queue_len)
-			return ERR_PTR(cursor->ring_only ? -E2BIG : -ENOENT);
+	ring = swap_queue[queue_idx];
+	if (cursor->nr_iter >= ring->size) {
+		if (cursor->ring_only)
+			return ERR_PTR(-E2BIG);
+		queue_idx = ++cursor->ring_idx;
+		if (queue_idx >= swap_queue_len)
+			return ERR_PTR(-ENOENT);
+		cursor->nr_iter = 0;
+		ring = swap_queue[queue_idx];
 	}
-	if (cursor->ring_only && cursor->ring_idx != queue_idx)
-		return ERR_PTR(-E2BIG);
 
-	ring = swap_queue[queue_idx];
+	/* Consume masked peers too, so every walk remains bounded. */
+	nr_iter = cursor->nr_iter++;
 	local_lock(&swap_queue_readers->lock);
 	ri = this_cpu_ptr(&swap_queue_readers->ri[queue_idx]);
 	/*
@@ -336,9 +340,7 @@ static struct swap_info_struct *swap_queue_get_device(long nr_alloc, int nr_iter
 	 * iterator can move between retries, but the cursor must visit every
 	 * device in the ring exactly once before falling through.
 	 */
-	if (!cursor->valid || cursor->ring_idx != queue_idx) {
-		cursor->valid = true;
-		cursor->ring_idx = queue_idx;
+	if (!nr_iter) {
 		if (ri->rr_counter < nr_alloc)
 			rotate = true;
 		else if (ri->offset >= ring->size)
@@ -1794,29 +1796,25 @@ static bool get_swap_device_info(struct swap_info_struct *si)
 	return true;
 }
 
-static int swap_alloc_entry(struct folio *folio)
+static void swap_alloc_entry(struct folio *folio)
 {
 	struct swap_queue_cursor cursor = {};
 	long nr_pages = folio_nr_pages(folio);
 	struct swap_info_struct *si;
-	int nr_iter, ret;
 
 	percpu_down_read(&swapon_rwsem);
 	migrate_disable();
-	for (nr_iter = 0;; nr_iter++) {
-		si = swap_queue_get_device(nr_pages, nr_iter, &cursor);
+	for (;;) {
+		si = swap_queue_get_device(nr_pages, &cursor);
 		if (IS_ERR(si)) {
-			ret = PTR_ERR(si);
-			if (ret == -EBUSY)
+			if (PTR_ERR(si) == -EBUSY)
 				continue;
 			break;
 		}
 		cluster_alloc_swap_entry(si, folio);
 
-		if (folio_test_swapcache(folio)) {
-			ret = 0;
+		if (folio_test_swapcache(folio))
 			break;
-		}
 
 		/*
 		 * For large allocations, try every device at the same priority,
@@ -1831,7 +1829,6 @@ static int swap_alloc_entry(struct folio *folio)
 
 	migrate_enable();
 	percpu_up_read(&swapon_rwsem);
-	return ret;
 }
 
 static int swap_extend_table_alloc(struct swap_info_struct *si,

  parent reply	other threads:[~2026-10-27 16:13 UTC|newest]

Thread overview: 19+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-10-27 15:58 [RFC PATCH v3 00/18] mm/swap: introduce priority queues for swap device allocation Lian Wang
2026-10-27 16:03 ` [RFC PATCH v3 01/18] mm/swap: remove unused parameter for reading swap header Lian Wang
2026-10-27 16:03 ` [RFC PATCH v3 02/18] mm/swap: slightly cleanup the code for hibernation error handling Lian Wang
2026-10-27 16:04 ` [RFC PATCH v3 03/18] mm/swap: cleanup and document swap device availability flag usage Lian Wang
2026-10-27 16:05 ` [RFC PATCH v3 04/18] mm/swap: introduce swap device iteration helper Lian Wang
2026-10-27 16:05 ` [RFC PATCH v3 05/18] mm/swap: change the swapon lock into a percpu rwsem Lian Wang
2026-10-27 16:06 ` [RFC PATCH v3 06/18] mm/swap: serialize swap publication before removing swapon mutex Lian Wang
2026-10-27 16:07 ` [RFC PATCH v3 07/18] mm/swap: consolidate swap inuse accounting helpers Lian Wang
2026-10-27 16:07 ` [RFC PATCH v3 08/18] mm/swap: change back to use each swap device's percpu cluster Lian Wang
2026-10-27 16:08 ` [RFC PATCH v3 09/18] mm/swap: avoid direct reclaim when populating swap cluster metadata Lian Wang
2026-10-27 16:08 ` [RFC PATCH v3 10/18] mm/swap: add priority queue for swap device allocation Lian Wang
2026-10-27 16:09 ` [RFC PATCH v3 11/18] mm/swap: remove available list Lian Wang
2026-10-27 16:10 ` [RFC PATCH v3 12/18] mm/swap: bound synchronous discard during allocation Lian Wang
2026-10-27 16:11 ` [RFC PATCH v3 13/18] mm/swap: drop swap active plist Lian Wang
2026-10-27 16:11 ` [RFC PATCH v3 14/18] lib/plist.c: remove requeue function Lian Wang
2026-10-27 16:12 ` [RFC PATCH v3 15/18] mm/swap: reuse per-CPU queue readers across priority changes Lian Wang
2026-10-27 16:12 ` Lian Wang [this message]
2026-10-27 16:13 ` [RFC PATCH v3 17/18] mm/swap: skip reader rotation for single-device priority rings Lian Wang
2026-10-27 16:14 ` [RFC PATCH v3 18/18] mm/swap: redistribute queue readers after membership changes Lian Wang

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=03c9f51439f1b457d7f9af40d6bbdf846f2776ed.1793106426.git.lianux.mm@gmail.com \
    --to=lianux.mm@gmail.com \
    --cc=akpm@linux-foundation.org \
    --cc=baohua@kernel.org \
    --cc=baoquan.he@linux.dev \
    --cc=chrisl@kernel.org \
    --cc=gustavoars@kernel.org \
    --cc=kasong@tencent.com \
    --cc=kees@kernel.org \
    --cc=kunwu.chan@linux.dev \
    --cc=linjh22s@gmail.com \
    --cc=linux-hardening@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=nphamcs@gmail.com \
    --cc=shikemeng@huaweicloud.com \
    --cc=tglx@kernel.org \
    --cc=youngjun.park@lge.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®