mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Baoquan He <baoquan.he@linux.dev>
To: linux-mm@kvack.org
Cc: akpm@linux-foundation.org, david@kernel.org, urezki@gmail.com,
	chrisl@kernel.org, kasong@tencent.com, nphamcs@gmail.com,
	baohua@kernel.org, youngjun.park@lge.com,
	shikemeng@huaweicloud.com, hannes@cmpxchg.org, yosry@kernel.org,
	chengming.zhou@linux.dev, corbet@lwn.net, kunwu.chan@gmail.com,
	klarasmodin@gmail.com, linux-kernel@vger.kernel.org,
	baoquan.he@linux.dev, Baoquan He <hebaoquan@kylinos.cn>
Subject: [PATCH v6 11/14] mm, swap: add xswap grow trigger on cluster allocation
Date: Sun, 22 Nov 2026 20:48:11 +0800	[thread overview]
Message-ID: <20261122124814.3489077-2-baoquan.he@linux.dev> (raw)
In-Reply-To: <20261122124814.3489077-1-baoquan.he@linux.dev>

From: Baoquan He <hebaoquan@kylinos.cn>

Grow the mapped xswap range before it runs out. When usage reaches 85%
of the mapped range, map more clusters from cluster_alloc_swap_entry().

Grow to about 65% usage. This leaves free space in the mapped range
before the next grow at 85%, so one grow is not immediately followed by
another.

Mapping pages can sleep. Grow before the allocation that needs them,
not after it fails.

Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
 include/linux/swap.h |   1 +
 mm/swapfile.c        | 124 ++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 118 insertions(+), 7 deletions(-)

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 02c3861a336d..5d9e5b6f2bd6 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -246,6 +246,7 @@ struct swap_info_struct {
 	unsigned long		nr_clusters_max;/* total clusters in the xswap address space */
 	unsigned long		nr_clusters_mapped; /* currently mapped cluster count */
 	struct mutex		xswap_lock;	/* serialize map/unmap operations */
+	atomic_long_t		xswap_grows;	/* grow operations so far */
 #endif
 	struct list_head free_clusters; /* free clusters list */
 	struct list_head full_clusters; /* full clusters list */
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 074c5114d129..213b66bb1d6d 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -68,6 +68,21 @@
 #define XSWAP_GROW_CLUSTERS \
 	max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
 
+/*
+ * Grow thresholds, as a percentage of the mapped range in use.  The target
+ * sits below the next grow trigger, so a grow that follows a rise in usage
+ * does not immediately trigger another one.
+ */
+#define XSWAP_GROW_UNTIL	65
+#define XSWAP_GROW_WHEN		85
+#define XSWAP_GROW_CHUNKS	4
+
+static bool xswap_should_grow(struct swap_info_struct *si);
+static unsigned long xswap_grow(struct swap_info_struct *si);
+static int xswap_map_clusters(struct swap_info_struct *si,
+			      unsigned long start_idx, unsigned long nr,
+			      bool link_free);
+
 static int xswap_create(int prio);
 static int xswap_create_by_name(const char *name, int swap_flags);
 static int xswap_destroy(void);
@@ -1325,6 +1340,13 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
 	if (order && !(si->flags & SWP_BLKDEV))
 		return 0;
 
+#ifdef CONFIG_XSWAP
+	/* Grow before allocating: mapping pages can sleep. */
+	if ((si->flags & SWP_XSWAP) && (si->flags & SWP_WRITEOK) &&
+	    xswap_should_grow(si))
+		xswap_grow(si);
+#endif
+
 	if (!(si->flags & SWP_SOLIDSTATE)) {
 		/* Serialize HDD SWAP allocation for each device. */
 		spin_lock(&si->global_cluster_lock);
@@ -1397,6 +1419,12 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
 		if (found)
 			goto done;
 	}
+
+#ifdef CONFIG_XSWAP
+	/* A concurrent free or grow may have added clusters; retry once. */
+	if (!found && (si->flags & SWP_XSWAP))
+		found = alloc_swap_scan_list(si, &si->free_clusters, folio, false);
+#endif
 done:
 	if (!(si->flags & SWP_SOLIDSTATE))
 		spin_unlock(&si->global_cluster_lock);
@@ -3986,8 +4014,39 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
 }
 
 #ifdef CONFIG_XSWAP
+/*
+ * Link the new clusters and publish the larger range in one si->lock
+ * section: linking first makes a cluster visible to an allocator only
+ * after it is on the free list.  @link_free is false for the first
+ * chunk, whose free list is not initialized yet.
+ */
+static void xswap_publish_clusters(struct swap_info_struct *si,
+				   unsigned long start_idx, unsigned long nr,
+				   bool link_free)
+{
+	unsigned long i;
+
+	spin_lock(&si->lock);
+	if (link_free) {
+		for (i = start_idx; i < start_idx + nr; i++) {
+			struct swap_cluster_info *ci = &si->cluster_info[i];
+
+			/* A concurrent grower cannot take these: xswap_lock. */
+			if (ci->flags != CLUSTER_FLAG_NONE)
+				continue;
+			list_add_tail(&ci->list, &si->free_clusters);
+			WRITE_ONCE(ci->flags, CLUSTER_FLAG_FREE);
+			ci->order = 0;
+		}
+	}
+	/* Pairs with the smp_load_acquire() in the readers of the range. */
+	smp_store_release(&si->nr_clusters_mapped, start_idx + nr);
+	spin_unlock(&si->lock);
+}
+
 static int xswap_map_clusters(struct swap_info_struct *si,
-			      unsigned long start_idx, unsigned long nr)
+			      unsigned long start_idx, unsigned long nr,
+			      bool link_free)
 {
 	/* vm_area_grow_pages() takes an offset into the area, not an address. */
 	unsigned long vm_end = PAGE_ALIGN((size_t)(start_idx + nr) *
@@ -4023,14 +4082,62 @@ static int xswap_map_clusters(struct swap_info_struct *si,
 	for (i = start_idx; i < start_idx + nr; i++)
 		spin_lock_init(&si->cluster_info[i].lock);
 
-	/*
-	 * Publish the new mappings and cluster lock initialization before
-	 * the count; walkers without xswap_lock use smp_load_acquire().
-	 */
-	smp_store_release(&si->nr_clusters_mapped, start_idx + nr);
+	/* The mappings and the locks are in place; make them visible. */
+	xswap_publish_clusters(si, start_idx, nr, link_free);
 	mutex_unlock(&si->xswap_lock);
 	return 0;
 }
+static bool xswap_should_grow(struct swap_info_struct *si)
+{
+	unsigned long mapped = READ_ONCE(si->nr_clusters_mapped);
+
+	if (mapped >= READ_ONCE(si->nr_clusters_max))
+		return false;
+
+	return swap_usage_in_pages(si) * 100 >=
+	       mapped * SWAPFILE_CLUSTER * XSWAP_GROW_WHEN;
+}
+
+/*
+ * Map more of the cluster_info array, up to GROW_UNTIL in use.  The caller
+ * holds percpu_swap_cluster.lock, which this drops while it sleeps.
+ */
+static unsigned long xswap_grow(struct swap_info_struct *si)
+{
+	unsigned long mapped = READ_ONCE(si->nr_clusters_mapped);
+	unsigned long nr_new, want;
+	int ret;
+
+	lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
+
+	want = DIV_ROUND_UP(swap_usage_in_pages(si) * 100,
+			    XSWAP_GROW_UNTIL * SWAPFILE_CLUSTER);
+	if (want <= mapped)
+		return 0;
+
+	nr_new = rounddown(want - mapped, XSWAP_GROW_CLUSTERS);
+	if (!nr_new)
+		nr_new = XSWAP_GROW_CLUSTERS;
+	if (nr_new > XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS)
+		nr_new = XSWAP_GROW_CHUNKS * XSWAP_GROW_CLUSTERS;
+	nr_new = min(nr_new, READ_ONCE(si->nr_clusters_max) - mapped);
+	if (!nr_new)
+		return 0;
+
+	/*
+	 * Mapping pages can sleep.  The lock only guards the per-cpu
+	 * cluster cache, which this path does not touch.
+	 */
+	local_unlock(&percpu_swap_cluster.lock);
+	ret = xswap_map_clusters(si, mapped, nr_new, true);
+	local_lock(&percpu_swap_cluster.lock);
+	if (ret)
+		return 0;
+
+	atomic_long_inc(&si->xswap_grows);
+	return nr_new;
+}
+
 #endif /* CONFIG_XSWAP */
 
 static int setup_swap_clusters_info(struct swap_info_struct *si,
@@ -4060,7 +4167,8 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
 		mutex_init(&si->xswap_lock);
 
 		err = xswap_map_clusters(si, 0, min_t(unsigned long,
-					XSWAP_GROW_CLUSTERS, nr_clusters));
+					XSWAP_GROW_CLUSTERS, nr_clusters),
+					false);
 		if (err)
 			goto err_free_vm;
 
@@ -4252,6 +4360,8 @@ static int xswap_create(int prio)
 	 */
 	si->ops = &swap_bdev_ops;
 
+	atomic_long_set(&si->xswap_grows, 0);
+
 	error = setup_swap_clusters_info(si, NULL, maxpages);
 	if (error)
 		goto bad_swap;
-- 
2.54.0


  reply	other threads:[~2026-11-22 12:48 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-22 12:39 [PATCH v6 00/14] mm, swap: extendable swap devices (xswap phase I) Baoquan He
2026-11-22 12:39 ` [PATCH v6 01/14] mm/vmalloc: support vmalloc-backed VM_SPARSE areas Baoquan He
2026-11-22 12:39 ` [PATCH v6 02/14] mm: xswap support for zswap Baoquan He
2026-11-22 12:39 ` [PATCH v6 03/14] mm, swap: refactor free_swap_cluster_info to take swap_info_struct Baoquan He
2026-11-22 12:39 ` [PATCH v6 04/14] mm, swap: use a VM_SPARSE cluster array for xswap devices Baoquan He
2026-11-22 12:39 ` [PATCH v6 05/14] mm, swap: widen swap_info_struct max/pages to unsigned long Baoquan He
2026-11-22 12:39 ` [PATCH v6 06/14] mm, swap: create the xswap device from swapon(2) Baoquan He
2026-11-22 12:39 ` [PATCH v6 07/14] mm, swap: refactor swapoff and add xswap_destroy Baoquan He
2026-11-22 12:39 ` [PATCH v6 08/14] mm, swap: require zswap for xswap devices Baoquan He
2026-11-22 12:39 ` [PATCH v6 09/14] mm, swap: let the command line set the xswap device size Baoquan He
2026-11-22 12:48 ` [PATCH v6 10/14] mm, swap: add CONFIG_XSWAP_DEFAULT_MAX Baoquan He
2026-11-22 12:48   ` Baoquan He [this message]
2026-11-22 12:48   ` [PATCH v6 12/14] mm, swap: add debugfs counters for xswap Baoquan He
2026-11-22 12:48   ` [PATCH v6 13/14] mm, swap: do not give an xswap slot to a cgroup that cannot zswap Baoquan He
2026-11-22 12:48   ` [PATCH v6 14/14] mm, swap: skip the overcommit probe when removing xswap Baoquan He

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261122124814.3489077-2-baoquan.he@linux.dev \
    --to=baoquan.he@linux.dev \
    --cc=akpm@linux-foundation.org \
    --cc=baohua@kernel.org \
    --cc=chengming.zhou@linux.dev \
    --cc=chrisl@kernel.org \
    --cc=corbet@lwn.net \
    --cc=david@kernel.org \
    --cc=hannes@cmpxchg.org \
    --cc=hebaoquan@kylinos.cn \
    --cc=kasong@tencent.com \
    --cc=klarasmodin@gmail.com \
    --cc=kunwu.chan@gmail.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=nphamcs@gmail.com \
    --cc=shikemeng@huaweicloud.com \
    --cc=urezki@gmail.com \
    --cc=yosry@kernel.org \
    --cc=youngjun.park@lge.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®