From: Baoquan He <baoquan.he@linux.dev>
To: linux-mm@kvack.org
Cc: akpm@linux-foundation.org, david@kernel.org, chrisl@kernel.org,
kasong@tencent.com, nphamcs@gmail.com, baohua@kernel.org,
youngjun.park@lge.com, corbet@lwn.net, hannes@cmpxchg.org,
chengming.zhou@linux.dev, kunwu.chan@gmail.com,
klarasmodin@gmail.com, linux-kernel@vger.kernel.org,
Baoquan He <hebaoquan@kylinos.cn>
Subject: [PATCH v5 04/19] mm, swap: use a VM_SPARSE cluster array for xswap devices
Date: Mon, 9 Nov 2026 13:54:40 +0800 [thread overview]
Message-ID: <20261109055455.536925-5-baoquan.he@linux.dev> (raw)
In-Reply-To: <20261109055455.536925-1-baoquan.he@linux.dev>
From: Baoquan He <hebaoquan@kylinos.cn>
An xswap device does not have to allocate cluster_info for its whole
address space at once. Back it with a VM_SPARSE area and map it in
chunks. At setup, map only the first chunk. The grow path maps more
chunks when they are needed.
Setup, free, and the two cluster_info scans must change together. If
only setup changes, free would free the wrong object, and the scans
would read cluster_info entries that are not mapped yet.
The map helper rejects a stale start_idx and cleans up a partial mapping
if it fails. The grow path added later depends on this check.
Signed-off-by: Baoquan He <hebaoquan@kylinos.cn>
---
mm/swapfile.c | 152 ++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 141 insertions(+), 11 deletions(-)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index afd0e8718544..00c7b398bafe 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -50,6 +50,17 @@
#include "swap.h"
#ifdef CONFIG_XSWAP
+/*
+ * xswap: dynamically grow the cluster_info array via a VM_SPARSE area.
+ *
+ * XSWAP_GROW_CLUSTERS is the number of clusters to map in one grow
+ * operation. It is set to the number of cluster_info structs that
+ * fit in a single page (at least 16), so that the vmalloc page table
+ * overhead is proportional to the number of clusters mapped.
+ */
+#define XSWAP_GROW_CLUSTERS \
+ max_t(unsigned long, PAGE_SIZE / sizeof(struct swap_cluster_info), 16)
+
/* Batch size for the fallback unmap; keeps the stack array small. */
#define XSWAP_UNMAP_BATCH_PAGES 16
#define XSWAP_UNMAP_BATCH_CLUSTERS \
@@ -2825,10 +2836,27 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
unsigned int prev)
{
struct swap_cluster_info *ci;
- unsigned long i, end;
+ unsigned long i, cluster_end, end;
unsigned int ci_off;
unsigned long swp_tb;
+ end = si->max;
+#ifdef CONFIG_XSWAP
+ /*
+ * An xswap device maps its cluster_info in chunks as it grows, so
+ * the tail past nr_clusters_mapped has nothing behind it.
+ */
+ if (si->flags & SWP_XSWAP) {
+ unsigned long mapped_end;
+
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ mapped_end = smp_load_acquire(&si->nr_clusters_mapped) *
+ SWAPFILE_CLUSTER;
+ if (mapped_end < end)
+ end = mapped_end;
+ }
+#endif
+
/*
* No need for swap_lock here: we're just looking
* for whether an entry is in use, not modifying it; false
@@ -2836,11 +2864,11 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
* allocations from this area (while holding swap_lock).
*/
i = prev + 1;
- while (i < si->max) {
+ while (i < end) {
ci = __swap_offset_to_cluster(si, i);
- end = min_t(unsigned long,
- ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER,
- si->max);
+ cluster_end = min_t(unsigned long,
+ ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER,
+ end);
/*
* An empty cluster has no slot in use, so skip it whole.
@@ -2850,13 +2878,13 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
* enough, unlike in every other cluster_is_empty() caller.
*/
if (!READ_ONCE(ci->count)) {
- i = end;
+ i = cluster_end;
cond_resched();
continue;
}
ci_off = i % SWAPFILE_CLUSTER;
- for (; i < end; ci_off++, i++) {
+ for (; i < cluster_end; ci_off++, i++) {
swp_tb = swap_table_get(ci, ci_off);
if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb))
return i;
@@ -2866,7 +2894,6 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
return 0;
}
-
static int try_to_unuse(unsigned int type)
{
struct mm_struct *prev_mm;
@@ -3164,6 +3191,14 @@ static void wait_for_allocation(struct swap_info_struct *si)
BUG_ON(si->flags & SWP_WRITEOK);
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ end = min(end, smp_load_acquire(&si->nr_clusters_mapped) *
+ SWAPFILE_CLUSTER);
+ }
+#endif
+
for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
ci = swap_cluster_lock(si, offset);
swap_cluster_unlock(ci);
@@ -3175,11 +3210,43 @@ static void free_swap_cluster_info(struct swap_info_struct *si)
struct swap_cluster_info *cluster_info = si->cluster_info;
unsigned long maxpages = si->max;
struct swap_cluster_info *ci;
- int i, nr_clusters;
+ unsigned long i, nr_clusters;
if (!cluster_info)
return;
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ unsigned long nr_mapped;
+
+ /*
+ * Cluster 0 keeps the bad header slot, so it never empties
+ * and __free_cluster() never frees its table.
+ */
+ /* Pairs with the smp_store_release() in xswap_map_clusters(). */
+ nr_mapped = smp_load_acquire(&si->nr_clusters_mapped);
+ for (i = 0; i < nr_mapped; i++) {
+ ci = &cluster_info[i];
+ spin_lock(&ci->lock);
+ if (cluster_table_is_alloced(ci)) {
+ swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
+ swap_cluster_free_table(ci);
+ }
+ spin_unlock(&ci->lock);
+ }
+ /*
+ * free_vm_area() drops the mapping without freeing the
+ * pages, and the unmap helper handles an empty range.
+ */
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->cluster_info = NULL;
+ si->nr_clusters_mapped = 0;
+ return;
+ }
+#endif
+
nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
for (i = 0; i < nr_clusters; i++) {
ci = cluster_info + i;
@@ -3678,7 +3745,7 @@ static unsigned long read_swap_header(struct swap_info_struct *si,
}
#ifdef CONFIG_XSWAP
-static int __maybe_unused xswap_map_clusters(struct swap_info_struct *si,
+static int xswap_map_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
{
unsigned long start_addr = (unsigned long)si->cluster_info +
@@ -3824,7 +3891,7 @@ static void xswap_unmap_range(struct swap_info_struct *si,
}
/* Caller must hold si->xswap_lock. Cannot fail. */
-static void __maybe_unused xswap_unmap_clusters(struct swap_info_struct *si,
+static void xswap_unmap_clusters(struct swap_info_struct *si,
unsigned long start_idx, unsigned long nr)
{
unsigned long start_addr = (unsigned long)si->cluster_info +
@@ -3902,6 +3969,69 @@ static int setup_swap_clusters_info(struct swap_info_struct *si,
int err = -ENOMEM;
unsigned long i;
+#ifdef CONFIG_XSWAP
+ if (si->flags & SWP_XSWAP) {
+ unsigned long size = PAGE_ALIGN(nr_clusters * sizeof(*cluster_info));
+ struct vm_struct *vm;
+
+ vm = get_vm_area(size, VM_SPARSE);
+ if (!vm)
+ goto err;
+
+ cluster_info = vm->addr;
+ si->cluster_vm = vm;
+ si->nr_clusters_max = nr_clusters;
+ si->cluster_info = cluster_info;
+
+ /* Must be initialized before xswap_map_clusters() locks it. */
+ mutex_init(&si->xswap_lock);
+
+ if (xswap_map_clusters(si, 0, min_t(unsigned long,
+ XSWAP_GROW_CLUSTERS, nr_clusters)))
+ goto err_free_vm;
+
+ /* xswap: only cluster 0 slot 0 is bad */
+ err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
+ if (err)
+ goto err_unmap;
+
+ INIT_LIST_HEAD(&si->free_clusters);
+ INIT_LIST_HEAD(&si->full_clusters);
+ INIT_LIST_HEAD(&si->discard_clusters);
+ for (i = 0; i < SWAP_NR_ORDERS; i++) {
+ INIT_LIST_HEAD(&si->nonfull_clusters[i]);
+ INIT_LIST_HEAD(&si->frag_clusters[i]);
+ }
+
+ /*
+ * Cluster 0 holds the header slot, which is marked bad, so it
+ * is not entirely free. si->max is cluster aligned, so no
+ * cluster holds a slot past the end of the device.
+ */
+ for (i = 0; i < si->nr_clusters_mapped; i++) {
+ struct swap_cluster_info *ci = &cluster_info[i];
+
+ if (ci->count) {
+ ci->flags = CLUSTER_FLAG_NONFULL;
+ list_add_tail(&ci->list, &si->nonfull_clusters[0]);
+ } else {
+ ci->flags = CLUSTER_FLAG_FREE;
+ list_add_tail(&ci->list, &si->free_clusters);
+ }
+ }
+
+ return 0;
+
+err_unmap:
+ xswap_unmap_clusters(si, 0, si->nr_clusters_mapped);
+err_free_vm:
+ free_vm_area(si->cluster_vm);
+ si->cluster_vm = NULL;
+ si->cluster_info = NULL;
+ return err;
+ }
+#endif /* CONFIG_XSWAP */
+
cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
if (!cluster_info)
goto err;
--
2.54.0
next prev parent reply other threads:[~2026-11-09 5:56 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-09 5:54 [PATCH v5 00/19] mm, swap: extendable swap devices (xswap phase I) Baoquan He
2026-11-09 5:54 ` [PATCH v5 01/19] mm: xswap support for zswap Baoquan He
2026-11-09 5:54 ` [PATCH v5 02/19] mm, swap: refactor free_swap_cluster_info to take swap_info_struct Baoquan He
2026-11-09 5:54 ` [PATCH v5 03/19] mm, swap: add VM_SPARSE cluster-array helpers for xswap Baoquan He
2026-11-09 5:54 ` Baoquan He [this message]
2026-11-09 5:54 ` [PATCH v5 05/19] mm, swap: widen swap_info_struct max/pages to unsigned long Baoquan He
2026-11-09 5:54 ` [PATCH v5 06/19] mm, swap: add a sysfs interface to create xswap devices Baoquan He
2026-11-09 5:54 ` [PATCH v5 07/19] mm, swap: refactor swapoff and add xswap_destroy Baoquan He
2026-11-09 5:54 ` [PATCH v5 08/19] mm, swap: require zswap for xswap devices Baoquan He
2026-11-09 5:54 ` [PATCH v5 09/19] mm, swap: use a stable id " Baoquan He
2026-11-09 5:54 ` [PATCH v5 10/19] mm, swap: let the command line set the xswap device size Baoquan He
2026-11-09 5:54 ` [PATCH v5 11/19] mm, swap: add CONFIG_XSWAP_DEFAULT_MAX Baoquan He
2026-11-09 5:54 ` [PATCH v5 12/19] mm, swap: let swapoff destroy xswap devices by name Baoquan He
2026-11-09 5:54 ` [PATCH v5 13/19] mm, swap: add xswap grow trigger on cluster allocation Baoquan He
2026-11-09 5:54 ` [PATCH v5 14/19] mm, swap: add debugfs counters for xswap Baoquan He
2026-11-09 5:54 ` [PATCH v5 15/19] mm, swap: make xswap cluster_info accesses RCU-safe Baoquan He
2026-11-09 5:54 ` [PATCH v5 16/19] mm, swap: prepare xswap_unmap_clusters for range shrinking Baoquan He
2026-11-09 5:54 ` [PATCH v5 17/19] mm, swap: add xswap shrink trigger and defer it to a workqueue Baoquan He
2026-11-09 5:54 ` [PATCH v5 18/19] mm, swap: do not give an xswap slot to a cgroup that cannot zswap Baoquan He
2026-11-09 5:54 ` [PATCH v5 19/19] mm, swap: skip the overcommit probe when removing xswap Baoquan He
2026-11-09 6:15 ` [PATCH v5 00/19] mm, swap: extendable swap devices (xswap phase I) Baoquan He
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261109055455.536925-5-baoquan.he@linux.dev \
--to=baoquan.he@linux.dev \
--cc=akpm@linux-foundation.org \
--cc=baohua@kernel.org \
--cc=chengming.zhou@linux.dev \
--cc=chrisl@kernel.org \
--cc=corbet@lwn.net \
--cc=david@kernel.org \
--cc=hannes@cmpxchg.org \
--cc=hebaoquan@kylinos.cn \
--cc=kasong@tencent.com \
--cc=klarasmodin@gmail.com \
--cc=kunwu.chan@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=nphamcs@gmail.com \
--cc=youngjun.park@lge.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®