From: Huan Nguyen <nhuhuan@google.com>
To: tglx@kernel.org, mingo@redhat.com, bp@alien8.de,
dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com,
akpm@linux-foundation.org, david@kernel.org, ljs@kernel.org,
Liam.Howlett@oracle.com, vbabka@kernel.org, rppt@kernel.org,
surenb@google.com, mhocko@suse.com, trondmy@kernel.org,
anna@kernel.org, ziy@nvidia.com, matthew.brost@intel.com,
joshua.hahnjy@gmail.com, rakie.kim@sk.com, byungchul@sk.com,
gourry@gourry.net, ying.huang@linux.alibaba.com,
apopple@nvidia.com, rostedt@goodmis.org, mhiramat@kernel.org,
mathieu.desnoyers@efficios.com, jackmanb@google.com,
hannes@cmpxchg.org, shivankg@amd.com
Cc: rientjes@google.com, dave@stgolabs.net, nifan.cxl@gmail.com,
fvdl@google.com, jic23@kernel.org, rkodsara@amd.com,
bharata@amd.com, sj@kernel.org, weixugc@google.com,
xuezhengchu@huawei.com, yiannis@zptcorp.com,
yiannis.nikolakop@gmail.com, dimitrios.palyvos@zptcorp.com,
alirad.malek@zptcorp.com, ryan.roberts@arm.com,
nhuhuan@google.com, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, linux-nfs@vger.kernel.org,
linux-trace-kernel@vger.kernel.org
Subject: [RFC PATCH v3 2/3] mm/migrate: introduce async non-temporal migration mode
Date: Tue, 3 Nov 2026 14:30:24 -0800 [thread overview]
Message-ID: <20261103223025.2482227-3-nhuhuan@google.com> (raw)
In-Reply-To: <20261103223025.2482227-1-nhuhuan@google.com>
Introduce MIGRATE_ASYNC_NONTEMPORAL to migrate cold folios using
non-temporal stores, avoiding CPU cache pollution.
Existing callers check "mode == MIGRATE_ASYNC" directly, which would
cause new asynchronous modes to fall back to synchronous handling.
Introduce migrate_is_async() and update callers across compaction,
NFS, and migration.
In __migrate_folio(), invoke folio_mc_copy_nt() when the migration
mode is MIGRATE_ASYNC_NONTEMPORAL.
Suggested-by: Alirad Malek <alirad.malek@zptcorp.com>
Suggested-by: Dimitrios Palyvos-Giannas <dimitrios.palyvos@zptcorp.com>
Signed-off-by: Huan Nguyen <nhuhuan@google.com>
---
fs/nfs/write.c | 2 +-
include/linux/migrate.h | 5 +++++
include/linux/migrate_mode.h | 1 +
include/trace/events/migrate.h | 1 +
mm/compaction.c | 18 +++++++++---------
mm/migrate.c | 29 ++++++++++++++++++-----------
6 files changed, 35 insertions(+), 21 deletions(-)
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 0b42d31f7a75..ad28ff48e541 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -2128,7 +2128,7 @@ int nfs_migrate_folio(struct address_space *mapping, struct folio *dst,
}
if (folio_test_private_2(src)) { /* [DEPRECATED] */
- if (mode == MIGRATE_ASYNC)
+ if (migrate_is_async(mode))
return -EBUSY;
folio_wait_private_2(src);
}
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 78424b3824c2..188f9d891a0a 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -51,6 +51,11 @@ struct movable_operations {
/* Defined in mm/debug.c: */
extern const char *migrate_reason_names[MR_TYPES];
+static inline bool migrate_is_async(enum migrate_mode mode)
+{
+ return mode == MIGRATE_ASYNC || mode == MIGRATE_ASYNC_NONTEMPORAL;
+}
+
#ifdef CONFIG_MIGRATION
void putback_movable_pages(struct list_head *l);
diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h
index 05102d4d2490..fd752fddc972 100644
--- a/include/linux/migrate_mode.h
+++ b/include/linux/migrate_mode.h
@@ -12,6 +12,7 @@ enum migrate_mode {
MIGRATE_ASYNC,
MIGRATE_SYNC_LIGHT,
MIGRATE_SYNC,
+ MIGRATE_ASYNC_NONTEMPORAL,
};
enum migrate_reason {
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 15ee2ef201b5..0bb2444211af 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -9,6 +9,7 @@
#define MIGRATE_MODE \
EM( MIGRATE_ASYNC, "MIGRATE_ASYNC") \
+ EM( MIGRATE_ASYNC_NONTEMPORAL, "MIGRATE_ASYNC_NONTEMPORAL") \
EM( MIGRATE_SYNC_LIGHT, "MIGRATE_SYNC_LIGHT") \
EMe(MIGRATE_SYNC, "MIGRATE_SYNC")
diff --git a/mm/compaction.c b/mm/compaction.c
index f1b2060eb201..c11739e6891f 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -445,7 +445,7 @@ static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
/* Update where async and sync compaction should restart */
if (pfn > zone->compact_cached_migrate_pfn[0])
zone->compact_cached_migrate_pfn[0] = pfn;
- if (cc->mode != MIGRATE_ASYNC &&
+ if (!migrate_is_async(cc->mode) &&
pfn > zone->compact_cached_migrate_pfn[1])
zone->compact_cached_migrate_pfn[1] = pfn;
}
@@ -508,7 +508,7 @@ static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
__acquires(lock)
{
/* Track if the lock is contended in async mode */
- if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
+ if (migrate_is_async(cc->mode) && !cc->contended) {
if (spin_trylock_irqsave(lock, *flags))
return true;
@@ -882,7 +882,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
return -EAGAIN;
/* async migration should just abort */
- if (cc->mode == MIGRATE_ASYNC)
+ if (migrate_is_async(cc->mode))
return -EAGAIN;
reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
@@ -893,7 +893,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
cond_resched();
- if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
+ if (cc->direct_compaction && migrate_is_async(cc->mode)) {
skip_on_failure = true;
next_skip_pfn = block_end_pfn(low_pfn, cc->order);
}
@@ -1409,7 +1409,7 @@ static bool suitable_migration_source(struct compact_control *cc,
* allocator fallbacks. For non-movable requests, this helps a
* lot, as they can finally scan movable blocks.
*/
- if (cc->mode != MIGRATE_ASYNC)
+ if (!migrate_is_async(cc->mode))
return true;
/*
@@ -1514,7 +1514,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn)
return;
/* Minimise scanning during async compaction */
- if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
+ if (cc->direct_compaction && migrate_is_async(cc->mode))
return;
/* Pageblock boundaries */
@@ -1753,7 +1753,7 @@ static void isolate_freepages(struct compact_control *cc)
block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
zone_end_pfn(zone));
low_pfn = pageblock_end_pfn(cc->migrate_pfn);
- stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
+ stride = migrate_is_async(cc->mode) ? COMPACT_CLUSTER_MAX : 1;
/*
* Isolate free pages until enough are available to migrate the
@@ -2565,7 +2565,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
unsigned long start_pfn = cc->zone->zone_start_pfn;
unsigned long end_pfn = zone_end_pfn(cc->zone);
unsigned long last_migrated_pfn;
- const bool sync = cc->mode != MIGRATE_ASYNC;
+ const bool sync = !migrate_is_async(cc->mode);
bool update_cached;
unsigned int nr_succeeded = 0, nr_migratepages;
int order;
@@ -2588,7 +2588,7 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
ret = compaction_suit_allocation_order(cc->zone, cc->order,
cc->highest_zoneidx,
cc->alloc_flags,
- cc->mode == MIGRATE_ASYNC,
+ migrate_is_async(cc->mode),
!cc->direct_compaction);
if (ret != COMPACT_CONTINUE)
return ret;
diff --git a/mm/migrate.c b/mm/migrate.c
index 7bdcdb57652f..18470de10857 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -876,9 +876,15 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst,
if (folio_ref_count(src) != expected_count)
return -EAGAIN;
- rc = folio_mc_copy(dst, src);
- if (unlikely(rc))
- return rc;
+ if (mode == MIGRATE_ASYNC_NONTEMPORAL) {
+ rc = folio_mc_copy_nt(dst, src);
+ if (unlikely(rc))
+ return rc;
+ } else {
+ rc = folio_mc_copy(dst, src);
+ if (unlikely(rc))
+ return rc;
+ }
rc = __folio_migrate_mapping(mapping, dst, src, expected_count);
if (rc)
@@ -921,7 +927,7 @@ static bool buffer_migrate_lock_buffers(struct buffer_head *head,
do {
if (!trylock_buffer(bh)) {
- if (mode == MIGRATE_ASYNC)
+ if (migrate_is_async(mode))
goto unlock;
if (mode == MIGRATE_SYNC_LIGHT && !buffer_uptodate(bh))
goto unlock;
@@ -1229,7 +1235,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
dst->migrate_info = 0;
if (!folio_trylock(src)) {
- if (mode == MIGRATE_ASYNC)
+ if (migrate_is_async(mode))
goto out;
/*
@@ -1334,7 +1340,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
/* Establish migration ptes */
VM_BUG_ON_FOLIO(folio_test_anon(src) &&
!folio_test_ksm(src) && !anon_vma, src);
- try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
+ try_to_migrate(src, migrate_is_async(mode) ? TTU_BATCH_FLUSH : 0);
old_folio_state |= FOLIO_WAS_MAPPED;
}
@@ -1592,7 +1598,7 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f
{
int rc;
- if (mode == MIGRATE_ASYNC) {
+ if (migrate_is_async(mode)) {
if (!folio_trylock(folio))
return -EAGAIN;
} else {
@@ -1826,7 +1832,7 @@ static int migrate_pages_batch(struct list_head *from,
LIST_HEAD(dst_folios);
bool nosplit = (reason == MR_NUMA_MISPLACED);
- VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
+ VM_WARN_ON_ONCE(!migrate_is_async(mode) &&
!list_empty(from) && !list_is_singular(from));
for (pass = 0; pass < nr_pass && retry; pass++) {
@@ -2134,7 +2140,7 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
list_cut_before(&folios, from, &folio2->lru);
else
list_splice_init(from, &folios);
- if (mode == MIGRATE_ASYNC)
+ if (migrate_is_async(mode))
rc = migrate_pages_batch(&folios, get_new_folio, put_new_folio,
private, mode, reason, &ret_folios,
&split_folios, &stats,
@@ -2156,8 +2162,9 @@ int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
* with minimal effort, force MIGRATE_ASYNC mode and retry once.
*/
migrate_pages_batch(&split_folios, get_new_folio,
- put_new_folio, private, MIGRATE_ASYNC, reason,
- &ret_folios, NULL, &stats, 1);
+ put_new_folio, private,
+ migrate_is_async(mode) ? mode : MIGRATE_ASYNC,
+ reason, &ret_folios, NULL, &stats, 1);
list_splice_tail_init(&split_folios, &ret_folios);
}
rc_gather += rc;
--
2.56.0.398.g882cbad275-goog
next prev parent reply other threads:[~2026-11-03 22:31 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-07-30 15:02 [PATCH RFC v2 0/3] Demote to lower tier using non-temporal stores Yiannis Nikolakopoulos
2026-07-30 15:02 ` [PATCH RFC v2 1/3] mm, x86: support copying a folio " Yiannis Nikolakopoulos
2026-07-31 21:53 ` Huan Nguyen
2026-08-17 16:34 ` Garg, Shivank
2026-07-30 15:02 ` [PATCH RFC v2 2/3] mm: new migrate_mode flag for async " Yiannis Nikolakopoulos
2026-08-05 14:14 ` Huang, Ying
2026-07-30 15:02 ` [PATCH RFC v2 3/3] mm: use non-temporal stores for demotion Yiannis Nikolakopoulos
2026-07-30 17:30 ` [PATCH RFC v2 0/3] Demote to lower tier using non-temporal stores Gregory Price
2026-07-30 17:40 ` Frank van der Linden
2026-07-30 18:08 ` Johannes Weiner
2026-07-30 20:49 ` Gregory Price
2026-11-03 22:30 ` [RFC PATCH v3 0/3] mm: use non-temporal stores for demotion Huan Nguyen
2026-11-03 22:30 ` [RFC PATCH v3 1/3] mm, x86: support copying a folio using non-temporal stores Huan Nguyen
2026-11-03 22:30 ` Huan Nguyen [this message]
2026-11-03 22:30 ` [RFC PATCH v3 3/3] mm/vmscan: demote folios " Huan Nguyen
2026-11-04 0:09 ` [RFC PATCH v3 0/3] mm: use non-temporal stores for demotion Zi Yan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261103223025.2482227-3-nhuhuan@google.com \
--to=nhuhuan@google.com \
--cc=Liam.Howlett@oracle.com \
--cc=akpm@linux-foundation.org \
--cc=alirad.malek@zptcorp.com \
--cc=anna@kernel.org \
--cc=apopple@nvidia.com \
--cc=bharata@amd.com \
--cc=bp@alien8.de \
--cc=byungchul@sk.com \
--cc=dave.hansen@linux.intel.com \
--cc=dave@stgolabs.net \
--cc=david@kernel.org \
--cc=dimitrios.palyvos@zptcorp.com \
--cc=fvdl@google.com \
--cc=gourry@gourry.net \
--cc=hannes@cmpxchg.org \
--cc=hpa@zytor.com \
--cc=jackmanb@google.com \
--cc=jic23@kernel.org \
--cc=joshua.hahnjy@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-nfs@vger.kernel.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=ljs@kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=matthew.brost@intel.com \
--cc=mhiramat@kernel.org \
--cc=mhocko@suse.com \
--cc=mingo@redhat.com \
--cc=nifan.cxl@gmail.com \
--cc=rakie.kim@sk.com \
--cc=rientjes@google.com \
--cc=rkodsara@amd.com \
--cc=rostedt@goodmis.org \
--cc=rppt@kernel.org \
--cc=ryan.roberts@arm.com \
--cc=shivankg@amd.com \
--cc=sj@kernel.org \
--cc=surenb@google.com \
--cc=tglx@kernel.org \
--cc=trondmy@kernel.org \
--cc=vbabka@kernel.org \
--cc=weixugc@google.com \
--cc=x86@kernel.org \
--cc=xuezhengchu@huawei.com \
--cc=yiannis.nikolakop@gmail.com \
--cc=yiannis@zptcorp.com \
--cc=ying.huang@linux.alibaba.com \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®