[PATCH 3/6] mm/folio, mm/mlock: Don't batch folios in per-CPU LRU caches on isolated CPUs

[Matthew Ruffell <matthew.ruffell@canonical.com>]

From: Caron Wills <caron.wills@pulsar.com>

Folios are collected in per-CPU folio batches (cpu_fbatches and
mlock_fbatch) before being added to or moved on the LRU lists. Another
CPU can only empty such a batch by queueing lru_add_drain_per_cpu() on
the owning CPU and waiting for it in __lru_add_drain_all(), which is
what every lru_add_drain_all() caller (compaction, khugepaged, memory
hotplug, memfd, drop_caches, ...) and every lru_cache_disable() caller
(page migration through cpuset.mems, mbind() and move_pages(),
alloc_contig_range(), memory offlining) does.

On an isolated CPU that per-CPU kworker may not run for a very long
time, or ever: low latency deployments pin a SCHED_FIFO busy-polling
thread to each isolated CPU, and since RT bandwidth throttling is gone
nothing forces such a thread to yield to a SCHED_NORMAL kworker. Any
folio left in that CPU's batch then blocks the waiting caller.

LP: #2165410 hits exactly this. Writing cpuset.mems for a cgroup on a
192-CPU AWS instance with nohz_full CPUs 18-95,108-178 running FIFO
spinners makes cpuset_migrate_mm_workfn() -> do_migrate_pages() ->
lru_cache_disable() -> __lru_add_drain_all() -> flush_work() wait for
the kworkers on those CPUs, and PID 1 (systemd), which flushes the
migration workqueue from task_work before returning to userspace, stays
in D state for 2 to 10 minutes. logind cannot create sessions meanwhile
and SSH logins fail. The reproducer, a FIFO/80 thread per isolated CPU
that touches and MADV_DONTNEEDs a 24 MB buffer in a loop, keeps 1 to 14
folios in the lru_add batch of every isolated CPU at all times.

Stop batching on isolated CPUs: when the current CPU is isolated, move
the folio to the LRU immediately through an on-stack batch, without
touching the per-CPU batch at all. The latter matters: __lru_add_drain_all()
samples the per-CPU counters without the owning CPU's local_lock, so a
folio that sits in the batch even for the few instructions between
folio_batch_add() and an immediate drain is enough to get work queued on
that CPU. A first version of this patch reused the lru_cache_disabled()
add-then-drain path and was caught doing exactly that in testing.
cpu_needs_drain() now stays false for isolated CPUs and
__lru_add_drain_all() never queues work on them. The per-CPU buffer_head
LRU has been skipping isolated CPUs the same way since commit 8a237adf213d 
("fs/buffer.c: disable per-CPU buffer_head cache for isolated CPUs"),
and mm/memcontrol.c schedule_drain_work() does not queue memcg stock drains
on them either.

The cost is one lruvec lock acquisition per folio instead of one per
batch of 15 on isolated CPUs, which are not expected to do bulk memory
work. Nothing changes on housekeeping CPUs or on systems without CPU
isolation, where cpu_is_isolated() is a static-key guarded test.

Link: https://bugs.launchpad.net/bugs/2165410
Signed-off-by: Caron Wills <caron.wills@pulsar.com>
Signed-off-by: Matthew Ruffell <matthew.ruffell@canonical.com>
---
 mm/folio.c    | 21 ++++++++++++++++++++-
 mm/internal.h | 28 ++++++++++++++++++++++++++++
 mm/mlock.c    | 28 ++++++++++++++++++++++------
 3 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/mm/folio.c b/mm/folio.c
index 35e242b48870..7f85747910d7 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -208,6 +208,16 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
 	folios_put(fbatch);
 }
 
+/* Move a single folio to the LRU without touching the per-CPU batches. */
+static void folio_move_lru_now(struct folio *folio, move_fn_t move_fn)
+{
+	struct folio_batch fbatch;
+
+	folio_batch_init(&fbatch);
+	folio_batch_add(&fbatch, folio);
+	folio_batch_move_lru(&fbatch, move_fn);
+}
+
 static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
 		struct folio *folio, move_fn_t move_fn, bool disable_irq)
 {
@@ -220,7 +230,9 @@ static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
 	else
 		local_lock(&cpu_fbatches.lock);
 
-	if (!folio_batch_add(this_cpu_ptr(fbatch), folio) ||
+	if (lru_batching_bypassed())
+		folio_move_lru_now(folio, move_fn);
+	else if (!folio_batch_add(this_cpu_ptr(fbatch), folio) ||
 			!folio_may_be_lru_cached(folio) || lru_cache_disabled())
 		folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
 
@@ -856,6 +868,13 @@ static inline void __lru_add_drain_all(bool force_all_cpus)
 	WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
 	smp_mb();
 
+	/*
+	 * Isolated CPUs don't batch folios (see lru_batching_bypassed()), so
+	 * their counters stay at zero and no work gets queued to them.  This
+	 * matters because the per-CPU kworker of an isolated CPU may not get
+	 * to run for a very long time, if ever, and flush_work() below would
+	 * wait for it.
+	 */
 	cpumask_clear(&has_work);
 	for_each_online_cpu(cpu) {
 		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
diff --git a/mm/internal.h b/mm/internal.h
index 566cbd2dce48..5888f7b5ea5c 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -17,6 +17,7 @@
 #include <linux/pagewalk.h>
 #include <linux/rmap.h>
 #include <linux/swap.h>
+#include <linux/sched/isolation.h>
 #include <linux/leafops.h>
 #include <linux/tracepoint-defs.h>
 
@@ -1006,6 +1007,33 @@ bool need_mlock_drain(int cpu);
 void mlock_drain_local(void);
 void mlock_drain_remote(int cpu);
 
+/*
+ * Folios are normally parked in per-CPU batches (cpu_fbatches and
+ * mlock_fbatch) before they are moved to the LRU lists. A remote CPU can
+ * only empty such a batch by queueing lru_add_drain_per_cpu() on the owning
+ * CPU and waiting for it, see __lru_add_drain_all(). On an isolated CPU that
+ * kworker may not get to run for a long time, e.g. because a SCHED_FIFO task
+ * busy-loops there, and every lru_add_drain_all() or lru_cache_disable()
+ * caller in the system then blocks behind it.
+ *
+ * Isolated CPUs therefore bypass the per-CPU batches entirely and move
+ * folios to the LRU lists right away, through an on-stack batch. The folio
+ * must not even transiently sit in the per-CPU batch: __lru_add_drain_all()
+ * samples the batch counters without the owning CPU's lock, so the few
+ * instructions between folio_batch_add() and an immediate drain are enough
+ * to get work queued on the isolated CPU. The per-CPU buffer_head LRU
+ * (bh_lru_install()) and the memcg stock drain (schedule_drain_work())
+ * already stay away from isolated CPUs the same way.
+ *
+ * Callers hold the batch's local_lock, so smp_processor_id() is stable. The
+ * housekeeping cpumask consulted by cpu_is_isolated() is RCU protected.
+ */
+static inline bool lru_batching_bypassed(void)
+{
+	guard(rcu)();
+	return cpu_is_isolated(smp_processor_id());
+}
+
 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
 
 static inline unsigned long __vma_address(const struct vm_area_struct *vma,
diff --git a/mm/mlock.c b/mm/mlock.c
index 67ae9b18b923..3bb8195e5c5f 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -209,6 +209,16 @@ static void mlock_folio_batch(struct folio_batch *fbatch)
 	folios_put(fbatch);
 }
 
+/* Handle a single folio right away, without touching the per-CPU batch. */
+static void mlock_folio_now(struct folio *folio)
+{
+	struct folio_batch fbatch;
+
+	folio_batch_init(&fbatch);
+	folio_batch_add(&fbatch, folio);
+	mlock_folio_batch(&fbatch);
+}
+
 void mlock_drain_local(void)
 {
 	struct folio_batch *fbatch;
@@ -254,8 +264,10 @@ void mlock_folio(struct folio *folio)
 	}
 
 	folio_get(folio);
-	if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
-	    !folio_may_be_lru_cached(folio) || lru_cache_disabled())
+	if (lru_batching_bypassed())
+		mlock_folio_now(mlock_lru(folio));
+	else if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
+		 !folio_may_be_lru_cached(folio) || lru_cache_disabled())
 		mlock_folio_batch(fbatch);
 	local_unlock(&mlock_fbatch.lock);
 }
@@ -277,8 +289,10 @@ void mlock_new_folio(struct folio *folio)
 	__count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
 
 	folio_get(folio);
-	if (!folio_batch_add(fbatch, mlock_new(folio)) ||
-	    !folio_may_be_lru_cached(folio) || lru_cache_disabled())
+	if (lru_batching_bypassed())
+		mlock_folio_now(mlock_new(folio));
+	else if (!folio_batch_add(fbatch, mlock_new(folio)) ||
+		 !folio_may_be_lru_cached(folio) || lru_cache_disabled())
 		mlock_folio_batch(fbatch);
 	local_unlock(&mlock_fbatch.lock);
 }
@@ -298,8 +312,10 @@ void munlock_folio(struct folio *folio)
 	 * which will check whether the folio is multiply mlocked.
 	 */
 	folio_get(folio);
-	if (!folio_batch_add(fbatch, folio) ||
-	    !folio_may_be_lru_cached(folio) || lru_cache_disabled())
+	if (lru_batching_bypassed())
+		mlock_folio_now(folio);
+	else if (!folio_batch_add(fbatch, folio) ||
+		 !folio_may_be_lru_cached(folio) || lru_cache_disabled())
 		mlock_folio_batch(fbatch);
 	local_unlock(&mlock_fbatch.lock);
 }
-- 
2.55.0