[PATCH 0/2] x86/sev: Per-node RMPOPT windows for >2TB systems

[PATCH 0/2] x86/sev: Per-node RMPOPT windows for >2TB systems

[Ashish Kalra]

From: Ashish Kalra <ashish.kalra@amd.com>

RMPOPT_BASE is a per-core MSR whose optimization window covers at most 2 TB
of physical address space from its base.  The merged RMPOPT series programs
every core with the same base (start of RAM), so on a machine with more
than 2 TB of RAM only the first 2 TB is ever RMP-check-skipped; everything
above 2 TB pays full RMP checks for the life of the system.

This series extends the optimization across all of physical memory on >2 TB
systems by giving each NUMA node's cores a 2 TB window placed over that
node's own local memory.  A core overwhelmingly accesses its local node's
RAM, so spending each node's 2 TB budget on that memory puts the
optimization where the accesses land; whole-machine coverage is the union
of the per-node windows.  The <=2 TB case is unchanged: the window search
returns start-of-RAM for every node, so all cores share one window.

  Patch 1 generalizes the optimization scan to read each core's own
    RMPOPT_BASE (base and window size straight from the MSR) and walk only
    that window.  No behavioral change while every core shares one window.
  Patch 2 adds the per-node window selection and programming, and warms
    every window before the interrupts-disabled fan-out.

HW-validated on a ~3 TB 2-socket NPS2 box: every node optimizes its own
local memory and all populated memory is covered.

Ashish Kalra (2):
  x86/sev: Scan each core's own RMPOPT window
  x86/sev: Add per-node RMPOPT windows for >2TB systems

 arch/x86/include/asm/msr-index.h |   2 +
 arch/x86/virt/svm/sev.c          | 166 +++++++++++++++++++++++++++----
 2 files changed, 148 insertions(+), 20 deletions(-)


base-commit: 9f74092fda91ef93c2b07585785c1dc83cd7971f
-- 
2.43.0

[PATCH 1/2] x86/sev: Scan each core's own RMPOPT window

[Ashish Kalra]

From: Ashish Kalra <ashish.kalra@amd.com>

rmpopt_scan_range() walked a single global [rmpopt_pa_start, rmpopt_pa_end]
range, which is only correct while every core shares one window.  Read the
executing core's own RMPOPT_BASE instead and walk only that window, so each
core optimizes the window its own RMPOPT_BASE points at.

The window is described entirely by the RMPOPT_BASE MSR: RMPOPT_BASE_ADDR
(bits 51:30) is the 1 GB-aligned base and RMPOPT_TABLE_SIZE (bits 22:1) is
the window size in GB.  Walk [base, base + size), clamped to the top of
populated memory.

Signed-off-by: Ashish Kalra <ashish.kalra@amd.com>
---
 arch/x86/include/asm/msr-index.h |  2 ++
 arch/x86/virt/svm/sev.c          | 29 +++++++++++++++--------------
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index c0914829a925..6d453bb13621 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -773,6 +773,8 @@
 #define MSR_AMD64_RMPOPT_BASE		0xc0010139
 #define MSR_AMD64_RMPOPT_ENABLE_BIT	0
 #define MSR_AMD64_RMPOPT_ENABLE		BIT_ULL(MSR_AMD64_RMPOPT_ENABLE_BIT)
+#define MSR_AMD64_RMPOPT_TABLE_SIZE	GENMASK_ULL(22, 1)
+#define MSR_AMD64_RMPOPT_BASE_ADDR	GENMASK_ULL(51, 30)
 
 #define MSR_SVSM_CAA			0xc001f000
 
diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index 4de9681cc2f8..a2756b18d5f7 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -8,6 +8,7 @@
  *
  */
 
+#include <linux/bitfield.h>
 #include <linux/cc_platform.h>
 #include <linux/printk.h>
 #include <linux/mm_types.h>
@@ -125,8 +126,6 @@ static void *rmp_bookkeeping __ro_after_init;
 
 static u64 probed_rmp_base, probed_rmp_size;
 
-static u64 rmpopt_pa_start, rmpopt_pa_end;
-
 enum rmpopt_op_type {
 	RMPOPT_OP_VERIFY_AND_REPORT_STATUS,
 	RMPOPT_OP_REPORT_STATUS
@@ -618,11 +617,18 @@ static void rmpopt(u64 pa)
 		     : "memory", "cc");
 }
 
+/* A core can only RMPOPT within its own window, [base, base + size). */
 static void rmpopt_scan_range(void *arg)
 {
-	u64 pa;
+	u64 msr, base, end, pa;
+	unsigned int size_gb;
+
+	rdmsrq(MSR_AMD64_RMPOPT_BASE, msr);
+	base = msr & MSR_AMD64_RMPOPT_BASE_ADDR;
+	size_gb = FIELD_GET(MSR_AMD64_RMPOPT_TABLE_SIZE, msr);
+	end = min_t(u64, base + (u64)size_gb * SZ_1G, ALIGN(PFN_PHYS(max_pfn), SZ_1G));
 
-	for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G)
+	for (pa = base; pa < end; pa += SZ_1G)
 		rmpopt(pa);
 }
 
@@ -650,12 +656,6 @@ static int __init rmpopt_init(void)
 
 	INIT_DELAYED_WORK(&rmpopt_delayed_work, do_rmpopt_work);
 
-	/* The optimization range is fixed at boot; compute it once. */
-	rmpopt_pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G);
-	rmpopt_pa_end = ALIGN(PFN_PHYS(max_pfn), SZ_1G);
-	if ((rmpopt_pa_end - rmpopt_pa_start) > SZ_2T)
-		rmpopt_pa_end = rmpopt_pa_start + SZ_2T;
-
 	pr_info("RMPOPT optimizations enabled\n");
 
 	rmpopt_soft_init = true;
@@ -665,7 +665,7 @@ static int __init rmpopt_init(void)
 
 void snp_enable_rmpopt(void)
 {
-	u64 base;
+	u64 base, pa_start;
 	int cpu;
 
 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
@@ -678,14 +678,15 @@ void snp_enable_rmpopt(void)
 		return;
 
 	/*
-	 * Per-CPU RMPOPT tables cover at most 2 TB.  Program each core's
-	 * RMPOPT_BASE with the start of RAM to optimize up to 2 TB.
+	 * A core's RMPOPT_BASE covers at most 2 TB.  Program every core with
+	 * the start of RAM to optimize up to 2 TB.
 	 */
+	pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G);
 	rdmsrq(MSR_AMD64_RMPOPT_BASE, base);
 	if (!(base & MSR_AMD64_RMPOPT_ENABLE))
 		for_each_cpu(cpu, cpu_primary_thread_mask)
 			wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE,
-				      rmpopt_pa_start | MSR_AMD64_RMPOPT_ENABLE);
+				      pa_start | MSR_AMD64_RMPOPT_ENABLE);
 
 	mod_delayed_work(rmpopt_wq, &rmpopt_delayed_work,
 			 msecs_to_jiffies(RMPOPT_WORK_TIMEOUT));
-- 
2.43.0

[PATCH 2/2] x86/sev: Add per-node RMPOPT windows for >2TB systems

[Ashish Kalra]

From: Ashish Kalra <ashish.kalra@amd.com>

A core's RMPOPT_BASE covers at most 2 TB of physical address space, so on
a system with more than 2 TB of RAM a single shared base leaves everything
above 2 TB unoptimized.  Give each NUMA node its own window, placed over
that node's local memory, and program it on that node's cores.  A core
overwhelmingly accesses its local node, so spending each node's 2 TB budget
on its own memory puts the optimization where the accesses land;
whole-machine coverage is the union of the per-node windows.

rmpopt_find_best_window() picks each node's base with a sliding-window
search: it slides a 2 TB window across physical memory a 1 GB at a time and
keeps the 1 GB-aligned base whose window best covers that node's own
memory.  When all of RAM fits in a single window the search returns the
start of RAM, so all cores share one window as before.  Because the
per-node search is bounded to the populated physical address range and
accounts for unpopulated regions explicitly, a node's window never falls
off the end of RAM into empty physical address space.

The warm-up then covers every window.  A <=2TB system has the single
shared window and is warmed once; a >2TB system warms each node's window in
parallel on a node-local core via schedule_work_on().  Either way the
warm-up runs with interrupts enabled, so the expensive first (uncached)
pass never runs in the interrupts-disabled fan-out that follows.

Signed-off-by: Ashish Kalra <ashish.kalra@amd.com>
---
 arch/x86/virt/svm/sev.c | 149 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 137 insertions(+), 12 deletions(-)

diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index a2756b18d5f7..1ed1fba3b62d 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -134,6 +134,9 @@ enum rmpopt_op_type {
 static struct workqueue_struct *rmpopt_wq;
 static struct delayed_work rmpopt_delayed_work;
 
+/* Per-node warm-up work, used to warm each node's window in parallel (>2TB). */
+static struct work_struct *rmpopt_node_work;
+
 /* Software RMPOPT facilities initialized */
 static bool rmpopt_soft_init;
 
@@ -146,6 +149,9 @@ static bool rmpopt_soft_init;
  */
 #define RMPOPT_WORK_TIMEOUT	(10 * MSEC_PER_SEC)
 
+#define RMPOPT_GB_WINDOW_SIZE	(SZ_2T / SZ_1G)		/* 2 TB, in 1 GB units */
+#define RMPOPT_EMPTY_DISTANCE	255			/* SLIT "unreachable" */
+
 static LIST_HEAD(snp_leaked_pages_list);
 static DEFINE_SPINLOCK(snp_leaked_pages_list_lock);
 
@@ -632,19 +638,125 @@ static void rmpopt_scan_range(void *arg)
 		rmpopt(pa);
 }
 
+/* NUMA node backing the 1 GB region at index @gb, or NUMA_NO_NODE if empty. */
+static int rmpopt_gb_to_node(u64 gb)
+{
+	u64 pfn = (gb * SZ_1G) >> PAGE_SHIFT;
+	int nid;
+
+	for_each_online_node(nid) {
+		if (pfn >= node_start_pfn(nid) && pfn < node_end_pfn(nid))
+			return nid;
+	}
+
+	return NUMA_NO_NODE;
+}
+
+/* NUMA distance from @node to the 1 GB region at index @gb. */
+static unsigned int rmpopt_gb_distance(int node, u64 gb)
+{
+	int nid = rmpopt_gb_to_node(gb);
+
+	return nid == NUMA_NO_NODE ? RMPOPT_EMPTY_DISTANCE
+				   : node_distance(node, nid);
+}
+
+/* True when all of RAM fits in one window, so every node shares it. */
+static bool rmpopt_single_window(void)
+{
+	u64 first_gb = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G) / SZ_1G;
+	u64 last_gb = ALIGN(PFN_PHYS(max_pfn), SZ_1G) / SZ_1G;
+
+	return last_gb - first_gb <= RMPOPT_GB_WINDOW_SIZE;
+}
+
+/*
+ * Return the 1 GB-aligned RMPOPT_BASE for @node: slide a window across physical
+ * memory and keep the base whose window covers the most of @node's own memory.
+ * When all of RAM fits in one window, use the start of RAM.
+ */
+static u64 rmpopt_find_best_window(int node)
+{
+	u64 first_gb = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G) / SZ_1G;
+	unsigned long own = 0, dist = 0, best_own, best_dist;
+	u64 base_gb, best_base_gb, gb, last_gb;
+
+	if (rmpopt_single_window())
+		return first_gb * SZ_1G;
+
+	last_gb = ALIGN(PFN_PHYS(max_pfn), SZ_1G) / SZ_1G;
+
+	/* Prime the first window. */
+	for (gb = first_gb; gb < first_gb + RMPOPT_GB_WINDOW_SIZE; gb++) {
+		if (rmpopt_gb_to_node(gb) == node)
+			own++;
+		dist += rmpopt_gb_distance(node, gb);
+	}
+	best_own = own;
+	best_dist = dist;
+	best_base_gb = first_gb;
+
+	for (base_gb = first_gb + 1; base_gb < last_gb; base_gb++) {
+		/*
+		 * Sliding-window optimization: the window advanced by 1 GB, so
+		 * rather than rescan it, drop the region at @leave that fell off
+		 * the bottom and add the region at @enter that came in at the top.
+		 */
+		u64 leave = base_gb - 1;
+		u64 enter = base_gb + RMPOPT_GB_WINDOW_SIZE - 1;
+
+		if (rmpopt_gb_to_node(leave) == node)
+			own--;
+		if (rmpopt_gb_to_node(enter) == node)
+			own++;
+		dist -= rmpopt_gb_distance(node, leave);
+		dist += rmpopt_gb_distance(node, enter);
+
+		if (own > best_own || (own == best_own && dist < best_dist)) {
+			best_own = own;
+			best_dist = dist;
+			best_base_gb = base_gb;
+		}
+	}
+
+	return best_base_gb * SZ_1G;
+}
+
+static void rmpopt_warmup_fn(struct work_struct *work)
+{
+	rmpopt_scan_range(NULL);
+}
+
 static void do_rmpopt_work(struct work_struct *work)
 {
+	unsigned int cpu;
+	int nid;
+
 	/*
-	 * Warm up the RMPOPT cache on this pinned per-CPU worker with interrupts
-	 * enabled, so the IRQ-disabled fan-out below only issues cache-hit RMPOPTs.
+	 * Warm up each window on a node-local core with interrupts enabled, so
+	 * the expensive first (uncached) pass never runs in the interrupts-
+	 * disabled fan-out below.  A <=2TB system shares one window and is warmed
+	 * once; a >2TB system warms each node's window in parallel.
 	 */
-	rmpopt_scan_range(NULL);
+	if (rmpopt_single_window()) {
+		rmpopt_scan_range(NULL);
+	} else {
+		for_each_online_node(nid) {
+			cpu = cpumask_first_and(cpu_primary_thread_mask, cpumask_of_node(nid));
+			if (cpu < nr_cpu_ids)
+				schedule_work_on(cpu, &rmpopt_node_work[nid]);
+		}
+		for_each_online_node(nid)
+			flush_work(&rmpopt_node_work[nid]);
+	}
 
 	on_each_cpu_mask(cpu_primary_thread_mask, rmpopt_scan_range, NULL, true);
 }
 
 static int __init rmpopt_init(void)
 {
+	int nid;
+
 	if (!cpu_feature_enabled(X86_FEATURE_RMPOPT))
 		return -ENODEV;
 
@@ -654,6 +766,14 @@ static int __init rmpopt_init(void)
 		return -ENOMEM;
 	}
 
+	rmpopt_node_work = kcalloc(nr_node_ids, sizeof(*rmpopt_node_work), GFP_KERNEL);
+	if (!rmpopt_node_work) {
+		destroy_workqueue(rmpopt_wq);
+		return -ENOMEM;
+	}
+	for_each_online_node(nid)
+		INIT_WORK(&rmpopt_node_work[nid], rmpopt_warmup_fn);
+
 	INIT_DELAYED_WORK(&rmpopt_delayed_work, do_rmpopt_work);
 
 	pr_info("RMPOPT optimizations enabled\n");
@@ -665,8 +785,9 @@ static int __init rmpopt_init(void)
 
 void snp_enable_rmpopt(void)
 {
-	u64 base, pa_start;
-	int cpu;
+	unsigned int cpu;
+	u64 base;
+	int nid;
 
 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
 		return;
@@ -678,15 +799,19 @@ void snp_enable_rmpopt(void)
 		return;
 
 	/*
-	 * A core's RMPOPT_BASE covers at most 2 TB.  Program every core with
-	 * the start of RAM to optimize up to 2 TB.
+	 * A core's RMPOPT_BASE covers at most 2 TB.  Give each NUMA node the
+	 * window that best covers its own memory and program it on that node's
+	 * cores; for <=2TB systems every node resolves to the start of RAM.
 	 */
-	pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G);
 	rdmsrq(MSR_AMD64_RMPOPT_BASE, base);
-	if (!(base & MSR_AMD64_RMPOPT_ENABLE))
-		for_each_cpu(cpu, cpu_primary_thread_mask)
-			wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE,
-				      pa_start | MSR_AMD64_RMPOPT_ENABLE);
+	if (!(base & MSR_AMD64_RMPOPT_ENABLE)) {
+		for_each_online_node(nid) {
+			u64 nbase = rmpopt_find_best_window(nid) | MSR_AMD64_RMPOPT_ENABLE;
+
+			for_each_cpu_and(cpu, cpu_primary_thread_mask, cpumask_of_node(nid))
+				wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE, nbase);
+		}
+	}
 
 	mod_delayed_work(rmpopt_wq, &rmpopt_delayed_work,
 			 msecs_to_jiffies(RMPOPT_WORK_TIMEOUT));
-- 
2.43.0