[PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk

[PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk

[Joshua Machauer]

From: Joshua Machauer <machaujo@amazon.com>

On systems that give every CPU its own redistributor frame, a CPU coming
online finds its frame by walking the frames from the first one until
GICR_TYPER matches its affinity. CPU n therefore performs O(n) MMIO
reads, and bringing up N CPUs costs O(N^2). Secondary CPUs boot
serially, so all of it adds to boot time.

gic_update_rdist_properties() already walks every frame once at boot.
Record each possible CPU's frame there and have gic_populate_rdist() try
it first, so bringing a CPU online costs one MMIO read. A CPU without a
recorded frame, such as an ACPI online-capable CPU absent at init, still
scans.

Recording compares each frame's affinity against that of every possible
CPU. That is still O(N^2) work at init, but it reads cpu_logical_map[],
which stays in the cache, instead of device memory.

The frame is kept apart from rd_base, which
inherit_vpe_l1_table_from_rd() reads as "this CPU has booted".

NVIDIA Vera, 352 CPUs in two redistributor regions, 69 ms faster.
Without this patch:

  [    0.007006] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.463026] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000

With it:

  [    0.007794] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.394395] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000

KVM guest with 352 vCPUs on that Vera system, where every GICR_TYPER
read traps to the hypervisor, 139 ms faster. Without this patch:

  [    0.023231] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.432895] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

With it:

  [    0.023456] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.293807] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

Graviton4, 192 CPUs with one redistributor region each, 5.6 ms faster.
Without this patch:

  [    0.009300] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
  [    0.091853] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000

With it:

  [    0.009343] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
  [    0.086299] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000

KVM guest with 192 vCPUs on that Graviton4 system, 33 ms faster.
Without this patch:

  [    0.008751] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
  [    0.115862] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000

With it:

  [    0.009036] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
  [    0.082898] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000

Assisted-by: LLM
Signed-off-by: Joshua Machauer <machaujo@amazon.de>
---
 drivers/irqchip/irq-gic-v3.c | 66 ++++++++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 11 deletions(-)

diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 6e1fa5b247fc..8f1007facda9 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1017,25 +1017,43 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 	return ret ? -ENODEV : 0;
 }
 
+/*
+ * Convert a CPU's affinity to a 32bit value that can be matched to
+ * GICR_TYPER bits [63:32].
+ */
+static u32 gic_cpu_to_rdist_aff(int cpu)
+{
+	u64 mpidr = gic_cpu_to_affinity(cpu);
+
+	return (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 0));
+}
+
+/*
+ * The redistributor frame of each possible CPU, as found by the walk in
+ * gic_update_rdist_properties(). A CPU coming up checks this frame first
+ * instead of scanning every frame from the start, which on a system with
+ * one redistributor per CPU makes bringing up N CPUs cost O(N^2) MMIO
+ * reads.
+ */
+struct gic_rdist_frame {
+	struct redist_region	*region;
+	void __iomem		*ptr;
+};
+
+static DEFINE_PER_CPU(struct gic_rdist_frame, gic_rdist_frame);
+
 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 {
 	unsigned long mpidr;
 	u64 typer;
-	u32 aff;
 
-	/*
-	 * Convert affinity to a 32bit value that can be matched to
-	 * GICR_TYPER bits [63:32].
-	 */
 	mpidr = gic_cpu_to_affinity(smp_processor_id());
 
-	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
-
 	typer = gic_read_typer(ptr + GICR_TYPER);
-	if ((typer >> 32) == aff) {
+	if ((typer >> 32) == gic_cpu_to_rdist_aff(smp_processor_id())) {
 		u64 offset = ptr - region->redist_base;
 		raw_spin_lock_init(&gic_data_rdist()->rd_lock);
 		gic_data_rdist_rd_base() = ptr;
@@ -1054,6 +1072,16 @@ static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 
 static int gic_populate_rdist(void)
 {
+	struct gic_rdist_frame *frame = this_cpu_ptr(&gic_rdist_frame);
+
+	/*
+	 * A CPU that was not present when the frames were walked, such as an
+	 * ACPI online-capable CPU added later, has no recorded frame and falls
+	 * back to the scan.
+	 */
+	if (frame->ptr && !__gic_populate_rdist(frame->region, frame->ptr))
+		return 0;
+
 	if (gic_iterate_rdists(__gic_populate_rdist) == 0)
 		return 0;
 
@@ -1064,12 +1092,28 @@ static int gic_populate_rdist(void)
 	return -ENODEV;
 }
 
+static void gic_record_rdist_frame(struct redist_region *region,
+				   void __iomem *ptr, u64 typer)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		if (gic_cpu_to_rdist_aff(cpu) == (typer >> 32)) {
+			per_cpu(gic_rdist_frame, cpu).region = region;
+			per_cpu(gic_rdist_frame, cpu).ptr = ptr;
+			return;
+		}
+	}
+}
+
 static int __gic_update_rdist_properties(struct redist_region *region,
 					 void __iomem *ptr)
 {
 	u64 typer = gic_read_typer(ptr + GICR_TYPER);
 	u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
 
+	gic_record_rdist_frame(region, ptr, typer);
+
 	/* Boot-time cleanup */
 	if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
 		u64 val;
-- 
2.47.3