mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
* [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk
@ 2026-10-23 18:15 Joshua Machauer
  2026-10-25 10:48 ` Marc Zyngier
  0 siblings, 1 reply; 3+ messages in thread
From: Joshua Machauer @ 2026-10-23 18:15 UTC (permalink / raw)
  To: Marc Zyngier, Thomas Gleixner
  Cc: Radu Rendec, linux-arm-kernel, linux-kernel, nh-open-source,
	Joshua Machauer

From: Joshua Machauer <machaujo@amazon.com>

On systems that give every CPU its own redistributor frame, a CPU coming
online finds its frame by walking the frames from the first one until
GICR_TYPER matches its affinity. CPU n therefore performs O(n) MMIO
reads, and bringing up N CPUs costs O(N^2). Secondary CPUs boot
serially, so all of it adds to boot time.

gic_update_rdist_properties() already walks every frame once at boot.
Record each possible CPU's frame there and have gic_populate_rdist() try
it first, so bringing a CPU online costs one MMIO read. A CPU without a
recorded frame, such as an ACPI online-capable CPU absent at init, still
scans.

Recording compares each frame's affinity against that of every possible
CPU. That is still O(N^2) work at init, but it reads cpu_logical_map[],
which stays in the cache, instead of device memory.

The frame is kept apart from rd_base, which
inherit_vpe_l1_table_from_rd() reads as "this CPU has booted".

NVIDIA Vera, 352 CPUs in two redistributor regions, 69 ms faster.
Without this patch:

  [    0.007006] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.463026] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000

With it:

  [    0.007794] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.394395] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000

KVM guest with 352 vCPUs on that Vera system, where every GICR_TYPER
read traps to the hypervisor, 139 ms faster. Without this patch:

  [    0.023231] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.432895] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

With it:

  [    0.023456] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.293807] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

Graviton4, 192 CPUs with one redistributor region each, 5.6 ms faster.
Without this patch:

  [    0.009300] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
  [    0.091853] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000

With it:

  [    0.009343] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
  [    0.086299] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000

KVM guest with 192 vCPUs on that Graviton4 system, 33 ms faster.
Without this patch:

  [    0.008751] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
  [    0.115862] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000

With it:

  [    0.009036] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
  [    0.082898] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000

Assisted-by: LLM
Signed-off-by: Joshua Machauer <machaujo@amazon.de>
---
 drivers/irqchip/irq-gic-v3.c | 66 ++++++++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 11 deletions(-)

diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 6e1fa5b247fc..8f1007facda9 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1017,25 +1017,43 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 	return ret ? -ENODEV : 0;
 }
 
+/*
+ * Convert a CPU's affinity to a 32bit value that can be matched to
+ * GICR_TYPER bits [63:32].
+ */
+static u32 gic_cpu_to_rdist_aff(int cpu)
+{
+	u64 mpidr = gic_cpu_to_affinity(cpu);
+
+	return (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 0));
+}
+
+/*
+ * The redistributor frame of each possible CPU, as found by the walk in
+ * gic_update_rdist_properties(). A CPU coming up checks this frame first
+ * instead of scanning every frame from the start, which on a system with
+ * one redistributor per CPU makes bringing up N CPUs cost O(N^2) MMIO
+ * reads.
+ */
+struct gic_rdist_frame {
+	struct redist_region	*region;
+	void __iomem		*ptr;
+};
+
+static DEFINE_PER_CPU(struct gic_rdist_frame, gic_rdist_frame);
+
 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 {
 	unsigned long mpidr;
 	u64 typer;
-	u32 aff;
 
-	/*
-	 * Convert affinity to a 32bit value that can be matched to
-	 * GICR_TYPER bits [63:32].
-	 */
 	mpidr = gic_cpu_to_affinity(smp_processor_id());
 
-	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
-
 	typer = gic_read_typer(ptr + GICR_TYPER);
-	if ((typer >> 32) == aff) {
+	if ((typer >> 32) == gic_cpu_to_rdist_aff(smp_processor_id())) {
 		u64 offset = ptr - region->redist_base;
 		raw_spin_lock_init(&gic_data_rdist()->rd_lock);
 		gic_data_rdist_rd_base() = ptr;
@@ -1054,6 +1072,16 @@ static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 
 static int gic_populate_rdist(void)
 {
+	struct gic_rdist_frame *frame = this_cpu_ptr(&gic_rdist_frame);
+
+	/*
+	 * A CPU that was not present when the frames were walked, such as an
+	 * ACPI online-capable CPU added later, has no recorded frame and falls
+	 * back to the scan.
+	 */
+	if (frame->ptr && !__gic_populate_rdist(frame->region, frame->ptr))
+		return 0;
+
 	if (gic_iterate_rdists(__gic_populate_rdist) == 0)
 		return 0;
 
@@ -1064,12 +1092,28 @@ static int gic_populate_rdist(void)
 	return -ENODEV;
 }
 
+static void gic_record_rdist_frame(struct redist_region *region,
+				   void __iomem *ptr, u64 typer)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		if (gic_cpu_to_rdist_aff(cpu) == (typer >> 32)) {
+			per_cpu(gic_rdist_frame, cpu).region = region;
+			per_cpu(gic_rdist_frame, cpu).ptr = ptr;
+			return;
+		}
+	}
+}
+
 static int __gic_update_rdist_properties(struct redist_region *region,
 					 void __iomem *ptr)
 {
 	u64 typer = gic_read_typer(ptr + GICR_TYPER);
 	u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
 
+	gic_record_rdist_frame(region, ptr, typer);
+
 	/* Boot-time cleanup */
 	if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
 		u64 val;
-- 
2.47.3


^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk
  2026-10-23 18:15 [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk Joshua Machauer
@ 2026-10-25 10:48 ` Marc Zyngier
  2026-11-19 18:42   ` [PATCH v2] irqchip/gic-v3: Fill in " Joshua Machauer
  0 siblings, 1 reply; 3+ messages in thread
From: Marc Zyngier @ 2026-10-25 10:48 UTC (permalink / raw)
  To: Joshua Machauer
  Cc: Thomas Gleixner, Radu Rendec, linux-arm-kernel, linux-kernel,
	nh-open-source, Joshua Machauer

On Fri, 23 Oct 2026 19:15:10 +0100,
Joshua Machauer <machaujo@amazon.de> wrote:
> 
> From: Joshua Machauer <machaujo@amazon.com>
> 
> On systems that give every CPU its own redistributor frame, a CPU coming

Is there *any* sort of GICv3-based systems that do not match this
description?

> online finds its frame by walking the frames from the first one until
> GICR_TYPER matches its affinity. CPU n therefore performs O(n) MMIO
> reads, and bringing up N CPUs costs O(N^2). Secondary CPUs boot
> serially, so all of it adds to boot time.
> 
> gic_update_rdist_properties() already walks every frame once at boot.
> Record each possible CPU's frame there and have gic_populate_rdist() try
> it first, so bringing a CPU online costs one MMIO read. A CPU without a
> recorded frame, such as an ACPI online-capable CPU absent at init, still
> scans.
> 
> Recording compares each frame's affinity against that of every possible
> CPU. That is still O(N^2) work at init, but it reads cpu_logical_map[],
> which stays in the cache, instead of device memory.
> 
> The frame is kept apart from rd_base, which
> inherit_vpe_l1_table_from_rd() reads as "this CPU has booted".

But this is a pointless duplication of the existing tracking, which
could equally be solved with a flag.

I'm not opposed to this, even if the gains are vanishingly small. But
it can't come at the cost of duplicated state.

Thanks,

	M.

-- 
Without deviation from the norm, progress is not possible.

^ permalink raw reply	[flat|nested] 3+ messages in thread

* [PATCH v2] irqchip/gic-v3: Fill in each CPU's redistributor from the init walk
  2026-10-25 10:48 ` Marc Zyngier
@ 2026-11-19 18:42   ` Joshua Machauer
  0 siblings, 0 replies; 3+ messages in thread
From: Joshua Machauer @ 2026-11-19 18:42 UTC (permalink / raw)
  To: Marc Zyngier, Thomas Gleixner
  Cc: Radu Rendec, linux-arm-kernel, linux-kernel, nh-open-source

A CPU coming online finds its redistributor by walking the frames from
the first one until GICR_TYPER matches its affinity. CPU n therefore
performs O(n) MMIO reads, and bringing up N CPUs costs O(N^2).
Secondary CPUs boot serially, so all of it adds to boot time.

gic_update_rdist_properties() already walks every frame once at boot.
Give each frame to the possible CPU it belongs to there, by setting that
CPU's rd_base and phys_base and initialising its rd_lock. A CPU coming
online then finds its frame assigned and does no MMIO at all. A CPU
absent at init, such as an ACPI online-capable CPU, still scans.

Finding the CPU compares each frame's affinity against that of every
possible CPU. That is still O(N^2) work at init, but it reads
cpu_logical_map[], which stays in the cache, instead of device memory.

inherit_vpe_l1_table_from_rd() and allocate_vpe_l2_table() take a
non-NULL rd_base to mean that the CPU has come up. Mark that with a new
RD_LOCAL_POPULATED flag instead, set in gic_populate_rdist(), and move
the RD_LOCAL_* flags to irq-gic-common.h so that both drivers see them.

NVIDIA Vera, 352 CPUs in two redistributor regions, 73 ms faster.
Without this patch:

  [    0.007441] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.873847] GICv3: CPU351: found redistributor 100600001 region 1:0x0000fe40430c0000

With it:

  [    0.006401] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
  [    0.800722] GICv3: CPU351: found redistributor 100600001 region 1:0x0000fe40430c0000

KVM guest with 352 vCPUs on that Vera system, where every GICR_TYPER
read traps to the hypervisor, 139 ms faster. Without this patch:

  [    0.023483] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.433820] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

With it:

  [    0.023817] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
  [    0.295205] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000

Signed-off-by: Joshua Machauer <machaujo@amazon.de>
---
An LLM coding assistant drafted the code and this changelog and ran the
boot measurements. I reviewed and edited the result.

v2:
  - Fill in rd_base and phys_base instead of keeping a separate per-CPU
    copy of the frame (Marc)
  - Add RD_LOCAL_POPULATED for the two users that took a non-NULL
    rd_base to mean the CPU has come up
  - Initialise rd_lock once, where the frame is assigned, instead of
    each time the CPU comes online
  - Drop the qualifier about systems with one redistributor per CPU,
    which is every GICv3 system (Marc)
  - Rebase onto tip irq/core (v7.4-rc1)
  - Trim the test results to the NVIDIA Vera runs, remeasured on v2
---
 drivers/irqchip/irq-gic-common.h |   7 ++
 drivers/irqchip/irq-gic-v3-its.c |  11 ++-
 drivers/irqchip/irq-gic-v3.c     | 113 +++++++++++++++++++++----------
 3 files changed, 90 insertions(+), 41 deletions(-)

diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h
index 710cab61d9195..ffe5546d6189b 100644
--- a/drivers/irqchip/irq-gic-common.h
+++ b/drivers/irqchip/irq-gic-common.h
@@ -33,4 +33,11 @@ void gic_enable_of_quirks(const struct device_node *np,
 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED     (1 << 1)
 #define RDIST_FLAGS_FORCE_NON_SHAREABLE        (1 << 2)
 
+/* Per-CPU redistributor flags, in struct rdists::rdist->flags */
+#define RD_LOCAL_LPI_ENABLED                    BIT(0)
+#define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
+#define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
+/* The CPU has come up */
+#define RD_LOCAL_POPULATED                      BIT(3)
+
 #endif /* _IRQ_GIC_COMMON_H */
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 282e4bafb084f..ba6a6edf34230 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -50,10 +50,6 @@
 #define ITS_FLAGS_FORCE_NON_SHAREABLE		(1ULL << 3)
 #define ITS_FLAGS_WORKAROUND_HISILICON_162100801	(1ULL << 4)
 
-#define RD_LOCAL_LPI_ENABLED                    BIT(0)
-#define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
-#define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
-
 static u32 lpi_id_bits;
 
 /*
@@ -2802,7 +2798,8 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
 	for_each_possible_cpu(cpu) {
 		void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
 
-		if (!base || cpu == smp_processor_id())
+		if (!(gic_data_rdist_cpu(cpu)->flags & RD_LOCAL_POPULATED) ||
+		    cpu == smp_processor_id())
 			continue;
 
 		val = gic_read_typer(base + GICR_TYPER);
@@ -2839,8 +2836,8 @@ static bool allocate_vpe_l2_table(int cpu, u32 id)
 	if (!gic_rdists->has_rvpeid)
 		return true;
 
-	/* Skip non-present CPUs */
-	if (!base)
+	/* Skip CPUs that have not come up yet */
+	if (!(gic_data_rdist_cpu(cpu)->flags & RD_LOCAL_POPULATED))
 		return true;
 
 	val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 94873646d46fb..a507d84bf8dbc 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1019,51 +1019,94 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 	return ret ? -ENODEV : 0;
 }
 
-static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
+/*
+ * Convert a CPU's affinity to a 32bit value that can be matched to
+ * GICR_TYPER bits [63:32].
+ */
+static u32 gic_cpu_to_rdist_aff(int cpu)
 {
-	unsigned long mpidr;
-	u64 typer;
-	u32 aff;
+	u64 mpidr = gic_cpu_to_affinity(cpu);
 
-	/*
-	 * Convert affinity to a 32bit value that can be matched to
-	 * GICR_TYPER bits [63:32].
-	 */
-	mpidr = gic_cpu_to_affinity(smp_processor_id());
+	return (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+		MPIDR_AFFINITY_LEVEL(mpidr, 0));
+}
 
-	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
+/* Give @cpu the redistributor frame at @ptr */
+static void gic_assign_rdist(int cpu, struct redist_region *region,
+			     void __iomem *ptr)
+{
+	raw_spin_lock_init(&per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_lock);
+	per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base = ptr;
+	per_cpu_ptr(gic_data.rdists.rdist, cpu)->phys_base =
+		region->phys_base + (ptr - region->redist_base);
+}
 
-	typer = gic_read_typer(ptr + GICR_TYPER);
-	if ((typer >> 32) == aff) {
-		u64 offset = ptr - region->redist_base;
-		raw_spin_lock_init(&gic_data_rdist()->rd_lock);
-		gic_data_rdist_rd_base() = ptr;
-		gic_data_rdist()->phys_base = region->phys_base + offset;
-
-		pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
-			smp_processor_id(), mpidr,
-			(int)(region - gic_data.redist_regions),
-			&gic_data_rdist()->phys_base);
-		return 0;
+static int __gic_find_rdist(struct redist_region *region, void __iomem *ptr)
+{
+	u64 typer = gic_read_typer(ptr + GICR_TYPER);
+
+	if ((typer >> 32) != gic_cpu_to_rdist_aff(smp_processor_id()))
+		return 1;	/* Try next one */
+
+	gic_assign_rdist(smp_processor_id(), region, ptr);
+	return 0;
+}
+
+/* The region holding @ptr: the one with the highest base not above it */
+static int gic_rdist_region_index(void __iomem *ptr)
+{
+	int i, idx = 0;
+
+	for (i = 1; i < gic_data.nr_redist_regions; i++) {
+		if (gic_data.redist_regions[i].redist_base <= ptr &&
+		    gic_data.redist_regions[i].redist_base >
+		    gic_data.redist_regions[idx].redist_base)
+			idx = i;
 	}
 
-	/* Try next one */
-	return 1;
+	return idx;
 }
 
 static int gic_populate_rdist(void)
 {
-	if (gic_iterate_rdists(__gic_populate_rdist) == 0)
-		return 0;
+	/*
+	 * The frame is normally assigned at init by
+	 * gic_update_rdist_properties(). A CPU that was absent then, such as
+	 * an ACPI online-capable CPU, scans for its frame instead.
+	 */
+	if (!gic_data_rdist_rd_base() &&
+	    gic_iterate_rdists(__gic_find_rdist)) {
+		/* We couldn't even deal with ourselves... */
+		WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
+		     smp_processor_id(),
+		     (unsigned long)cpu_logical_map(smp_processor_id()));
+		return -ENODEV;
+	}
 
-	/* We couldn't even deal with ourselves... */
-	WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
-	     smp_processor_id(),
-	     (unsigned long)cpu_logical_map(smp_processor_id()));
-	return -ENODEV;
+	gic_data_rdist()->flags |= RD_LOCAL_POPULATED;
+
+	pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
+		smp_processor_id(),
+		(unsigned long)gic_cpu_to_affinity(smp_processor_id()),
+		gic_rdist_region_index(gic_data_rdist_rd_base()),
+		&gic_data_rdist()->phys_base);
+	return 0;
+}
+
+/* Give the frame at @ptr to the possible CPU whose affinity it reports */
+static void gic_find_rdist_cpu(struct redist_region *region, void __iomem *ptr,
+			       u64 typer)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		if (gic_cpu_to_rdist_aff(cpu) == (typer >> 32)) {
+			gic_assign_rdist(cpu, region, ptr);
+			return;
+		}
+	}
 }
 
 static int __gic_update_rdist_properties(struct redist_region *region,
@@ -1072,6 +1115,8 @@ static int __gic_update_rdist_properties(struct redist_region *region,
 	u64 typer = gic_read_typer(ptr + GICR_TYPER);
 	u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
 
+	gic_find_rdist_cpu(region, ptr, typer);
+
 	/* Boot-time cleanup */
 	if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
 		u64 val;
-- 
2.47.3


^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2026-11-19 18:42 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-10-23 18:15 [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk Joshua Machauer
2026-10-25 10:48 ` Marc Zyngier
2026-11-19 18:42   ` [PATCH v2] irqchip/gic-v3: Fill in " Joshua Machauer

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®