* [PATCH v2 1/2] sparc64: flush only the aliased page, not its whole folio
2026-09-28 13:21 [PATCH v2 0/2] sparc64: D-cache alias flushing fixes Imre Kaloz
@ 2026-09-28 13:21 ` Imre Kaloz
2026-09-28 13:21 ` [PATCH v2 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap Imre Kaloz
1 sibling, 0 replies; 3+ messages in thread
From: Imre Kaloz @ 2026-09-28 13:21 UTC (permalink / raw)
To: Andreas Larsson, David S. Miller
Cc: Matthew Wilcox (Oracle), Mike Rapoport (IBM),
Andrew Morton, sparclinux, linux-kernel, stable
Since the conversion to folios, move_pte() and tlb_batch_add() flush the
D-cache of every page in the folio a single PTE maps. Both run once per
PTE, so moving or unmapping a range backed by a large folio flushes each
of its pages once for every PTE of that folio, and each page flush is a
cross-call to all other online CPUs.
Only the page the PTE maps can hold lines at the old colour, so flush
just that page, as before the conversion. flush_dcache_folio_all() has
no other users and becomes flush_dcache_page_all() again.
On SMP QEMU guests, apt rebuilding its caches through mremap() triggers
RCU stalls and soft lockups in move_pte().
Fixes: 1a10a44dfc1d ("sparc64: implement the new page table range API")
Cc: stable@vger.kernel.org
Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
arch/sparc/include/asm/cacheflush_64.h | 3 +--
arch/sparc/include/asm/pgtable_64.h | 7 +++++--
arch/sparc/kernel/smp_64.c | 24 +++++++++++++-----------
arch/sparc/mm/init_64.c | 22 ++++++++++++++++++++++
arch/sparc/mm/tlb.c | 2 +-
5 files changed, 42 insertions(+), 16 deletions(-)
diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 06092572c045..02c969417e7b 100644
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -38,11 +38,10 @@ void __flush_dcache_page(void *addr, int flush_icache);
void flush_dcache_folio_impl(struct folio *folio);
#ifdef CONFIG_SMP
void smp_flush_dcache_folio_impl(struct folio *folio, int cpu);
-void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio);
#else
#define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio)
-#define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio)
#endif
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page);
void __flush_dcache_range(unsigned long start, unsigned long end);
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 0837ebbc5dce..6712eac4baf1 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -946,6 +946,9 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
set_pte_at((mm), (addr), (ptep), __pte(0UL))
#ifdef DCACHE_ALIASING_POSSIBLE
+/* Without pte_batch_hint(), move_ptes() calls this once per PTE, so
+ * only the page this PTE maps can have lines at the old colour.
+ */
#define __HAVE_ARCH_MOVE_PTE
#define move_pte(pte, old_addr, new_addr) \
({ \
@@ -955,8 +958,8 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
\
if (pfn_valid(this_pfn) && \
(((old_addr) ^ (new_addr)) & (1 << 13))) \
- flush_dcache_folio_all(current->mm, \
- page_folio(pfn_to_page(this_pfn))); \
+ flush_dcache_page_all(current->mm, \
+ pfn_to_page(this_pfn)); \
} \
newpte; \
})
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 371460e34484..18b6145da591 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -982,8 +982,9 @@ void smp_flush_dcache_folio_impl(struct folio *folio, int cpu)
put_cpu();
}
-void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
{
+ struct folio *folio = page_folio(page);
void *pg_addr;
u64 data0;
@@ -996,7 +997,7 @@ void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
atomic_inc(&dcpage_flushes);
#endif
data0 = 0;
- pg_addr = folio_address(folio);
+ pg_addr = page_address(page);
if (tlb_type == spitfire) {
data0 = ((u64)&xcall_flush_dcache_page_spitfire);
if (folio_flush_mapping(folio) != NULL)
@@ -1007,18 +1008,19 @@ void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio)
#endif
}
if (data0) {
- unsigned int i, nr = folio_nr_pages(folio);
-
- for (i = 0; i < nr; i++) {
- xcall_deliver(data0, __pa(pg_addr),
- (u64) pg_addr, cpu_online_mask);
+ xcall_deliver(data0, __pa(pg_addr),
+ (u64)pg_addr, cpu_online_mask);
#ifdef CONFIG_DEBUG_DCFLUSH
- atomic_inc(&dcpage_flushes_xcall);
+ atomic_inc(&dcpage_flushes_xcall);
#endif
- pg_addr += PAGE_SIZE;
- }
}
- __local_flush_dcache_folio(folio);
+#ifdef DCACHE_ALIASING_POSSIBLE
+ __flush_dcache_page(pg_addr,
+ tlb_type == spitfire && folio_flush_mapping(folio));
+#else
+ if (tlb_type == spitfire && folio_flush_mapping(folio))
+ __flush_icache_page(__pa(pg_addr));
+#endif
preempt_enable();
}
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 103db4683b16..8792e5d92517 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -214,6 +214,28 @@ inline void flush_dcache_folio_impl(struct folio *folio)
#endif
}
+#ifndef CONFIG_SMP
+void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
+{
+ struct folio *folio = page_folio(page);
+
+ if (tlb_type == hypervisor)
+ return;
+
+#ifdef CONFIG_DEBUG_DCFLUSH
+ atomic_inc(&dcpage_flushes);
+#endif
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+ __flush_dcache_page(page_address(page),
+ tlb_type == spitfire && folio_flush_mapping(folio));
+#else
+ if (tlb_type == spitfire && folio_flush_mapping(folio))
+ __flush_icache_page(page_to_phys(page));
+#endif
+}
+#endif
+
#define PG_dcache_dirty PG_arch_1
#define PG_dcache_cpu_shift 32UL
#define PG_dcache_cpu_mask \
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 6d9dd5eb1328..1221814ca0e1 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -144,7 +144,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
paddr = (unsigned long) page_address(page);
if ((paddr ^ vaddr) & (1 << 13))
- flush_dcache_folio_all(mm, folio);
+ flush_dcache_page_all(mm, page);
}
no_cache_flush:
--
2.47.3
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH v2 2/2] sparc64: flush vmalloc ranges from the D-cache on map and unmap
2026-09-28 13:21 [PATCH v2 0/2] sparc64: D-cache alias flushing fixes Imre Kaloz
2026-09-28 13:21 ` [PATCH v2 1/2] sparc64: flush only the aliased page, not its whole folio Imre Kaloz
@ 2026-09-28 13:21 ` Imre Kaloz
1 sibling, 0 replies; 3+ messages in thread
From: Imre Kaloz @ 2026-09-28 13:21 UTC (permalink / raw)
To: Andreas Larsson, David S. Miller
Cc: Matthew Wilcox (Oracle), Mike Rapoport (IBM),
Andrew Morton, sparclinux, linux-kernel, nsafran1217, stable,
Stian Halseth
The L1 D-cache of Spitfire and Cheetah is indexed with VA bit 13, but
flush_cache_vmap() and flush_cache_vunmap() are no-ops. Lines loaded
through a vmalloc address survive the unmap, and once the page is
written through the other colour, e.g. zeroed through the linear
mapping, a later mapping of the old colour reads stale data. Anonymous
user pages arrive the same way, as tlb_batch_add() does not flush them.
On UltraSPARC III this corrupts BPF programs and module data.
Flush each page on every CPU on map and unmap, or the whole D-cache in
one cross call once the range exceeds its size.
flush_cache_vmap() also runs on the vmemmap, from sparse_init() before
init_IRQ() sets up the cross-call mondo blocks; with the mondo block PA
still zero, flush_dcache_all()'s cross-call lands on physical address 0
instead. The vmemmap has no aliases to flush, so skip addresses outside
vmalloc and module space.
Link: https://github.com/sparclinux/issues/issues/29#issuecomment-5041121375
Closes: https://github.com/sparclinux/issues/issues/29
Suggested-by: nsafran1217 <54966414+nsafran1217@users.noreply.github.com>
Cc: stable@vger.kernel.org
Tested-by: Stian Halseth <stian@itx.no> # Sun Fire V240, SPARC T4-1
Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
v2: skip the flush outside vmalloc/module space, since
flush_cache_vmap() also runs on the vmemmap during sparse_init()
before cross-calls are set up (Stian Halseth); picked up his
Tested-by.
arch/sparc/include/asm/cacheflush_64.h | 6 ++--
arch/sparc/kernel/smp_64.c | 18 +++++++++++
arch/sparc/mm/init_64.c | 43 ++++++++++++++++++++++++++
arch/sparc/mm/ultra.S | 25 +++++++++++++++
4 files changed, 90 insertions(+), 2 deletions(-)
diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 02c969417e7b..54a2d37008e6 100644
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -42,6 +42,8 @@ void smp_flush_dcache_folio_impl(struct folio *folio, int cpu);
#define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio)
#endif
void flush_dcache_page_all(struct mm_struct *mm, struct page *page);
+void __flush_dcache_all(unsigned long size, unsigned long line_size);
+void flush_dcache_all(void);
void __flush_dcache_range(unsigned long start, unsigned long end);
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
@@ -73,9 +75,9 @@ void flush_ptrace_access(struct vm_area_struct *, struct page *,
#define flush_dcache_mmap_lock(mapping) do { } while (0)
#define flush_dcache_mmap_unlock(mapping) do { } while (0)
-#define flush_cache_vmap(start, end) do { } while (0)
+void flush_cache_vmap(unsigned long start, unsigned long end);
#define flush_cache_vmap_early(start, end) do { } while (0)
-#define flush_cache_vunmap(start, end) do { } while (0)
+#define flush_cache_vunmap(start, end) flush_cache_vmap(start, end)
#endif /* !__ASSEMBLER__ */
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 18b6145da591..bebfc44241b3 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -915,6 +915,7 @@ extern unsigned long xcall_kgdb_capture;
#ifdef DCACHE_ALIASING_POSSIBLE
extern unsigned long xcall_flush_dcache_page_cheetah;
+extern unsigned long xcall_flush_dcache_all;
#endif
extern unsigned long xcall_flush_dcache_page_spitfire;
@@ -1025,6 +1026,23 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
preempt_enable();
}
+#ifdef DCACHE_ALIASING_POSSIBLE
+void flush_dcache_all(void)
+{
+ unsigned long size, line_size;
+
+ if (tlb_type == hypervisor)
+ return;
+
+ preempt_disable();
+ size = local_cpu_data().dcache_size;
+ line_size = local_cpu_data().dcache_line_size;
+ smp_cross_call(&xcall_flush_dcache_all, 0, size, line_size);
+ __flush_dcache_all(size, line_size);
+ preempt_enable();
+}
+#endif
+
#ifdef CONFIG_KGDB
void kgdb_roundup_cpus(void)
{
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 8792e5d92517..3e2096545e52 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -27,6 +27,7 @@
#include <linux/percpu.h>
#include <linux/mmzone.h>
#include <linux/gfp.h>
+#include <linux/vmalloc.h>
#include <asm/head.h>
#include <asm/page.h>
@@ -234,6 +235,17 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
__flush_icache_page(page_to_phys(page));
#endif
}
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+void flush_dcache_all(void)
+{
+ if (tlb_type == hypervisor)
+ return;
+
+ __flush_dcache_all(local_cpu_data().dcache_size,
+ local_cpu_data().dcache_line_size);
+}
+#endif
#endif
#define PG_dcache_dirty PG_arch_1
@@ -3072,6 +3084,37 @@ arch_initcall(report_memory);
#define do_flush_tlb_kernel_range __flush_tlb_kernel_range
#endif
+/*
+ * The L1 D-cache is indexed with VA bit 13, so lines loaded through a
+ * vmalloc address outlive the mapping. Stores through a mapping of the
+ * other colour do not update them, and a later load through their colour
+ * returns stale data. Flush at unmap so vmalloc's lines do not follow
+ * the page back to the allocator, and at map time since anonymous user
+ * pages are freed without a flush (see tlb_batch_add()). Past the
+ * D-cache size, one whole-cache flush is cheaper than a cross call per
+ * page.
+ */
+void flush_cache_vmap(unsigned long start, unsigned long end)
+{
+#ifdef DCACHE_ALIASING_POSSIBLE
+ /* The vmemmap has no aliases and is mapped before cross calls work. */
+ if (tlb_type == hypervisor || !is_vmalloc_or_module_addr((void *)start))
+ return;
+
+ if (end - start > cpu_data(raw_smp_processor_id()).dcache_size) {
+ flush_dcache_all();
+ return;
+ }
+
+ for (; start < end; start += PAGE_SIZE) {
+ struct page *page = vmalloc_to_page((void *)start);
+
+ if (page)
+ flush_dcache_page_all(NULL, page);
+ }
+#endif
+}
+
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) {
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index 70e658d107e0..ff190670a235 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -242,6 +242,20 @@ __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
retl
nop
+ /* Zeroing every D-cache tag invalidates the whole cache on
+ * Spitfire and Cheetah alike.
+ */
+ .align 32
+ .globl __flush_dcache_all
+__flush_dcache_all: /* %o0=D-cache size, %o1=D-cache line size */
+1: subcc %o0, %o1, %o0
+ stxa %g0, [%o0] ASI_DCACHE_TAG
+ membar #Sync
+ bne,pt %icc, 1b
+ nop
+ retl
+ nop
+
#endif /* DCACHE_ALIASING_POSSIBLE */
.previous
@@ -801,6 +815,17 @@ xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
nop
retry
nop
+
+ .align 32
+ .globl xcall_flush_dcache_all
+xcall_flush_dcache_all: /* %g1 == D-cache size, %g7 == D-cache line size */
+1: subcc %g1, %g7, %g1
+ stxa %g0, [%g1] ASI_DCACHE_TAG
+ membar #Sync
+ bne,pt %icc, 1b
+ nop
+ retry
+ nop
#endif /* DCACHE_ALIASING_POSSIBLE */
.globl xcall_flush_dcache_page_spitfire
--
2.47.3
^ permalink raw reply [flat|nested] 3+ messages in thread