[PATCH v8 11/15] s390: Distinguish hardware and software PTEs

[Alexander Gordeev <agordeev@linux.ibm.com>]

This is s390 version of ARM64 commit XXXXXXXXXXXX ("arm64:
distinguish HW PTE pointers from SW PTE value pointers").

It distinguishes hardware and software PTEs, introduces hw_pte_t
type for hardware PTEs, while software PTEs still use pte_t type
and enables ARCH_HAS_HW_PTE_T configuration for s390.

Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>
---
 arch/s390/Kconfig                    |  1 +
 arch/s390/boot/startup.c             |  2 +-
 arch/s390/boot/vmem.c                | 17 ++++-----
 arch/s390/include/asm/gmap_helpers.h |  2 +-
 arch/s390/include/asm/hugetlb.h      | 18 +++++-----
 arch/s390/include/asm/maccess.h      |  2 +-
 arch/s390/include/asm/page.h         |  3 +-
 arch/s390/include/asm/pgalloc.h      |  6 ++--
 arch/s390/include/asm/pgtable.h      | 54 +++++++++++++++-------------
 arch/s390/kernel/uv.c                |  4 +--
 arch/s390/kvm/s390/pv.c              |  2 +-
 arch/s390/mm/gmap_helpers.c          | 18 +++++-----
 arch/s390/mm/hugetlbpage.c           | 20 +++++------
 arch/s390/mm/maccess.c               |  2 +-
 arch/s390/mm/pageattr.c              | 10 +++---
 arch/s390/mm/pgtable.c               | 26 +++++++-------
 arch/s390/mm/vmem.c                  | 26 +++++++-------
 17 files changed, 112 insertions(+), 101 deletions(-)

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 4b51bc6e8948..b9bc0e5e7d7d 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -100,6 +100,7 @@ config S390
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_GIGANTIC_PAGE
+	select ARCH_HAS_HW_PTE_T
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_MEM_ENCRYPT
diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
index c1dcf8b1b579..d8732b416161 100644
--- a/arch/s390/boot/startup.c
+++ b/arch/s390/boot/startup.c
@@ -30,7 +30,7 @@
 struct vm_layout __bootdata_preserved(vm_layout);
 unsigned long __bootdata_preserved(__abs_lowcore);
 unsigned long __bootdata_preserved(__memcpy_real_area);
-pte_t *__bootdata_preserved(memcpy_real_ptep);
+hw_pte_t *__bootdata_preserved(memcpy_real_ptep);
 unsigned long __bootdata_preserved(VMALLOC_START);
 unsigned long __bootdata_preserved(VMALLOC_END);
 struct page *__bootdata_preserved(vmemmap);
diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c
index ff6d58a476ba..cc619cc85d37 100644
--- a/arch/s390/boot/vmem.c
+++ b/arch/s390/boot/vmem.c
@@ -75,7 +75,7 @@ static void pgtable_populate(unsigned long addr, unsigned long end, enum populat
 #ifdef CONFIG_KASAN
 
 #define kasan_early_shadow_page	vmlinux.kasan_early_shadow_page_off
-#define kasan_early_shadow_pte	((pte_t *)vmlinux.kasan_early_shadow_pte_off)
+#define kasan_early_shadow_pte	((hw_pte_t *)vmlinux.kasan_early_shadow_pte_off)
 #define kasan_early_shadow_pmd	((pmd_t *)vmlinux.kasan_early_shadow_pmd_off)
 #define kasan_early_shadow_pud	((pud_t *)vmlinux.kasan_early_shadow_pud_off)
 #define kasan_early_shadow_p4d	((p4d_t *)vmlinux.kasan_early_shadow_p4d_off)
@@ -178,7 +178,7 @@ static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,
 	return false;
 }
 
-static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
+static bool kasan_pte_populate_zero_shadow(hw_pte_t *pte, enum populate_mode mode)
 {
 	if (mode == POPULATE_KASAN_ZERO_SHADOW) {
 		set_pte(pte, pte_z);
@@ -216,7 +216,7 @@ static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr
 	return false;
 }
 
-static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
+static bool kasan_pte_populate_zero_shadow(hw_pte_t *pte, enum populate_mode mode)
 {
 	return false;
 }
@@ -226,7 +226,7 @@ static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)
 /*
  * Mimic virt_to_kpte() in lack of init_mm symbol. Skip pmd NULL check though.
  */
-static inline pte_t *__virt_to_kpte(unsigned long va)
+static inline hw_pte_t *__virt_to_kpte(unsigned long va)
 {
 	return pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va), va);
 }
@@ -242,9 +242,9 @@ static void *boot_crst_alloc(unsigned long val)
 	return table;
 }
 
-static pte_t *boot_pte_alloc(void)
+static hw_pte_t *boot_pte_alloc(void)
 {
-	pte_t *pte;
+	hw_pte_t *pte;
 
 	pte = (void *)physmem_alloc_or_die(RR_VMEM, PAGE_SIZE, PAGE_SIZE);
 	__arch_set_page_dat(pte, 1);
@@ -334,7 +334,8 @@ static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long e
 				 enum populate_mode mode)
 {
 	unsigned long pages = 0;
-	pte_t *pte, entry;
+	hw_pte_t *pte;
+	pte_t entry;
 
 	pte = pte_offset_kernel(pmd, addr);
 	for (; addr < end; addr += PAGE_SIZE, pte++) {
@@ -356,7 +357,7 @@ static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long e
 {
 	unsigned long pa, next, pages = 0;
 	pmd_t *pmd, entry, large_entry;
-	pte_t *pte;
+	hw_pte_t *pte;
 
 	pmd = pmd_offset(pud, addr);
 	for (; addr < end; addr = next, pmd++) {
diff --git a/arch/s390/include/asm/gmap_helpers.h b/arch/s390/include/asm/gmap_helpers.h
index d2b616604a46..d23907ed78f3 100644
--- a/arch/s390/include/asm/gmap_helpers.h
+++ b/arch/s390/include/asm/gmap_helpers.h
@@ -12,6 +12,6 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr);
 void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned long end);
 int gmap_helper_disable_cow_sharing(void);
 void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr);
-pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
+hw_pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl);
 
 #endif /* _ASM_S390_GMAP_HELPERS_H */
diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index aea754b67c89..7c1cf08725c5 100644
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -19,19 +19,19 @@
 
 #define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-		     pte_t *ptep, pte_t pte, unsigned long sz);
+		     hw_pte_t *ptep, pte_t pte, unsigned long sz);
 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-		       pte_t *ptep, pte_t pte);
+		       hw_pte_t *ptep, pte_t pte);
 
 #define __HAVE_ARCH_HUGE_PTEP_GET
-pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
+pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, hw_pte_t *ptep);
 
 pte_t __huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
-				pte_t *ptep);
+				hw_pte_t *ptep);
 
 #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
-					    unsigned long addr, pte_t *ptep,
+					    unsigned long addr, hw_pte_t *ptep,
 					    unsigned long sz)
 {
 	return __huge_ptep_get_and_clear(mm, addr, ptep);
@@ -39,7 +39,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 
 #define __HAVE_ARCH_HUGE_PTE_CLEAR
 static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
-				  pte_t *ptep, unsigned long sz)
+				  hw_pte_t *ptep, unsigned long sz)
 {
 	if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 		set_pud((pud_t *)ptep, __pud(_REGION3_ENTRY_EMPTY));
@@ -49,14 +49,14 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
 
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
-					  unsigned long address, pte_t *ptep)
+					  unsigned long address, hw_pte_t *ptep)
 {
 	return __huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
 }
 
 #define  __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
 static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
-					     unsigned long addr, pte_t *ptep,
+					     unsigned long addr, hw_pte_t *ptep,
 					     pte_t pte, int dirty)
 {
 	int changed = !pte_same(huge_ptep_get(vma->vm_mm, addr, ptep), pte);
@@ -70,7 +70,7 @@ static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 
 #define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT
 static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
-					   unsigned long addr, pte_t *ptep)
+					   unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t pte = __huge_ptep_get_and_clear(mm, addr, ptep);
 
diff --git a/arch/s390/include/asm/maccess.h b/arch/s390/include/asm/maccess.h
index 50225940d971..dafd9fe290ce 100644
--- a/arch/s390/include/asm/maccess.h
+++ b/arch/s390/include/asm/maccess.h
@@ -10,7 +10,7 @@
 struct iov_iter;
 
 extern unsigned long __memcpy_real_area;
-extern pte_t *memcpy_real_ptep;
+extern hw_pte_t *memcpy_real_ptep;
 size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count);
 int memcpy_real(void *dest, unsigned long src, size_t count);
 #ifdef CONFIG_CRASH_DUMP
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h
index 56da819a79e6..42b4fecd2de3 100644
--- a/arch/s390/include/asm/page.h
+++ b/arch/s390/include/asm/page.h
@@ -113,7 +113,8 @@ DEFINE_PGVAL_FUNC(pud)
 DEFINE_PGVAL_FUNC(p4d)
 DEFINE_PGVAL_FUNC(pgd)
 
-typedef pte_t *pgtable_t;
+struct __hw_pte_t;
+typedef struct __hw_pte_t *pgtable_t;
 
 #define __pgprot(x)	((pgprot_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index a5de9e61ea9e..83a732898819 100644
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -159,8 +159,8 @@ static inline void pmd_populate(struct mm_struct *mm,
 /*
  * page table entry allocation/free routines.
  */
-#define pte_alloc_one_kernel(mm) ((pte_t *)page_table_alloc(mm))
-#define pte_alloc_one(mm) ((pte_t *)page_table_alloc(mm))
+#define pte_alloc_one_kernel(mm) ((hw_pte_t *)page_table_alloc(mm))
+#define pte_alloc_one(mm) ((hw_pte_t *)page_table_alloc(mm))
 
 #define pte_free_kernel(mm, pte) page_table_free(mm, (unsigned long *) pte)
 #define pte_free(mm, pte) page_table_free(mm, (unsigned long *) pte)
@@ -171,7 +171,7 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
 
 void vmem_map_init(void);
 void *vmem_crst_alloc(unsigned long val);
-pte_t *vmem_pte_alloc(void);
+hw_pte_t *vmem_pte_alloc(void);
 
 unsigned long base_asce_alloc(unsigned long addr, unsigned long num_pages);
 void base_asce_free(unsigned long asce);
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index e882663a58e7..c47264f3abf2 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -978,17 +978,19 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 	WRITE_ONCE(*pmdp, pmd);
 }
 
-static inline void set_pte(pte_t *ptep, pte_t pte)
+static inline void set_pte(hw_pte_t *ptep, pte_t pte)
 {
+	hw_pte_t hwpte = (hw_pte_t) { (pte) };
+
 	if (pte_present(pte))
 		pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED));
-	WRITE_ONCE(*ptep, pte);
+	WRITE_ONCE(*ptep, hwpte);
 }
 
 #define ptep_get ptep_get
-static inline pte_t ptep_get(pte_t *ptep)
+static inline pte_t ptep_get(hw_pte_t *ptep)
 {
-	return READ_ONCE(*ptep);
+	return __pte_from_hw(READ_ONCE(*ptep));
 }
 
 #define pmdp_get pmdp_get
@@ -1015,7 +1017,7 @@ static inline pgd_t pgdp_get(pgd_t *pgdp)
 	return READ_ONCE(*pgdp);
 }
 
-static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+static inline void pte_clear(struct mm_struct *mm, unsigned long addr, hw_pte_t *ptep)
 {
 	set_pte(ptep, __pte(_PAGE_INVALID));
 }
@@ -1133,7 +1135,7 @@ static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
 #define IPTE_NODAT	0x400
 #define IPTE_GUEST_ASCE	0x800
 
-static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep, int local)
+static __always_inline void __ptep_rdp(unsigned long addr, hw_pte_t *ptep, int local)
 {
 	unsigned long pto;
 
@@ -1144,7 +1146,7 @@ static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep, int loca
 		       [m4] "i" (local));
 }
 
-static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep,
+static __always_inline void __ptep_ipte(unsigned long address, hw_pte_t *ptep,
 					unsigned long opt, unsigned long asce,
 					int local)
 {
@@ -1168,7 +1170,7 @@ static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep,
 }
 
 static __always_inline void __ptep_ipte_range(unsigned long address, int nr,
-					      pte_t *ptep, int local)
+					      hw_pte_t *ptep, int local)
 {
 	unsigned long pto = __pa(ptep);
 
@@ -1194,12 +1196,12 @@ static __always_inline void __ptep_ipte_range(unsigned long address, int nr,
  * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
  * is a nop.
  */
-pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
-pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
+pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, hw_pte_t *, pte_t);
+pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, hw_pte_t *, pte_t);
 
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma,
-		unsigned long addr, pte_t *ptep)
+		unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t pte = ptep_get(ptep);
 
@@ -1209,14 +1211,14 @@ static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma,
 
 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
 static inline bool ptep_clear_flush_young(struct vm_area_struct *vma,
-		unsigned long address, pte_t *ptep)
+		unsigned long address, hw_pte_t *ptep)
 {
 	return ptep_test_and_clear_young(vma, address, ptep);
 }
 
 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
-				       unsigned long addr, pte_t *ptep)
+				       unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t res;
 
@@ -1229,13 +1231,13 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 }
 
 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
-pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
+pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, hw_pte_t *);
 void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
-			     pte_t *, pte_t, pte_t);
+			     hw_pte_t *, pte_t, pte_t);
 
 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
-				     unsigned long addr, pte_t *ptep)
+				     unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t res;
 
@@ -1257,7 +1259,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 					    unsigned long addr,
-					    pte_t *ptep, int full)
+					    hw_pte_t *ptep, int full)
 {
 	pte_t res;
 
@@ -1289,7 +1291,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 
 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
 static inline void ptep_set_wrprotect(struct mm_struct *mm,
-				      unsigned long addr, pte_t *ptep)
+				      unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t pte = ptep_get(ptep);
 
@@ -1315,7 +1317,7 @@ static inline int pte_allow_rdp(pte_t old, pte_t new)
 
 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 						unsigned long address,
-						pte_t *ptep)
+						hw_pte_t *ptep)
 {
 	/*
 	 * RDP might not have propagated the PTE protection reset to all CPUs,
@@ -1332,17 +1334,19 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 }
 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
 
-void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
+void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, hw_pte_t *ptep,
 			 pte_t new);
 
 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
-					unsigned long addr, pte_t *ptep,
+					unsigned long addr, hw_pte_t *ptep,
 					pte_t entry, int dirty)
 {
-	if (pte_same(*ptep, entry))
+	pte_t pte = ptep_get(ptep);
+
+	if (pte_same(pte, entry))
 		return 0;
-	if (cpu_has_rdp() && pte_allow_rdp(*ptep, entry))
+	if (cpu_has_rdp() && pte_allow_rdp(pte, entry))
 		ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);
 	else
 		ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
@@ -1359,7 +1363,7 @@ pgprot_t pgprot_writecombine(pgprot_t prot);
  * are within the same folio, PMD and VMA.
  */
 static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
-			      pte_t *ptep, pte_t entry, unsigned int nr)
+		hw_pte_t *ptep, pte_t entry, unsigned int nr)
 {
 	page_table_check_ptes_set(mm, addr, ptep, entry, nr);
 	for (;;) {
@@ -1997,7 +2001,7 @@ extern void vmem_remove_mapping(unsigned long start, unsigned long size);
 extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc);
 extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot);
 extern void vmem_unmap_4k_page(unsigned long addr);
-extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
+extern hw_pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
 
 /* s390 has a private copy of get unmapped area to deal with cache synonyms */
 #define HAVE_ARCH_UNMAPPED_AREA
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
index 8ea9dd7704ff..f81c47db421e 100644
--- a/arch/s390/kernel/uv.c
+++ b/arch/s390/kernel/uv.c
@@ -210,7 +210,7 @@ int uv_convert_from_secure_pte(pte_t pte)
 	return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte)));
 }
 
-static int uv_free_range_cb(pte_t *ptep, unsigned long addr, void *data)
+static int uv_free_range_cb(hw_pte_t *ptep, unsigned long addr, void *data)
 {
 	pte_t pte = ptep_get(ptep);
 
@@ -242,7 +242,7 @@ void uv_free_stor_var(void *stor_var)
 }
 EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm");
 
-static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data)
+static int uv_alloc_range_cb(hw_pte_t *ptep, unsigned long addr, void *data)
 {
 	struct page *page;
 	pte_t pte;
diff --git a/arch/s390/kvm/s390/pv.c b/arch/s390/kvm/s390/pv.c
index b18abd0e29ef..bec85e434514 100644
--- a/arch/s390/kvm/s390/pv.c
+++ b/arch/s390/kvm/s390/pv.c
@@ -106,7 +106,7 @@ static void _kvm_s390_pv_make_secure(struct guest_fault *f)
 	struct pv_make_secure *priv = f->priv;
 	struct folio *folio;
 	spinlock_t *ptl;	/* pte lock from try_get_locked_pte() */
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	folio = pfn_folio(f->pfn);
 	priv->rc = -EAGAIN;
diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c
index ff63ffb1dbd2..b39f939b308e 100644
--- a/arch/s390/mm/gmap_helpers.c
+++ b/arch/s390/mm/gmap_helpers.c
@@ -39,14 +39,14 @@
  * * the pointer to the pte corresponding to @addr in @mm, if it can be reached
  *   and locked.
  */
-pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t **ptl)
+hw_pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t **ptl)
 __context_unsafe(/* Returns nonnull if lock taken or not taken */)
 {
 	pmd_t *pmdp, pmd, pmdval;
 	pud_t *pudp, pud;
 	p4d_t *p4dp, p4d;
 	pgd_t *pgdp, pgd;
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	pgdp = pgd_offset(mm, vmaddr);
 	pgd = pgdp_get(pgdp);
@@ -96,7 +96,7 @@ __context_unsafe(/* pte_unmap_unlock() not instrumented */)
 	struct vm_area_struct *vma;
 	spinlock_t *ptl;	/* Lock for the host (userspace) page table */
 	softleaf_t sl;
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	mmap_assert_locked(mm);
 
@@ -109,8 +109,8 @@ __context_unsafe(/* pte_unmap_unlock() not instrumented */)
 	ptep = try_get_locked_pte(mm, vmaddr, &ptl);
 	if (IS_ERR_OR_NULL(ptep))
 		return;
-	sl = softleaf_from_pte(*ptep);
-	if (pte_swap(*ptep) && softleaf_is_swap(sl)) {
+	sl = softleaf_from_pte(ptep_get(ptep));
+	if (pte_swap(ptep_get(ptep)) && softleaf_is_swap(sl)) {
 		dec_mm_counter(mm, MM_SWAPENTS);
 		swap_put_entries_direct(sl, 1);
 		pte_clear(mm, vmaddr, ptep);
@@ -166,7 +166,7 @@ void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr)
 __context_unsafe(/* pte_unmap_unlock() not instrumented */)
 {
 	spinlock_t *ptl;	/* Lock for the host (userspace) page table */
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	/*
 	 * Several paths exists that takes the ptl lock and then call the
@@ -184,19 +184,19 @@ __context_unsafe(/* pte_unmap_unlock() not instrumented */)
 	if (IS_ERR_OR_NULL(ptep))
 		return;
 
-	if (pte_present(*ptep))
+	if (pte_present(ptep_get(ptep)))
 		__atomic64_or(_PAGE_UNUSED, (long *)ptep);
 	pte_unmap_unlock(ptep, ptl);
 }
 EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused);
 
-static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr,
+static int find_zeropage_pte_entry(hw_pte_t *pte, unsigned long addr,
 				   unsigned long end, struct mm_walk *walk)
 {
 	unsigned long *found_addr = walk->private;
 
 	/* Return 1 of the page is a zeropage. */
-	if (is_zero_pfn(pte_pfn(*pte))) {
+	if (is_zero_pfn(pte_pfn(ptep_get(pte)))) {
 		/*
 		 * Shared zeropage in e.g., a FS DAX mapping? We cannot do the
 		 * right thing and likely don't care: FAULT_FLAG_UNSHARE
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index f84aa9265430..35b7ae166fe6 100644
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -136,7 +136,7 @@ static inline pte_t __rste_to_pte(unsigned long rste)
 }
 
 void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-		     pte_t *ptep, pte_t pte)
+		     hw_pte_t *ptep, pte_t pte)
 {
 	unsigned long rste;
 
@@ -156,18 +156,18 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
-		     pte_t *ptep, pte_t pte, unsigned long sz)
+		     hw_pte_t *ptep, pte_t pte, unsigned long sz)
 {
 	__set_huge_pte_at(mm, addr, ptep, pte);
 }
 
-pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, hw_pte_t *ptep)
 {
 	return __rste_to_pte(pte_val(ptep_get(ptep)));
 }
 
 pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
-				unsigned long addr, pte_t *ptep)
+				unsigned long addr, hw_pte_t *ptep)
 {
 	pte_t pte = huge_ptep_get(mm, addr, ptep);
 	pmd_t *pmdp = (pmd_t *) ptep;
@@ -180,7 +180,7 @@ pte_t __huge_ptep_get_and_clear(struct mm_struct *mm,
 	return pte;
 }
 
-pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
+hw_pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
 			unsigned long addr, unsigned long sz)
 {
 	pgd_t *pgdp;
@@ -194,15 +194,15 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
 		pudp = pud_alloc(mm, p4dp, addr);
 		if (pudp) {
 			if (sz == PUD_SIZE)
-				return (pte_t *) pudp;
+				return (hw_pte_t *) pudp;
 			else if (sz == PMD_SIZE)
 				pmdp = pmd_alloc(mm, pudp, addr);
 		}
 	}
-	return (pte_t *) pmdp;
+	return (hw_pte_t *) pmdp;
 }
 
-pte_t *huge_pte_offset(struct mm_struct *mm,
+hw_pte_t *huge_pte_offset(struct mm_struct *mm,
 		       unsigned long addr, unsigned long sz)
 {
 	pgd_t *pgdp;
@@ -216,12 +216,12 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
 		if (p4d_present(p4dp_get(p4dp))) {
 			pudp = pud_offset(p4dp, addr);
 			if (sz == PUD_SIZE)
-				return (pte_t *)pudp;
+				return (hw_pte_t *)pudp;
 			if (pud_present(pudp_get(pudp)))
 				pmdp = pmd_offset(pudp, addr);
 		}
 	}
-	return (pte_t *) pmdp;
+	return (hw_pte_t *) pmdp;
 }
 
 bool __init arch_hugetlb_valid_size(unsigned long size)
diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c
index f39968dd8063..9026d6a3e657 100644
--- a/arch/s390/mm/maccess.c
+++ b/arch/s390/mm/maccess.c
@@ -22,7 +22,7 @@
 #include <asm/ctlreg.h>
 
 unsigned long __bootdata_preserved(__memcpy_real_area);
-pte_t *__bootdata_preserved(memcpy_real_ptep);
+hw_pte_t *__bootdata_preserved(memcpy_real_ptep);
 static DEFINE_MUTEX(memcpy_real_mutex);
 
 static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size)
diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index 1e202e3d08e7..b5a055791611 100644
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -79,7 +79,8 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr,
 static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
 			  unsigned long flags)
 {
-	pte_t *ptep, new;
+	hw_pte_t *ptep;
+	pte_t new;
 
 	if (flags == SET_MEMORY_4K)
 		return 0;
@@ -112,7 +113,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end,
 static int split_pmd_page(pmd_t *pmdp, unsigned long addr)
 {
 	unsigned long pte_addr, prot;
-	pte_t *pt_dir, *ptep;
+	hw_pte_t *pt_dir, *ptep;
 	pmd_t new, pmd;
 	int i, ro, nx;
 
@@ -421,7 +422,7 @@ bool kernel_page_present(struct page *page)
 
 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
 
-static void ipte_range(pte_t *pte, unsigned long address, int nr)
+static void ipte_range(hw_pte_t *pte, unsigned long address, int nr)
 {
 	int i;
 
@@ -439,7 +440,8 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr)
 void __kernel_map_pages(struct page *page, int numpages, int enable)
 {
 	unsigned long address;
-	pte_t *ptep, pte;
+	hw_pte_t *ptep;
+	pte_t pte;
 	int nr, i, j;
 
 	for (i = 0; i < numpages;) {
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index da076928b63b..b5eca8bfb539 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -37,7 +37,7 @@ pgprot_t pgprot_writecombine(pgprot_t prot)
 EXPORT_SYMBOL_GPL(pgprot_writecombine);
 
 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
-				   pte_t *ptep, int nodat)
+				   hw_pte_t *ptep, int nodat)
 {
 	unsigned long opt, asce;
 
@@ -57,7 +57,7 @@ static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
 }
 
 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
-				    pte_t *ptep, int nodat)
+				    hw_pte_t *ptep, int nodat)
 {
 	unsigned long opt, asce;
 
@@ -77,12 +77,12 @@ static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
 }
 
 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
-				      unsigned long addr, pte_t *ptep,
+				      unsigned long addr, hw_pte_t *ptep,
 				      int nodat)
 {
 	pte_t old;
 
-	old = *ptep;
+	old = ptep_get(ptep);
 	if (unlikely(pte_val(old) & _PAGE_INVALID))
 		return old;
 	atomic_inc(&mm->context.flush_count);
@@ -96,12 +96,12 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm,
 }
 
 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
-				    unsigned long addr, pte_t *ptep,
+				    unsigned long addr, hw_pte_t *ptep,
 				    int nodat)
 {
 	pte_t old;
 
-	old = *ptep;
+	old = ptep_get(ptep);
 	if (unlikely(pte_val(old) & _PAGE_INVALID))
 		return old;
 	atomic_inc(&mm->context.flush_count);
@@ -116,7 +116,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
 }
 
 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
-		       pte_t *ptep, pte_t new)
+		       hw_pte_t *ptep, pte_t new)
 {
 	pte_t old;
 
@@ -132,7 +132,7 @@ EXPORT_SYMBOL(ptep_xchg_direct);
  * Caller must check that new PTE only differs in _PAGE_PROTECT HW bit, so that
  * RDP can be used instead of IPTE. See also comments at pte_allow_rdp().
  */
-void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
+void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, hw_pte_t *ptep,
 			 pte_t new)
 {
 	preempt_disable();
@@ -154,7 +154,7 @@ void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
 EXPORT_SYMBOL(ptep_reset_dat_prot);
 
 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
-		     pte_t *ptep, pte_t new)
+		     hw_pte_t *ptep, pte_t new)
 {
 	pte_t old;
 
@@ -167,13 +167,13 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
 EXPORT_SYMBOL(ptep_xchg_lazy);
 
 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t *ptep)
+			     hw_pte_t *ptep)
 {
 	return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1);
 }
 
 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t *ptep, pte_t old_pte, pte_t pte)
+			     hw_pte_t *ptep, pte_t old_pte, pte_t pte)
 {
 	set_pte(ptep, pte);
 }
@@ -333,7 +333,7 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 {
 	struct list_head *lh;
 	pgtable_t pgtable;
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	assert_spin_locked(pmd_lockptr(mm, pmdp));
 
@@ -346,7 +346,7 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 		list_del(lh);
 	}
-	ptep = (pte_t *) pgtable;
+	ptep = (hw_pte_t *) pgtable;
 	set_pte(ptep, __pte(_PAGE_INVALID));
 	ptep++;
 	set_pte(ptep, __pte(_PAGE_INVALID));
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index d2879ce860a1..22737319a4c8 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -66,14 +66,14 @@ void *vmem_crst_alloc(unsigned long val)
 	return table;
 }
 
-pte_t __ref *vmem_pte_alloc(void)
+hw_pte_t __ref *vmem_pte_alloc(void)
 {
-	pte_t *pte;
+	hw_pte_t *pte;
 
 	if (slab_is_available())
-		pte = (pte_t *)page_table_alloc(&init_mm);
+		pte = (hw_pte_t *)page_table_alloc(&init_mm);
 	else
-		pte = (pte_t *)memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+		pte = (hw_pte_t *)memblock_alloc(PAGE_SIZE, PAGE_SIZE);
 	if (!pte)
 		return NULL;
 	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
@@ -168,7 +168,8 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr,
 {
 	unsigned long prot, pages = 0;
 	int ret = -ENOMEM;
-	pte_t *pte, entry;
+	hw_pte_t *pte;
+	pte_t entry;
 
 	prot = pgprot_val(PAGE_KERNEL);
 	pte = pte_offset_kernel(pmd, addr);
@@ -204,7 +205,7 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr,
 
 static void try_free_pte_table(pmd_t *pmd, unsigned long start)
 {
-	pte_t *pte;
+	hw_pte_t *pte;
 	int i;
 
 	/* We can safely assume this is fully in 1:1 mapping & vmemmap area */
@@ -226,7 +227,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
 	int ret = -ENOMEM;
 	pmd_t entry;
 	pmd_t *pmd;
-	pte_t *pte;
+	hw_pte_t *pte;
 
 	prot = pgprot_val(SEGMENT_KERNEL);
 	pmd = pmd_offset(pud, addr);
@@ -575,16 +576,16 @@ int vmem_add_mapping(unsigned long start, unsigned long size)
  * while traversing is an error, since the function is expected to be
  * called against virtual regions reserved for 4KB mappings only.
  */
-pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
+hw_pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
 {
-	pte_t *ptep = NULL;
+	hw_pte_t *ptep = NULL;
 	pud_t pud_entry;
 	pmd_t pmd_entry;
 	pgd_t *pgd;
 	p4d_t *p4d;
 	pud_t *pud;
 	pmd_t *pmd;
-	pte_t *pte;
+	hw_pte_t *pte;
 
 	pgd = pgd_offset_k(addr);
 	if (pgd_none(pgdp_get(pgd))) {
@@ -635,7 +636,8 @@ pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
 
 int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc)
 {
-	pte_t *ptep, pte;
+	hw_pte_t *ptep;
+	pte_t pte;
 
 	if (!IS_ALIGNED(addr, PAGE_SIZE))
 		return -EINVAL;
@@ -660,7 +662,7 @@ int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot)
 
 void vmem_unmap_4k_page(unsigned long addr)
 {
-	pte_t *ptep;
+	hw_pte_t *ptep;
 
 	mutex_lock(&vmem_mutex);
 	ptep = virt_to_kpte(addr);
-- 
2.53.0