[RFC PATCH 1/4] mm/mprotect: add MM_CP_PERM_UPGRADE

[RFC PATCH 1/4] mm/mprotect: add MM_CP_PERM_UPGRADE

[Carlos López]

Add a change_protection() flag for changes that only add permissions:
the PFN stays the same and no access is revoked. mprotect() sets it when
the new VMA permissions are a superset of the old, and the protection
key is unchanged.

This will allow MMU notifier users to perform optimizations, as they
can avoid invalidating PTEs whose permissions can be faulted in.

Signed-off-by: Carlos López <clopez@suse.de>
---
 include/linux/mm.h |  2 ++
 mm/mprotect.c      | 13 +++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 691cef038d1e..f406ce9d1032 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3415,6 +3415,8 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 #define  MM_CP_UFFD_RWP_RESOLVE            (1UL << 5) /* resolve rwp */
 #define  MM_CP_UFFD_RWP_ALL                (MM_CP_UFFD_RWP | \
 					    MM_CP_UFFD_RWP_RESOLVE)
+/* Whether this change only upgrades/adds permissions */
+#define  MM_CP_PERM_UPGRADE                (1UL << 6)
 
 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 			     pte_t pte);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 2888ee638d87..e9b636809c42 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -756,6 +756,17 @@ static const struct mm_walk_ops prot_none_walk_ops = {
 	.walk_lock		= PGWALK_WRLOCK,
 };
 
+static bool vma_flags_is_perm_upgrade(vma_flags_t old_vma_flags,
+				       vma_flags_t new_vma_flags)
+{
+	vm_flags_t old = vma_flags_to_legacy(old_vma_flags);
+	vm_flags_t new = vma_flags_to_legacy(new_vma_flags);
+	vm_flags_t rwx = VM_READ | VM_WRITE | VM_EXEC;
+
+	return !(old & rwx & ~new) &&
+	       (old & ARCH_VM_PKEY_FLAGS) == (new & ARCH_VM_PKEY_FLAGS);
+}
+
 int
 mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 	       struct vm_area_struct *vma, struct vm_area_struct **pprev,
@@ -837,6 +848,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 	vma_flags_reset_once(vma, &new_vma_flags);
 	if (vma_wants_manual_pte_write_upgrade(vma))
 		mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
+	if (vma_flags_is_perm_upgrade(old_vma_flags, new_vma_flags))
+		mm_cp_flags |= MM_CP_PERM_UPGRADE;
 	vma_set_page_prot(vma);
 
 	change_protection(tlb, vma, start, end, mm_cp_flags);
-- 
2.51.0

[RFC PATCH 2/4] mm/mmu_notifier: add MMU_NOTIFIER_RANGE_PERM_UPGRADE

[Carlos López]

Add a range flag marking an invalidation that only upgrades permissions
(that is, no PFN change and no revoked access). change_protection() sets
it for MM_CP_PERM_UPGRADE and for the userfaultfd permission resolves,
which unprotect a page and thus only restore access.

A secondary MMU can use the flag to keep its mappings rather than zap
them. Consumers that ignore it are unaffected.

Signed-off-by: Carlos López <clopez@suse.de>
---
 include/linux/mm.h           | 6 ++++++
 include/linux/mmu_notifier.h | 5 +++++
 mm/hugetlb.c                 | 6 +++++-
 mm/mprotect.c                | 7 ++++++-
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index f406ce9d1032..c4fc3876a4b7 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3418,6 +3418,12 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 /* Whether this change only upgrades/adds permissions */
 #define  MM_CP_PERM_UPGRADE                (1UL << 6)
 
+static inline bool mm_cp_is_perm_upgrade(unsigned long cp_flags)
+{
+	return cp_flags & (MM_CP_PERM_UPGRADE | MM_CP_UFFD_WP_RESOLVE |
+			   MM_CP_UFFD_RWP_RESOLVE);
+}
+
 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 			     pte_t pte);
 extern long change_protection(struct mmu_gather *tlb,
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index a11a44eef521..112be28cdf6b 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -60,6 +60,11 @@ enum mmu_notifier_event {
 };
 
 #define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
+/*
+ * Set on an invalidation that only expands permissions without
+ * revoking any existing access, nor freeing/moving the underlying page.
+ */
+#define MMU_NOTIFIER_RANGE_PERM_UPGRADE (1 << 1)
 
 struct mmu_notifier_ops {
 	/*
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index b017ae1dfd6f..68d44044fdd2 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6548,19 +6548,23 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 	long pages = 0, psize = huge_page_size(h);
 	struct mmu_notifier_range range;
 	unsigned long last_addr_mask;
+	unsigned int mn_flags = 0;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
 	bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
 	bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE;
 	struct mmu_gather tlb;
 
+	if (mm_cp_is_perm_upgrade(cp_flags))
+		mn_flags |= MMU_NOTIFIER_RANGE_PERM_UPGRADE;
+
 	/*
 	 * In the case of shared PMDs, the area to flush could be beyond
 	 * start/end.  Set range.start/range.end to cover the maximum possible
 	 * range if PMD sharing is possible.
 	 */
 	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
-				0, mm, start, end);
+				mn_flags, mm, start, end);
 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
 
 	BUG_ON(address >= end);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index e9b636809c42..b48d2fd3b7b5 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -597,8 +597,13 @@ static inline long change_pud_range(struct mmu_gather *tlb,
 			continue;
 
 		if (!range.start) {
+			unsigned int mn_flags = 0;
+
+			if (mm_cp_is_perm_upgrade(cp_flags))
+				mn_flags |= MMU_NOTIFIER_RANGE_PERM_UPGRADE;
+
 			mmu_notifier_range_init(&range,
-						MMU_NOTIFY_PROTECTION_VMA, 0,
+						MMU_NOTIFY_PROTECTION_VMA, mn_flags,
 						vma->vm_mm, addr, end);
 			mmu_notifier_invalidate_range_start(&range);
 		}
-- 
2.51.0

[RFC PATCH 3/4] KVM: plumb MMU_NOTIFIER_RANGE_PERM_UPGRADE into kvm_gfn_range

[Carlos López]

Propagate the permission upgrade flag from the mmu_notifier range into
struct kvm_gfn_range, so the unmap handler can react accordingly. The
value is not consumed yet, so no functional change.

Signed-off-by: Carlos López <clopez@suse.de>
---
 include/linux/kvm_host.h | 1 +
 virt/kvm/kvm_main.c      | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index e7fc94e2910a..aa109818cede 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -274,6 +274,7 @@ struct kvm_gfn_range {
 	enum kvm_gfn_range_filter attr_filter;
 	bool may_block;
 	bool lockless;
+	bool perm_upgrade;
 };
 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index cf4cb586f140..0d0073615fa8 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -532,6 +532,7 @@ struct kvm_mmu_notifier_range {
 	bool flush_on_ret;
 	bool may_block;
 	bool lockless;
+	bool perm_upgrade;
 };
 
 /*
@@ -613,6 +614,7 @@ static __always_inline kvm_mn_ret_t kvm_handle_hva_range(struct kvm *kvm,
 			 */
 			gfn_range.arg = range->arg;
 			gfn_range.may_block = range->may_block;
+			gfn_range.perm_upgrade = range->perm_upgrade;
 			/*
 			 * HVA-based notifications aren't relevant to private
 			 * mappings as they don't have a userspace mapping.
@@ -738,6 +740,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 		.on_lock	= kvm_mmu_invalidate_start,
 		.flush_on_ret	= true,
 		.may_block	= mmu_notifier_range_blockable(range),
+		.perm_upgrade = !!(range->flags & MMU_NOTIFIER_RANGE_PERM_UPGRADE),
 	};
 
 	trace_kvm_unmap_hva_range(range->start, range->end);
@@ -2552,6 +2555,7 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
 
 	gfn_range.arg = range->arg;
 	gfn_range.may_block = range->may_block;
+	gfn_range.perm_upgrade = range->perm_upgrade;
 
 	/*
 	 * If/when KVM supports more attributes beyond private .vs shared, this
-- 
2.51.0

[RFC PATCH 4/4] KVM: x86/mmu: don't zap the secondary MMU on permission upgrade

[Carlos López]

When an mmu_notifier invalidation only upgrades permissions
(MMU_NOTIFIER_RANGE_PERM_UPGRADE), keep the existing SPTEs instead of
zapping them. The PFN is unchanged and no access is revoked, so a held
SPTE is at most as permissive as before. Permissions for the SPTE are
updated on the next fault, so dropping them early gains nothing.

The motivating case is a userfaultfd-wp dirty tracker such as QEMU's
background snapshot, which unprotects every guest page as it saves it.
Zapping on each unprotect makes the guest refault its whole working
set. On a 64GB guest under a background snapshot, a read-mostly workload
drops secondary-MMU refaults from ~1M/s to ~2.5K/s and cross-vCPU TLB
shootdowns from ~1400/s to near zero. Uses of mprotect() that only add
access benefit likewise.

Signed-off-by: Carlos López <clopez@suse.de>
---
 arch/x86/kvm/mmu/mmu.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index dc2aef72ffdd..a4258a8c0c89 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -1719,6 +1719,13 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
 	lockdep_assert_once(kvm->mmu_invalidate_in_progress ||
 			    lockdep_is_held(&kvm->slots_lock));
 
+	/*
+	 * Do not bother zapping SPTEs if this is only a protection upgrade.
+	 * Permissions will be faulted-in on demand.
+	 */
+	if (range->perm_upgrade)
+		return false;
+
 	if (kvm_memslots_have_rmaps(kvm))
 		flush = __kvm_rmap_zap_gfn_range(kvm, range->slot,
 						 range->start, range->end,
-- 
2.51.0