From: Paolo Bonzini <pbonzini@redhat.com>
To: linux-kernel@vger.kernel.org, kvm@vger.kernel.org
Cc: bgardon@google.com
Subject: [PATCH 14/22] kvm: x86/mmu: Support invalidate range MMU notifier for TDP MMU
Date: Fri, 23 Oct 2020 12:30:16 -0400 [thread overview]
Message-ID: <20201023163024.2765558-15-pbonzini@redhat.com> (raw)
In-Reply-To: <20201023163024.2765558-1-pbonzini@redhat.com>
From: Ben Gardon <bgardon@google.com>
In order to interoperate correctly with the rest of KVM and other Linux
subsystems, the TDP MMU must correctly handle various MMU notifiers. Add
hooks to handle the invalidate range family of MMU notifiers.
Tested by running kvm-unit-tests and KVM selftests on an Intel Haswell
machine. This series introduced no new failures.
This series can be viewed in Gerrit at:
https://linux-review.googlesource.com/c/virt/kvm/kvm/+/2538
Signed-off-by: Ben Gardon <bgardon@google.com>
Message-Id: <20201014182700.2888246-13-bgardon@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
arch/x86/kvm/mmu/mmu.c | 9 ++++-
arch/x86/kvm/mmu/tdp_mmu.c | 80 +++++++++++++++++++++++++++++++++++---
arch/x86/kvm/mmu/tdp_mmu.h | 3 ++
3 files changed, 86 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 31d7ba716b44..35c277ed6c78 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -1497,7 +1497,14 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end,
unsigned flags)
{
- return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
+ int r;
+
+ r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
+
+ if (kvm->arch.tdp_mmu_enabled)
+ r |= kvm_tdp_mmu_zap_hva_range(kvm, start, end);
+
+ return r;
}
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index f06802289c1f..96bc6aa39628 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -52,7 +52,7 @@ bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
}
static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
- gfn_t start, gfn_t end);
+ gfn_t start, gfn_t end, bool can_yield);
void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root)
{
@@ -65,7 +65,7 @@ void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root)
list_del(&root->link);
- zap_gfn_range(kvm, root, 0, max_gfn);
+ zap_gfn_range(kvm, root, 0, max_gfn, false);
free_page((unsigned long)root->spt);
kmem_cache_free(mmu_page_header_cache, root);
@@ -303,9 +303,14 @@ static bool tdp_mmu_iter_flush_cond_resched(struct kvm *kvm, struct tdp_iter *it
* non-root pages mapping GFNs strictly within that range. Returns true if
* SPTEs have been cleared and a TLB flush is needed before releasing the
* MMU lock.
+ * If can_yield is true, will release the MMU lock and reschedule if the
+ * scheduler needs the CPU or there is contention on the MMU lock. If this
+ * function cannot yield, it will not release the MMU lock or reschedule and
+ * the caller must ensure it does not supply too large a GFN range, or the
+ * operation can cause a soft lockup.
*/
static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
- gfn_t start, gfn_t end)
+ gfn_t start, gfn_t end, bool can_yield)
{
struct tdp_iter iter;
bool flush_needed = false;
@@ -326,7 +331,10 @@ static bool zap_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
tdp_mmu_set_spte(kvm, &iter, 0);
- flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
+ if (can_yield)
+ flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
+ else
+ flush_needed = true;
}
return flush_needed;
}
@@ -349,7 +357,7 @@ bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, gfn_t start, gfn_t end)
*/
kvm_mmu_get_root(kvm, root);
- flush |= zap_gfn_range(kvm, root, start, end);
+ flush |= zap_gfn_range(kvm, root, start, end, true);
kvm_mmu_put_root(kvm, root);
}
@@ -496,3 +504,65 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
return ret;
}
+
+static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
+ unsigned long end, unsigned long data,
+ int (*handler)(struct kvm *kvm, struct kvm_memory_slot *slot,
+ struct kvm_mmu_page *root, gfn_t start,
+ gfn_t end, unsigned long data))
+{
+ struct kvm_memslots *slots;
+ struct kvm_memory_slot *memslot;
+ struct kvm_mmu_page *root;
+ int ret = 0;
+ int as_id;
+
+ for_each_tdp_mmu_root(kvm, root) {
+ /*
+ * Take a reference on the root so that it cannot be freed if
+ * this thread releases the MMU lock and yields in this loop.
+ */
+ kvm_mmu_get_root(kvm, root);
+
+ as_id = kvm_mmu_page_as_id(root);
+ slots = __kvm_memslots(kvm, as_id);
+ kvm_for_each_memslot(memslot, slots) {
+ unsigned long hva_start, hva_end;
+ gfn_t gfn_start, gfn_end;
+
+ hva_start = max(start, memslot->userspace_addr);
+ hva_end = min(end, memslot->userspace_addr +
+ (memslot->npages << PAGE_SHIFT));
+ if (hva_start >= hva_end)
+ continue;
+ /*
+ * {gfn(page) | page intersects with [hva_start, hva_end)} =
+ * {gfn_start, gfn_start+1, ..., gfn_end-1}.
+ */
+ gfn_start = hva_to_gfn_memslot(hva_start, memslot);
+ gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
+
+ ret |= handler(kvm, memslot, root, gfn_start,
+ gfn_end, data);
+ }
+
+ kvm_mmu_put_root(kvm, root);
+ }
+
+ return ret;
+}
+
+static int zap_gfn_range_hva_wrapper(struct kvm *kvm,
+ struct kvm_memory_slot *slot,
+ struct kvm_mmu_page *root, gfn_t start,
+ gfn_t end, unsigned long unused)
+{
+ return zap_gfn_range(kvm, root, start, end, false);
+}
+
+int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start,
+ unsigned long end)
+{
+ return kvm_tdp_mmu_handle_hva_range(kvm, start, end, 0,
+ zap_gfn_range_hva_wrapper);
+}
diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h
index aed21a7a3bd6..af25d2462cb8 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.h
+++ b/arch/x86/kvm/mmu/tdp_mmu.h
@@ -18,4 +18,7 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm);
int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code,
int map_writable, int max_level, kvm_pfn_t pfn,
bool prefault);
+
+int kvm_tdp_mmu_zap_hva_range(struct kvm *kvm, unsigned long start,
+ unsigned long end);
#endif /* __KVM_X86_MMU_TDP_MMU_H */
--
2.26.2
next prev parent reply other threads:[~2020-10-23 16:31 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-10-23 16:30 [PATCH 00/22] Introduce the " Paolo Bonzini
2020-10-23 16:30 ` [PATCH 01/22] kvm: mmu: Separate making non-leaf sptes from link_shadow_page Paolo Bonzini
2020-10-23 16:30 ` [PATCH 02/22] kvm: x86/mmu: Separate making SPTEs from set_spte Paolo Bonzini
2020-10-23 16:30 ` [PATCH 03/22] KVM: mmu: Separate updating a PTE from kvm_set_pte_rmapp Paolo Bonzini
2020-10-23 16:30 ` [PATCH 04/22] KVM: mmu: extract spte.h and spte.c Paolo Bonzini
2020-10-27 14:46 ` Valdis Klētnieks
2020-10-28 17:36 ` Paolo Bonzini
2020-10-23 16:30 ` [PATCH 05/22] kvm: x86/mmu: Introduce tdp_iter Paolo Bonzini
2020-10-23 16:30 ` [PATCH 06/22] kvm: x86/mmu: Init / Uninit the TDP MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 07/22] kvm: x86/mmu: Allocate and free TDP MMU roots Paolo Bonzini
2020-10-23 16:30 ` [PATCH 08/22] kvm: x86/mmu: Add functions to handle changed TDP SPTEs Paolo Bonzini
2020-10-23 16:30 ` [PATCH 09/22] KVM: Cache as_id in kvm_memory_slot Paolo Bonzini
2020-10-23 16:30 ` [PATCH 10/22] kvm: x86/mmu: Support zapping SPTEs in the TDP MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 11/22] kvm: x86/mmu: Remove disallowed_hugepage_adjust shadow_walk_iterator arg Paolo Bonzini
2020-10-23 16:30 ` [PATCH 12/22] kvm: x86/mmu: Add TDP MMU PF handler Paolo Bonzini
2020-10-23 16:30 ` [PATCH 13/22] kvm: x86/mmu: Allocate struct kvm_mmu_pages for all pages in TDP MMU Paolo Bonzini
2020-10-23 16:30 ` Paolo Bonzini [this message]
2020-10-23 16:30 ` [PATCH 15/22] kvm: x86/mmu: Add access tracking for tdp_mmu Paolo Bonzini
2020-10-23 16:30 ` [PATCH 16/22] kvm: x86/mmu: Support changed pte notifier in tdp MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 17/22] kvm: x86/mmu: Support dirty logging for the TDP MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 18/22] kvm: x86/mmu: Support disabling dirty logging for the tdp MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 19/22] kvm: x86/mmu: Support write protection for nesting in " Paolo Bonzini
2020-10-23 16:30 ` [PATCH 20/22] kvm: x86/mmu: Support MMIO in the TDP MMU Paolo Bonzini
2020-10-23 16:30 ` [PATCH 21/22] kvm: x86/mmu: Don't clear write flooding count for direct roots Paolo Bonzini
2020-10-23 16:30 ` [PATCH 22/22] kvm: x86/mmu: NX largepage recovery for TDP MMU Paolo Bonzini
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201023163024.2765558-15-pbonzini@redhat.com \
--to=pbonzini@redhat.com \
--cc=bgardon@google.com \
--cc=kvm@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®