From: isaku.yamahata@intel.com
To: kvm@vger.kernel.org, linux-kernel@vger.kernel.org
Cc: isaku.yamahata@intel.com, isaku.yamahata@gmail.com,
Paolo Bonzini <pbonzini@redhat.com>,
erdemaktas@google.com, Sean Christopherson <seanjc@google.com>,
Sagi Shahar <sagis@google.com>, Kai Huang <kai.huang@intel.com>,
chen.bo@intel.com, hang.yuan@intel.com, tina.zhang@intel.com,
Xiaoyao Li <xiaoyao.li@intel.com>
Subject: [PATCH v7 08/13] KVM: x86/tdp_mmu: Split the large page when zap leaf
Date: Mon, 22 Jan 2024 16:22:23 -0800 [thread overview]
Message-ID: <3391bf2cf96df8744e0abc023d8af9ec677ee4e8.1705965958.git.isaku.yamahata@intel.com> (raw)
In-Reply-To: <cover.1705965958.git.isaku.yamahata@intel.com>
From: Xiaoyao Li <xiaoyao.li@intel.com>
When TDX enabled, a large page cannot be zapped if it contains mixed
pages. In this case, it has to split the large page.
Signed-off-by: Xiaoyao Li <xiaoyao.li@intel.com>
---
v7:
- remote unnecessary tlb shoot down in tdp_mmu_zap_leafs() to free unused
split_sp.
---
arch/x86/kvm/Kconfig | 1 +
arch/x86/kvm/mmu/mmu.c | 6 ++--
arch/x86/kvm/mmu/mmu_internal.h | 9 +++++
arch/x86/kvm/mmu/tdp_mmu.c | 60 ++++++++++++++++++++++++++++++---
4 files changed, 69 insertions(+), 7 deletions(-)
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index fa00abb9ab39..1aa37d494ae9 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -89,6 +89,7 @@ config KVM_INTEL
tristate "KVM for Intel (and compatible) processors support"
depends on KVM && IA32_FEAT_CTL
select KVM_SW_PROTECTED_VM if INTEL_TDX_HOST
+ select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST
select KVM_PRIVATE_MEM if INTEL_TDX_HOST
help
Provides support for KVM on processors equipped with Intel's VT
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 971dbd9c95cc..a9e7a3d2d362 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -7461,8 +7461,8 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
return kvm_unmap_gfn_range(kvm, range);
}
-static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
- int level)
+bool kvm_hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
+ int level)
{
return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;
}
@@ -7489,7 +7489,7 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
return kvm_range_has_memory_attributes(kvm, start, end, attrs);
for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {
- if (hugepage_test_mixed(slot, gfn, level - 1) ||
+ if (kvm_hugepage_test_mixed(slot, gfn, level - 1) ||
attrs != kvm_get_memory_attributes(kvm, gfn))
return false;
}
diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h
index 9888ea0046ea..cc0a95e554b5 100644
--- a/arch/x86/kvm/mmu/mmu_internal.h
+++ b/arch/x86/kvm/mmu/mmu_internal.h
@@ -461,4 +461,13 @@ void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
void track_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp);
void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp);
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+bool kvm_hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level);
+#else
+static inline bool kvm_hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level)
+{
+ return false;
+}
+#endif
+
#endif /* __KVM_X86_MMU_INTERNAL_H */
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 7991934b3f37..98de2c093815 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -953,6 +953,14 @@ bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
return true;
}
+
+static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm,
+ struct tdp_iter *iter,
+ bool shared);
+
+static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter,
+ struct kvm_mmu_page *sp, bool shared);
+
/*
* If can_yield is true, will release the MMU lock and reschedule if the
* scheduler needs the CPU or there is contention on the MMU lock. If this
@@ -964,14 +972,16 @@ static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
gfn_t start, gfn_t end, bool can_yield, bool flush,
bool zap_private)
{
+ bool is_private = is_private_sp(root);
+ struct kvm_mmu_page *split_sp = NULL;
struct tdp_iter iter;
end = min(end, tdp_mmu_max_gfn_exclusive());
lockdep_assert_held_write(&kvm->mmu_lock);
- WARN_ON_ONCE(zap_private && !is_private_sp(root));
- if (!zap_private && is_private_sp(root))
+ WARN_ON_ONCE(zap_private && !is_private);
+ if (!zap_private && is_private)
return false;
/*
@@ -995,12 +1005,56 @@ static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
!is_last_spte(iter.old_spte, iter.level))
continue;
+ if (is_private && kvm_gfn_shared_mask(kvm) &&
+ is_large_pte(iter.old_spte)) {
+ gfn_t gfn = iter.gfn & ~kvm_gfn_shared_mask(kvm);
+ gfn_t mask = KVM_PAGES_PER_HPAGE(iter.level) - 1;
+ struct kvm_memory_slot *slot;
+ struct kvm_mmu_page *sp;
+
+ slot = gfn_to_memslot(kvm, gfn);
+ if (kvm_hugepage_test_mixed(slot, gfn, iter.level) ||
+ (gfn & mask) < start ||
+ end < (gfn & mask) + KVM_PAGES_PER_HPAGE(iter.level)) {
+ WARN_ON_ONCE(!can_yield);
+ if (split_sp) {
+ sp = split_sp;
+ split_sp = NULL;
+ sp->role = tdp_iter_child_role(&iter);
+ } else {
+ WARN_ON(iter.yielded);
+ if (flush && can_yield) {
+ kvm_flush_remote_tlbs(kvm);
+ flush = false;
+ }
+ sp = tdp_mmu_alloc_sp_for_split(kvm, &iter, false);
+ if (iter.yielded) {
+ split_sp = sp;
+ continue;
+ }
+ }
+ KVM_BUG_ON(!sp, kvm);
+
+ tdp_mmu_init_sp(sp, iter.sptep, iter.gfn);
+ if (tdp_mmu_split_huge_page(kvm, &iter, sp, false)) {
+ /* force retry on this gfn. */
+ iter.yielded = true;
+ split_sp = sp;
+ } else
+ flush = true;
+ continue;
+ }
+ }
+
tdp_mmu_iter_set_spte(kvm, &iter, SHADOW_NONPRESENT_VALUE);
flush = true;
}
rcu_read_unlock();
+ if (split_sp)
+ tdp_mmu_free_sp(split_sp);
+
/*
* Because this flow zaps _only_ leaf SPTEs, the caller doesn't need
* to provide RCU protection as no 'struct kvm_mmu_page' will be freed.
@@ -1617,8 +1671,6 @@ static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm,
kvm_lockdep_assert_mmu_lock_held(kvm, shared);
KVM_BUG_ON(kvm_mmu_page_role_is_private(role) !=
is_private_sptep(iter->sptep), kvm);
- /* TODO: Large page isn't supported for private SPTE yet. */
- KVM_BUG_ON(kvm_mmu_page_role_is_private(role), kvm);
/*
* Since we are allocating while under the MMU lock we have to be
--
2.25.1
next prev parent reply other threads:[~2024-01-23 0:22 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-01-23 0:22 [PATCH v7 00/13] KVM TDX: TDP MMU: large page support isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 01/13] KVM: TDX: Flush cache based on page size before TDX SEAMCALL isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 02/13] KVM: TDX: Pass KVM page level to tdh_mem_page_aug() isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 03/13] KVM: TDX: Pass size to reclaim_page() isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 04/13] KVM: TDX: Update tdx_sept_{set,drop}_private_spte() to support large page isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 05/13] KVM: MMU: Introduce level info in PFERR code isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 06/13] KVM: TDX: Pass desired page level in err code for page fault handler isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 07/13] KVM: x86/tdp_mmu: Allocate private page table for large page split isaku.yamahata
2024-01-23 0:22 ` isaku.yamahata [this message]
2024-01-23 0:22 ` [PATCH v7 09/13] KVM: x86/tdp_mmu, TDX: Split a large page when 4KB page within it converted to shared isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 10/13] KVM: x86/tdp_mmu: Try to merge pages into a large page isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 11/13] KVM: TDX: Implement " isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 12/13] KVM: x86/mmu: Make kvm fault handler aware of large page of private memslot isaku.yamahata
2024-01-23 0:22 ` [PATCH v7 13/13] KVM: TDX: Allow 2MB large page for TD GUEST isaku.yamahata
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=3391bf2cf96df8744e0abc023d8af9ec677ee4e8.1705965958.git.isaku.yamahata@intel.com \
--to=isaku.yamahata@intel.com \
--cc=chen.bo@intel.com \
--cc=erdemaktas@google.com \
--cc=hang.yuan@intel.com \
--cc=isaku.yamahata@gmail.com \
--cc=kai.huang@intel.com \
--cc=kvm@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=pbonzini@redhat.com \
--cc=sagis@google.com \
--cc=seanjc@google.com \
--cc=tina.zhang@intel.com \
--cc=xiaoyao.li@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®