[PATCH v2 0/4] KVM: VMX: enable LBR virtualization

[PATCH v2 0/4] KVM: VMX: enable LBR virtualization

[Jian Zhou]

Changelog in v2:
  (1) move the implementation into vmx.c
  (2) migraton is supported
  (3) add arrays in kvm_vcpu_arch struct to save/restore
      LBR MSRs at vm exit/entry time.
  (3) add a parameter of kvm_intel module to permanently
      disable LBRV
  (4) table of supported CPUs is reorgnized, LBRV
      can be enabled or not according to the guest CPUID

Jian Zhou (4):
  KVM: X86: Add arrays to save/restore LBR MSRs
  KVM: X86: LBR MSRs of supported CPU types
  KVM: X86: Migration is supported
  KVM: VMX: details of LBR virtualization implementation

 arch/x86/include/asm/kvm_host.h  |  26 ++++-
 arch/x86/include/asm/msr-index.h |  26 ++++-
 arch/x86/kvm/vmx.c               | 245 +++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/x86.c               |  88 ++++++++++++--
 4 files changed, 366 insertions(+), 19 deletions(-)

--
1.7.12.4

[PATCH v2 1/4] KVM: X86: Add arrays to save/restore LBR MSRs

[Jian Zhou]

Add arrays in kvm_vcpu_arch struct to save/restore
LBR MSRs at vm exit/entry time.
Add new hooks to set/get DEBUGCTLMSR and LBR MSRs.

Signed-off-by: Jian Zhou <jianjay.zhou@huawei.com>
Signed-off-by: Stephen He <herongguang.he@huawei.com>
---
 arch/x86/include/asm/kvm_host.h | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3a36ee7..dc2c120 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -376,6 +376,12 @@ struct kvm_vcpu_hv {
 	u64 hv_vapic;
 };

+struct msr_data {
+	bool host_initiated;
+	u32 index;
+	u64 data;
+};
+
 struct kvm_vcpu_arch {
 	/*
 	 * rip and regs accesses must go through
@@ -516,6 +522,15 @@ struct kvm_vcpu_arch {
 	unsigned long eff_db[KVM_NR_DB_REGS];
 	unsigned long guest_debug_dr7;

+	int lbr_status;
+	int lbr_used;
+
+	struct lbr_msr {
+		unsigned nr;
+		struct msr_data guest[MAX_NUM_LBR_MSRS];
+		struct msr_data host[MAX_NUM_LBR_MSRS];
+	}lbr_msr;
+
 	u64 mcg_cap;
 	u64 mcg_status;
 	u64 mcg_ctl;
@@ -728,12 +743,6 @@ struct kvm_vcpu_stat {

 struct x86_instruction_info;

-struct msr_data {
-	bool host_initiated;
-	u32 index;
-	u64 data;
-};
-
 struct kvm_lapic_irq {
 	u32 vector;
 	u16 delivery_mode;
@@ -887,6 +896,11 @@ struct kvm_x86_ops {
 					   gfn_t offset, unsigned long mask);
 	/* pmu operations of sub-arch */
 	const struct kvm_pmu_ops *pmu_ops;
+
+	int (*set_debugctlmsr)(struct kvm_vcpu *vcpu, u64 value);
+	u64 (*get_debugctlmsr)(void);
+	void (*set_lbr_msr)(struct kvm_vcpu *vcpu, u32 msr, u64 data);
+	u64 (*get_lbr_msr)(struct kvm_vcpu *vcpu, u32 msr);
 };

 struct kvm_arch_async_pf {
--
1.7.12.4

[PATCH v2 2/4] KVM: X86: LBR MSRs of supported CPU types

[Jian Zhou]

Macros about LBR MSRs.

Signed-off-by: Jian Zhou <jianjay.zhou@huawei.com>
Signed-off-by: Stephen He <herongguang.he@huawei.com>
---
 arch/x86/include/asm/msr-index.h | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index b98b471..2afcacd 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -68,10 +68,32 @@

 #define MSR_LBR_SELECT			0x000001c8
 #define MSR_LBR_TOS			0x000001c9
+#define MSR_LBR_CORE_FROM		0x00000040
+#define MSR_LBR_CORE_TO 		0x00000060
+#define MAX_NUM_LBR_MSRS		128
+/* Pentium4/Xeon(based on NetBurst) LBR */
+#define MSR_PENTIUM4_LER_FROM_LIP	0x000001d7
+#define MSR_PENTIUM4_LER_TO_LIP		0x000001d8
+#define MSR_PENTIUM4_LBR_TOS		0x000001da
+#define MSR_LBR_PENTIUM4_FROM		0x00000680
+#define MSR_LBR_PENTIUM4_TO		0x000006c0
+#define SIZE_PENTIUM4_LBR_STACK 	16
+/* Core2 LBR */
+#define MSR_LBR_CORE2_FROM		MSR_LBR_CORE_FROM
+#define MSR_LBR_CORE2_TO		MSR_LBR_CORE_TO
+#define SIZE_CORE2_LBR_STACK		4
+/* Atom LBR */
+#define MSR_LBR_ATOM_FROM		MSR_LBR_CORE_FROM
+#define MSR_LBR_ATOM_TO 		MSR_LBR_CORE_TO
+#define SIZE_ATOM_LBR_STACK		8
+/* Nehalem LBR */
 #define MSR_LBR_NHM_FROM		0x00000680
 #define MSR_LBR_NHM_TO			0x000006c0
-#define MSR_LBR_CORE_FROM		0x00000040
-#define MSR_LBR_CORE_TO			0x00000060
+#define SIZE_NHM_LBR_STACK		16
+/* Skylake LBR */
+#define MSR_LBR_SKYLAKE_FROM		MSR_LBR_NHM_FROM
+#define MSR_LBR_SKYLAKE_TO		MSR_LBR_NHM_TO
+#define SIZE_SKYLAKE_LBR_STACK		32

 #define MSR_LBR_INFO_0			0x00000dc0 /* ... 0xddf for _31 */
 #define LBR_INFO_MISPRED		BIT_ULL(63)
--
1.7.12.4

[PATCH v2 3/4] KVM: X86: Migration is supported

[Jian Zhou]

Supported bits of MSR_IA32_DEBUGCTLMSR are DEBUGCTLMSR_LBR(bit 0),
DEBUGCTLMSR_BTF(bit 1) and DEBUGCTLMSR_FREEZE_LBRS_ON_PMI(bit 11).
Qemu can get/set contents of LBR MSRs and LBR status in order to
support migration.

Signed-off-by: Jian Zhou <jianjay.zhou@huawei.com>
Signed-off-by: Stephen He <herongguang.he@huawei.com>
---
 arch/x86/kvm/x86.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 77 insertions(+), 11 deletions(-)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 9a9a198..a3c72db 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -136,6 +136,8 @@ struct kvm_shared_msrs {
 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
 static struct kvm_shared_msrs __percpu *shared_msrs;

+#define MSR_LBR_STATUS 0xd6
+
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "pf_fixed", VCPU_STAT(pf_fixed) },
 	{ "pf_guest", VCPU_STAT(pf_guest) },
@@ -1917,6 +1919,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	bool pr = false;
 	u32 msr = msr_info->index;
 	u64 data = msr_info->data;
+	u64 supported = 0;

 	switch (msr) {
 	case MSR_AMD64_NB_CFG:
@@ -1948,16 +1951,25 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		}
 		break;
 	case MSR_IA32_DEBUGCTLMSR:
-		if (!data) {
-			/* We support the non-activated case already */
-			break;
-		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
-			/* Values other than LBR and BTF are vendor-specific,
-			   thus reserved and should throw a #GP */
+		supported = DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF |
+				DEBUGCTLMSR_FREEZE_LBRS_ON_PMI;
+
+		if (data & ~supported) {
+			/*
+			 * Values other than LBR/BTF/FREEZE_LBRS_ON_PMI
+			 * are not supported, thus reserved and should throw a #GP
+			 */
+			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
+					__func__, data);
 			return 1;
 		}
-		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
-			    __func__, data);
+		if (kvm_x86_ops->set_debugctlmsr) {
+			if (kvm_x86_ops->set_debugctlmsr(vcpu, data))
+				return 1;
+		}
+		else
+			return 1;
+
 		break;
 	case 0x200 ... 0x2ff:
 		return kvm_mtrr_set_msr(vcpu, msr, data);
@@ -2078,6 +2090,33 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
 				    "0x%x data 0x%llx\n", msr, data);
 		break;
+	case MSR_LBR_STATUS:
+		if (kvm_x86_ops->set_debugctlmsr) {
+			vcpu->arch.lbr_status = (data == 0) ? 0 : 1;
+			if (data)
+				kvm_x86_ops->set_debugctlmsr(vcpu,
+						DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+		} else
+			vcpu_unimpl(vcpu, "lbr is disabled, ignored wrmsr: "
+					"0x%x data 0x%llx\n", msr, data);
+		break;
+	case MSR_LBR_SELECT:
+	case MSR_LBR_TOS:
+	case MSR_PENTIUM4_LER_FROM_LIP:
+	case MSR_PENTIUM4_LER_TO_LIP:
+	case MSR_PENTIUM4_LBR_TOS:
+	case MSR_IA32_LASTINTFROMIP:
+	case MSR_IA32_LASTINTTOIP:
+	case MSR_LBR_CORE2_FROM ... MSR_LBR_CORE2_FROM + 0x7:
+	case MSR_LBR_CORE2_TO ... MSR_LBR_CORE2_TO + 0x7:
+	case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 0x1f:
+	case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 0x1f:
+		if (kvm_x86_ops->set_lbr_msr)
+			kvm_x86_ops->set_lbr_msr(vcpu, msr, data);
+		else
+			vcpu_unimpl(vcpu, "lbr is disabled, ignored wrmsr: "
+					"0x%x data 0x%llx\n", msr, data);
+		break;
 	case MSR_K7_CLK_CTL:
 		/*
 		 * Ignore all writes to this no longer documented MSR.
@@ -2178,13 +2217,16 @@ static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
 	switch (msr_info->index) {
+	case MSR_IA32_DEBUGCTLMSR:
+		if (kvm_x86_ops->get_debugctlmsr)
+			msr_info->data = kvm_x86_ops->get_debugctlmsr();
+		else
+			msr_info->data = 0;
+		break;
 	case MSR_IA32_PLATFORM_ID:
 	case MSR_IA32_EBL_CR_POWERON:
-	case MSR_IA32_DEBUGCTLMSR:
 	case MSR_IA32_LASTBRANCHFROMIP:
 	case MSR_IA32_LASTBRANCHTOIP:
-	case MSR_IA32_LASTINTFROMIP:
-	case MSR_IA32_LASTINTTOIP:
 	case MSR_K8_SYSCFG:
 	case MSR_K8_TSEG_ADDR:
 	case MSR_K8_TSEG_MASK:
@@ -2204,6 +2246,26 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
 		msr_info->data = 0;
 		break;
+	case MSR_LBR_STATUS:
+		msr_info->data = vcpu->arch.lbr_status;
+		break;
+	case MSR_LBR_SELECT:
+	case MSR_LBR_TOS:
+	case MSR_PENTIUM4_LER_FROM_LIP:
+	case MSR_PENTIUM4_LER_TO_LIP:
+	case MSR_PENTIUM4_LBR_TOS:
+	case MSR_IA32_LASTINTFROMIP:
+	case MSR_IA32_LASTINTTOIP:
+	case MSR_LBR_CORE2_FROM ... MSR_LBR_CORE2_FROM + 0x7:
+	case MSR_LBR_CORE2_TO ... MSR_LBR_CORE2_TO + 0x7:
+	case MSR_LBR_SKYLAKE_FROM ... MSR_LBR_SKYLAKE_FROM + 0x1f:
+	case MSR_LBR_SKYLAKE_TO ... MSR_LBR_SKYLAKE_TO + 0x1f:
+		if (kvm_x86_ops->get_lbr_msr)
+			msr_info->data = kvm_x86_ops->get_lbr_msr(vcpu,
+					msr_info->index);
+		else
+			msr_info->data = 0;
+		break;
 	case MSR_IA32_UCODE_REV:
 		msr_info->data = 0x100000000ULL;
 		break;
@@ -7376,6 +7438,10 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 	kvm_async_pf_hash_reset(vcpu);
 	kvm_pmu_init(vcpu);

+	vcpu->arch.lbr_status = 0;
+	vcpu->arch.lbr_used = 0;
+	vcpu->arch.lbr_msr.nr = 0;
+
 	return 0;

 fail_free_mce_banks:
--
1.7.12.4

Re: [PATCH v2 1/4] KVM: X86: Add arrays to save/restore LBR MSRs

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 1383 bytes --]

Hi Jian,

[auto build test ERROR on v4.3-rc6 -- if it's inappropriate base, please suggest rules for selecting the more suitable base]

url:    https://github.com/0day-ci/linux/commits/Jian-Zhou/KVM-X86-Add-arrays-to-save-restore-LBR-MSRs/20151023-172601
config: x86_64-lkp (attached as .config)
reproduce:
        # save the attached .config to linux build tree
        make ARCH=x86_64 

Note: the linux-review/Jian-Zhou/KVM-X86-Add-arrays-to-save-restore-LBR-MSRs/20151023-172601 HEAD d402c03a709c1dff60e2800becbafaf3b2d86dcd builds fine.
      It only hurts bisectibility.

All errors (new ones prefixed by >>):

   In file included from include/linux/kvm_host.h:34:0,
                    from arch/x86/kvm/../../../virt/kvm/kvm_main.c:21:
>> arch/x86/include/asm/kvm_host.h:530:25: error: 'MAX_NUM_LBR_MSRS' undeclared here (not in a function)
      struct msr_data guest[MAX_NUM_LBR_MSRS];
                            ^

vim +/MAX_NUM_LBR_MSRS +530 arch/x86/include/asm/kvm_host.h

   524	
   525		int lbr_status;
   526		int lbr_used;
   527	
   528		struct lbr_msr {
   529			unsigned nr;
 > 530			struct msr_data guest[MAX_NUM_LBR_MSRS];
   531			struct msr_data host[MAX_NUM_LBR_MSRS];
   532		}lbr_msr;
   533	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 21911 bytes --]

[PATCH v2 1/4] KVM: X86: Add arrays to save/restore LBR MSRs

[Jian Zhou]

Add arrays in kvm_vcpu_arch struct to save/restore
LBR MSRs at vm exit/entry time.
Add new hooks to set/get DEBUGCTLMSR and LBR MSRs.

Signed-off-by: Jian Zhou <jianjay.zhou@huawei.com>
Signed-off-by: Stephen He <herongguang.he@huawei.com>
---
 arch/x86/include/asm/kvm_host.h | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3a36ee7..dc2c120 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -376,6 +376,12 @@ struct kvm_vcpu_hv {
 	u64 hv_vapic;
 };

+struct msr_data {
+	bool host_initiated;
+	u32 index;
+	u64 data;
+};
+
 struct kvm_vcpu_arch {
 	/*
 	 * rip and regs accesses must go through
@@ -516,6 +522,15 @@ struct kvm_vcpu_arch {
 	unsigned long eff_db[KVM_NR_DB_REGS];
 	unsigned long guest_debug_dr7;

+	int lbr_status;
+	int lbr_used;
+
+	struct lbr_msr {
+		unsigned nr;
+		struct msr_data guest[MAX_NUM_LBR_MSRS];
+		struct msr_data host[MAX_NUM_LBR_MSRS];
+	}lbr_msr;
+
 	u64 mcg_cap;
 	u64 mcg_status;
 	u64 mcg_ctl;
@@ -728,12 +743,6 @@ struct kvm_vcpu_stat {

 struct x86_instruction_info;

-struct msr_data {
-	bool host_initiated;
-	u32 index;
-	u64 data;
-};
-
 struct kvm_lapic_irq {
 	u32 vector;
 	u16 delivery_mode;
@@ -887,6 +896,11 @@ struct kvm_x86_ops {
 					   gfn_t offset, unsigned long mask);
 	/* pmu operations of sub-arch */
 	const struct kvm_pmu_ops *pmu_ops;
+
+	int (*set_debugctlmsr)(struct kvm_vcpu *vcpu, u64 value);
+	u64 (*get_debugctlmsr)(void);
+	void (*set_lbr_msr)(struct kvm_vcpu *vcpu, u32 msr, u64 data);
+	u64 (*get_lbr_msr)(struct kvm_vcpu *vcpu, u32 msr);
 };

 struct kvm_arch_async_pf {
--
1.7.12.4