[RFC PATCH v2 5/9] KVM: arm64: nv: selftests: Enable vgic tests to run in vEL2

[Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>]

Extend the vgic_init, vgic_irq and vgic_lpi_stress to run with NV
enabled(vEL2). NV enabled using command line argument and it is
disabled by default. The NV mode is applicable to GICv3 tests only.

Signed-off-by: Ganapatrao Kulkarni <gankulkarni@os.amperecomputing.com>
---
 tools/testing/selftests/kvm/arm64/vgic_init.c | 54 +++++++++++++++++--
 tools/testing/selftests/kvm/arm64/vgic_irq.c  | 27 ++++++----
 .../selftests/kvm/arm64/vgic_lpi_stress.c     | 19 +++++--
 3 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c
index b3b5fb0ff0a9..174350291c96 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_init.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_init.c
@@ -13,6 +13,7 @@
 #include "kvm_util.h"
 #include "processor.h"
 #include "vgic.h"
+#include "nv_util.h"
 
 #define NR_VCPUS		4
 
@@ -29,6 +30,7 @@ struct vm_gic {
 	uint32_t gic_dev_type;
 };
 
+static bool is_nested;
 static uint64_t max_phys_size;
 
 /*
@@ -75,9 +77,19 @@ static struct vm_gic vm_gic_create_with_vcpus(uint32_t gic_dev_type,
 					      struct kvm_vcpu *vcpus[])
 {
 	struct vm_gic v;
+	struct kvm_vcpu_init init;
+	int i;
 
 	v.gic_dev_type = gic_dev_type;
-	v.vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
+
+	v.vm = vm_create(nr_vcpus);
+	vm_ioctl(v.vm, KVM_ARM_PREFERRED_TARGET, &init);
+	if (is_nested)
+		init_vcpu_nested(&init);
+
+	for (i = 0; i < nr_vcpus; i++)
+		vcpus[i] = aarch64_vcpu_add(v.vm, i, &init, guest_code);
+
 	v.gic_fd = kvm_create_device(v.vm, gic_dev_type);
 
 	return v;
@@ -336,14 +348,19 @@ static void test_vgic_then_vcpus(uint32_t gic_dev_type)
 	struct kvm_vcpu *vcpus[NR_VCPUS];
 	struct vm_gic v;
 	int ret, i;
+	struct kvm_vcpu_init init;
 
 	v = vm_gic_create_with_vcpus(gic_dev_type, 1, vcpus);
 
 	subtest_dist_rdist(&v);
 
 	/* Add the rest of the VCPUs */
+	vm_ioctl(v.vm, KVM_ARM_PREFERRED_TARGET, &init);
+	if (is_nested)
+		init_vcpu_nested(&init);
+
 	for (i = 1; i < NR_VCPUS; ++i)
-		vcpus[i] = vm_vcpu_add(v.vm, i, guest_code);
+		vcpus[i] = aarch64_vcpu_add(v.vm, i, &init, guest_code);
 
 	ret = run_vcpu(vcpus[3]);
 	TEST_ASSERT(ret == -EINVAL, "dist/rdist overlap detected on 1st vcpu run");
@@ -606,6 +623,7 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void)
 	struct vm_gic v;
 	int ret, i;
 	uint64_t addr;
+	struct kvm_vcpu_init init;
 
 	v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, vcpus);
 
@@ -619,8 +637,12 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void)
 			    KVM_VGIC_V3_ADDR_TYPE_DIST, &addr);
 
 	/* Add the rest of the VCPUs */
-	for (i = 1; i < NR_VCPUS; ++i)
-		vcpus[i] = vm_vcpu_add(v.vm, i, guest_code);
+	vm_ioctl(v.vm, KVM_ARM_PREFERRED_TARGET, &init);
+	if (is_nested)
+		init_vcpu_nested(&init);
+
+	for (i = 1; i < NR_VCPUS; i++)
+		vcpus[i] = aarch64_vcpu_add(v.vm, i, &init, guest_code);
 
 	kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
 			    KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
@@ -733,11 +755,33 @@ void run_tests(uint32_t gic_dev_type)
 	}
 }
 
-int main(int ac, char **av)
+static void pr_usage(const char *name)
+{
+	pr_info("%s [-g nv] -h\n", name);
+	pr_info("  -g:\tEnable Nested Virtualization, run guest code as guest hypervisor (default: Disabled)\n");
+}
+
+int main(int argc, char **argv)
 {
 	int ret;
 	int pa_bits;
 	int cnt_impl = 0;
+	int opt;
+
+	while ((opt = getopt(argc, argv, "g:")) != -1) {
+		switch (opt) {
+		case 'g':
+			is_nested = atoi_non_negative("Is Nested", optarg);
+			break;
+		case 'h':
+		default:
+			pr_usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (is_nested)
+		TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_EL2));
 
 	pa_bits = vm_guest_mode_params[VM_MODE_DEFAULT].pa_bits;
 	max_phys_size = 1ULL << pa_bits;
diff --git a/tools/testing/selftests/kvm/arm64/vgic_irq.c b/tools/testing/selftests/kvm/arm64/vgic_irq.c
index f4ac28d53747..e4319f91f7cd 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_irq.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_irq.c
@@ -15,6 +15,7 @@
 #include "processor.h"
 #include "test_util.h"
 #include "kvm_util.h"
+#include "nv_util.h"
 #include "gic.h"
 #include "gic_v3.h"
 #include "vgic.h"
@@ -728,7 +729,7 @@ static void print_args(struct test_args *args)
 			args->eoi_split);
 }
 
-static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
+static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split, bool is_nested)
 {
 	struct ucall uc;
 	int gic_fd;
@@ -747,7 +748,10 @@ static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
 
 	print_args(&args);
 
-	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+	if (is_nested)
+		vm = nv_vm_create_with_vcpus_gic(1, &vcpu, NULL, guest_code);
+	else
+		vm = vm_create_with_one_vcpu(&vcpu, guest_code);
 
 	vm_init_descriptor_tables(vm);
 	vcpu_init_descriptor_tables(vcpu);
@@ -795,7 +799,8 @@ static void help(const char *name)
 		"It has to be a multiple of 32 and between 64 and 1024.\n");
 	printf(" -e: if 1 then EOI is split into a write to DIR on top "
 		"of writing EOI.\n");
-	printf(" -l: specify whether the IRQs are level-sensitive (1) or not (0).");
+	printf(" -l: specify whether the IRQs are level-sensitive (1) or not (0).\n");
+	printf(" -g: Enable Nested Virtualization, run guest code as guest hypervisor (default: Disabled)\n");
 	puts("");
 	exit(1);
 }
@@ -807,8 +812,9 @@ int main(int argc, char **argv)
 	bool level_sensitive = false;
 	int opt;
 	bool eoi_split = false;
+	bool is_nested = false;
 
-	while ((opt = getopt(argc, argv, "hn:e:l:")) != -1) {
+	while ((opt = getopt(argc, argv, "hn:e:l:g:")) != -1) {
 		switch (opt) {
 		case 'n':
 			nr_irqs = atoi_non_negative("Number of IRQs", optarg);
@@ -823,6 +829,9 @@ int main(int argc, char **argv)
 			level_sensitive = (bool)atoi_paranoid(optarg);
 			default_args = false;
 			break;
+		case 'g':
+			is_nested = atoi_non_negative("Is Nested", optarg);
+			break;
 		case 'h':
 		default:
 			help(argv[0]);
@@ -835,12 +844,12 @@ int main(int argc, char **argv)
 	 * combinations.
 	 */
 	if (default_args) {
-		test_vgic(nr_irqs, false /* level */, false /* eoi_split */);
-		test_vgic(nr_irqs, false /* level */, true /* eoi_split */);
-		test_vgic(nr_irqs, true /* level */, false /* eoi_split */);
-		test_vgic(nr_irqs, true /* level */, true /* eoi_split */);
+		test_vgic(nr_irqs, false /* level */, false /* eoi_split */, is_nested);
+		test_vgic(nr_irqs, false /* level */, true /* eoi_split */, is_nested);
+		test_vgic(nr_irqs, true /* level */, false /* eoi_split */, is_nested);
+		test_vgic(nr_irqs, true /* level */, true /* eoi_split */, is_nested);
 	} else {
-		test_vgic(nr_irqs, level_sensitive, eoi_split);
+		test_vgic(nr_irqs, level_sensitive, eoi_split, is_nested);
 	}
 
 	return 0;
diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
index fc4fe52fb6f8..63de3903b2c8 100644
--- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
+++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c
@@ -11,6 +11,7 @@
 #include <sys/sysinfo.h>
 
 #include "kvm_util.h"
+#include "nv_util.h"
 #include "gic.h"
 #include "gic_v3.h"
 #include "gic_v3_its.h"
@@ -43,10 +44,12 @@ static struct test_data {
 
 	vm_paddr_t	lpi_prop_table;
 	vm_paddr_t	lpi_pend_tables;
+	bool		is_nested;
 } test_data =  {
 	.nr_cpus	= 1,
 	.nr_devices	= 1,
 	.nr_event_ids	= 16,
+	.is_nested      = false,
 };
 
 static void guest_irq_handler(struct ex_regs *regs)
@@ -333,14 +336,20 @@ static void run_test(void)
 static void setup_vm(void)
 {
 	int i;
+	bool is_nested = test_data.is_nested;
+	u32 nr_cpus = test_data.nr_cpus;
 
 	vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu));
 	TEST_ASSERT(vcpus, "Failed to allocate vCPU array");
 
-	vm = vm_create_with_vcpus(test_data.nr_cpus, guest_code, vcpus);
+
+	if (is_nested)
+		vm = nv_vm_create_with_vcpus_gic(nr_cpus, vcpus, NULL, guest_code);
+	else
+		vm = vm_create_with_vcpus(nr_cpus, guest_code, vcpus);
 
 	vm_init_descriptor_tables(vm);
-	for (i = 0; i < test_data.nr_cpus; i++)
+	for (i = 0; i < nr_cpus; i++)
 		vcpu_init_descriptor_tables(vcpus[i]);
 
 	vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT, guest_irq_handler);
@@ -367,6 +376,7 @@ static void pr_usage(const char *name)
 	pr_info("  -d:\tnumber of devices (default: %u)\n", test_data.nr_devices);
 	pr_info("  -e:\tnumber of event IDs per device (default: %u)\n", test_data.nr_event_ids);
 	pr_info("  -i:\tnumber of iterations (default: %lu)\n", nr_iterations);
+	pr_info("  -g:\tEnable Nested Virtualization, run guest code as guest hypervisor (default: Disabled)\n");
 }
 
 int main(int argc, char **argv)
@@ -374,7 +384,7 @@ int main(int argc, char **argv)
 	u32 nr_threads;
 	int c;
 
-	while ((c = getopt(argc, argv, "hv:d:e:i:")) != -1) {
+	while ((c = getopt(argc, argv, "hv:d:e:i:g:")) != -1) {
 		switch (c) {
 		case 'v':
 			test_data.nr_cpus = atoi(optarg);
@@ -388,6 +398,9 @@ int main(int argc, char **argv)
 		case 'i':
 			nr_iterations = strtoul(optarg, NULL, 0);
 			break;
+		case 'g':
+			test_data.is_nested = atoi_non_negative("Is Nested", optarg);
+			break;
 		case 'h':
 		default:
 			pr_usage(argv[0]);
-- 
2.48.1