[PATCH 09/19] nitro_enclaves: Do not return enclave threads to a replaced CPU pool

[Alexander Graf <graf@amazon.com>]

A write to ne_cpus checks that no enclave exists, drops
enclaves_list_mutex and then replaces the CPU pool, so an NE_CREATE_VM
in between gets an enclave whose release writes into the freed pool.
Hold the mutex across the check, the teardown and the setup, and take it
and ne_devs.lock killable, as the ioctls do.

Fixes: 9c8eb50fe9e2 ("nitro_enclaves: Add logic for terminating an enclave")
Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_misc_dev.c | 76 +++++++++++------------
 1 file changed, 37 insertions(+), 39 deletions(-)

diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
index ac33fbcd41f35..0a1080443fc72 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
@@ -137,37 +137,6 @@ struct ne_phys_contig_mem_regions {
 	struct range  *regions;
 };
 
-/**
- * ne_check_enclaves_created() - Verify if at least one enclave has been created.
- * @void:	No parameters provided.
- *
- * Context: Process context.
- * Return:
- * * True if at least one enclave is created.
- * * False otherwise.
- */
-static bool ne_check_enclaves_created(void)
-{
-	struct ne_pci_dev *ne_pci_dev = NULL;
-	bool ret = false;
-
-	down_read(&ne_devs.lock);
-
-	ne_pci_dev = ne_devs.ne_pci_dev;
-	if (ne_pci_dev) {
-		mutex_lock(&ne_pci_dev->enclaves_list_mutex);
-
-		if (!list_empty(&ne_pci_dev->enclaves_list))
-			ret = true;
-
-		mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
-	}
-
-	up_read(&ne_devs.lock);
-
-	return ret;
-}
-
 /**
  * ne_setup_cpu_pool() - Set the NE CPU pool after handling sanity checks such
  *			 as not sharing CPU cores with the primary / parent VM
@@ -176,7 +145,8 @@ static bool ne_check_enclaves_created(void)
  *			 pool after the checks passed.
  * @ne_cpu_list:	The CPU list used for setting NE CPU pool.
  *
- * Context: Process context.
+ * Context: Process context. This function is called with the ne_pci_dev enclave
+ *	    mutex held, if there is a NE PCI device.
  * Return:
  * * 0 on success.
  * * Negative return value on failure.
@@ -385,7 +355,8 @@ static int ne_setup_cpu_pool(const char *ne_cpu_list)
  *			    CPU pool.
  * @void:	No parameters provided.
  *
- * Context: Process context.
+ * Context: Process context. This function is called with the ne_pci_dev enclave
+ *	    mutex held, if there is a NE PCI device.
  */
 static void ne_teardown_cpu_pool(void)
 {
@@ -435,15 +406,37 @@ static void ne_teardown_cpu_pool(void)
 static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
 {
 	char error_val[] = "";
+	struct ne_pci_dev *ne_pci_dev = NULL;
 	int rc = -EINVAL;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if (ne_check_enclaves_created()) {
+	rc = down_read_killable(&ne_devs.lock);
+	if (rc)
+		return rc;
+
+	ne_pci_dev = ne_devs.ne_pci_dev;
+
+	/*
+	 * Keep the enclave list from gaining an entry while the pool is being
+	 * replaced, so that no enclave ends up holding a core count that
+	 * describes a pool that is already gone. The list lives in the PCI
+	 * device, which is not bound yet when this runs for a value given on
+	 * the kernel command line; nothing can create an enclave then either.
+	 */
+	if (ne_pci_dev) {
+		rc = mutex_lock_killable(&ne_pci_dev->enclaves_list_mutex);
+		if (rc)
+			goto unlock_devs;
+	}
+
+	if (ne_pci_dev && !list_empty(&ne_pci_dev->enclaves_list)) {
 		pr_err("%s: The CPU pool is used by enclave(s)\n", ne_misc_dev.name);
 
-		return -EPERM;
+		rc = -EPERM;
+
+		goto unlock;
 	}
 
 	ne_teardown_cpu_pool();
@@ -454,7 +447,7 @@ static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
 
 		param_set_copystring(error_val, kp);
 
-		return rc;
+		goto unlock;
 	}
 
 	rc = param_set_copystring(val, kp);
@@ -464,11 +457,16 @@ static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
 		ne_teardown_cpu_pool();
 
 		param_set_copystring(error_val, kp);
-
-		return rc;
 	}
 
-	return 0;
+unlock:
+	if (ne_pci_dev)
+		mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
+
+unlock_devs:
+	up_read(&ne_devs.lock);
+
+	return rc;
 }
 
 /**
-- 
2.47.1