[PATCH 14/19] nitro_enclaves: Build the boot-time CPU pool once the CPUs are online

[Alexander Graf <graf@amazon.com>]

An admin who passes nitro_enclaves.ne_cpus= to a kernel with the driver
built in gets no CPU pool, and this not very informative error message:

  Booting kernel: `2-5' invalid for parameter `nitro_enclaves.ne_cpus'

The parameter setter runs from parse_args() in start_kernel(), before
the page allocator and the secondary CPUs are up. Building a pool needs
both.

Keep a value that arrives while system_state is SYSTEM_BOOTING in
ne_cpus, note its parameter in ne_cpus_kp, and build the pool from it in
ne_init(), which runs once both are up. The pool building moves into
ne_replace_cpu_pool() for that. That way a valid list on the command
line gets its pool, and an invalid one reports why.

Fixes: ff8a4d3e3a99 ("nitro_enclaves: Add logic for setting an enclave vCPU")
Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_misc_dev.c | 63 +++++++++++++++++++++--
 1 file changed, 58 insertions(+), 5 deletions(-)

diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
index 44a4432be8164..a897036dee2dd 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
@@ -14,6 +14,7 @@
 #include <linux/device.h>
 #include <linux/file.h>
 #include <linux/hugetlb.h>
+#include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/list.h>
 #include <linux/miscdevice.h>
@@ -102,6 +103,13 @@ module_param_cb(ne_cpus, &ne_cpu_pool_ops, &ne_cpus_arg, 0644);
 /* https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists */
 MODULE_PARM_DESC(ne_cpus, "<cpu-list> - CPU pool used for Nitro Enclaves");
 
+/*
+ * The parameter a value from the kernel command line is kept in until ne_init()
+ * builds its pool. Nothing else writes ne_cpus, so it is set whenever ne_cpus
+ * holds a value, and no sysfs write can reach the setter before ne_init() runs.
+ */
+static const struct kernel_param *ne_cpus_kp;
+
 /**
  * struct ne_cpu_pool - CPU pool used for Nitro Enclaves.
  * @avail_threads_per_core:	Available full CPU cores to be dedicated to
@@ -403,7 +411,7 @@ static void ne_snapshot_cpu_pool(struct cpumask *cpu_pool)
 }
 
 /**
- * ne_set_kernel_param() - Set the NE CPU pool value via the NE kernel parameter.
+ * ne_replace_cpu_pool() - Replace the NE CPU pool with the one a CPU list names.
  * @val:	NE CPU pool string value.
  * @kp :	NE kernel parameter associated with the NE CPU pool.
  *
@@ -412,7 +420,7 @@ static void ne_snapshot_cpu_pool(struct cpumask *cpu_pool)
  * * 0 on success.
  * * Negative return value on failure.
  */
-static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
+static int ne_replace_cpu_pool(const char *val, const struct kernel_param *kp)
 {
 	char error_val[] = "";
 	cpumask_var_t cpu_pool;
@@ -420,9 +428,6 @@ static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
 	struct ne_pci_dev *ne_pci_dev = NULL;
 	int rc = -EINVAL;
 
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
 	if (!zalloc_cpumask_var(&cpu_pool, GFP_KERNEL))
 		return -ENOMEM;
 
@@ -509,6 +514,39 @@ static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
 	return rc;
 }
 
+/**
+ * ne_set_kernel_param() - Set the NE CPU pool value via the NE kernel parameter.
+ * @val:	NE CPU pool string value.
+ * @kp :	NE kernel parameter associated with the NE CPU pool.
+ *
+ * Context: Process context.
+ * Return:
+ * * 0 on success.
+ * * Negative return value on failure.
+ */
+static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)
+{
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	/*
+	 * A value on the kernel command line reaches this from the parse_args()
+	 * call in start_kernel(), which runs before mm_core_init() has brought
+	 * up the allocator a cpumask comes from and long before smp_init() has
+	 * brought online the CPUs a pool is built out of. Keep the value and
+	 * leave the pool to ne_init(), which runs from do_basic_setup() with
+	 * both of those behind it. A value from modprobe or from sysfs arrives
+	 * with the system already up and is applied here.
+	 */
+	if (system_state == SYSTEM_BOOTING) {
+		ne_cpus_kp = kp;
+
+		return param_set_copystring(val, kp);
+	}
+
+	return ne_replace_cpu_pool(val, kp);
+}
+
 /**
  * ne_donated_cpu() - Check if the provided CPU is already used by the enclave.
  * @ne_enclave :	Private data associated with the current enclave.
@@ -1904,6 +1942,21 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
 static int __init ne_init(void)
 {
+	static char cpu_list[NE_CPUS_SIZE] __initdata;
+
+	/*
+	 * Build the pool of a CPU list that came from the kernel command line,
+	 * which the setter only kept. The value is taken out of the parameter
+	 * string first, because building the pool writes that string.
+	 */
+	if (ne_cpus_kp && ne_cpus[0]) {
+		memcpy(cpu_list, ne_cpus, sizeof(cpu_list));
+
+		if (ne_replace_cpu_pool(cpu_list, ne_cpus_kp) < 0)
+			pr_err("%s: No CPU pool from nitro_enclaves.ne_cpus=%s\n",
+			       ne_misc_dev.name, cpu_list);
+	}
+
 	return pci_register_driver(&ne_pci_driver);
 }
 
-- 
2.47.1