[PATCH 36/40] nitro_enclaves: Add NE_ENCLAVE_PCIE_HOTPLUG_MODE

[Alexander Graf <graf@amazon.com>]

Users want to add devices to a Nitro Sandbox while it runs. The
hypervisor supports that for a slot it sets up for it when it allocates
the slot.

Add NE_ENCLAVE_PCIE_HOTPLUG_MODE, which asks for such a slot, and let
NE_ADD_DEVICE take a device in the running state for a slot that carries
the flag.

Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_misc_dev.c |  9 ++++++++-
 drivers/virt/nitro_enclaves/ne_misc_dev.h |  1 +
 include/uapi/linux/nitro_enclaves.h       | 24 ++++++++++++++++++++---
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
index cccfceeb575cc..870b3a6368cf9 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
@@ -2967,7 +2967,14 @@ static long ne_enclave_do_ioctl(struct file *file, unsigned int cmd, unsigned lo
 		if (rc)
 			return rc;
 
-		if (ne_enclave->state != NE_STATE_INIT) {
+		/*
+		 * A slot allocated to take a device once its enclave runs
+		 * takes one in the running state as well. An enclave that has
+		 * stopped takes nothing either way.
+		 */
+		if (ne_enclave->state != NE_STATE_INIT &&
+		    !(ne_enclave->state == NE_STATE_RUNNING &&
+		      (ne_enclave->launch_flags & NE_ENCLAVE_PCIE_HOTPLUG_MODE))) {
 			dev_err_ratelimited(ne_misc_dev.this_device,
 					    "Enclave is not in init state\n");
 
diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.h b/drivers/virt/nitro_enclaves/ne_misc_dev.h
index 1fd93bddf7766..ed1b29997e1bb 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.h
@@ -26,6 +26,7 @@
 					 NE_ENCLAVE_PCIE_MODE | \
 					 NE_ENCLAVE_PCIE_VSOCK_MODE | \
 					 NE_ENCLAVE_PCIE_VIRTIO_MODE | \
+					 NE_ENCLAVE_PCIE_HOTPLUG_MODE | \
 					 NE_ENCLAVE_PCIE_NUMA_MODE)
 
 struct cma;
diff --git a/include/uapi/linux/nitro_enclaves.h b/include/uapi/linux/nitro_enclaves.h
index b346e0582c5fa..f204d0f6e7efd 100644
--- a/include/uapi/linux/nitro_enclaves.h
+++ b/include/uapi/linux/nitro_enclaves.h
@@ -85,6 +85,17 @@
  */
 #define NE_ENCLAVE_PCIE_VIRTIO_MODE	(1u << 9)
 
+/**
+ * NE_ENCLAVE_PCIE_HOTPLUG_MODE - Allocate a slot whose PCIe bus can take
+ *				  a device once the enclave is running. The
+ *				  hypervisor decides it at SLOT_ALLOC time, and
+ *				  %NE_ADD_DEVICE takes a device in the running
+ *				  state for a slot that carries it. Requires
+ *				  NE_ENCLAVE_PCIE_MODE and excludes
+ *				  NE_ENCLAVE_PCIE_VSOCK_MODE.
+ */
+#define NE_ENCLAVE_PCIE_HOTPLUG_MODE	(1u << 8)
+
 /**
  * NE_ENCLAVE_PCIE_NUMA_MODE - Allocate a slot whose memory and vCPUs may come
  *			       from more than one host NUMA node. Without it the
@@ -701,7 +712,11 @@ struct ne_add_device_args {
  *		   An argument that ends at @vq_info is taken and gets no
  *		   @info_notify_index back.
  *		   The ioctl can be invoked on the enclave fd, before an
- *		   enclave is started.
+ *		   enclave is started, and while it is running if the slot
+ *		   was allocated with %NE_ENCLAVE_PCIE_HOTPLUG_MODE. A
+ *		   started enclave refuses %NE_ALLOC_MEMORY, so a device
+ *		   given to one works from allocations made before it
+ *		   started.
  *
  * Context: Process context.
  * Return:
@@ -738,8 +753,11 @@ struct ne_add_device_args {
  * * EPROTO				- The device replied for a different number of
  *					  virtqueues than it was asked for, or placed
  *					  their entries outside the info page.
- * * NE_ERR_NOT_IN_INIT_STATE		- The enclave is not in init state (init =
- *					  before being started).
+ * * NE_ERR_NOT_IN_INIT_STATE		- The enclave left its init state (init =
+ *					  before being started) and the slot was
+ *					  not allocated with
+ *					  %NE_ENCLAVE_PCIE_HOTPLUG_MODE, or the
+ *					  enclave has stopped.
  * * NE_ERR_INVALID_FLAG_VALUE		- The value of the provided flag is invalid.
  * * NE_ERR_MEM_REGION_ALREADY_USED	- An allocation was already given to the
  *					  enclave, or is its info page, or is the
-- 
2.47.1