[PATCH 27/40] nitro_enclaves: Dispatch multiplexed interrupts to per-queue bindings

[Alexander Graf <graf@amazon.com>]

Nitro Sandboxes provide a way for user space to receive
per-device-virtqueue events. These are multiplexed through a
nitro_enclaves device built-in interrupt multiplexer. Implement logic to
support this multiplexer and distribute the respective interrupt events.

Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_pci_dev.c | 179 ++++++++++++++++++++++-
 drivers/virt/nitro_enclaves/ne_pci_dev.h |  47 ++++++
 2 files changed, 225 insertions(+), 1 deletion(-)

diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
index 0fafd42162f3a..de96d68906bbf 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
@@ -7,6 +7,7 @@
  * DOC: Nitro Enclaves (NE) PCI device driver.
  */
 
+#include <linux/bitops.h>
 #include <linux/cma.h>
 #include <linux/delay.h>
 #include <linux/device.h>
@@ -27,6 +28,7 @@
 #include <linux/sysfs.h>
 #include <linux/types.h>
 #include <linux/wait.h>
+#include <linux/xarray.h>
 
 #include "ne_misc_dev.h"
 #include "ne_pci_dev.h"
@@ -551,6 +553,129 @@ static void ne_teardown_msix(struct pci_dev *pdev)
 	pci_free_irq_vectors(pdev);
 }
 
+/**
+ * ne_mux_binding_rcu_free() - RCU callback freeing one enclave queue binding.
+ * @head:	RCU head of the binding to free.
+ *
+ * Context: Softirq or process context.
+ */
+static void ne_mux_binding_rcu_free(struct rcu_head *head)
+{
+	struct ne_mux_binding *binding = container_of(head,
+						      struct ne_mux_binding,
+						      rcu);
+
+	eventfd_ctx_put(binding->ctx);
+
+	kfree(binding);
+}
+
+void ne_mux_binding_free_rcu(struct ne_mux_binding *binding)
+{
+	call_rcu(&binding->rcu, ne_mux_binding_rcu_free);
+}
+
+/**
+ * ne_mux_scan() - Deliver the bits set in the bitmap of a pooled vector that it
+ *		   has a binding for.
+ * @mux_vec:	Multiplexer state of the pooled vector.
+ *
+ * A bit leaves the bitmap by the same atomic read-modify-write that decides to
+ * deliver it, so a bit this vector has no binding for stays set. It is left set
+ * on purpose: a queue can be kicked before user space has bound an eventfd to
+ * it, and a bit taken out of the bitmap on the strength of a binding that is not
+ * there yet is a kick nobody can ever be told about. The bit waits instead, and
+ * the bind of that queue takes it and delivers it.
+ *
+ * So a set bit is consumed at most once, and exactly once if a scan finds a
+ * binding for it or a bind for its index runs. A bit whose index is never bound
+ * is consumed never, and stays set until the page is reclaimed when the device
+ * unbinds. A bit whose bind is still coming looks the same to the driver, so it
+ * holds both.
+ *
+ * The bound only ever rises, so a scan reads every word a binding has reached
+ * into and not only the words the bindings of the moment need.
+ *
+ * Context: Interrupt context. Called with the RCU read-side held.
+ * Return:
+ * * true if the scan found no bit it could deliver, false if it found at least
+ *   one.
+ */
+static bool ne_mux_scan(struct ne_mux_vec *mux_vec)
+{
+	struct ne_pci_dev *ne_pci_dev = mux_vec->ne_pci_dev;
+	u32 bits_per_word = BITS_PER_TYPE(*mux_vec->bitmap);
+	u32 nr_words = READ_ONCE(mux_vec->max_bit) / bits_per_word + 1;
+	bool empty = true;
+	u32 i = 0;
+
+	for (i = 0; i < nr_words; i++) {
+		u64 word = READ_ONCE(mux_vec->bitmap[i]);
+
+		while (word) {
+			u32 bit = i * bits_per_word + __ffs64(word);
+			unsigned long notify_index = bit * ne_pci_dev->mux_nr_vecs +
+						     mux_vec->vec_rel;
+			struct ne_mux_binding *binding = NULL;
+
+			word &= word - 1;
+
+			binding = xa_load(&ne_pci_dev->notify_xa, notify_index);
+			if (!binding)
+				continue;
+
+			empty = false;
+
+			if (test_and_clear_bit(bit, (unsigned long *)mux_vec->bitmap))
+				eventfd_signal(binding->ctx);
+		}
+	}
+
+	return empty;
+}
+
+/**
+ * ne_mux_handler() - Interrupt handler for one pooled MSI-X vector.
+ * @irq:	Interrupt line the pooled vector is on.
+ * @data:	Multiplexer state of the pooled vector.
+ *
+ * Context: Interrupt context.
+ * Return:
+ * * IRQ_HANDLED on interrupt handled.
+ */
+static irqreturn_t ne_mux_handler(int irq, void *data)
+{
+	struct ne_mux_vec *mux_vec = data;
+	u32 pass = 0;
+
+	/*
+	 * xa_load() drops its own RCU read-side before it returns, so take one
+	 * here to keep a binding alive for as long as it is dereferenced.
+	 */
+	rcu_read_lock();
+
+	/*
+	 * The device sets a bit and then raises the interrupt, and one interrupt
+	 * can stand for several bits, so a scan can miss a bit set while it runs
+	 * or one past the bound it read. The device raises for every bit it sets
+	 * whatever the bitmap holds, so such a bit has an interrupt of its own
+	 * still to come and reaches its eventfd on that one. Scanning again,
+	 * reading the bound again, delivers it on this interrupt instead, so stop
+	 * on the first scan that delivers nothing. Stopping on an empty bitmap
+	 * instead would wait for a read a bit no bind is coming for never gives.
+	 * NE_MUX_MAX_PASSES caps the scans either way, so a device that goes on
+	 * setting bits cannot hold the handler.
+	 */
+	do {
+		if (ne_mux_scan(mux_vec))
+			break;
+	} while (++pass < NE_MUX_MAX_PASSES);
+
+	rcu_read_unlock();
+
+	return IRQ_HANDLED;
+}
+
 /**
  * ne_mux_teardown() - Take the interrupt multiplexer bitmaps back from the
  *		       device.
@@ -561,6 +686,8 @@ static void ne_teardown_msix(struct pci_dev *pdev)
 static void ne_mux_teardown(struct pci_dev *pdev)
 {
 	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
+	struct ne_mux_binding *binding = NULL;
+	unsigned long notify_index = 0;
 	u32 i = 0;
 
 	if (!ne_pci_dev->mux)
@@ -568,13 +695,40 @@ static void ne_mux_teardown(struct pci_dev *pdev)
 
 	iowrite32(NE_MUX_SHUTDOWN_ON, ne_pci_dev->iomem_base + NE_MUX_SHUTDOWN);
 
-	for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++)
+	for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++) {
+		if (!ne_pci_dev->mux[i].bitmap)
+			continue;
+
+		if (ne_pci_dev->mux[i].irq_requested)
+			free_irq(pci_irq_vector(pdev,
+						ne_pci_dev->mux_vec_base + i),
+				 &ne_pci_dev->mux[i]);
+
 		free_page((unsigned long)ne_pci_dev->mux[i].bitmap);
+	}
 
 	kfree(ne_pci_dev->mux);
 
 	ne_pci_dev->mux = NULL;
 	ne_pci_dev->mux_nr_vecs = 0;
+
+	xa_for_each(&ne_pci_dev->notify_xa, notify_index, binding) {
+		xa_erase(&ne_pci_dev->notify_xa, notify_index);
+
+		eventfd_ctx_put(binding->ctx);
+
+		kfree(binding);
+	}
+
+	/*
+	 * Freeing the handlers above quiesced the readers that hold no lock,
+	 * and the loop above freed every binding that outlived them. Wait for
+	 * the callbacks a replaced binding already queued, so that none of
+	 * them runs once the module text is gone.
+	 */
+	rcu_barrier();
+
+	xa_destroy(&ne_pci_dev->notify_xa);
 }
 
 /**
@@ -631,8 +785,11 @@ static int ne_mux_setup(struct pci_dev *pdev)
 	ne_pci_dev->mux_nr_vecs = nr_vecs;
 	ne_pci_dev->mux_vec_base = vec_base;
 
+	xa_init(&ne_pci_dev->notify_xa);
+
 	for (i = 0; i < nr_vecs; i++) {
 		u16 donate_err = 0;
+		int irq = 0;
 
 		ne_pci_dev->mux[i].bitmap = (u64 *)get_zeroed_page(GFP_KERNEL);
 		if (!ne_pci_dev->mux[i].bitmap) {
@@ -663,6 +820,26 @@ static int ne_mux_setup(struct pci_dev *pdev)
 
 			goto teardown_mux;
 		}
+
+		ne_pci_dev->mux[i].ne_pci_dev = ne_pci_dev;
+		ne_pci_dev->mux[i].vec_rel = i;
+
+		irq = pci_irq_vector(pdev, vec_base + i);
+		if (irq < 0) {
+			rc = irq;
+
+			goto teardown_mux;
+		}
+
+		rc = request_irq(irq, ne_mux_handler, 0, "enclave_mux",
+				 &ne_pci_dev->mux[i]);
+		if (rc < 0) {
+			dev_err(&pdev->dev, "Error in request irq mux [rc=%d]\n", rc);
+
+			goto teardown_mux;
+		}
+
+		ne_pci_dev->mux[i].irq_requested = true;
 	}
 
 	return 0;
diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.h b/drivers/virt/nitro_enclaves/ne_pci_dev.h
index 2cef40c627f9f..0dbce83cb06bd 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.h
@@ -9,12 +9,15 @@
 #include <linux/atomic.h>
 #include <linux/bits.h>
 #include <linux/build_bug.h>
+#include <linux/eventfd.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
+#include <linux/rcupdate.h>
 #include <linux/stddef.h>
 #include <linux/wait.h>
+#include <linux/xarray.h>
 
 /**
  * DOC: Nitro Enclaves (NE) PCI device
@@ -201,6 +204,12 @@
 static_assert(NE_MUX_MAX_BITS / BITS_PER_TYPE(u64) * sizeof(u64) <= PAGE_SIZE,
 	      "the bitmap of a pooled vector has to fit its donated page");
 
+/**
+ * NE_MUX_MAX_PASSES - Largest number of times the handler of a pooled vector
+ *		       scans its bitmap in one interrupt.
+ */
+#define NE_MUX_MAX_PASSES	(4)
+
 /**
  * enum ne_pci_dev_cmd_type - Device command types.
  * @INVALID_CMD:		Invalid command.
@@ -648,9 +657,35 @@ struct ne_slot_pool_run {
  * @bitmap:			Page-sized bitmap in which the device sets a bit
  *				for every enclave queue that signalled through
  *				this vector.
+ * @max_bit:			Highest bitmap bit an enclave queue has been
+ *				bound to, bounding the scan of @bitmap. It only
+ *				rises, so a scan reads the words of bindings that
+ *				are gone as well.
+ * @ne_pci_dev:			NE PCI device this vector belongs to.
+ * @vec_rel:			Index of this vector within the pool.
+ * @irq_requested:		Whether the interrupt handler of this vector is
+ *				installed, which is not implied by @bitmap: a
+ *				vector whose bitmap the device already owns may
+ *				have no handler yet.
  */
 struct ne_mux_vec {
 	u64			*bitmap;
+	u32			max_bit;
+	struct ne_pci_dev	*ne_pci_dev;
+	u32			vec_rel;
+	bool			irq_requested;
+};
+
+/**
+ * struct ne_mux_binding - Eventfd bound to one enclave queue.
+ * @ctx:			Eventfd signalled when the queue is kicked.
+ * @rcu:			Head used to free the binding after a grace
+ *				period, so that an interrupt handler which
+ *				already loaded it can still dereference it.
+ */
+struct ne_mux_binding {
+	struct eventfd_ctx	*ctx;
+	struct rcu_head		rcu;
 };
 
 /**
@@ -672,6 +707,8 @@ struct ne_mux_vec {
  *				capability, and with it the multiplexer and slot
  *				pool registers.
  * @iomem_base :		MMIO region of the PCI device.
+ * @notify_xa:			Enclave queue bindings, keyed by the notify index of
+ *				the queue.
  * @mux:			Per-pooled-vector interrupt multiplexer state.
  * @mux_nr_vecs:		Number of MSI-X vectors in the interrupt
  *				multiplexer pool.
@@ -706,6 +743,7 @@ struct ne_pci_dev {
 	struct workqueue_struct	*event_wq;
 	bool			has_vendor_cap;
 	void __iomem		*iomem_base;
+	struct xarray		notify_xa;
 	struct ne_mux_vec	*mux;
 	u32			mux_nr_vecs;
 	u32			mux_vec_base;
@@ -750,6 +788,15 @@ int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
 		  struct ne_pci_dev_cmd_reply *cmd_reply,
 		  size_t cmd_reply_size);
 
+/**
+ * ne_mux_binding_free_rcu() - Free an enclave queue binding after a grace
+ *			       period.
+ * @binding:	Binding that has been removed from the queue bindings xarray.
+ *
+ * Context: Process context.
+ */
+void ne_mux_binding_free_rcu(struct ne_mux_binding *binding);
+
 /* Nitro Enclaves (NE) PCI device driver */
 extern struct pci_driver ne_pci_driver;
 
-- 
2.47.1