[PATCH 26/40] nitro_enclaves: Donate a bitmap page per multiplexed vector

[Alexander Graf <graf@amazon.com>]

In preparation to support the interrupt multiplexer, give the device the
bitmaps it records events in. At probe, donate one zeroed page per
pooled vector by writing its address to the matching NE_MUX_DONATE_BASE
slot, and read the outcome from NE_MUX_DONATE_ERR. A page holds
NE_MUX_MAX_BITS queue bits. The device sets the bit of a queue and then
raises the vector even when other bits are set already, which the
dispatch code relies on.

On teardown, write NE_MUX_SHUTDOWN before the pages go back to the
allocator.

Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_pci_dev.c | 87 +++++++++++++++++++++++-
 drivers/virt/nitro_enclaves/ne_pci_dev.h | 50 ++++++++++++++
 2 files changed, 136 insertions(+), 1 deletion(-)

diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
index 05dd3040afbc3..0fafd42162f3a 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
@@ -11,6 +11,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/iopoll.h>
 #include <linux/jiffies.h>
 #include <linux/kstrtox.h>
@@ -550,6 +551,32 @@ static void ne_teardown_msix(struct pci_dev *pdev)
 	pci_free_irq_vectors(pdev);
 }
 
+/**
+ * ne_mux_teardown() - Take the interrupt multiplexer bitmaps back from the
+ *		       device.
+ * @pdev:	PCI device to take the multiplexer bitmaps back from.
+ *
+ * Context: Process context.
+ */
+static void ne_mux_teardown(struct pci_dev *pdev)
+{
+	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
+	u32 i = 0;
+
+	if (!ne_pci_dev->mux)
+		return;
+
+	iowrite32(NE_MUX_SHUTDOWN_ON, ne_pci_dev->iomem_base + NE_MUX_SHUTDOWN);
+
+	for (i = 0; i < ne_pci_dev->mux_nr_vecs; i++)
+		free_page((unsigned long)ne_pci_dev->mux[i].bitmap);
+
+	kfree(ne_pci_dev->mux);
+
+	ne_pci_dev->mux = NULL;
+	ne_pci_dev->mux_nr_vecs = 0;
+}
+
 /**
  * ne_mux_setup() - Read the interrupt multiplexer geometry from the device.
  * @pdev:	PCI device to read the multiplexer geometry of.
@@ -561,6 +588,7 @@ static void ne_teardown_msix(struct pci_dev *pdev)
  * Context: Process context.
  * Return:
  * * 0 on success.
+ * * Negative return value on failure.
  */
 static int ne_mux_setup(struct pci_dev *pdev)
 {
@@ -568,6 +596,8 @@ static int ne_mux_setup(struct pci_dev *pdev)
 	int msix_nr_vecs = 0;
 	u32 nr_vecs = 0;
 	u32 vec_base = 0;
+	int rc = -EINVAL;
+	u32 i = 0;
 
 	if (!ne_pci_dev->has_vendor_cap)
 		return 0;
@@ -593,10 +623,54 @@ static int ne_mux_setup(struct pci_dev *pdev)
 		return 0;
 	}
 
+	ne_pci_dev->mux = kcalloc(nr_vecs, sizeof(*ne_pci_dev->mux),
+				  GFP_KERNEL);
+	if (!ne_pci_dev->mux)
+		return -ENOMEM;
+
 	ne_pci_dev->mux_nr_vecs = nr_vecs;
 	ne_pci_dev->mux_vec_base = vec_base;
 
+	for (i = 0; i < nr_vecs; i++) {
+		u16 donate_err = 0;
+
+		ne_pci_dev->mux[i].bitmap = (u64 *)get_zeroed_page(GFP_KERNEL);
+		if (!ne_pci_dev->mux[i].bitmap) {
+			rc = -ENOMEM;
+
+			goto teardown_mux;
+		}
+
+		/*
+		 * On a kernel without a 64-bit MMIO accessor this write is two
+		 * 32-bit writes, low half first, so the device sees a complete
+		 * address only once the high half has arrived. The device is
+		 * expected to take the donation from the high half and not from
+		 * the low one, which is also what lets the outcome be read back
+		 * right after.
+		 */
+		writeq(virt_to_phys(ne_pci_dev->mux[i].bitmap),
+		       ne_pci_dev->iomem_base + NE_MUX_DONATE_BASE +
+		       i * sizeof(u64));
+
+		donate_err = ioread16(ne_pci_dev->iomem_base +
+				      NE_MUX_DONATE_ERR);
+		if (donate_err) {
+			dev_err(&pdev->dev, "Error in bitmap donate for vector %u [err=%u]\n",
+				i, donate_err);
+
+			rc = -EIO;
+
+			goto teardown_mux;
+		}
+	}
+
 	return 0;
+
+teardown_mux:
+	ne_mux_teardown(pdev);
+
+	return rc;
 }
 
 /**
@@ -1648,7 +1722,12 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto teardown_msix;
 	}
 
-	ne_mux_setup(pdev);
+	rc = ne_mux_setup(pdev);
+	if (rc < 0) {
+		dev_err(&pdev->dev, "Error in mux setup [rc=%d]\n", rc);
+
+		goto pci_dev_disable;
+	}
 
 	down_write(&ne_devs.lock);
 
@@ -1717,6 +1796,8 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	ne_devs.ne_pci_dev = NULL;
 unlock_ne_devs:
 	up_write(&ne_devs.lock);
+pci_dev_disable:
+	ne_mux_teardown(pdev);
 	ne_pci_dev_disable(pdev);
 teardown_msix:
 	ne_teardown_msix(pdev);
@@ -1755,6 +1836,8 @@ static void ne_pci_remove(struct pci_dev *pdev)
 
 	ne_devs.ne_pci_dev = NULL;
 
+	ne_mux_teardown(pdev);
+
 	ne_pci_dev_disable(pdev);
 
 	ne_teardown_msix(pdev);
@@ -1797,6 +1880,8 @@ static void ne_pci_shutdown(struct pci_dev *pdev)
 
 	ne_devs.ne_pci_dev = NULL;
 
+	ne_mux_teardown(pdev);
+
 	ne_pci_dev_disable(pdev);
 
 	ne_teardown_msix(pdev);
diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.h b/drivers/virt/nitro_enclaves/ne_pci_dev.h
index aee22b4ed8b6e..2cef40c627f9f 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.h
@@ -94,6 +94,30 @@
  */
 #define NE_MUX_NUM_VECTORS	(0x01f0)
 
+/**
+ * NE_MUX_DONATE_ERR - (2 bytes) Register holding the outcome of the last
+ *		       interrupt multiplexer bitmap donation (Read-Only).
+ */
+#define NE_MUX_DONATE_ERR	(0x01f4)
+
+/**
+ * NE_MUX_SHUTDOWN - (4 bytes) Register to tell the device to stop using every
+ *		     donated interrupt multiplexer bitmap (Write-Only).
+ */
+#define NE_MUX_SHUTDOWN		(0x01f8)
+#define NE_MUX_SHUTDOWN_ON	(0x01)
+
+/**
+ * NE_MUX_DONATE_BASE - (8 bytes per pooled vector) Array of registers to hand
+ *			the device the bitmap of one pooled vector (Read/Write).
+ *
+ * The device sets the bit of an enclave queue in the bitmap of the vector that
+ * queue shares, and then raises that vector's interrupt. It raises for every
+ * bit it sets, whatever the bitmap holds at the time, so bits a driver leaves
+ * in a bitmap never keep a later kick from arriving.
+ */
+#define NE_MUX_DONATE_BASE	(0x0800)
+
 /**
  * DOC: Device MMIO buffer sizes
  */
@@ -163,6 +187,20 @@
  */
 #define NE_MUX_MAX_VECTORS	(256)
 
+/**
+ * NE_MUX_MAX_BITS - Largest number of enclave queues one pooled vector
+ *		     multiplexes, which is how many bits the page donated as its
+ *		     bitmap holds.
+ */
+#define NE_MUX_MAX_BITS		(PAGE_SIZE * BITS_PER_BYTE)
+
+/*
+ * A bitmap is read in u64 words, so the words that span NE_MUX_MAX_BITS bits
+ * have to stay inside the one page donated for it.
+ */
+static_assert(NE_MUX_MAX_BITS / BITS_PER_TYPE(u64) * sizeof(u64) <= PAGE_SIZE,
+	      "the bitmap of a pooled vector has to fit its donated page");
+
 /**
  * enum ne_pci_dev_cmd_type - Device command types.
  * @INVALID_CMD:		Invalid command.
@@ -605,6 +643,16 @@ struct ne_slot_pool_run {
 	unsigned long		nr_pages;
 };
 
+/**
+ * struct ne_mux_vec - Interrupt multiplexer state of one pooled MSI-X vector.
+ * @bitmap:			Page-sized bitmap in which the device sets a bit
+ *				for every enclave queue that signalled through
+ *				this vector.
+ */
+struct ne_mux_vec {
+	u64			*bitmap;
+};
+
 /**
  * struct ne_pci_dev - Nitro Enclaves (NE) PCI device.
  * @cmd_reply_avail:		Variable set if a reply has been sent by the
@@ -624,6 +672,7 @@ struct ne_slot_pool_run {
  *				capability, and with it the multiplexer and slot
  *				pool registers.
  * @iomem_base :		MMIO region of the PCI device.
+ * @mux:			Per-pooled-vector interrupt multiplexer state.
  * @mux_nr_vecs:		Number of MSI-X vectors in the interrupt
  *				multiplexer pool.
  * @mux_vec_base:		First MSI-X vector of the interrupt multiplexer
@@ -657,6 +706,7 @@ struct ne_pci_dev {
 	struct workqueue_struct	*event_wq;
 	bool			has_vendor_cap;
 	void __iomem		*iomem_base;
+	struct ne_mux_vec	*mux;
 	u32			mux_nr_vecs;
 	u32			mux_vec_base;
 	struct work_struct	notify_work;
-- 
2.47.1