[PATCH 31/40] nitro_enclaves: Map the doorbell window at probe

[Alexander Graf <graf@amazon.com>]

In preparation to support notifying a sandbox virtqueue from the parent,
map the doorbell window of the NE PCI device. The doorbell window is a
second MMIO BAR of the device holding an array of two-byte doorbells; a
write to the doorbell of a virtqueue tells the device that the queue has
data waiting for the sandbox.

A device that does not offer the window serves every enclave operation
that does not involve a virtqueue, so keep the mapping optional and let
probe carry on without it. Record the length of the window so that a
doorbell offset can be bounded against it.

Signed-off-by: Alexander Graf <graf@amazon.com>
---
 drivers/virt/nitro_enclaves/ne_pci_dev.c | 17 +++++++++++++++++
 drivers/virt/nitro_enclaves/ne_pci_dev.h |  9 +++++++++
 2 files changed, 26 insertions(+)

diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
index 4678799998f4a..b5a7ecfffb4b6 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
@@ -1968,6 +1968,15 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto release_pci_regions;
 	}
 
+	/*
+	 * A device without a doorbell window serves every enclave operation
+	 * that does not involve a virtqueue, so carry on without one.
+	 */
+	ne_pci_dev->doorbell_base = pci_iomap(pdev, PCI_BAR_NE_DOORBELL, 0);
+	if (ne_pci_dev->doorbell_base)
+		ne_pci_dev->doorbell_len =
+			pci_resource_len(pdev, PCI_BAR_NE_DOORBELL);
+
 	pci_set_drvdata(pdev, ne_pci_dev);
 
 	atomic_set(&ne_pci_dev->cmd_reply_avail, 0);
@@ -2086,6 +2095,8 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	ne_teardown_msix(pdev);
 iounmap_pci_bar:
 	pci_set_drvdata(pdev, NULL);
+	if (ne_pci_dev->doorbell_base)
+		pci_iounmap(pdev, ne_pci_dev->doorbell_base);
 	pci_iounmap(pdev, ne_pci_dev->iomem_base);
 release_pci_regions:
 	pci_release_regions(pdev);
@@ -2127,6 +2138,9 @@ static void ne_pci_remove(struct pci_dev *pdev)
 
 	pci_set_drvdata(pdev, NULL);
 
+	if (ne_pci_dev->doorbell_base)
+		pci_iounmap(pdev, ne_pci_dev->doorbell_base);
+
 	pci_iounmap(pdev, ne_pci_dev->iomem_base);
 
 	pci_release_regions(pdev);
@@ -2171,6 +2185,9 @@ static void ne_pci_shutdown(struct pci_dev *pdev)
 
 	pci_set_drvdata(pdev, NULL);
 
+	if (ne_pci_dev->doorbell_base)
+		pci_iounmap(pdev, ne_pci_dev->doorbell_base);
+
 	pci_iounmap(pdev, ne_pci_dev->iomem_base);
 
 	pci_release_regions(pdev);
diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.h b/drivers/virt/nitro_enclaves/ne_pci_dev.h
index 71aaa8f557679..5a92e0c565ba0 100644
--- a/drivers/virt/nitro_enclaves/ne_pci_dev.h
+++ b/drivers/virt/nitro_enclaves/ne_pci_dev.h
@@ -31,6 +31,10 @@
  * PCI_BAR_NE - Nitro Enclaves PCI device MMIO BAR.
  */
 #define PCI_BAR_NE		(0x03)
+/**
+ * PCI_BAR_NE_DOORBELL - Nitro Enclaves PCI device virtqueue doorbell BAR.
+ */
+#define PCI_BAR_NE_DOORBELL	(0x04)
 
 /**
  * DOC: Device registers in the NE PCI device MMIO BAR
@@ -724,6 +728,9 @@ struct ne_mux_binding {
  *				could not be made, and the calls that take it
  *				turn such a pointer down themselves, so it is
  *				kept as it came.
+ * @doorbell_base:		Virtqueue doorbell window of the PCI device, NULL
+ *				on a device that does not offer one.
+ * @doorbell_len:		Length of the virtqueue doorbell window.
  * @enclaves_list:		List of the enclaves managed by the PCI device.
  * @enclaves_list_mutex:	Mutex for accessing the list of enclaves.
  * @event_wq:			Work queue for handling out-of-band events
@@ -766,6 +773,8 @@ struct ne_pci_dev {
 	bool			failed;
 	wait_queue_head_t	cmd_reply_wait_q;
 	struct dentry		*debugfs_dir;
+	void __iomem		*doorbell_base;
+	resource_size_t		doorbell_len;
 	struct list_head	enclaves_list;
 	struct mutex		enclaves_list_mutex;
 	struct workqueue_struct	*event_wq;
-- 
2.47.1