[PATCH v5 22/27] vfio/cxl: Clear the HDM access gate after a hot reset

[<mhonap@nvidia.com>]

From: Manish Honap <mhonap@nvidia.com>

The HDM memory fault inserts a PFN only when the decoder is in a
known-good state (cxl->hdm_valid), so the host CPU never faults the
coherent range into a disabled decoder.

VFIO_DEVICE_PCI_HOT_RESET drives a plain secondary bus reset rather than
the CXL reset sequence, so nothing restores the decoder in the
reset_done handler. A secondary bus reset only reaches a CXL endpoint
when the upstream port has SBR unmasked (CXL r3.1 sec 8.1.5.2); in that
case the reset decommits the decoder, so clear cxl->hdm_valid to keep a
later fault from inserting a PFN into a dead decoder.

A subsequent VFIO_DEVICE_RESET runs the CXL reset sequence and restores
it, and a masked SBR is a no-op that leaves the decoder intact.

vfio_cxl_sbr_unmasked() mirrors the cxl_sbr_masked() check the PCI core
uses (drivers/pci/pci.c) for its own CXL bus reset.

Assisted-by: LLM
Signed-off-by: Manish Honap <mhonap@nvidia.com>
---
 drivers/vfio/pci/cxl/vfio_cxl_core.c | 38 ++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c
index 37e8a3b54cfb..395b4a5b0956 100644
--- a/drivers/vfio/pci/cxl/vfio_cxl_core.c
+++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c
@@ -514,8 +514,46 @@ static void vfio_cxl_reset_prepare(struct vfio_pci_core_device *vdev)
 {
 }
 
+/*
+ * A secondary bus reset only reaches a CXL endpoint when the upstream port
+ * has SBR unmasked (CXL r3.1 sec 8.1.5.2); otherwise the Bridge Control SBR
+ * bit is ignored and the decoder is left intact. Mirrors the cxl_sbr_masked()
+ * check the PCI core uses for its own CXL bus reset.
+ */
+static bool vfio_cxl_sbr_unmasked(struct pci_dev *pdev)
+{
+	struct pci_dev *bridge = pci_upstream_bridge(pdev);
+	u16 dvsec, ctl;
+
+	if (!bridge)
+		return false;
+
+	dvsec = pci_find_dvsec_capability(bridge, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_PORT);
+	if (!dvsec)
+		return false;
+
+	if (pci_read_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, &ctl))
+		return false;
+
+	return ctl & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR;
+}
+
 static void vfio_cxl_reset_done(struct vfio_pci_core_device *vdev)
 {
+	struct vfio_cxl_state *cxl = vdev->cxl;
+
+	/*
+	 * VFIO_DEVICE_PCI_HOT_RESET drives a plain secondary bus reset, not the
+	 * CXL-aware cxl_reset_bus_function(), so nothing restores the HDM
+	 * decoder here. When the upstream port has SBR unmasked the reset
+	 * decommits the decoder; gate host access to the HDM range so a later
+	 * fault cannot insert a PFN into a dead decoder. A VFIO_DEVICE_RESET
+	 * then runs the CXL reset sequence and restores it. A masked SBR is a
+	 * no-op and leaves the decoder intact.
+	 */
+	if (vfio_cxl_sbr_unmasked(vdev->pdev))
+		cxl->hdm_valid = false;
 }
 
 static const struct vfio_cxl_ops vfio_cxl_ops = {
-- 
2.25.1