[RFC PATCH v2 5/5] cxl: Invalidate cached coherent range when Normalized Addressing is detected

[Ankit Agrawal <ankita@nvidia.com>]

In case of Normalized addressing, the HDM decoders report DPA instead
of HPA. Determine such case and ensure the pdev->coh_resource[] do not
cache it.

CXL_DECODER_F_NORMALIZED_ADDRESSING is only set on a decoder once the
endpoint is assembled into a region in cxl_prm_setup_root(). This is
well after the HDM decoders are snapshotted and populated in
pdev->coh_resource[]. So the DPA would be populated in the
pdev->coh_resource.

Add pci_cxl_hdm_coh_invalidate() and call it later when the normalized
addressing is identified to clear the cached value.

Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
---
 drivers/cxl/core/atl.c      |  6 ++++++
 drivers/cxl/core/core.h     |  9 +++++++++
 drivers/cxl/core/resource.c | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/drivers/cxl/core/atl.c b/drivers/cxl/core/atl.c
index 07ce7c50c47a..2bf4d73fd02f 100644
--- a/drivers/cxl/core/atl.c
+++ b/drivers/cxl/core/atl.c
@@ -175,6 +175,12 @@ static int cxl_prm_setup_root(struct cxl_root *cxl_root, void *data)
 	cxld->config.flags |= CXL_DECODER_F_LOCK;
 	cxld->config.flags |= CXL_DECODER_F_NORMALIZED_ADDRESSING;
 
+	/*
+	 * The PCI-core snapshot of this decoder was taken before the flag was
+	 * known. Invalidate it so a DPA is never exposed as a coherent HPA.
+	 */
+	pci_cxl_hdm_coh_invalidate(pci_dev, cxld->config.id);
+
 	ctx->hpa_range = hpa_range;
 	ctx->interleave_ways = ways;
 	ctx->interleave_granularity = gran;
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 1aabbbb4ae5e..97d27a27ab67 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -139,6 +139,15 @@ struct cxl_rwsem {
 
 extern struct cxl_rwsem cxl_rwsem;
 
+#ifdef CONFIG_CXL_RESET
+void pci_cxl_hdm_coh_invalidate(struct pci_dev *pdev, int decoder_id);
+#else
+static inline void pci_cxl_hdm_coh_invalidate(struct pci_dev *pdev,
+					      int decoder_id)
+{
+}
+#endif
+
 void cxl_commit_start(void __iomem *hdm, struct cxl_decoder_settings *settings);
 int cxld_await_commit(void __iomem *hdm, int id);
 
diff --git a/drivers/cxl/core/resource.c b/drivers/cxl/core/resource.c
index d837f8e952a9..d0774926da2d 100644
--- a/drivers/cxl/core/resource.c
+++ b/drivers/cxl/core/resource.c
@@ -527,6 +527,38 @@ void pci_cxl_hdm_cache_release(struct pci_dev *pdev)
 	kfree(info);
 }
 
+/*
+ * A decoder is found to use Normalized Addressing only once the endpoint is
+ * assembled into a region, after the snapshot. Record the flag in the cached
+ * settings and drop the coherent range so a DPA is never reported as an HPA.
+ */
+void pci_cxl_hdm_coh_invalidate(struct pci_dev *pdev, int decoder_id)
+{
+	struct cxl_hdm_info *info;
+	struct resource *res;
+
+	guard(rwsem_write)(&cxl_rwsem.dpa);
+	info = pdev->hdm;
+	if (!info || decoder_id < 0 || decoder_id >= info->decoder_count)
+		return;
+
+	info->settings[decoder_id].config.flags |=
+		CXL_DECODER_F_NORMALIZED_ADDRESSING;
+
+	if (decoder_id >= PCI_CXL_MAX_COHERENT_RANGES)
+		return;
+
+	res = &pdev->coh_resource[decoder_id];
+	if (!res->flags)
+		return;
+
+	pci_warn(pdev, "CXL coherent range: decoder%d uses Normalized Addressing, dropping %pR\n",
+		 decoder_id, res);
+	if (res->parent)
+		remove_resource(res);
+	memset(res, 0, sizeof(*res));
+}
+
 static void __iomem *cxl_pci_hdm_ioremap_current(struct pci_dev *pdev,
 						 int bar,
 						 resource_size_t offset,
-- 
2.53.0