[PATCH v10 06/10] PCI: endpoint: pci-epf-vntb: Allow DMA and MW to share a BAR

[Koichiro Den <den@valinux.co.jp>]

Some endpoint configurations have no spare BAR for DMA resources. Allow
dma_bar to select an MW BAR and place the DMA ranges after that MW.

DMA submaps have fixed BAR offsets, so a shared MW translation must
cover its configured size. Keep the power-of-two restriction used for
ordinary MWs. The size must also meet the EPC's alignment requirement
for the following DMA submaps.

Older hosts treat the whole BAR as an MW. Fall back to that layout when
LINK_UP does not advertise MW/DMA boundary support, so that NTB keeps
working without DMA: the enumerated BAR size is kept, the MW spans the
whole BAR and the DMA submaps are removed.

Reviewed-by: Frank Li <Frank.Li@nxp.com>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes in v10:
  - No code changes. Add Frank's Reviewed-by.
  - Link to v9: https://lore.kernel.org/r/20261110082459.1798763-7-den@valinux.co.jp/

 drivers/pci/endpoint/functions/pci-epf-vntb.c | 220 ++++++++++++++----
 include/linux/pci-epf-ntb.h                   |   3 +
 2 files changed, 178 insertions(+), 45 deletions(-)

diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index bbe43994078a..fd03fb45a5f2 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -84,15 +84,17 @@ enum epf_irq_slot {
 #define MIN_DB_COUNT			(EPF_IRQ_DB_START + 1)
 
 struct epf_ntb_dma {
+	struct mutex lock; /* Serialize DMA submap and MW updates */
 	struct epf_ntb_dma_ctrl ctrl;
 	struct dma_chan *dchan[EDMA_MAX_RD_CH];
 	void *bar_scratch;
 	dma_addr_t bar_scratch_phys;
 	size_t bar_scratch_size;
-	struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 2];
+	struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 3];
 	struct pci_epf_bar_submap *reg_submap;
 	unsigned int num_submap;
 	u16 rd_ch_cnt;
+	bool mw_fallback; /* The MW spans the whole shared BAR */
 };
 
 struct epf_ntb {
@@ -229,6 +231,22 @@ static bool epf_ntb_is_bar_used(struct epf_ntb *ntb,
 	return false;
 }
 
+static u64 epf_ntb_dma_bar_offset(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	unsigned int i;
+
+	for (i = 0; i < ntb->num_mws; i++)
+		if (ntb->epf_ntb_bar[BAR_MW1 + i] == barno)
+			return ntb->mws_size[i];
+
+	return 0;
+}
+
+static bool epf_ntb_dma_shares_bar(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	return ntb->dma && ntb->epf_ntb_bar[BAR_DMA] == barno;
+}
+
 struct epf_ntb_dma_filter {
 	struct device *dev;
 	int chan_id;
@@ -353,6 +371,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
 	unsigned int i;
 	size_t align;
 	u32 next = 0;
+	u64 offset;
 	int ret;
 
 	if (ctrl->u.dma_ctrl.reg_layout_data != EDMA_MF_EDMA_UNROLL)
@@ -371,10 +390,30 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
 	if (!is_power_of_2(align))
 		return -EINVAL;
 
-	if (epf_ntb_is_bar_used(ntb, barno) ||
-	    pci_epc_get_next_free_bar(features, barno) != barno)
+	if (barno == ntb->epf_ntb_bar[BAR_CONFIG] ||
+	    barno == ntb->epf_ntb_bar[BAR_DB])
 		return -EINVAL;
 
+	offset = epf_ntb_dma_bar_offset(ntb, barno);
+	if (pci_epc_get_next_free_bar(features, barno) != barno ||
+	    (!offset && epf_ntb_is_bar_used(ntb, barno)))
+		return -EINVAL;
+	if (offset > U32_MAX)
+		return -EOVERFLOW;
+	if (offset && (!is_power_of_2(offset) || !IS_ALIGNED(offset, align)))
+		return -EINVAL;
+	dma->ctrl.submap.offset = offset;
+	next = offset;
+	if (next) {
+		/*
+		 * submap[0] covers the MW prefix. Keep its temporary
+		 * target at address 0 until the MW translation is
+		 * installed.
+		 */
+		dma->submap[0].size = next;
+		dma->num_submap = 1;
+	}
+
 	dma->ctrl.magic = EPF_NTB_DMA_MAGIC;
 	dma->ctrl.revision = EPF_NTB_DMA_REVISION;
 	dma->ctrl.type = EPF_NTB_DMA_TYPE_DW_EDMA;
@@ -408,7 +447,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
 		chan->desc_addr_lo = lower_32_bits(dma_addr);
 		chan->desc_addr_hi = upper_32_bits(dma_addr);
 	}
-	dma->ctrl.submap.size = next;
+	dma->ctrl.submap.size = next - dma->ctrl.submap.offset;
 
 	dma_dev = ntb->epf->epc->dev.parent;
 	for (i = 0; i < dma->rd_ch_cnt; i++) {
@@ -481,6 +520,8 @@ static int epf_ntb_dma_collect(struct epf_ntb *ntb)
 	if (!dma)
 		return -ENOMEM;
 
+	mutex_init(&dma->lock);
+
 	switch (ctrl->u.dma_ctrl.reg_layout) {
 	case PCI_EPC_AUX_DMA_REG_LAYOUT_DW_EDMA:
 		ret = epf_ntb_dw_edma_collect(ntb, dma, ctrl, resources, count);
@@ -551,7 +592,7 @@ static int epf_ntb_dma_bar_init(struct epf_ntb *ntb)
 		return -EOPNOTSUPP;
 
 	barno = ntb->epf_ntb_bar[BAR_DMA];
-	mapped_size = dma->ctrl.submap.size;
+	mapped_size = dma->ctrl.submap.offset + dma->ctrl.submap.size;
 	/*
 	 * Submaps cannot be installed until the host assigns the BAR address.
 	 * Use address 0 for the temporary BAR Match Mode mapping, as is done
@@ -607,23 +648,29 @@ static void epf_ntb_dma_bar_clear(struct epf_ntb *ntb)
 	memset(bar, 0, sizeof(*bar));
 }
 
-static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
+static int epf_ntb_dma_set_bar_locked(struct epf_ntb *ntb, bool use_submap,
+				      const dma_addr_t *mw_addr)
 {
-	struct pci_epf_bar_submap *old_submap;
 	struct epf_ntb_dma *dma = ntb->dma;
-	unsigned int old_num_submap;
-	struct pci_epf_bar *bar;
+	struct pci_epf_bar *bar, old_bar;
+	dma_addr_t old_mw_addr;
 	int restore, ret;
 
-	if (!dma)
-		return -EOPNOTSUPP;
+	lockdep_assert_held(&dma->lock);
 
 	bar = &ntb->epf->bar[ntb->epf_ntb_bar[BAR_DMA]];
-
-	old_submap = bar->submap;
-	old_num_submap = bar->num_submap;
-	bar->submap = dma->submap;
-	bar->num_submap = dma->num_submap;
+	old_bar = *bar;
+	old_mw_addr = dma->submap[0].phys_addr;
+	if (use_submap) {
+		if (mw_addr)
+			dma->submap[0].phys_addr = *mw_addr;
+		bar->submap = dma->submap;
+		bar->num_submap = dma->num_submap;
+	} else {
+		bar->submap = NULL;
+		bar->num_submap = 0;
+		bar->phys_addr = mw_addr ? *mw_addr : 0;
+	}
 
 	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
 			      ntb->epf->vfunc_no, bar);
@@ -631,8 +678,8 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
 		return 0;
 
 	/* A failed dynamic update may have already removed the old mapping. */
-	bar->submap = old_submap;
-	bar->num_submap = old_num_submap;
+	dma->submap[0].phys_addr = old_mw_addr;
+	*bar = old_bar;
 	restore = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
 				  ntb->epf->vfunc_no, bar);
 	if (restore)
@@ -642,6 +689,51 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
 	return ret;
 }
 
+static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
+{
+	struct epf_ntb_dma *dma = ntb->dma;
+
+	if (!dma)
+		return -EOPNOTSUPP;
+
+	guard(mutex)(&dma->lock);
+
+	return epf_ntb_dma_set_bar_locked(ntb, true, NULL);
+}
+
+static int epf_ntb_dma_link_up(struct epf_ntb *ntb, u32 argument)
+{
+	struct epf_ntb_dma *dma = ntb->dma;
+	struct pci_epf_bar *bar;
+	bool fallback;
+	int ret;
+
+	if (!dma || !dma->ctrl.submap.offset)
+		return 0;
+
+	fallback = !(argument & EPF_NTB_LINK_UP_DMA_MW_SUPPORTED);
+
+	guard(mutex)(&dma->lock);
+
+	if (fallback == dma->mw_fallback)
+		return 0;
+
+	if (fallback) {
+		dev_warn(&ntb->epf->dev,
+			 "Host does not support MW/DMA BAR sharing, exposing the whole BAR as an MW\n");
+		/* Replace installed DMA submaps until the MW is set up. */
+		bar = &ntb->epf->bar[ntb->epf_ntb_bar[BAR_DMA]];
+		if (bar->num_submap) {
+			ret = epf_ntb_dma_set_bar_locked(ntb, false, NULL);
+			if (ret)
+				return ret;
+		}
+	}
+	dma->mw_fallback = fallback;
+
+	return 0;
+}
+
 /**
  * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
  *   to access the memory window of HOST
@@ -780,7 +872,9 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 					     EPF_NTB_COMMAND_STATUS_OK;
 		break;
 	case EPF_NTB_COMMAND_LINK_UP:
-		ret = epf_ntb_set_link_enabled(ntb, true);
+		ret = epf_ntb_dma_link_up(ntb, argument);
+		if (!ret)
+			ret = epf_ntb_set_link_enabled(ntb, true);
 		if (ret < 0)
 			ctrl->command_status = EPF_NTB_COMMAND_STATUS_ERROR;
 		else
@@ -1269,6 +1363,7 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
  */
 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 {
+	bool shared;
 	int ret = 0;
 	int i;
 	u64 size;
@@ -1278,22 +1373,25 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 	for (i = 0; i < ntb->num_mws; i++) {
 		size = ntb->mws_size[i];
 		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
-
-		ntb->epf->bar[barno].barno = barno;
-		ntb->epf->bar[barno].size = size;
-		ntb->epf->bar[barno].addr = NULL;
-		ntb->epf->bar[barno].phys_addr = 0;
-		ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
-				PCI_BASE_ADDRESS_MEM_TYPE_64 :
-				PCI_BASE_ADDRESS_MEM_TYPE_32;
-
-		ret = pci_epc_set_bar(ntb->epf->epc,
-				      ntb->epf->func_no,
-				      ntb->epf->vfunc_no,
-				      &ntb->epf->bar[barno]);
-		if (ret) {
-			dev_err(dev, "MW set failed\n");
-			goto err_alloc_mem;
+		shared = epf_ntb_dma_shares_bar(ntb, barno);
+
+		if (!shared) {
+			ntb->epf->bar[barno].barno = barno;
+			ntb->epf->bar[barno].size = size;
+			ntb->epf->bar[barno].addr = NULL;
+			ntb->epf->bar[barno].phys_addr = 0;
+			ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
+					PCI_BASE_ADDRESS_MEM_TYPE_64 :
+					PCI_BASE_ADDRESS_MEM_TYPE_32;
+
+			ret = pci_epc_set_bar(ntb->epf->epc,
+					      ntb->epf->func_no,
+					      ntb->epf->vfunc_no,
+					      &ntb->epf->bar[barno]);
+			if (ret) {
+				dev_err(dev, "MW set failed\n");
+				goto err_alloc_mem;
+			}
 		}
 
 		/* Allocate EPC outbound memory windows to vpci vntb device */
@@ -1310,10 +1408,11 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 	return ret;
 
 err_set_bar:
-	pci_epc_clear_bar(ntb->epf->epc,
-			  ntb->epf->func_no,
-			  ntb->epf->vfunc_no,
-			  &ntb->epf->bar[barno]);
+	if (!shared)
+		pci_epc_clear_bar(ntb->epf->epc,
+				  ntb->epf->func_no,
+				  ntb->epf->vfunc_no,
+				  &ntb->epf->bar[barno]);
 err_alloc_mem:
 	epf_ntb_mw_bar_clear(ntb, i);
 	return ret;
@@ -1331,10 +1430,11 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
 
 	for (i = 0; i < num_mws; i++) {
 		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
-		pci_epc_clear_bar(ntb->epf->epc,
-				  ntb->epf->func_no,
-				  ntb->epf->vfunc_no,
-				  &ntb->epf->bar[barno]);
+		if (!epf_ntb_dma_shares_bar(ntb, barno))
+			pci_epc_clear_bar(ntb->epf->epc,
+					  ntb->epf->func_no,
+					  ntb->epf->vfunc_no,
+					  &ntb->epf->bar[barno]);
 
 		pci_epc_mem_free_addr(ntb->epf->epc,
 				      ntb->vpci_mw_phy[i],
@@ -1936,6 +2036,28 @@ static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
 	dev = &ndev->dev;
 	barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
 	epf_bar = &ntb->epf->bar[barno];
+	if (epf_ntb_dma_shares_bar(ntb, barno)) {
+		guard(mutex)(&ntb->dma->lock);
+
+		if (ntb->dma->mw_fallback) {
+			if (size > epf_bar->size)
+				return -EINVAL;
+		} else if (size != ntb->mws_size[idx]) {
+			/*
+			 * The MW submap spans the full size configured
+			 * through configfs. DMA submaps follow it at
+			 * fixed BAR offsets. Supporting smaller MWs
+			 * would need a scratch-backed submap to fill
+			 * the gap. Let's add that complexity only when
+			 * it is really needed.
+			 */
+			return -EINVAL;
+		}
+
+		return epf_ntb_dma_set_bar_locked(ntb, !ntb->dma->mw_fallback,
+						  &addr);
+	}
+
 	epf_bar->phys_addr = addr;
 	epf_bar->barno = barno;
 	epf_bar->size = size;
@@ -2134,15 +2256,23 @@ static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
 			resource_size_t *size_max)
 {
 	struct epf_ntb *ntb = ntb_ndev(ndev);
+	enum pci_barno barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
+	resource_size_t aalign = SZ_4K;
+	u64 size = ntb->mws_size[idx];
+
+	if (epf_ntb_dma_shares_bar(ntb, barno) && ntb->dma->mw_fallback) {
+		size = ntb->epf->bar[barno].size;
+		aalign = size;
+	}
 
 	if (addr_align)
-		*addr_align = SZ_4K;
+		*addr_align = max_t(resource_size_t, SZ_4K, aalign);
 
 	if (size_align)
 		*size_align = 1;
 
 	if (size_max)
-		*size_max = ntb->mws_size[idx];
+		*size_max = size;
 
 	return 0;
 }
diff --git a/include/linux/pci-epf-ntb.h b/include/linux/pci-epf-ntb.h
index 5cc0f0badae0..8131ed4c6ac9 100644
--- a/include/linux/pci-epf-ntb.h
+++ b/include/linux/pci-epf-ntb.h
@@ -24,6 +24,9 @@
 #define EPF_NTB_DB_COUNT_MASK			GENMASK(15, 0)
 #define EPF_NTB_MSIX_ENABLE			BIT(16)
 
+/* LINK_UP argument: the host honors the MW/DMA BAR boundary. */
+#define EPF_NTB_LINK_UP_DMA_MW_SUPPORTED		BIT(0)
+
 #define EPF_NTB_COMMAND_STATUS_OK		1
 #define EPF_NTB_COMMAND_STATUS_ERROR		2
 
-- 
2.54.0