mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Koichiro Den <den@valinux.co.jp>
To: Manivannan Sadhasivam <mani@kernel.org>,
	Frank Li <Frank.Li@kernel.org>, Niklas Cassel <cassel@kernel.org>,
	Bjorn Helgaas <bhelgaas@google.com>
Cc: "Jon Mason" <jdmason@kudzu.us>,
	"Dave Jiang" <dave.jiang@intel.com>,
	"Allen Hubbe" <allenbh@gmail.com>,
	"Krzysztof Wilczyński" <kwilczynski@kernel.org>,
	"Kishon Vijay Abraham I" <kishon@kernel.org>,
	ntb@lists.linux.dev, linux-pci@vger.kernel.org,
	linux-kernel@vger.kernel.org
Subject: [PATCH v5 2/2] PCI: endpoint: pci-epf-vntb: Honor MSI-X selection
Date: Thu, 15 Oct 2026 10:44:02 +0900	[thread overview]
Message-ID: <20261015014402.2976188-3-den@valinux.co.jp> (raw)
In-Reply-To: <20261015014402.2976188-1-den@valinux.co.jp>

ntb_hw_epf tries MSI-X first and falls back to MSI. It reports the
result in COMMAND_CONFIGURE_DOORBELL. pci-epf-vntb ignores MSIX_ENABLE,
configures only MSI, and always raises peer interrupts with PCI_IRQ_MSI.
When MSI-X is selected, the host does not program MSI, so raising it can
issue a write to an invalid address. This was observed as IOMMU faults
on the RC, with no doorbell interrupts delivered.

Configure MSI-X when supported and use the selected type for peer
doorbells and link events. Allocate the MSI-X Table and PBA in the config
BAR and configure db_count entries. Use this EPF-owned layout even if the
EPC has fixed, hardware-owned Table/PBA regions, as pci-epf-test and
nvmet-pci-epf already do.

Fixes: e35f56bb0330 ("PCI: endpoint: Support NTB transfer between RC and EP")
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes since v4:
  - Split out the local epf cleanup into a prep patch. (Frank)
  - Keep the MSI labels in the interrupt-numbering comment. (Frank)
  - Rebase on the link-state fix and honor MSI-X for link events too.
  - Link to v4: https://lore.kernel.org/r/20261014053256.2378817-1-den@valinux.co.jp/

 drivers/pci/endpoint/functions/pci-epf-vntb.c | 47 ++++++++++++++++---
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index d74402ea10bf..f4eede5330c6 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -152,6 +152,7 @@ struct epf_ntb {
 	struct mutex link_lock;
 	bool host_linkup;
 	bool vhost_linkup;
+	bool peer_msix;
 
 	/*
 	 * True when doorbells are interrupt-driven (MSI or embedded), false
@@ -159,6 +160,7 @@ struct epf_ntb {
 	 */
 	bool msi_doorbell;
 	u32 spad_size;
+	u32 msix_table_offset;
 
 	enum pci_barno epf_ntb_bar[VNTB_BAR_NUM];
 
@@ -202,13 +204,16 @@ static void epf_ntb_update_link(struct epf_ntb *ntb)
 
 static int epf_ntb_notify_link(struct ntb_dev *ndev)
 {
-	struct pci_epf *epf = ntb_ndev(ndev)->epf;
+	struct epf_ntb *ntb = ntb_ndev(ndev);
+	struct pci_epf *epf = ntb->epf;
+	unsigned int irq_type;
 	int ret;
 
 	ntb_link_event(ndev);
 
+	irq_type = ntb->peer_msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
 	ret = pci_epc_raise_irq(epf->epc, epf->func_no, epf->vfunc_no,
-				PCI_IRQ_MSI, EPF_IRQ_LINK + 1);
+				irq_type, EPF_IRQ_LINK + 1);
 	if (ret)
 		dev_err(&epf->dev, "Failed to raise link event IRQ: %d\n", ret);
 
@@ -310,6 +315,7 @@ static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
  */
 static void epf_ntb_cmd_handler(struct work_struct *work)
 {
+	const struct pci_epc_features *epc_features;
 	struct epf_ntb_ctrl *ctrl;
 	u32 command, argument;
 	struct epf_ntb *ntb;
@@ -342,6 +348,15 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 
 	switch (command) {
 	case COMMAND_CONFIGURE_DOORBELL:
+		epc_features = pci_epc_get_features(ntb->epf->epc,
+						    ntb->epf->func_no,
+						    ntb->epf->vfunc_no);
+		if (((argument & MSIX_ENABLE) && !epc_features->msix_capable) ||
+		    (!(argument & MSIX_ENABLE) && !epc_features->msi_capable)) {
+			ctrl->command_status = COMMAND_STATUS_ERROR;
+			break;
+		}
+		ntb->peer_msix = argument & MSIX_ENABLE;
 		ctrl->command_status = COMMAND_STATUS_OK;
 		break;
 	case COMMAND_TEARDOWN_DOORBELL:
@@ -483,8 +498,9 @@ static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
  * @ntb: NTB device that facilitates communication between HOST and VHOST
  *
  * Allocate the Local Memory mentioned in the above diagram. The size of
- * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
- * is obtained from "spad-count" configfs entry.
+ * CONFIG REGION is sizeof(struct epf_ntb_ctrl), plus space for the MSI-X
+ * Table and PBA if supported. The size of SCRATCHPAD REGION is obtained
+ * from "spad-count" configfs entry.
  *
  * Returns: Zero for success, or an error code in case of failure
  */
@@ -505,6 +521,12 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
 	spad_count = ntb->spad_count;
 
 	ctrl_size = ALIGN(sizeof(struct epf_ntb_ctrl), sizeof(u32));
+	if (epc_features->msix_capable) {
+		ntb->msix_table_offset = ALIGN(ctrl_size, 8);
+		ctrl_size = ntb->msix_table_offset +
+			    ntb->db_count * PCI_MSIX_ENTRY_SIZE +
+			    BITS_TO_U64(ntb->db_count) * sizeof(u64);
+	}
 	spad_size = 2 * spad_count * sizeof(u32);
 
 	base = pci_epf_alloc_space(epf, ctrl_size + spad_size,
@@ -573,6 +595,17 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
 		}
 	}
 
+	if (epc_features->msix_capable) {
+		ret = pci_epc_set_msix(epf->epc, epf->func_no, epf->vfunc_no,
+				       ntb->db_count,
+				       ntb->epf_ntb_bar[BAR_CONFIG],
+				       ntb->msix_table_offset);
+		if (ret) {
+			dev_err(dev, "MSI-X configuration failed\n");
+			return ret;
+		}
+	}
+
 	return 0;
 }
 
@@ -1577,6 +1610,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
 	struct epf_ntb *ntb = container_of(work, struct epf_ntb, peer_db_work);
 	struct pci_epf *epf = ntb->epf;
 	unsigned int budget = VNTB_PEER_DB_WORK_BUDGET;
+	unsigned int irq_type;
 	u8 func_no, vfunc_no;
 	unsigned int db_bit;
 	u32 interrupt_num;
@@ -1588,6 +1622,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
 
 	func_no = epf->func_no;
 	vfunc_no = epf->vfunc_no;
+	irq_type = ntb->peer_msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
 
 	/*
 	 * Drain doorbells from peer_db_pending in snapshots (atomic64_xchg()).
@@ -1601,7 +1636,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
 
 		while (db_bits) {
 			/*
-			 * pci_epc_raise_irq() for MSI expects a 1-based
+			 * pci_epc_raise_irq() for MSI/MSI-X expects a 1-based
 			 * interrupt number. The first usable doorbell starts
 			 * at EPF_IRQ_DB_START in the legacy slot layout.
 			 *
@@ -1621,7 +1656,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
 			db_bits &= ~BIT_ULL(db_bit);
 
 			ret = pci_epc_raise_irq(epf->epc, func_no, vfunc_no,
-						PCI_IRQ_MSI, interrupt_num);
+						irq_type, interrupt_num);
 			if (ret)
 				dev_err(&epf->dev,
 					"Failed to raise IRQ for interrupt_num %u: %d\n",
-- 
2.54.0


  parent reply	other threads:[~2026-10-15  1:44 UTC|newest]

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-10-15  1:44 [PATCH v5 0/2] " Koichiro Den
2026-10-15  1:44 ` [PATCH v5 1/2] PCI: endpoint: pci-epf-vntb: Use a local epf pointer in interrupt setup Koichiro Den
2026-10-15  2:05   ` Frank Li
2026-10-15  1:44 ` Koichiro Den [this message]
2026-10-15  2:12   ` [PATCH v5 2/2] PCI: endpoint: pci-epf-vntb: Honor MSI-X selection Frank Li
2026-10-17  8:31 ` [PATCH v5 0/2] " Manivannan Sadhasivam

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261015014402.2976188-3-den@valinux.co.jp \
    --to=den@valinux.co.jp \
    --cc=Frank.Li@kernel.org \
    --cc=allenbh@gmail.com \
    --cc=bhelgaas@google.com \
    --cc=cassel@kernel.org \
    --cc=dave.jiang@intel.com \
    --cc=jdmason@kudzu.us \
    --cc=kishon@kernel.org \
    --cc=kwilczynski@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=mani@kernel.org \
    --cc=ntb@lists.linux.dev \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®