mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
* [PATCH v2] scsi: pm8001: Offset hardware queue index for MSI-X pre-vector
@ 2026-11-17 17:55 TJ Adams
  2026-11-17 19:27 ` Bart Van Assche
  0 siblings, 1 reply; 2+ messages in thread
From: TJ Adams @ 2026-11-17 17:55 UTC (permalink / raw)
  To: Jack Wang, James E . J . Bottomley, Martin K . Petersen
  Cc: Bart Van Assche, John Garry, linux-scsi, linux-kernel, Terrence Adams

From: Terrence Adams <tadamsjr@google.com>

When MSI-X interrupt vectors are configured in pm8001_setup_msix(), the
driver requests vector affinity with .pre_vectors = 1 to reserve Vector
0 for non-affinitized housekeeping interrupts.

blk_mq_map_hw_queues() maps hardware queues (hwq 0 to nr_hw_queues - 1)
with an offset of 1, mapping hwq 0 to Vector 1, hwq 1 to Vector 2, etc.

However, pm80xx_chip_get_q_index() retrieves the queue index directly
from blk_mq_unique_tag_to_hwq(tag) without adding 1 for the pre-vector
offset. Consequently, commands on hwq 0 are issued with responseQueue 0
(firing unmanaged Vector 0 on CPU 0), while hwq k fires Vector k (the
CPU affinitized to hwq k-1).

This results in 100% of I/O completion interrupts being delivered to the
wrong CPU core, causing heavy cross-CPU IPIs, cache line bouncing, and
increased kernel and IRQ CPU processing overhead.

Define PM8001_PRE_VECTORS for the reserved housekeeping vector and use
it consistently across MSI-X vector allocation, hardware queue setup and
mapping, and queue index lookup. Update pm80xx_chip_get_q_index() to
accept pm8001_ha and add PM8001_PRE_VECTORS to
blk_mq_unique_tag_to_hwq() whenever multi-queue is enabled
(number_of_intr > PM8001_PRE_VECTORS), routing completion interrupts to
their affinitized CPU cores.

Fixes: 42f22fe36d51 ("scsi: pm8001: Expose hardware queues for pm80xx")
Signed-off-by: Terrence Adams <tadamsjr@google.com>
---
Changes in v2:
- Define PM8001_PRE_VECTORS in pm8001_defs.h instead of hardcoding 1 and
  use it consistently across pm8001_setup_msix(), pm8001_pci_probe(),
  pm8001_map_queues(), and pm80xx_chip_get_q_index() (Bart Van Assche).

v1: https://lore.kernel.org/all/20261116224811.3362188-1-tadamsjr@google.com/

LLM assisted with drafting the fix, v2 refactor, and commit message.

 drivers/scsi/pm8001/pm8001_defs.h |  1 +
 drivers/scsi/pm8001/pm8001_init.c | 15 +++++++++------
 drivers/scsi/pm8001/pm80xx_hwi.c  | 13 +++++++++----
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/drivers/scsi/pm8001/pm8001_defs.h b/drivers/scsi/pm8001/pm8001_defs.h
index a9605ca317f3..ef7b6257cc8b 100644
--- a/drivers/scsi/pm8001/pm8001_defs.h
+++ b/drivers/scsi/pm8001/pm8001_defs.h
@@ -178,6 +178,7 @@ enum port_type {
 #define	PM8001_MAX_INB_NUM	 64
 #define	PM8001_MAX_OUTB_NUM	 64
 #define	PM8001_CAN_QUEUE	 508	/* SCSI Queue depth */
+#define PM8001_PRE_VECTORS	 1	/* non-affinitized housekeeping vector */
 
 /* Inbound/Outbound queue size */
 #define IOMB_SIZE_SPC		64
diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
index 5af81c73a8f8..b1788788b7ed 100644
--- a/drivers/scsi/pm8001/pm8001_init.c
+++ b/drivers/scsi/pm8001/pm8001_init.c
@@ -104,8 +104,9 @@ static void pm8001_map_queues(struct Scsi_Host *shost)
 	struct pm8001_hba_info *pm8001_ha = sha->lldd_ha;
 	struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
 
-	if (pm8001_ha->number_of_intr > 1) {
-		blk_mq_map_hw_queues(qmap, &pm8001_ha->pdev->dev, 1);
+	if (pm8001_ha->number_of_intr > PM8001_PRE_VECTORS) {
+		blk_mq_map_hw_queues(qmap, &pm8001_ha->pdev->dev,
+				     PM8001_PRE_VECTORS);
 		return;
 	}
 
@@ -977,10 +978,11 @@ static u32 pm8001_setup_msix(struct pm8001_hba_info *pm8001_ha)
 		 * include in the affinity spreading.
 		 */
 		struct irq_affinity desc = {
-			.pre_vectors = 1,
+			.pre_vectors = PM8001_PRE_VECTORS,
 		};
 		rc = pci_alloc_irq_vectors_affinity(
-				pm8001_ha->pdev, 2, PM8001_MAX_MSIX_VEC,
+				pm8001_ha->pdev, PM8001_PRE_VECTORS + 1,
+				PM8001_MAX_MSIX_VEC,
 				PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &desc);
 	}
 
@@ -1183,8 +1185,9 @@ static int pm8001_pci_probe(struct pci_dev *pdev,
 
 	PM8001_CHIP_DISP->chip_post_init(pm8001_ha);
 
-	if (pm8001_ha->number_of_intr > 1) {
-		shost->nr_hw_queues = pm8001_ha->number_of_intr - 1;
+	if (pm8001_ha->number_of_intr > PM8001_PRE_VECTORS) {
+		shost->nr_hw_queues = pm8001_ha->number_of_intr -
+				      PM8001_PRE_VECTORS;
 		/*
 		 * For now, ensure we're not sent too many commands by setting
 		 * host_tagset. This is also required if we start using request
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
index d2572dd66071..1d4f5ee861c1 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.c
+++ b/drivers/scsi/pm8001/pm80xx_hwi.c
@@ -4328,14 +4328,19 @@ static int check_enc_sat_cmd(struct sas_task *task)
 	return ret;
 }
 
-static u32 pm80xx_chip_get_q_index(struct sas_task *task)
+static u32 pm80xx_chip_get_q_index(struct pm8001_hba_info *pm8001_ha,
+				   struct sas_task *task)
 {
 	struct request *rq = sas_task_find_rq(task);
 
 	if (!rq)
 		return 0;
 
-	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(rq));
+	if (pm8001_ha->number_of_intr > PM8001_PRE_VECTORS)
+		return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(rq)) +
+		       PM8001_PRE_VECTORS;
+
+	return 0;
 }
 
 /**
@@ -4371,7 +4376,7 @@ static int pm80xx_chip_ssp_io_req(struct pm8001_hba_info *pm8001_ha,
 	ssp_cmd.ssp_iu.efb_prio_attr |= (task->ssp_task.task_attr & 7);
 	memcpy(ssp_cmd.ssp_iu.cdb, task->ssp_task.cmd->cmnd,
 		       task->ssp_task.cmd->cmd_len);
-	q_index = pm80xx_chip_get_q_index(task);
+	q_index = pm80xx_chip_get_q_index(pm8001_ha, task);
 
 	/* Check if encryption is set */
 	if (pm8001_ha->chip->encrypt &&
@@ -4510,7 +4515,7 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
 	u32 opc = OPC_INB_SATA_HOST_OPSTART;
 	memset(&sata_cmd, 0, sizeof(sata_cmd));
 
-	q_index = pm80xx_chip_get_q_index(task);
+	q_index = pm80xx_chip_get_q_index(pm8001_ha, task);
 
 	if (task->data_dir == DMA_NONE && !task->ata_task.use_ncq) {
 		ATAP = 0x04; /* no data*/

base-commit: 666539eecdf4db623f5e3c3e55cf9f2dfad24aa4
-- 
2.56.1.703.gf848ed807c-goog


^ permalink raw reply	[flat|nested] 2+ messages in thread

* Re: [PATCH v2] scsi: pm8001: Offset hardware queue index for MSI-X pre-vector
  2026-11-17 17:55 [PATCH v2] scsi: pm8001: Offset hardware queue index for MSI-X pre-vector TJ Adams
@ 2026-11-17 19:27 ` Bart Van Assche
  0 siblings, 0 replies; 2+ messages in thread
From: Bart Van Assche @ 2026-11-17 19:27 UTC (permalink / raw)
  To: TJ Adams, Jack Wang, James E . J . Bottomley, Martin K . Petersen
  Cc: John Garry, linux-scsi, linux-kernel

On 11/17/26 9:55 AM, TJ Adams wrote:
 > diff --git a/drivers/scsi/pm8001/pm8001_defs.h 
b/drivers/scsi/pm8001/pm8001_defs.h
 > index a9605ca317f3..ef7b6257cc8b 100644
 > --- a/drivers/scsi/pm8001/pm8001_defs.h
 > +++ b/drivers/scsi/pm8001/pm8001_defs.h
 > @@ -178,6 +178,7 @@ enum port_type {
 >  #define	PM8001_MAX_INB_NUM	 64
 >  #define	PM8001_MAX_OUTB_NUM	 64
 >  #define	PM8001_CAN_QUEUE	 508	/* SCSI Queue depth */
 > +#define PM8001_PRE_VECTORS	 1	/* non-affinitized housekeeping vector */

The surrounding definitions have a tab between "#define" and the
macro name, while this one has a space. Is this intentional?

 > -static u32 pm80xx_chip_get_q_index(struct sas_task *task)
 > +static u32 pm80xx_chip_get_q_index(struct pm8001_hba_info *pm8001_ha,
 > +				   struct sas_task *task)
 >  {
 >  	struct request *rq = sas_task_find_rq(task);
 >
 >  	if (!rq)
 >  		return 0;
 >
 > -	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(rq));
 > +	if (pm8001_ha->number_of_intr > PM8001_PRE_VECTORS)
 > +		return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(rq)) +
 > +		       PM8001_PRE_VECTORS;
 > +
 > +	return 0;
 >  }

A side effect of this change: completions for the highest hardware
queue now go to outbound queue max_q_num - 1. pm80xx_chip_init()
programs that vector as the fatal error interrupt. process_oq() reads
MSGU_SCRATCH_PAD_1 for every interrupt on that vector, and also
MSGU_SCRATCH_PAD_RSVD_0 if no fatal error is reported. Without this
patch, no I/O completions go to that queue, so those MMIO reads only
happen for fatal error interrupts. The code is still correct, and the
driver used that queue for I/O before commit 42f22fe36d51 too, but it
might be worth mentioning in the patch description. Alternatively, a
follow-up patch could move the fatal error interrupt to the
housekeeping vector.

Anyway:

Reviewed-by: Bart Van Assche <bvanassche@acm.org>

^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2026-11-17 19:27 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-17 17:55 [PATCH v2] scsi: pm8001: Offset hardware queue index for MSI-X pre-vector TJ Adams
2026-11-17 19:27 ` Bart Van Assche

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®