[PATCH v2 2/2] PCI/DOE: Poll briefly for a response before sleeping

[Michal Clapinski <mclapinski@google.com>]

doe_statemachine_work() and pci_doe_abort() wait for the mailbox by
sleeping PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128) between status
reads, which is 7 ms at HZ=1000 and 4 ms at HZ=250. Mailbox operations
are served from the device's local mailbox logic and complete much
faster: on NVIDIA B200 GPUs, an abort completes in ~70 us and a
discovery exchange in ~110 us, including the status reads polling for
them. Every DOE-capable endpoint pays the sleep once for the abort
issued during mailbox creation and once per supported feature during
enumeration.

Make pci_doe_wait(), which both paths use, back off exponentially:
start with a 1 us delay and double it on every poll until it reaches
PCI_DOE_POLL_INTERVAL, then fall back to the existing jiffies-based
wait.

Enumeration at boot runs before high resolution timers are enabled, so
any sleep lasts until the next tick, which at HZ=250 is no shorter than
PCI_DOE_POLL_INTERVAL. Until they are enabled, busy-wait instead of
sleeping for delays of up to 128 us. Operations that complete within
~255 us of accumulated delay, like the ones above, then finish without
sleeping, while slower ones waste at most that much CPU time before
sleeping anyway. Once high resolution timers are enabled, short sleeps
are accurate and no busy-wait beyond what fsleep() does is needed.

Time spent in pci_device_add() for a B200 GPU drops from 45.9 ms to
6.4 ms.

Signed-off-by: Michal Clapinski <mclapinski@google.com>
---
 drivers/pci/doe.c | 47 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c
index 6d258f751318..3f2f0b181f25 100644
--- a/drivers/pci/doe.c
+++ b/drivers/pci/doe.c
@@ -15,6 +15,7 @@
 #include <linux/bitfield.h>
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/hrtimer.h>
 #include <linux/jiffies.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
@@ -27,6 +28,8 @@
 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 #define PCI_DOE_TIMEOUT HZ
 #define PCI_DOE_POLL_INTERVAL	max(PCI_DOE_TIMEOUT / 128, 1)
+#define PCI_DOE_POLL_MIN_US	1
+#define PCI_DOE_POLL_SPIN_MAX_US	128
 
 #define PCI_DOE_FLAG_CANCEL	0
 #define PCI_DOE_FLAG_DEAD	1
@@ -244,11 +247,45 @@ void pci_doe_sysfs_init(struct pci_dev *pdev)
 }
 #endif
 
-static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
+/*
+ * Wait before the next poll of the DOE Status register.
+ *
+ * Mailbox operations are usually served from the device's local mailbox
+ * logic within a few hundred microseconds, far below the jiffies
+ * granularity of PCI_DOE_POLL_INTERVAL.  Start with a short delay and
+ * double it on every call, so that a response is noticed soon after it
+ * becomes available.
+ *
+ * Until high resolution timers are enabled, any sleep lasts until the next
+ * tick.  At boot, that is the case during PCI enumeration.  Busy-wait for
+ * delays of up to PCI_DOE_POLL_SPIN_MAX_US instead, so that typical
+ * operations complete without sleeping.  Slower operations waste at most
+ * that much CPU time before sleeping anyway.  Once the delay reaches
+ * PCI_DOE_POLL_INTERVAL, fall back to waiting on the wait queue.
+ *
+ * @delay_us must be initialized to PCI_DOE_POLL_MIN_US by the caller.
+ */
+static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned int *delay_us)
 {
+	if (test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags))
+		return -EIO;
+
+	if (*delay_us <= PCI_DOE_POLL_SPIN_MAX_US &&
+	    !hrtimer_highres_enabled()) {
+		udelay(*delay_us);
+		*delay_us *= 2;
+		return 0;
+	}
+
+	if (*delay_us < jiffies_to_usecs(PCI_DOE_POLL_INTERVAL)) {
+		fsleep(*delay_us);
+		*delay_us *= 2;
+		return 0;
+	}
+
 	if (wait_event_timeout(doe_mb->wq,
 			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
-			       timeout))
+			       PCI_DOE_POLL_INTERVAL))
 		return -EIO;
 	return 0;
 }
@@ -266,6 +303,7 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 	struct pci_dev *pdev = doe_mb->pdev;
 	int offset = doe_mb->cap_offset;
 	unsigned long timeout_jiffies;
+	unsigned int delay_us = PCI_DOE_POLL_MIN_US;
 
 	pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
 
@@ -276,7 +314,7 @@ static int pci_doe_abort(struct pci_doe_mb *doe_mb)
 		int rc;
 		u32 val;
 
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
+		rc = pci_doe_wait(doe_mb, &delay_us);
 		if (rc)
 			return rc;
 		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
@@ -486,6 +524,7 @@ static void doe_statemachine_work(struct work_struct *work)
 	struct pci_dev *pdev = doe_mb->pdev;
 	int offset = doe_mb->cap_offset;
 	unsigned long timeout_jiffies;
+	unsigned int delay_us = PCI_DOE_POLL_MIN_US;
 	u32 val;
 	int rc;
 
@@ -525,7 +564,7 @@ static void doe_statemachine_work(struct work_struct *work)
 			signal_task_abort(task, -EIO);
 			return;
 		}
-		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
+		rc = pci_doe_wait(doe_mb, &delay_us);
 		if (rc) {
 			signal_task_abort(task, rc);
 			return;
-- 
2.56.0.385.gac671c819a-goog