[PATCH RFC 0/3] PCI/TPH: Expose CPU-to-Steering Tag mappings via debugfs

[PATCH RFC 0/3] PCI/TPH: Expose CPU-to-Steering Tag mappings via debugfs

[Wei Huang]

Add reusable per-device PCI debugfs directories and use them to expose
the firmware-provided CPU-to-Steering Tag mapping for each Root Port.
Factor the ACPI _DSM query path for reuse and document the new read-only
interface.

Wei Huang (3):
  PCI/TPH: Factor out the ST _DSM evaluation
  PCI: Add per-device debugfs directories
  Expose the CPU/Steering Tag mapping via debugfs

 Documentation/PCI/tph.rst |  32 ++++++
 drivers/pci/Makefile      |   1 +
 drivers/pci/debugfs.c     |  60 ++++++++++++
 drivers/pci/pci.h         |  12 +++
 drivers/pci/remove.c      |   1 +
 drivers/pci/tph.c         | 201 ++++++++++++++++++++++++++++++++++----
 include/linux/pci.h       |   3 +
 7 files changed, 290 insertions(+), 20 deletions(-)
 create mode 100644 drivers/pci/debugfs.c

-- 
2.55.0

[PATCH RFC 1/3] PCI/TPH: Factor out the ST _DSM evaluation

[Wei Huang]

The ACPI _DSM that returns the CPU to Steering Tag (ST) mapping is
evaluated in one place today, with the lookup of the ACPI device providing
it open coded in pcie_tph_get_cpu_st().

Factor the handle lookup and the per-CPU query into tph_dsm_handle() and
tph_get_cpu_st_info() so other callers can reuse them, name the _DSM
revision instead of repeating the bare number, and constify the ST info
passed to tph_extract_tag().

No functional change introduced.

Signed-off-by: Wei Huang <wei.huang2@amd.com>
---
 drivers/pci/tph.c | 65 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c
index 655ffd60e62f..d2c6d3dda202 100644
--- a/drivers/pci/tph.c
+++ b/drivers/pci/tph.c
@@ -18,6 +18,9 @@
 static bool pci_tph_disabled;
 
 #ifdef CONFIG_ACPI
+#define TPH_ST_DSM_REV		7
+#define TPH_ST_DSM_FUNC_INDEX	0xF
+
 /*
  * The st_info struct defines the Steering Tag (ST) info returned by the
  * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache
@@ -54,7 +57,7 @@ union st_info {
 };
 
 static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type,
-			   union st_info *info)
+			   const union st_info *info)
 {
 	switch (req_type) {
 	case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */
@@ -88,13 +91,12 @@ static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type,
 	return 0;
 }
 
-#define TPH_ST_DSM_FUNC_INDEX	0xF
 static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
 				  union st_info *st_out)
 {
 	union acpi_object arg3[3], in_obj, *out_obj;
 
-	if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7,
+	if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, TPH_ST_DSM_REV,
 			    BIT(TPH_ST_DSM_FUNC_INDEX)))
 		return AE_ERROR;
 
@@ -114,7 +116,7 @@ static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
 	in_obj.package.count = ARRAY_SIZE(arg3);
 	in_obj.package.elements = arg3;
 
-	out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7,
+	out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, TPH_ST_DSM_REV,
 				    TPH_ST_DSM_FUNC_INDEX, &in_obj);
 	if (!out_obj)
 		return AE_ERROR;
@@ -130,6 +132,43 @@ static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
 
 	return AE_OK;
 }
+
+/*
+ * The ST _DSM is provided by the ACPI device of the Root Port's host bridge,
+ * so all Root Ports below the same host bridge share one ST mapping.
+ */
+static acpi_handle tph_dsm_handle(struct pci_dev *pdev)
+{
+	struct pci_dev *rp;
+
+	rp = pcie_find_root_port(pdev);
+	if (!rp || !rp->bus || !rp->bus->bridge)
+		return NULL;
+
+	return ACPI_HANDLE(rp->bus->bridge);
+}
+
+/* Query the firmware for the ST info of @cpu - Return 0 if OK, else -errno */
+static int tph_get_cpu_st_info(struct pci_dev *pdev, unsigned int cpu,
+			       union st_info *info)
+{
+	acpi_handle handle;
+	u32 cpu_uid;
+	int ret;
+
+	handle = tph_dsm_handle(pdev);
+	if (!handle)
+		return -ENODEV;
+
+	ret = acpi_get_cpu_uid(cpu, &cpu_uid);
+	if (ret)
+		return ret;
+
+	if (tph_invoke_dsm(handle, cpu_uid, info) != AE_OK)
+		return -EINVAL;
+
+	return 0;
+}
 #endif
 
 /* Update the TPH Requester Enable field of TPH Control Register */
@@ -247,27 +286,13 @@ int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type,
 			unsigned int cpu, u16 *tag)
 {
 #ifdef CONFIG_ACPI
-	struct pci_dev *rp;
-	acpi_handle rp_acpi_handle;
 	union st_info info;
-	u32 cpu_uid;
 	int ret;
 
-	ret = acpi_get_cpu_uid(cpu, &cpu_uid);
-	if (ret != 0)
+	ret = tph_get_cpu_st_info(pdev, cpu, &info);
+	if (ret)
 		return ret;
 
-	rp = pcie_find_root_port(pdev);
-	if (!rp || !rp->bus || !rp->bus->bridge)
-		return -ENODEV;
-
-	rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge);
-
-	if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) {
-		*tag = 0;
-		return -EINVAL;
-	}
-
 	*tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info);
 
 	pci_dbg(pdev, "get steering tag: mem_type=%s, cpu=%d, tag=%#04x\n",
-- 
2.55.0

[PATCH RFC 2/3] PCI: Add per-device debugfs directories

[Wei Huang]

PCI features that export debug information have no common place to put it.
The only existing user, PTM, creates a "pcie_ptm_<device>" directory in the
debugfs root, which does not scale as more features follow and leaves each
user to invent its own lifetime rules and leaf names.

Add /sys/kernel/debug/pci/<device>/ directories owned by the PCI core.
pci_dev_debugfs_dir() returns the directory of a device, creating it and
the PCI root directory on first use, so devices without debug content do
not get an empty directory. pci_destroy_dev() removes the directory
recursively, so a feature that pins data in a debugfs file needs no removal
hook of its own: debugfs_remove_recursive() returns only once no operation
on those files is still running.

No functional change for existing users; the next patch is the first
consumer.

Signed-off-by: Wei Huang <wei.huang2@amd.com>
---
 drivers/pci/Makefile  |  1 +
 drivers/pci/debugfs.c | 60 +++++++++++++++++++++++++++++++++++++++++++
 drivers/pci/pci.h     | 12 +++++++++
 drivers/pci/remove.c  |  1 +
 include/linux/pci.h   |  3 +++
 5 files changed, 77 insertions(+)
 create mode 100644 drivers/pci/debugfs.c

diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 41ebc3b9a518..dbc841c3e990 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -15,6 +15,7 @@ ifdef CONFIG_PCI
 obj-$(CONFIG_PROC_FS)		+= proc.o
 obj-$(CONFIG_SYSFS)		+= pci-sysfs.o slot.o
 obj-$(CONFIG_ACPI)		+= pci-acpi.o
+obj-$(CONFIG_DEBUG_FS)		+= debugfs.o
 obj-$(CONFIG_GENERIC_PCI_IOMAP) += iomap.o
 endif
 
diff --git a/drivers/pci/debugfs.c b/drivers/pci/debugfs.c
new file mode 100644
index 000000000000..682fb614d7cf
--- /dev/null
+++ b/drivers/pci/debugfs.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Per-device debugfs directories for the PCI core
+ *
+ * PCI features that want to export debug information create their files in
+ * the directory returned by pci_dev_debugfs_dir().  The directory itself is
+ * created on first use and is torn down, together with everything below it,
+ * when the device is destroyed.
+ */
+#include <linux/debugfs.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+
+#include "pci.h"
+
+/* Serializes creation of the root and of the per-device directories */
+static DEFINE_MUTEX(pci_debugfs_lock);
+static struct dentry *pci_debugfs_root;
+
+/**
+ * pci_dev_debugfs_dir - Return the debugfs directory of a PCI device
+ * @dev: PCI device
+ *
+ * Return the /sys/kernel/debug/pci/<device> directory of @dev, creating it
+ * and the PCI root directory if they do not exist yet.  Devices that have no
+ * debugfs content never get a directory.
+ *
+ * Return: the dentry of the directory, or an ERR_PTR() on failure.  The
+ * return value only needs to be checked with IS_ERR() if the caller cares;
+ * it can be passed to debugfs_create_*() as is.
+ */
+struct dentry *pci_dev_debugfs_dir(struct pci_dev *dev)
+{
+	guard(mutex)(&pci_debugfs_lock);
+
+	if (!pci_debugfs_root)
+		pci_debugfs_root = debugfs_create_dir("pci", NULL);
+
+	if (!dev->debugfs)
+		dev->debugfs = debugfs_create_dir(pci_name(dev),
+						  pci_debugfs_root);
+
+	return dev->debugfs;
+}
+
+/**
+ * pci_dev_debugfs_remove - Remove the debugfs directory of a PCI device
+ * @dev: PCI device
+ *
+ * Remove the device's debugfs directory and everything below it.  Returns
+ * once no debugfs operation on those files is running any more, so the
+ * owners of the files can drop the data the files refer to afterwards.
+ */
+void pci_dev_debugfs_remove(struct pci_dev *dev)
+{
+	guard(mutex)(&pci_debugfs_lock);
+
+	debugfs_remove_recursive(dev->debugfs);
+	dev->debugfs = NULL;
+}
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index ba3c3fddddc2..aead99a4ea34 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -1059,6 +1059,18 @@ static inline int pci_resource_num_to_vf_bar(int resno)
 }
 #endif /* CONFIG_PCI_IOV */
 
+#ifdef CONFIG_DEBUG_FS
+struct dentry *pci_dev_debugfs_dir(struct pci_dev *dev);
+void pci_dev_debugfs_remove(struct pci_dev *dev);
+#else
+static inline struct dentry *pci_dev_debugfs_dir(struct pci_dev *dev)
+{
+	return ERR_PTR(-ENODEV);
+}
+
+static inline void pci_dev_debugfs_remove(struct pci_dev *dev) { }
+#endif
+
 #ifdef CONFIG_PCIE_TPH
 void pci_restore_tph_state(struct pci_dev *dev);
 void pci_save_tph_state(struct pci_dev *dev);
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index e711ac1d4e38..e10139117d57 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -37,6 +37,7 @@ static void pci_destroy_dev(struct pci_dev *dev)
 	platform_pci_remove_wake(dev);
 	pci_doe_sysfs_teardown(dev);
 	pci_npem_remove(dev);
+	pci_dev_debugfs_remove(dev);
 
 	/*
 	 * While device is in D0 drop the device from TSM link operations
diff --git a/include/linux/pci.h b/include/linux/pci.h
index d31a8d107b1e..251d6eeef087 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -580,6 +580,9 @@ struct pci_dev {
 #endif
 #ifdef CONFIG_PCI_TSM
 	struct pci_tsm *tsm;		/* TSM operation state */
+#endif
+#ifdef CONFIG_DEBUG_FS
+	struct dentry	*debugfs;	/* debugfs directory, if any */
 #endif
 	u16		acs_cap;	/* ACS Capability offset */
 	u16		acs_capabilities; /* ACS Capabilities */
-- 
2.55.0

[PATCH RFC 3/3] Expose the CPU/Steering Tag mapping via debugfs

[Wei Huang]

Steering Tags (ST) are opaque, platform specific values obtained from the
ACPI ST _DSM, and are only visible to the driver that asks for them. That
makes it hard to validate whether firmware reports a sane mapping at all.

Expose the mapping read-only in the debugfs directory of each Root Port
that has ST _DSM support:

  # cat /sys/kernel/debug/pci/0000:00:01.1/tph_cpu_st
  cpu  vm_st  vm_xst vm_ph_ignore pm_st  pm_xst pm_ph_ignore
  0    0x03   0x0103 0            -      -      0
  1    0x03   0x0103 0            -      -      0

Each line holds the raw ST info the _DSM returns for one online CPU;
invalid tags and failed queries are reported as "-".

The interface is per Root Port because the _DSM is evaluated on the ACPI
device of the host bridge above it. A system with multiple host bridges may
return different tags for the same CPU, and the value that matters to a
driver is the one for the Root Port its device sits below. Root Ports below
the same host bridge report identical contents.

The file is created during enumeration and removed with the device's
debugfs directory in pci_destroy_dev(), which returns only once no read is
running, so the pci_dev the file refers to stays valid.

Signed-off-by: Wei Huang <wei.huang2@amd.com>
---
 Documentation/PCI/tph.rst |  32 +++++++++
 drivers/pci/tph.c         | 136 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 168 insertions(+)

diff --git a/Documentation/PCI/tph.rst b/Documentation/PCI/tph.rst
index b6cf22b9bd90..19239546af3b 100644
--- a/Documentation/PCI/tph.rst
+++ b/Documentation/PCI/tph.rst
@@ -125,6 +125,38 @@ been changed. Here is a sample code for IRQ affinity notifier:
              return;
     }
 
+Inspect the firmware ST mapping
+-------------------------------
+
+When firmware provides the ST _DSM, the CPU to ST mapping it returns is
+exposed read-only under debugfs, in the directory of each Root Port::
+
+  /sys/kernel/debug/pci/<domain:bus:device.function>/tph_cpu_st
+
+Each file holds one line per online CPU::
+
+  # cat /sys/kernel/debug/pci/0000:00:01.1/tph_cpu_st
+  cpu  vm_st  vm_xst vm_ph_ignore pm_st  pm_xst pm_ph_ignore
+  0    0x03   0x0103 0            -      -      0
+  1    0x03   0x0103 0            -      -      0
+
+``vm_*`` and ``pm_*`` refer to volatile and persistent memory, ``*_st`` and
+``*_xst`` are the 8-bit and 16-bit (extended) Steering Tags, and
+``*_ph_ignore`` reports whether the Processing Hint is ignored by the
+platform. Steering Tags that firmware marks as invalid, and CPUs whose
+query fails, are shown as ``-``.
+
+The _DSM is evaluated on the ACPI device of the host bridge above the Root
+Port, so all Root Ports below the same host bridge report the same mapping.
+The file is per Root Port because that is the granularity drivers see: the
+mapping that applies to a device is the one in the directory of the Root
+Port the device sits below.
+
+Tags are queried on read rather than cached, so the output always matches
+what pcie_tph_get_cpu_st() would return at that moment.  The list of CPUs
+is a snapshot; CPUs that come or go while the file is being read may be
+missing from the output.
+
 Disable TPH system-wide
 -----------------------
 
diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c
index d2c6d3dda202..bd682185ffbb 100644
--- a/drivers/pci/tph.c
+++ b/drivers/pci/tph.c
@@ -6,11 +6,14 @@
  *     Eric Van Tassell <Eric.VanTassell@amd.com>
  *     Wei Huang <wei.huang2@amd.com>
  */
+#include <linux/cpu.h>
+#include <linux/debugfs.h>
 #include <linux/pci.h>
 #include <linux/pci-acpi.h>
 #include <linux/msi.h>
 #include <linux/bitfield.h>
 #include <linux/pci-tph.h>
+#include <linux/seq_file.h>
 
 #include "pci.h"
 
@@ -171,6 +174,133 @@ static int tph_get_cpu_st_info(struct pci_dev *pdev, unsigned int cpu,
 }
 #endif
 
+#if defined(CONFIG_ACPI) && defined(CONFIG_DEBUG_FS)
+/* Longest Steering Tag string is "0xffff" */
+#define TPH_ST_STR_SIZE		7
+
+static bool tph_dsm_supported(struct pci_dev *pdev)
+{
+	acpi_handle handle = tph_dsm_handle(pdev);
+
+	if (!handle)
+		return false;
+
+	return acpi_check_dsm(handle, &pci_acpi_dsm_guid, TPH_ST_DSM_REV,
+			      BIT(TPH_ST_DSM_FUNC_INDEX));
+}
+
+static const char *tph_st_str(char *buf, bool valid, u16 st, int digits)
+{
+	if (!valid)
+		return "-";
+
+	snprintf(buf, TPH_ST_STR_SIZE, "0x%0*x", digits, st);
+
+	return buf;
+}
+
+/*
+ * Position 0 is the header, position N > 0 is the first online CPU with an
+ * id >= N - 1.  Encoding the CPU id in the position, instead of an index
+ * into the online mask, keeps the remaining output stable if a CPU goes
+ * offline between two read() calls.
+ *
+ * The iterator itself is the CPU id biased by 2, because 0 is the end of
+ * the sequence and 1 is SEQ_START_TOKEN.
+ */
+static void *tph_cpu_st_pos(loff_t *pos)
+{
+	unsigned int cpu;
+
+	if (!*pos)
+		return SEQ_START_TOKEN;
+
+	if (*pos > nr_cpu_ids)
+		return NULL;
+
+	cpu = cpumask_next((int)*pos - 2, cpu_online_mask);
+	if (cpu >= nr_cpu_ids)
+		return NULL;
+
+	*pos = cpu + 1;
+
+	return (void *)(unsigned long)(cpu + 2);
+}
+
+static void *tph_cpu_st_start(struct seq_file *s, loff_t *pos)
+{
+	return tph_cpu_st_pos(pos);
+}
+
+static void *tph_cpu_st_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	++*pos;
+
+	return tph_cpu_st_pos(pos);
+}
+
+static void tph_cpu_st_stop(struct seq_file *s, void *v)
+{
+}
+
+static int tph_cpu_st_show(struct seq_file *s, void *v)
+{
+	char vm_st[TPH_ST_STR_SIZE], vm_xst[TPH_ST_STR_SIZE];
+	char pm_st[TPH_ST_STR_SIZE], pm_xst[TPH_ST_STR_SIZE];
+	struct pci_dev *pdev = s->private;
+	union st_info info;
+	unsigned int cpu;
+
+	if (v == SEQ_START_TOKEN) {
+		seq_printf(s, "%-4s %-6s %-6s %-12s %-6s %-6s %s\n",
+			   "cpu", "vm_st", "vm_xst", "vm_ph_ignore",
+			   "pm_st", "pm_xst", "pm_ph_ignore");
+		return 0;
+	}
+
+	cpu = (unsigned long)v - 2;
+
+	if (tph_get_cpu_st_info(pdev, cpu, &info)) {
+		seq_printf(s, "%-4u %-6s %-6s %-12s %-6s %-6s %s\n", cpu,
+			   "-", "-", "-", "-", "-", "-");
+		return 0;
+	}
+
+	seq_printf(s, "%-4u %-6s %-6s %-12u %-6s %-6s %u\n", cpu,
+		   tph_st_str(vm_st, info.vm_st_valid, info.vm_st, 2),
+		   tph_st_str(vm_xst, info.vm_xst_valid, info.vm_xst, 4),
+		   (unsigned int)info.vm_ph_ignore,
+		   tph_st_str(pm_st, info.pm_st_valid, info.pm_st, 2),
+		   tph_st_str(pm_xst, info.pm_xst_valid, info.pm_xst, 4),
+		   (unsigned int)info.pm_ph_ignore);
+
+	return 0;
+}
+
+static const struct seq_operations tph_cpu_st_sops = {
+	.start	= tph_cpu_st_start,
+	.next	= tph_cpu_st_next,
+	.stop	= tph_cpu_st_stop,
+	.show	= tph_cpu_st_show,
+};
+DEFINE_SEQ_ATTRIBUTE(tph_cpu_st);
+
+static void tph_debugfs_add_cpu_st(struct pci_dev *pdev)
+{
+	if (!pci_is_pcie(pdev) ||
+	    pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT)
+		return;
+
+	if (!tph_dsm_supported(pdev))
+		return;
+
+	debugfs_create_file("tph_cpu_st", 0444, pci_dev_debugfs_dir(pdev),
+			    pdev, &tph_cpu_st_fops);
+}
+#else
+static void tph_debugfs_add_cpu_st(struct pci_dev *pdev) { }
+#endif /* CONFIG_ACPI && CONFIG_DEBUG_FS */
+
 /* Update the TPH Requester Enable field of TPH Control Register */
 static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type)
 {
@@ -536,6 +666,12 @@ void pci_tph_init(struct pci_dev *pdev)
 	int num_entries;
 	u32 save_size;
 
+	/*
+	 * A Root Port has no ST table of its own, but it is where the
+	 * firmware ST mapping of its host bridge is exposed.
+	 */
+	tph_debugfs_add_cpu_st(pdev);
+
 	pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH);
 	if (!pdev->tph_cap)
 		return;
-- 
2.55.0