[PATCH v3 0/2] RISC-V IOMMU HPM support

[PATCH v3 0/2] RISC-V IOMMU HPM support

[Zong Li]

This series implements support for the RISC-V IOMMU hardware performance
monitor.

The RISC-V IOMMU PMU driver is implemented as an auxiliary device driver
created by the parent RISC-V IOMMU driver. Therefore, the child driver
can obtain resources and information from the parent device, such as
the MMIO base address and IRQ number.

Changed in v2:
- Rebased onto v7.2-rc1
- Use hi-lo-hi mechanism to read counter.
  Suggested by Guo Ren and David Laight

Changed in v1:
- Rebased onto v6.19-rc8
- Pick all suggestions and feedbacks from v1 series
- Add cpu hotplug implementation to avoid race enablement
- Move PMU-related definition from header to c file
- Change PMU driver to auxiliary device driver

Changed in RFC:
- Rebase onto v6.13-rc7
- Clear interrupt pending before handling interrupt
- Fix the counter value issue caused by OF bit in the cycle counter.
- Invoke riscv_iommu_hpm_disable() instead of riscv_iommu_pmu_uninit()
  in riscv_iommu_remove()

Zong Li (2):
  drivers/perf: riscv-iommu: add risc-v iommu pmu driver
  iommu/riscv: create a auxiliary device for HPM

 drivers/iommu/riscv/iommu-bits.h |  61 ---
 drivers/iommu/riscv/iommu.c      |  19 +
 drivers/perf/Kconfig             |  12 +
 drivers/perf/Makefile            |   1 +
 drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
 5 files changed, 735 insertions(+), 61 deletions(-)
 create mode 100644 drivers/perf/riscv_iommu_pmu.c

-- 
2.43.7

[PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
device driver created by the parent RISC-V IOMMU driver.

The RISC-V IOMMU PMU separates the cycle counter from the event counters.
The cycle counter is not associated with iohpmevt0, so a software-defined
cycle event is required for the perf subsystem.

The number and width of the counters are hardware-implemented and must
be detected at runtime.

The performance monitor provides counters with filtering support to
collect events for specific device ID/process ID, or GSCID/PSCID.

PMU-related definitions are moved into the perf driver, where they are
used exclusively.

Suggested-by: David Laight <david.laight.linux@gmail.com>
Suggested-by: Guo Ren <guoren@kernel.org>
Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
Signed-off-by: Zong Li <zong.li@sifive.com>
---
 drivers/iommu/riscv/iommu-bits.h |  61 ---
 drivers/perf/Kconfig             |  12 +
 drivers/perf/Makefile            |   1 +
 drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
 4 files changed, 716 insertions(+), 61 deletions(-)
 create mode 100644 drivers/perf/riscv_iommu_pmu.c

diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
index f2ef9bd3cde9..6b5de913a032 100644
--- a/drivers/iommu/riscv/iommu-bits.h
+++ b/drivers/iommu/riscv/iommu-bits.h
@@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
 #define RISCV_IOMMU_IPSR_PMIP		BIT(RISCV_IOMMU_INTR_PM)
 #define RISCV_IOMMU_IPSR_PIP		BIT(RISCV_IOMMU_INTR_PQ)
 
-/* 5.19 Performance monitoring counter overflow status (32bits) */
-#define RISCV_IOMMU_REG_IOCOUNTOVF	0x0058
-#define RISCV_IOMMU_IOCOUNTOVF_CY	BIT(0)
-#define RISCV_IOMMU_IOCOUNTOVF_HPM	GENMASK_ULL(31, 1)
-
-/* 5.20 Performance monitoring counter inhibits (32bits) */
-#define RISCV_IOMMU_REG_IOCOUNTINH	0x005C
-#define RISCV_IOMMU_IOCOUNTINH_CY	BIT(0)
-#define RISCV_IOMMU_IOCOUNTINH_HPM	GENMASK(31, 1)
-
-/* 5.21 Performance monitoring cycles counter (64bits) */
-#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
-#define RISCV_IOMMU_IOHPMCYCLES_COUNTER	GENMASK_ULL(62, 0)
-#define RISCV_IOMMU_IOHPMCYCLES_OF	BIT_ULL(63)
-
-/* 5.22 Performance monitoring event counters (31 * 64bits) */
-#define RISCV_IOMMU_REG_IOHPMCTR_BASE	0x0068
-#define RISCV_IOMMU_REG_IOHPMCTR(_n)	(RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
-
-/* 5.23 Performance monitoring event selectors (31 * 64bits) */
-#define RISCV_IOMMU_REG_IOHPMEVT_BASE	0x0160
-#define RISCV_IOMMU_REG_IOHPMEVT(_n)	(RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
-#define RISCV_IOMMU_IOHPMEVT_EVENTID	GENMASK_ULL(14, 0)
-#define RISCV_IOMMU_IOHPMEVT_DMASK	BIT_ULL(15)
-#define RISCV_IOMMU_IOHPMEVT_PID_PSCID	GENMASK_ULL(35, 16)
-#define RISCV_IOMMU_IOHPMEVT_DID_GSCID	GENMASK_ULL(59, 36)
-#define RISCV_IOMMU_IOHPMEVT_PV_PSCV	BIT_ULL(60)
-#define RISCV_IOMMU_IOHPMEVT_DV_GSCV	BIT_ULL(61)
-#define RISCV_IOMMU_IOHPMEVT_IDT	BIT_ULL(62)
-#define RISCV_IOMMU_IOHPMEVT_OF		BIT_ULL(63)
-
-/* Number of defined performance-monitoring event selectors */
-#define RISCV_IOMMU_IOHPMEVT_CNT	31
-
-/**
- * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
- *
- * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
- * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
- * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
- * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
- * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
- * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
- * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
- * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
- * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
- * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
- */
-enum riscv_iommu_hpmevent_id {
-	RISCV_IOMMU_HPMEVENT_INVALID    = 0,
-	RISCV_IOMMU_HPMEVENT_URQ        = 1,
-	RISCV_IOMMU_HPMEVENT_TRQ        = 2,
-	RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
-	RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
-	RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
-	RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
-	RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
-	RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
-	RISCV_IOMMU_HPMEVENT_MAX        = 9
-};
-
 /* 5.24 Translation request IOVA (64bits) */
 #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
 #define RISCV_IOMMU_TR_REQ_IOVA_VPN	GENMASK_ULL(63, 12)
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 245e7bb763b9..8cce6c2ea626 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -105,6 +105,18 @@ config RISCV_PMU_SBI
 	  full perf feature support i.e. counter overflow, privilege mode
 	  filtering, counter configuration.
 
+config RISCV_IOMMU_PMU
+	depends on RISCV || COMPILE_TEST
+	depends on RISCV_IOMMU
+	bool "RISC-V IOMMU Hardware Performance Monitor"
+	default y
+	help
+	  Say Y if you want to use the RISC-V IOMMU performance monitor
+	  implementation. The performance monitor is an optional hardware
+	  feature, and whether it is actually enabled depends on IOMMU
+	  hardware support. If the underlying hardware does not implement
+	  the PMU, this option will have no effect.
+
 config STARFIVE_STARLINK_PMU
 	depends on ARCH_STARFIVE || COMPILE_TEST
 	depends on 64BIT
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index eb8a022dad9a..90c75f3c0ac1 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
 obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
 obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
 obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
+obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
 obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
 obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
 obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
new file mode 100644
index 000000000000..2e856289ddc9
--- /dev/null
+++ b/drivers/perf/riscv_iommu_pmu.c
@@ -0,0 +1,703 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 SiFive
+ *
+ * Authors
+ *	Zong Li <zong.li@sifive.com>
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+#include <linux/perf_event.h>
+
+#include "../iommu/riscv/iommu.h"
+
+/* 5.19 Performance monitoring counter overflow status (32bits) */
+#define RISCV_IOMMU_REG_IOCOUNTOVF	0x0058
+#define RISCV_IOMMU_IOCOUNTOVF_CY	BIT(0)
+#define RISCV_IOMMU_IOCOUNTOVF_HPM	GENMASK_ULL(31, 1)
+
+/* 5.20 Performance monitoring counter inhibits (32bits) */
+#define RISCV_IOMMU_REG_IOCOUNTINH	0x005C
+#define RISCV_IOMMU_IOCOUNTINH_CY	BIT(0)
+#define RISCV_IOMMU_IOCOUNTINH_HPM	GENMASK(31, 0)
+
+/* 5.21 Performance monitoring cycles counter (64bits) */
+#define RISCV_IOMMU_REG_IOHPMCYCLES	0x0060
+#define RISCV_IOMMU_IOHPMCYCLES_COUNTER	GENMASK_ULL(62, 0)
+#define RISCV_IOMMU_IOHPMCYCLES_OF	BIT_ULL(63)
+#define RISCV_IOMMU_REG_IOHPMCTR(_n)	(RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
+
+/* 5.22 Performance monitoring event counters (31 * 64bits) */
+#define RISCV_IOMMU_REG_IOHPMCTR_BASE	0x0068
+#define RISCV_IOMMU_IOHPMCTR_COUNTER	GENMASK_ULL(63, 0)
+
+/* 5.23 Performance monitoring event selectors (31 * 64bits) */
+#define RISCV_IOMMU_REG_IOHPMEVT_BASE	0x0160
+#define RISCV_IOMMU_REG_IOHPMEVT(_n)	(RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
+#define RISCV_IOMMU_IOHPMEVT_EVENTID	GENMASK_ULL(14, 0)
+#define RISCV_IOMMU_IOHPMEVT_DMASK	BIT_ULL(15)
+#define RISCV_IOMMU_IOHPMEVT_PID_PSCID	GENMASK_ULL(35, 16)
+#define RISCV_IOMMU_IOHPMEVT_DID_GSCID	GENMASK_ULL(59, 36)
+#define RISCV_IOMMU_IOHPMEVT_PV_PSCV	BIT_ULL(60)
+#define RISCV_IOMMU_IOHPMEVT_DV_GSCV	BIT_ULL(61)
+#define RISCV_IOMMU_IOHPMEVT_IDT	BIT_ULL(62)
+#define RISCV_IOMMU_IOHPMEVT_OF		BIT_ULL(63)
+#define RISCV_IOMMU_IOHPMEVT_EVENT	GENMASK_ULL(62, 0)
+
+/* The total number of counters is 31 event counters plus 1 cycle counter */
+#define RISCV_IOMMU_HPM_COUNTER_NUM	32
+
+static int cpuhp_state;
+
+/**
+ * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
+ *
+ * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
+ * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
+ * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
+ * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
+ * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
+ * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
+ * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
+ * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
+ * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
+ * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
+ *
+ * The specification does not define an event ID for counting the
+ * number of clock cycles, meaning there is no associated 'iohpmevt0'.
+ * Event ID 0 is an invalid event and does not overlap with any valid
+ * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
+ * event is not actually written into any register, it serves solely
+ * as an identifier.
+ */
+enum riscv_iommu_hpmevent_id {
+	RISCV_IOMMU_HPMEVENT_CYCLE	= 0,
+	RISCV_IOMMU_HPMEVENT_URQ        = 1,
+	RISCV_IOMMU_HPMEVENT_TRQ        = 2,
+	RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
+	RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
+	RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
+	RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
+	RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
+	RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
+	RISCV_IOMMU_HPMEVENT_MAX        = 9
+};
+
+struct riscv_iommu_pmu {
+	struct pmu pmu;
+	struct hlist_node node;
+	void __iomem *reg;
+	unsigned int on_cpu;
+	int num_counters;
+	u64 cycle_cntr_mask;
+	u64 event_cntr_mask;
+	struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
+	DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
+	u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
+	u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
+};
+
+#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
+
+#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)			\
+	static inline u32 get_##_name(struct perf_event *event)		\
+	{								\
+		return FIELD_GET(_mask, event->attr.config);		\
+	}								\
+
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
+RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
+
+/* Formats */
+PMU_FORMAT_ATTR(event,            "config:0-14");
+PMU_FORMAT_ATTR(partial_matching, "config:15");
+PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
+PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
+PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
+PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
+PMU_FORMAT_ATTR(filter_id_type,   "config:62");
+
+static struct attribute *riscv_iommu_pmu_formats[] = {
+	&format_attr_event.attr,
+	&format_attr_partial_matching.attr,
+	&format_attr_pid_pscid.attr,
+	&format_attr_did_gscid.attr,
+	&format_attr_filter_pid_pscid.attr,
+	&format_attr_filter_did_gscid.attr,
+	&format_attr_filter_id_type.attr,
+	NULL,
+};
+
+static const struct attribute_group riscv_iommu_pmu_format_group = {
+	.name = "format",
+	.attrs = riscv_iommu_pmu_formats,
+};
+
+/* Events */
+static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
+					  struct device_attribute *attr,
+					  char *page)
+{
+	struct perf_pmu_events_attr *pmu_attr;
+
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
+
+	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
+}
+
+#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)			\
+	PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
+
+static struct attribute *riscv_iommu_pmu_events[] = {
+	RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
+	RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
+	RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
+	RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
+	RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
+	RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
+	RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
+	RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
+	RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
+	NULL,
+};
+
+static const struct attribute_group riscv_iommu_pmu_events_group = {
+	.name = "events",
+	.attrs = riscv_iommu_pmu_events,
+};
+
+/* cpumask */
+static ssize_t riscv_iommu_cpumask_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
+
+	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
+}
+
+static struct device_attribute riscv_iommu_cpumask_attr =
+	__ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
+
+static struct attribute *riscv_iommu_cpumask_attrs[] = {
+	&riscv_iommu_cpumask_attr.attr,
+	NULL
+};
+
+static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
+	.attrs = riscv_iommu_cpumask_attrs,
+};
+
+static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
+	&riscv_iommu_pmu_cpumask_group,
+	&riscv_iommu_pmu_format_group,
+	&riscv_iommu_pmu_events_group,
+	NULL,
+};
+
+/* PMU Operations */
+static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
+					u64 value)
+{
+	u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
+
+	writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
+}
+
+/*
+ * As stated in the RISC-V IOMMU Specification, Chapter 6:
+ * Whether an 8 byte access to an IOMMU register is single-copy atomic
+ * is UNSPECIFIED, and such an access may appear, internally to the
+ * IOMMU, as if two separate 4 byte accesses - first to the high half
+ * and second to the low half - were performed
+ *
+ * To make sure the driver works correctly on different hardware,
+ * the software will always use two 4-byte access for the counter.
+ *
+ * This function implements the hi-lo-hi pattern to detect and handle
+ * wraparound during the read operation:
+ *   1. Read high half (hi)
+ *   2. Read low half (lo)
+ *   3. Check if low half wrapped or high half changed:
+ *      - If lo <= lo_prev: possible wraparound occurred
+ *      - If hi != hi_prev: high half changed during read
+ *   4. If wraparound detected, re-read high half and assume low half is 0
+ *   5. Update previous values for next read
+ */
+static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
+{
+	void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
+	u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
+	u32 hi, lo;
+
+	hi = readl(addr + 4);
+	lo = readl(addr);
+
+	if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
+		u32 hi_tmp = readl(addr + 4);
+
+		/*
+		 * If hi changes, then hi+1:0 must have happened while
+		 * the code was running so it is a safe return value
+		 */
+		if (hi_tmp != hi) {
+			hi = hi_tmp;
+			lo = 0;
+		}
+		pmu->lo_prev[idx] = ~0u;
+		pmu->hi_prev[idx] = hi;
+	}
+	pmu->lo_prev[idx] = lo;
+
+	value = (((u64)hi << 32) | lo) & counter_mask;
+
+	/* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
+	return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
+}
+
+static bool is_cycle_event(u64 event)
+{
+	return event == RISCV_IOMMU_HPMEVENT_CYCLE;
+}
+
+static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
+				      u64 value)
+{
+	/* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
+	if (is_cycle_event(value))
+		return;
+
+	/* Event counter start from idx 1 */
+	writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
+	       pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
+}
+
+static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
+{
+	void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
+	u32 value = readl(addr);
+
+	writel(value & ~BIT(idx), addr);
+}
+
+static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
+{
+	void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
+	u32 value = readl(addr);
+
+	writel(value | BIT(idx), addr);
+}
+
+static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
+{
+	void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
+	u32 used_cntr = 0;
+
+	/* The performance-monitoring counter inhibits is a 32-bit WARL register */
+	bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
+
+	writel(~used_cntr, addr);
+}
+
+static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
+{
+	writel(GENMASK_ULL(pmu->num_counters - 1, 0),
+	       pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
+}
+
+/* PMU APIs */
+static void riscv_iommu_pmu_set_period(struct perf_event *event)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
+	u64 period;
+
+	/*
+	 * Limit the maximum period to prevent the counter value
+	 * from overtaking the one we are about to program.
+	 * In effect we are reducing max_period to account for
+	 * interrupt latency (and we are being very conservative).
+	 */
+	period = counter_mask >> 1;
+	riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
+	local64_set(&hwc->prev_count, period);
+}
+
+static int riscv_iommu_pmu_event_init(struct perf_event *event)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	struct perf_event *sibling;
+	int total_event_counters = pmu->num_counters - 1;
+	int counters = 0;
+
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	if (hwc->sample_period)
+		return -EOPNOTSUPP;
+
+	if (event->cpu < 0)
+		return -EOPNOTSUPP;
+
+	event->cpu = pmu->on_cpu;
+
+	hwc->idx = -1;
+	hwc->config = event->attr.config;
+
+	if (event->group_leader == event)
+		return 0;
+
+	if (is_cycle_event(get_event(event->group_leader)))
+		if (++counters > total_event_counters)
+			return -EINVAL;
+
+	for_each_sibling_event(sibling, event->group_leader) {
+		if (is_cycle_event(get_event(sibling)))
+			continue;
+
+		if (sibling->pmu != event->pmu && !is_software_event(sibling))
+			return -EINVAL;
+
+		if (++counters > total_event_counters)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void riscv_iommu_pmu_update(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	u64 delta, prev, now;
+	u32 idx = hwc->idx;
+	u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
+
+	do {
+		prev = local64_read(&hwc->prev_count);
+		now = riscv_iommu_pmu_get_counter(pmu, idx);
+	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
+
+	delta = (now - prev) & counter_mask;
+	local64_add(delta, &event->count);
+}
+
+static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+
+	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
+		return;
+
+	if (flags & PERF_EF_RELOAD)
+		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+
+	hwc->state = 0;
+	riscv_iommu_pmu_set_period(event);
+	riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
+	riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
+
+	perf_event_update_userpage(event);
+}
+
+static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	if (hwc->state & PERF_HES_STOPPED)
+		return;
+
+	riscv_iommu_pmu_disable_counter(pmu, idx);
+
+	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
+		riscv_iommu_pmu_update(event);
+
+	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+}
+
+static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	unsigned int num_counters = pmu->num_counters;
+	int idx;
+
+	/* Reserve index zero for iohpmcycles */
+	if (is_cycle_event(get_event(event)))
+		idx = 0;
+	else
+		idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
+
+	/* All event counters or cycle counter are in use */
+	if (idx == num_counters || pmu->events[idx])
+		return -EAGAIN;
+
+	set_bit(idx, pmu->used_counters);
+
+	pmu->events[idx] = event;
+	hwc->idx = idx;
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+	local64_set(&hwc->prev_count, 0);
+
+	if (flags & PERF_EF_START)
+		riscv_iommu_pmu_start(event, flags);
+
+	/* Propagate changes to the userspace mapping. */
+	perf_event_update_userpage(event);
+
+	return 0;
+}
+
+static void riscv_iommu_pmu_read(struct perf_event *event)
+{
+	riscv_iommu_pmu_update(event);
+}
+
+static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
+{
+	struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
+	pmu->events[idx] = NULL;
+	clear_bit(idx, pmu->used_counters);
+
+	perf_event_update_userpage(event);
+}
+
+static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct riscv_iommu_pmu *iommu_pmu;
+
+	iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
+
+	if (iommu_pmu->on_cpu == -1)
+		iommu_pmu->on_cpu = cpu;
+
+	return 0;
+}
+
+static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct riscv_iommu_pmu *iommu_pmu;
+	unsigned int target_cpu;
+
+	iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
+
+	if (cpu != iommu_pmu->on_cpu)
+		return 0;
+
+	iommu_pmu->on_cpu = -1;
+
+	target_cpu = cpumask_any_but(cpu_online_mask, cpu);
+	if (target_cpu >= nr_cpu_ids)
+		return 0;
+
+	perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
+	iommu_pmu->on_cpu = target_cpu;
+
+	return 0;
+}
+
+static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
+{
+	u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
+	int idx;
+
+	if (!ovf)
+		return IRQ_NONE;
+
+	riscv_iommu_pmu_stop_all(pmu);
+
+	for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
+		struct perf_event *event = pmu->events[idx];
+
+		if (WARN_ON_ONCE(!event))
+			continue;
+
+		riscv_iommu_pmu_update(event);
+		riscv_iommu_pmu_set_period(event);
+	}
+
+	riscv_iommu_pmu_start_all(pmu);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
+{
+	struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
+	irqreturn_t ret;
+
+	/* Check whether this interrupt is for PMU */
+	if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
+		return IRQ_NONE;
+
+	/* Process PMU IRQ */
+	ret = riscv_iommu_pmu_handle_irq(pmu);
+
+	/* Clear performance monitoring interrupt pending bit */
+	writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
+
+	return ret;
+}
+
+static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
+{
+	/* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
+	int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
+
+	return iommu->irqs[vec];
+}
+
+static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
+				       struct riscv_iommu_pmu *pmu)
+{
+	unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
+
+	/*
+	 * Set the IRQF_ONESHOT flag because this IRQ can be shared with
+	 * other threaded IRQs by other queues.
+	 */
+	return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
+				IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
+}
+
+static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
+				     struct riscv_iommu_pmu *pmu)
+{
+	unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
+
+	free_irq(irq, pmu);
+}
+
+static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
+				 const struct auxiliary_device_id *id)
+{
+	struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
+	struct riscv_iommu_pmu *iommu_pmu;
+	void __iomem *addr;
+	char *name;
+	int ret;
+
+	iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
+	if (!iommu_pmu)
+		return -ENOMEM;
+
+	iommu_pmu->reg = iommu_dev->reg;
+
+	/* Counter number and width are hardware-implemented. Detect them by write 1s */
+	addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
+	writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
+	iommu_pmu->num_counters = hweight32(readl(addr));
+
+	addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
+	writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
+	iommu_pmu->cycle_cntr_mask = readq(addr);
+
+	/* Assume the width of all event counters are the same */
+	addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
+	writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
+	iommu_pmu->event_cntr_mask = readq(addr);
+
+	iommu_pmu->pmu = (struct pmu) {
+		.module		= THIS_MODULE,
+		.parent		= &auxdev->dev,
+		.task_ctx_nr	= perf_invalid_context,
+		.event_init	= riscv_iommu_pmu_event_init,
+		.add		= riscv_iommu_pmu_add,
+		.del		= riscv_iommu_pmu_del,
+		.start		= riscv_iommu_pmu_start,
+		.stop		= riscv_iommu_pmu_stop,
+		.read		= riscv_iommu_pmu_read,
+		.attr_groups	= riscv_iommu_pmu_attr_grps,
+		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+	};
+
+	auxiliary_set_drvdata(auxdev, iommu_pmu);
+
+	name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
+			      "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
+	if (!name) {
+		dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
+			dev_name(iommu_dev->dev));
+		return -ENOMEM;
+	}
+
+	/* Bind all events to the same cpu context to avoid race enabling */
+	iommu_pmu->on_cpu = raw_smp_processor_id();
+
+	ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
+	if (ret) {
+		dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
+		return ret;
+	}
+
+	ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
+	if (ret) {
+		dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
+		goto err_cpuhp_remove;
+	}
+
+	ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
+	if (ret) {
+		dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
+		goto err_free_irq;
+	}
+
+	dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
+		 name, iommu_pmu->num_counters);
+
+	return 0;
+
+err_free_irq:
+	riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
+err_cpuhp_remove:
+	cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
+	return ret;
+}
+
+static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
+	{ .name = "iommu.pmu" },
+	{}
+};
+MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
+
+static struct auxiliary_driver iommu_pmu_driver = {
+	.probe		= riscv_iommu_pmu_probe,
+	.id_table	= riscv_iommu_pmu_id_table,
+};
+
+static int __init riscv_iommu_pmu_init(void)
+{
+	int ret;
+
+	cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+					      "perf/riscv/iommu:online",
+					      riscv_iommu_pmu_online_cpu,
+					      riscv_iommu_pmu_offline_cpu);
+	if (cpuhp_state < 0)
+		return cpuhp_state;
+
+	ret = auxiliary_driver_register(&iommu_pmu_driver);
+	if (ret)
+		cpuhp_remove_multi_state(cpuhp_state);
+
+	return ret;
+}
+module_init(riscv_iommu_pmu_init);
+
+MODULE_DESCRIPTION("RISC-V IOMMU PMU");
+MODULE_LICENSE("GPL");
-- 
2.43.7

[PATCH v3 2/2] iommu/riscv: create a auxiliary device for HPM

[Zong Li]

Create an auxiliary device for HPM when the IOMMU supports a
hardware performance monitor.

Suggested-by: Samuel Holland <samuel.holland@sifive.com>
Signed-off-by: Zong Li <zong.li@sifive.com>
---
 drivers/iommu/riscv/iommu.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index cec3ddd7ab10..36ad4aa81465 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -14,6 +14,7 @@
 
 #include <linux/acpi.h>
 #include <linux/acpi_rimt.h>
+#include <linux/auxiliary_bus.h>
 #include <linux/compiler.h>
 #include <linux/crash_dump.h>
 #include <linux/init.h>
@@ -565,6 +566,21 @@ static irqreturn_t riscv_iommu_fltq_process(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+/*
+ * IOMMU Hardware performance monitor
+ */
+static int riscv_iommu_hpm_enable(struct riscv_iommu_device *iommu)
+{
+	struct auxiliary_device *auxdev;
+
+	auxdev = __devm_auxiliary_device_create(iommu->dev, KBUILD_MODNAME,
+						"pmu", iommu, 0);
+	if (!auxdev)
+		return -ENODEV;
+
+	return 0;
+}
+
 /* Lookup and initialize device context info structure. */
 static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,
 						 unsigned int devid)
@@ -1613,6 +1629,9 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu)
 		goto err_remove_sysfs;
 	}
 
+	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_HPM)
+		riscv_iommu_hpm_enable(iommu);
+
 	return 0;
 
 err_remove_sysfs:
-- 
2.43.7

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Guo Ren]

Hi Zong Li,

On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
>
> Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> device driver created by the parent RISC-V IOMMU driver.
>
> The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> The cycle counter is not associated with iohpmevt0, so a software-defined
> cycle event is required for the perf subsystem.
>
> The number and width of the counters are hardware-implemented and must
> be detected at runtime.
>
> The performance monitor provides counters with filtering support to
> collect events for specific device ID/process ID, or GSCID/PSCID.
>
> PMU-related definitions are moved into the perf driver, where they are
> used exclusively.
>
> Suggested-by: David Laight <david.laight.linux@gmail.com>
> Suggested-by: Guo Ren <guoren@kernel.org>
> Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> Signed-off-by: Zong Li <zong.li@sifive.com>
> ---
>  drivers/iommu/riscv/iommu-bits.h |  61 ---
>  drivers/perf/Kconfig             |  12 +
>  drivers/perf/Makefile            |   1 +
>  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
>  4 files changed, 716 insertions(+), 61 deletions(-)
>  create mode 100644 drivers/perf/riscv_iommu_pmu.c
>
> diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> index f2ef9bd3cde9..6b5de913a032 100644
> --- a/drivers/iommu/riscv/iommu-bits.h
> +++ b/drivers/iommu/riscv/iommu-bits.h
> @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
>  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
>  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
>
> -/* 5.19 Performance monitoring counter overflow status (32bits) */
> -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> -
> -/* 5.20 Performance monitoring counter inhibits (32bits) */
> -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> -
> -/* 5.21 Performance monitoring cycles counter (64bits) */
> -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> -
> -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> -
> -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> -
> -/* Number of defined performance-monitoring event selectors */
> -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> -
> -/**
> - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> - *
> - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> - */
> -enum riscv_iommu_hpmevent_id {
> -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> -};
> -
>  /* 5.24 Translation request IOVA (64bits) */
>  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
>  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> index 245e7bb763b9..8cce6c2ea626 100644
> --- a/drivers/perf/Kconfig
> +++ b/drivers/perf/Kconfig
> @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
>           full perf feature support i.e. counter overflow, privilege mode
>           filtering, counter configuration.
>
> +config RISCV_IOMMU_PMU
> +       depends on RISCV || COMPILE_TEST
> +       depends on RISCV_IOMMU
> +       bool "RISC-V IOMMU Hardware Performance Monitor"
> +       default y
> +       help
> +         Say Y if you want to use the RISC-V IOMMU performance monitor
> +         implementation. The performance monitor is an optional hardware
> +         feature, and whether it is actually enabled depends on IOMMU
> +         hardware support. If the underlying hardware does not implement
> +         the PMU, this option will have no effect.
> +
>  config STARFIVE_STARLINK_PMU
>         depends on ARCH_STARFIVE || COMPILE_TEST
>         depends on 64BIT
> diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> index eb8a022dad9a..90c75f3c0ac1 100644
> --- a/drivers/perf/Makefile
> +++ b/drivers/perf/Makefile
> @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
>  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
>  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
>  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
>  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
>  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
>  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> new file mode 100644
> index 000000000000..2e856289ddc9
> --- /dev/null
> +++ b/drivers/perf/riscv_iommu_pmu.c
> @@ -0,0 +1,703 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (C) 2026 SiFive
> + *
> + * Authors
> + *     Zong Li <zong.li@sifive.com>
> + */
> +
> +#include <linux/auxiliary_bus.h>
> +#include <linux/io-64-nonatomic-hi-lo.h>
> +#include <linux/perf_event.h>
> +
> +#include "../iommu/riscv/iommu.h"
> +
> +/* 5.19 Performance monitoring counter overflow status (32bits) */
> +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> +
> +/* 5.20 Performance monitoring counter inhibits (32bits) */
> +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> +
> +/* 5.21 Performance monitoring cycles counter (64bits) */
> +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> +
> +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> +
> +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> +
> +/* The total number of counters is 31 event counters plus 1 cycle counter */
> +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> +
> +static int cpuhp_state;
> +
> +/**
> + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> + *
> + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> + *
> + * The specification does not define an event ID for counting the
> + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> + * Event ID 0 is an invalid event and does not overlap with any valid
> + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> + * event is not actually written into any register, it serves solely
> + * as an identifier.
> + */
> +enum riscv_iommu_hpmevent_id {
> +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> +};
> +
> +struct riscv_iommu_pmu {
> +       struct pmu pmu;
> +       struct hlist_node node;
> +       void __iomem *reg;
> +       unsigned int on_cpu;
> +       int num_counters;
> +       u64 cycle_cntr_mask;
> +       u64 event_cntr_mask;
> +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> +};
> +
> +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> +
> +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> +       static inline u32 get_##_name(struct perf_event *event)         \
> +       {                                                               \
> +               return FIELD_GET(_mask, event->attr.config);            \
> +       }                                                               \
> +
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> +
> +/* Formats */
> +PMU_FORMAT_ATTR(event,            "config:0-14");
> +PMU_FORMAT_ATTR(partial_matching, "config:15");
> +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> +
> +static struct attribute *riscv_iommu_pmu_formats[] = {
> +       &format_attr_event.attr,
> +       &format_attr_partial_matching.attr,
> +       &format_attr_pid_pscid.attr,
> +       &format_attr_did_gscid.attr,
> +       &format_attr_filter_pid_pscid.attr,
> +       &format_attr_filter_did_gscid.attr,
> +       &format_attr_filter_id_type.attr,
> +       NULL,
> +};
> +
> +static const struct attribute_group riscv_iommu_pmu_format_group = {
> +       .name = "format",
> +       .attrs = riscv_iommu_pmu_formats,
> +};
> +
> +/* Events */
> +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> +                                         struct device_attribute *attr,
> +                                         char *page)
> +{
> +       struct perf_pmu_events_attr *pmu_attr;
> +
> +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> +
> +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> +}
> +
> +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> +
> +static struct attribute *riscv_iommu_pmu_events[] = {
> +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> +       NULL,
> +};
> +
> +static const struct attribute_group riscv_iommu_pmu_events_group = {
> +       .name = "events",
> +       .attrs = riscv_iommu_pmu_events,
> +};
> +
> +/* cpumask */
> +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> +                                       struct device_attribute *attr,
> +                                       char *buf)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> +
> +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> +}
> +
> +static struct device_attribute riscv_iommu_cpumask_attr =
> +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> +
> +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> +       &riscv_iommu_cpumask_attr.attr,
> +       NULL
> +};
> +
> +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> +       .attrs = riscv_iommu_cpumask_attrs,
> +};
> +
> +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> +       &riscv_iommu_pmu_cpumask_group,
> +       &riscv_iommu_pmu_format_group,
> +       &riscv_iommu_pmu_events_group,
> +       NULL,
> +};
> +
> +/* PMU Operations */
> +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> +                                       u64 value)
> +{
> +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> +
> +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
#include <include/linux/io-64-nonatomic-lo-hi.h>
lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));

For better compatibility, it is recommended to use writel and readl
instead of readq and writeq. Sorry for overlooking this last time —
could you please make the change again?
In the future, your driver could support RV32 directly, as the RISC-V
IOMMU specification also supports RV32; it's just that the current
RISC-V IOMMU driver's Kconfig depends on 64BIT.

> +}
> +
> +/*
> + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> + * is UNSPECIFIED, and such an access may appear, internally to the
> + * IOMMU, as if two separate 4 byte accesses - first to the high half
> + * and second to the low half - were performed
> + *
> + * To make sure the driver works correctly on different hardware,
> + * the software will always use two 4-byte access for the counter.
> + *
> + * This function implements the hi-lo-hi pattern to detect and handle
> + * wraparound during the read operation:
> + *   1. Read high half (hi)
> + *   2. Read low half (lo)
> + *   3. Check if low half wrapped or high half changed:
> + *      - If lo <= lo_prev: possible wraparound occurred
> + *      - If hi != hi_prev: high half changed during read
> + *   4. If wraparound detected, re-read high half and assume low half is 0
> + *   5. Update previous values for next read
> + */
> +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> +{
> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> +       u32 hi, lo;
> +
> +       hi = readl(addr + 4);
> +       lo = readl(addr);
> +
> +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> +               u32 hi_tmp = readl(addr + 4);
> +
> +               /*
> +                * If hi changes, then hi+1:0 must have happened while
> +                * the code was running so it is a safe return value
> +                */
> +               if (hi_tmp != hi) {
> +                       hi = hi_tmp;
> +                       lo = 0;
> +               }
> +               pmu->lo_prev[idx] = ~0u;
> +               pmu->hi_prev[idx] = hi;
> +       }
> +       pmu->lo_prev[idx] = lo;
> +
> +       value = (((u64)hi << 32) | lo) & counter_mask;
> +
> +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> +}
Yes! Thanks for your implementation.

> +
> +static bool is_cycle_event(u64 event)
> +{
> +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> +}
> +
> +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> +                                     u64 value)
> +{
> +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> +       if (is_cycle_event(value))
> +               return;
> +
> +       /* Event counter start from idx 1 */
> +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
       lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));

> +}
> +
> +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> +{
> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> +       u32 value = readl(addr);
> +
> +       writel(value & ~BIT(idx), addr);
> +}
> +
> +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> +{
> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> +       u32 value = readl(addr);
> +
> +       writel(value | BIT(idx), addr);
> +}
> +
> +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> +{
> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> +       u32 used_cntr = 0;
> +
> +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> +
> +       writel(~used_cntr, addr);
> +}
> +
> +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> +{
> +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> +}
> +
> +/* PMU APIs */
> +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> +       u64 period;
> +
> +       /*
> +        * Limit the maximum period to prevent the counter value
> +        * from overtaking the one we are about to program.
> +        * In effect we are reducing max_period to account for
> +        * interrupt latency (and we are being very conservative).
> +        */
> +       period = counter_mask >> 1;
> +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> +       local64_set(&hwc->prev_count, period);
> +}
> +
> +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +       struct perf_event *sibling;
> +       int total_event_counters = pmu->num_counters - 1;
> +       int counters = 0;
> +
> +       if (event->attr.type != event->pmu->type)
> +               return -ENOENT;
> +
> +       if (hwc->sample_period)
> +               return -EOPNOTSUPP;
> +
> +       if (event->cpu < 0)
> +               return -EOPNOTSUPP;
> +
> +       event->cpu = pmu->on_cpu;
> +
> +       hwc->idx = -1;
> +       hwc->config = event->attr.config;
> +
> +       if (event->group_leader == event)
> +               return 0;
> +
> +       if (is_cycle_event(get_event(event->group_leader)))
> +               if (++counters > total_event_counters)
> +                       return -EINVAL;
> +
> +       for_each_sibling_event(sibling, event->group_leader) {
> +               if (is_cycle_event(get_event(sibling)))
> +                       continue;
> +
> +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> +                       return -EINVAL;
> +
> +               if (++counters > total_event_counters)
> +                       return -EINVAL;
> +       }
> +
> +       return 0;
> +}
> +
> +static void riscv_iommu_pmu_update(struct perf_event *event)
> +{
> +       struct hw_perf_event *hwc = &event->hw;
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       u64 delta, prev, now;
> +       u32 idx = hwc->idx;
> +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> +
> +       do {
> +               prev = local64_read(&hwc->prev_count);
> +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> +
> +       delta = (now - prev) & counter_mask;
> +       local64_add(delta, &event->count);
> +}
> +
> +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +
> +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> +               return;
> +
> +       if (flags & PERF_EF_RELOAD)
> +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> +
> +       hwc->state = 0;
> +       riscv_iommu_pmu_set_period(event);
> +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> +
> +       perf_event_update_userpage(event);
> +}
> +
> +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +       int idx = hwc->idx;
> +
> +       if (hwc->state & PERF_HES_STOPPED)
> +               return;
> +
> +       riscv_iommu_pmu_disable_counter(pmu, idx);
> +
> +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> +               riscv_iommu_pmu_update(event);
> +
> +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> +}
> +
> +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +       unsigned int num_counters = pmu->num_counters;
> +       int idx;
> +
> +       /* Reserve index zero for iohpmcycles */
> +       if (is_cycle_event(get_event(event)))
> +               idx = 0;
> +       else
> +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> +
> +       /* All event counters or cycle counter are in use */
> +       if (idx == num_counters || pmu->events[idx])
> +               return -EAGAIN;
> +
> +       set_bit(idx, pmu->used_counters);
> +
> +       pmu->events[idx] = event;
> +       hwc->idx = idx;
> +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> +       local64_set(&hwc->prev_count, 0);
> +
> +       if (flags & PERF_EF_START)
> +               riscv_iommu_pmu_start(event, flags);
> +
> +       /* Propagate changes to the userspace mapping. */
> +       perf_event_update_userpage(event);
> +
> +       return 0;
> +}
> +
> +static void riscv_iommu_pmu_read(struct perf_event *event)
> +{
> +       riscv_iommu_pmu_update(event);
> +}
> +
> +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> +{
> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> +       struct hw_perf_event *hwc = &event->hw;
> +       int idx = hwc->idx;
> +
> +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> +       pmu->events[idx] = NULL;
> +       clear_bit(idx, pmu->used_counters);
> +
> +       perf_event_update_userpage(event);
> +}
> +
> +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> +{
> +       struct riscv_iommu_pmu *iommu_pmu;
> +
> +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> +
> +       if (iommu_pmu->on_cpu == -1)
> +               iommu_pmu->on_cpu = cpu;
> +
> +       return 0;
> +}
> +
> +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> +{
> +       struct riscv_iommu_pmu *iommu_pmu;
> +       unsigned int target_cpu;
> +
> +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> +
> +       if (cpu != iommu_pmu->on_cpu)
> +               return 0;
> +
> +       iommu_pmu->on_cpu = -1;
> +
> +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> +       if (target_cpu >= nr_cpu_ids)
> +               return 0;
> +
> +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> +       iommu_pmu->on_cpu = target_cpu;
> +
> +       return 0;
> +}
> +
> +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> +{
> +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> +       int idx;
> +
> +       if (!ovf)
> +               return IRQ_NONE;
> +
> +       riscv_iommu_pmu_stop_all(pmu);
> +
> +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> +               struct perf_event *event = pmu->events[idx];
> +
> +               if (WARN_ON_ONCE(!event))
> +                       continue;
> +
> +               riscv_iommu_pmu_update(event);
> +               riscv_iommu_pmu_set_period(event);
> +       }
> +
> +       riscv_iommu_pmu_start_all(pmu);
> +
> +       return IRQ_HANDLED;
> +}
> +
> +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> +{
> +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> +       irqreturn_t ret;
> +
> +       /* Check whether this interrupt is for PMU */
> +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> +               return IRQ_NONE;
> +
> +       /* Process PMU IRQ */
> +       ret = riscv_iommu_pmu_handle_irq(pmu);
> +
> +       /* Clear performance monitoring interrupt pending bit */
> +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> +
> +       return ret;
> +}
> +
> +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> +{
> +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> +
> +       return iommu->irqs[vec];
> +}
> +
> +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> +                                      struct riscv_iommu_pmu *pmu)
> +{
> +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> +
> +       /*
> +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> +        * other threaded IRQs by other queues.
> +        */
> +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> +}
> +
> +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> +                                    struct riscv_iommu_pmu *pmu)
> +{
> +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> +
> +       free_irq(irq, pmu);
> +}
> +
> +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> +                                const struct auxiliary_device_id *id)
> +{
> +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> +       struct riscv_iommu_pmu *iommu_pmu;
> +       void __iomem *addr;
> +       char *name;
> +       int ret;
> +
> +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> +       if (!iommu_pmu)
> +               return -ENOMEM;
> +
> +       iommu_pmu->reg = iommu_dev->reg;
> +
> +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> +       iommu_pmu->num_counters = hweight32(readl(addr));
> +
> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
    lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);

> +       iommu_pmu->cycle_cntr_mask = readq(addr);
iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);

> +
> +       /* Assume the width of all event counters are the same */
> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
    lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);

> +       iommu_pmu->event_cntr_mask = readq(addr);
 iommu_pmu->event_cntr_mask = lo_hi_readq(addr);

> +
> +       iommu_pmu->pmu = (struct pmu) {
> +               .module         = THIS_MODULE,
> +               .parent         = &auxdev->dev,
> +               .task_ctx_nr    = perf_invalid_context,
> +               .event_init     = riscv_iommu_pmu_event_init,
> +               .add            = riscv_iommu_pmu_add,
> +               .del            = riscv_iommu_pmu_del,
> +               .start          = riscv_iommu_pmu_start,
> +               .stop           = riscv_iommu_pmu_stop,
> +               .read           = riscv_iommu_pmu_read,
> +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> +       };
> +
> +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> +
> +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> +       if (!name) {
> +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> +                       dev_name(iommu_dev->dev));
> +               return -ENOMEM;
> +       }
> +
> +       /* Bind all events to the same cpu context to avoid race enabling */
> +       iommu_pmu->on_cpu = raw_smp_processor_id();
> +
> +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> +       if (ret) {
> +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> +               return ret;
> +       }
> +
> +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> +       if (ret) {
> +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> +               goto err_cpuhp_remove;
> +       }
> +
> +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> +       if (ret) {
> +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> +               goto err_free_irq;
> +       }
> +
> +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> +                name, iommu_pmu->num_counters);
> +
> +       return 0;
> +
> +err_free_irq:
> +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> +err_cpuhp_remove:
> +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> +       return ret;
> +}
> +
> +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> +       { .name = "iommu.pmu" },
> +       {}
> +};
> +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> +
> +static struct auxiliary_driver iommu_pmu_driver = {
> +       .probe          = riscv_iommu_pmu_probe,
> +       .id_table       = riscv_iommu_pmu_id_table,
> +};
> +
> +static int __init riscv_iommu_pmu_init(void)
> +{
> +       int ret;
> +
> +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> +                                             "perf/riscv/iommu:online",
> +                                             riscv_iommu_pmu_online_cpu,
> +                                             riscv_iommu_pmu_offline_cpu);
> +       if (cpuhp_state < 0)
> +               return cpuhp_state;
> +
> +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> +       if (ret)
> +               cpuhp_remove_multi_state(cpuhp_state);
> +
> +       return ret;
> +}
> +module_init(riscv_iommu_pmu_init);
> +
> +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> +MODULE_LICENSE("GPL");
> --
> 2.43.7
>

-- 
Best Regards
 Guo Ren

Re: [PATCH v3 2/2] iommu/riscv: create a auxiliary device for HPM

[Guo Ren]

On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
>
> Create an auxiliary device for HPM when the IOMMU supports a
> hardware performance monitor.
>
> Suggested-by: Samuel Holland <samuel.holland@sifive.com>
> Signed-off-by: Zong Li <zong.li@sifive.com>
> ---
>  drivers/iommu/riscv/iommu.c | 19 +++++++++++++++++++
>  1 file changed, 19 insertions(+)
>
> diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
> index cec3ddd7ab10..36ad4aa81465 100644
> --- a/drivers/iommu/riscv/iommu.c
> +++ b/drivers/iommu/riscv/iommu.c
> @@ -14,6 +14,7 @@
>
>  #include <linux/acpi.h>
>  #include <linux/acpi_rimt.h>
> +#include <linux/auxiliary_bus.h>
>  #include <linux/compiler.h>
>  #include <linux/crash_dump.h>
>  #include <linux/init.h>
> @@ -565,6 +566,21 @@ static irqreturn_t riscv_iommu_fltq_process(int irq, void *data)
>         return IRQ_HANDLED;
>  }
>
> +/*
> + * IOMMU Hardware performance monitor
> + */
> +static int riscv_iommu_hpm_enable(struct riscv_iommu_device *iommu)
> +{
> +       struct auxiliary_device *auxdev;
> +
> +       auxdev = __devm_auxiliary_device_create(iommu->dev, KBUILD_MODNAME,
> +                                               "pmu", iommu, 0);

Using an auxiliary device makes sense here — it keeps perf PMU logic
out of the IOMMU driver and lets a dedicated driver handle
perf_pmu_register() independently. The devm-managed creation also ties
the auxiliary device lifetime to the parent IOMMU device, so no
explicit teardown is needed.

Reviewed-by: Guo Ren <guoren@kernel.org>


> +       if (!auxdev)
> +               return -ENODEV;
> +
> +       return 0;
> +}
> +
>  /* Lookup and initialize device context info structure. */
>  static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,
>                                                  unsigned int devid)
> @@ -1613,6 +1629,9 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu)
>                 goto err_remove_sysfs;
>         }
>
> +       if (iommu->caps & RISCV_IOMMU_CAPABILITIES_HPM)
> +               riscv_iommu_hpm_enable(iommu);
> +
>         return 0;
>
>  err_remove_sysfs:
> --
> 2.43.7
>


-- 
Best Regards
 Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
>
> Hi Zong Li,
>
> On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> >
> > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > device driver created by the parent RISC-V IOMMU driver.
> >
> > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > The cycle counter is not associated with iohpmevt0, so a software-defined
> > cycle event is required for the perf subsystem.
> >
> > The number and width of the counters are hardware-implemented and must
> > be detected at runtime.
> >
> > The performance monitor provides counters with filtering support to
> > collect events for specific device ID/process ID, or GSCID/PSCID.
> >
> > PMU-related definitions are moved into the perf driver, where they are
> > used exclusively.
> >
> > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > Suggested-by: Guo Ren <guoren@kernel.org>
> > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > Signed-off-by: Zong Li <zong.li@sifive.com>
> > ---
> >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> >  drivers/perf/Kconfig             |  12 +
> >  drivers/perf/Makefile            |   1 +
> >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> >  4 files changed, 716 insertions(+), 61 deletions(-)
> >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> >
> > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > index f2ef9bd3cde9..6b5de913a032 100644
> > --- a/drivers/iommu/riscv/iommu-bits.h
> > +++ b/drivers/iommu/riscv/iommu-bits.h
> > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> >
> > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > -
> > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > -
> > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > -
> > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > -
> > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > -
> > -/* Number of defined performance-monitoring event selectors */
> > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > -
> > -/**
> > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > - *
> > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > - */
> > -enum riscv_iommu_hpmevent_id {
> > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > -};
> > -
> >  /* 5.24 Translation request IOVA (64bits) */
> >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > index 245e7bb763b9..8cce6c2ea626 100644
> > --- a/drivers/perf/Kconfig
> > +++ b/drivers/perf/Kconfig
> > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> >           full perf feature support i.e. counter overflow, privilege mode
> >           filtering, counter configuration.
> >
> > +config RISCV_IOMMU_PMU
> > +       depends on RISCV || COMPILE_TEST
> > +       depends on RISCV_IOMMU
> > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > +       default y
> > +       help
> > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > +         implementation. The performance monitor is an optional hardware
> > +         feature, and whether it is actually enabled depends on IOMMU
> > +         hardware support. If the underlying hardware does not implement
> > +         the PMU, this option will have no effect.
> > +
> >  config STARFIVE_STARLINK_PMU
> >         depends on ARCH_STARFIVE || COMPILE_TEST
> >         depends on 64BIT
> > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > index eb8a022dad9a..90c75f3c0ac1 100644
> > --- a/drivers/perf/Makefile
> > +++ b/drivers/perf/Makefile
> > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > new file mode 100644
> > index 000000000000..2e856289ddc9
> > --- /dev/null
> > +++ b/drivers/perf/riscv_iommu_pmu.c
> > @@ -0,0 +1,703 @@
> > +// SPDX-License-Identifier: GPL-2.0-only
> > +/*
> > + * Copyright (C) 2026 SiFive
> > + *
> > + * Authors
> > + *     Zong Li <zong.li@sifive.com>
> > + */
> > +
> > +#include <linux/auxiliary_bus.h>
> > +#include <linux/io-64-nonatomic-hi-lo.h>
> > +#include <linux/perf_event.h>
> > +
> > +#include "../iommu/riscv/iommu.h"
> > +
> > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > +
> > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > +
> > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > +
> > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > +
> > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > +
> > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > +
> > +static int cpuhp_state;
> > +
> > +/**
> > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > + *
> > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > + *
> > + * The specification does not define an event ID for counting the
> > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > + * Event ID 0 is an invalid event and does not overlap with any valid
> > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > + * event is not actually written into any register, it serves solely
> > + * as an identifier.
> > + */
> > +enum riscv_iommu_hpmevent_id {
> > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > +};
> > +
> > +struct riscv_iommu_pmu {
> > +       struct pmu pmu;
> > +       struct hlist_node node;
> > +       void __iomem *reg;
> > +       unsigned int on_cpu;
> > +       int num_counters;
> > +       u64 cycle_cntr_mask;
> > +       u64 event_cntr_mask;
> > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > +};
> > +
> > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > +
> > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > +       static inline u32 get_##_name(struct perf_event *event)         \
> > +       {                                                               \
> > +               return FIELD_GET(_mask, event->attr.config);            \
> > +       }                                                               \
> > +
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > +
> > +/* Formats */
> > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > +
> > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > +       &format_attr_event.attr,
> > +       &format_attr_partial_matching.attr,
> > +       &format_attr_pid_pscid.attr,
> > +       &format_attr_did_gscid.attr,
> > +       &format_attr_filter_pid_pscid.attr,
> > +       &format_attr_filter_did_gscid.attr,
> > +       &format_attr_filter_id_type.attr,
> > +       NULL,
> > +};
> > +
> > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > +       .name = "format",
> > +       .attrs = riscv_iommu_pmu_formats,
> > +};
> > +
> > +/* Events */
> > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > +                                         struct device_attribute *attr,
> > +                                         char *page)
> > +{
> > +       struct perf_pmu_events_attr *pmu_attr;
> > +
> > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > +
> > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > +}
> > +
> > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > +
> > +static struct attribute *riscv_iommu_pmu_events[] = {
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > +       NULL,
> > +};
> > +
> > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > +       .name = "events",
> > +       .attrs = riscv_iommu_pmu_events,
> > +};
> > +
> > +/* cpumask */
> > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > +                                       struct device_attribute *attr,
> > +                                       char *buf)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > +
> > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > +}
> > +
> > +static struct device_attribute riscv_iommu_cpumask_attr =
> > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > +
> > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > +       &riscv_iommu_cpumask_attr.attr,
> > +       NULL
> > +};
> > +
> > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > +       .attrs = riscv_iommu_cpumask_attrs,
> > +};
> > +
> > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > +       &riscv_iommu_pmu_cpumask_group,
> > +       &riscv_iommu_pmu_format_group,
> > +       &riscv_iommu_pmu_events_group,
> > +       NULL,
> > +};
> > +
> > +/* PMU Operations */
> > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > +                                       u64 value)
> > +{
> > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > +
> > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> #include <include/linux/io-64-nonatomic-lo-hi.h>
> lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
>
> For better compatibility, it is recommended to use writel and readl
> instead of readq and writeq. Sorry for overlooking this last time —
> could you please make the change again?
> In the future, your driver could support RV32 directly, as the RISC-V
> IOMMU specification also supports RV32; it's just that the current
> RISC-V IOMMU driver's Kconfig depends on 64BIT.

Hi Guo,
Thank you for your review. In my implementation for RV32, readq and
writeq are replaced by hi_lo_readq and hi_lo_writeq (from
io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
However, as you mentioned before: 64-bit access might be undefined on
some hardware. To be safe and compatible with all hardware, we should
use 32-bit access instead of 64-bit access.
I will modify them in the next version

I'd like to make sure I didn't miss anything important. My original
code uses the "hi-lo" order. May I ask if there is a specific reason
why you prefer the "lo-hi" order?
Thanks

>
> > +}
> > +
> > +/*
> > + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> > + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> > + * is UNSPECIFIED, and such an access may appear, internally to the
> > + * IOMMU, as if two separate 4 byte accesses - first to the high half
> > + * and second to the low half - were performed
> > + *
> > + * To make sure the driver works correctly on different hardware,
> > + * the software will always use two 4-byte access for the counter.
> > + *
> > + * This function implements the hi-lo-hi pattern to detect and handle
> > + * wraparound during the read operation:
> > + *   1. Read high half (hi)
> > + *   2. Read low half (lo)
> > + *   3. Check if low half wrapped or high half changed:
> > + *      - If lo <= lo_prev: possible wraparound occurred
> > + *      - If hi != hi_prev: high half changed during read
> > + *   4. If wraparound detected, re-read high half and assume low half is 0
> > + *   5. Update previous values for next read
> > + */
> > +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > +{
> > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> > +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > +       u32 hi, lo;
> > +
> > +       hi = readl(addr + 4);
> > +       lo = readl(addr);
> > +
> > +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> > +               u32 hi_tmp = readl(addr + 4);
> > +
> > +               /*
> > +                * If hi changes, then hi+1:0 must have happened while
> > +                * the code was running so it is a safe return value
> > +                */
> > +               if (hi_tmp != hi) {
> > +                       hi = hi_tmp;
> > +                       lo = 0;
> > +               }
> > +               pmu->lo_prev[idx] = ~0u;
> > +               pmu->hi_prev[idx] = hi;
> > +       }
> > +       pmu->lo_prev[idx] = lo;
> > +
> > +       value = (((u64)hi << 32) | lo) & counter_mask;
> > +
> > +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> > +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> > +}
> Yes! Thanks for your implementation.
>
> > +
> > +static bool is_cycle_event(u64 event)
> > +{
> > +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> > +}
> > +
> > +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> > +                                     u64 value)
> > +{
> > +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> > +       if (is_cycle_event(value))
> > +               return;
> > +
> > +       /* Event counter start from idx 1 */
> > +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
>        lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
>               pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
>
> > +}
> > +
> > +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > +{
> > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > +       u32 value = readl(addr);
> > +
> > +       writel(value & ~BIT(idx), addr);
> > +}
> > +
> > +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > +{
> > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > +       u32 value = readl(addr);
> > +
> > +       writel(value | BIT(idx), addr);
> > +}
> > +
> > +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> > +{
> > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > +       u32 used_cntr = 0;
> > +
> > +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> > +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> > +
> > +       writel(~used_cntr, addr);
> > +}
> > +
> > +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> > +{
> > +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> > +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> > +}
> > +
> > +/* PMU APIs */
> > +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > +       u64 period;
> > +
> > +       /*
> > +        * Limit the maximum period to prevent the counter value
> > +        * from overtaking the one we are about to program.
> > +        * In effect we are reducing max_period to account for
> > +        * interrupt latency (and we are being very conservative).
> > +        */
> > +       period = counter_mask >> 1;
> > +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> > +       local64_set(&hwc->prev_count, period);
> > +}
> > +
> > +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       struct perf_event *sibling;
> > +       int total_event_counters = pmu->num_counters - 1;
> > +       int counters = 0;
> > +
> > +       if (event->attr.type != event->pmu->type)
> > +               return -ENOENT;
> > +
> > +       if (hwc->sample_period)
> > +               return -EOPNOTSUPP;
> > +
> > +       if (event->cpu < 0)
> > +               return -EOPNOTSUPP;
> > +
> > +       event->cpu = pmu->on_cpu;
> > +
> > +       hwc->idx = -1;
> > +       hwc->config = event->attr.config;
> > +
> > +       if (event->group_leader == event)
> > +               return 0;
> > +
> > +       if (is_cycle_event(get_event(event->group_leader)))
> > +               if (++counters > total_event_counters)
> > +                       return -EINVAL;
> > +
> > +       for_each_sibling_event(sibling, event->group_leader) {
> > +               if (is_cycle_event(get_event(sibling)))
> > +                       continue;
> > +
> > +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> > +                       return -EINVAL;
> > +
> > +               if (++counters > total_event_counters)
> > +                       return -EINVAL;
> > +       }
> > +
> > +       return 0;
> > +}
> > +
> > +static void riscv_iommu_pmu_update(struct perf_event *event)
> > +{
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       u64 delta, prev, now;
> > +       u32 idx = hwc->idx;
> > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > +
> > +       do {
> > +               prev = local64_read(&hwc->prev_count);
> > +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> > +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> > +
> > +       delta = (now - prev) & counter_mask;
> > +       local64_add(delta, &event->count);
> > +}
> > +
> > +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +
> > +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> > +               return;
> > +
> > +       if (flags & PERF_EF_RELOAD)
> > +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> > +
> > +       hwc->state = 0;
> > +       riscv_iommu_pmu_set_period(event);
> > +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> > +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> > +
> > +       perf_event_update_userpage(event);
> > +}
> > +
> > +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       int idx = hwc->idx;
> > +
> > +       if (hwc->state & PERF_HES_STOPPED)
> > +               return;
> > +
> > +       riscv_iommu_pmu_disable_counter(pmu, idx);
> > +
> > +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> > +               riscv_iommu_pmu_update(event);
> > +
> > +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > +}
> > +
> > +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       unsigned int num_counters = pmu->num_counters;
> > +       int idx;
> > +
> > +       /* Reserve index zero for iohpmcycles */
> > +       if (is_cycle_event(get_event(event)))
> > +               idx = 0;
> > +       else
> > +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> > +
> > +       /* All event counters or cycle counter are in use */
> > +       if (idx == num_counters || pmu->events[idx])
> > +               return -EAGAIN;
> > +
> > +       set_bit(idx, pmu->used_counters);
> > +
> > +       pmu->events[idx] = event;
> > +       hwc->idx = idx;
> > +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > +       local64_set(&hwc->prev_count, 0);
> > +
> > +       if (flags & PERF_EF_START)
> > +               riscv_iommu_pmu_start(event, flags);
> > +
> > +       /* Propagate changes to the userspace mapping. */
> > +       perf_event_update_userpage(event);
> > +
> > +       return 0;
> > +}
> > +
> > +static void riscv_iommu_pmu_read(struct perf_event *event)
> > +{
> > +       riscv_iommu_pmu_update(event);
> > +}
> > +
> > +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> > +{
> > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > +       struct hw_perf_event *hwc = &event->hw;
> > +       int idx = hwc->idx;
> > +
> > +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> > +       pmu->events[idx] = NULL;
> > +       clear_bit(idx, pmu->used_counters);
> > +
> > +       perf_event_update_userpage(event);
> > +}
> > +
> > +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> > +{
> > +       struct riscv_iommu_pmu *iommu_pmu;
> > +
> > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > +
> > +       if (iommu_pmu->on_cpu == -1)
> > +               iommu_pmu->on_cpu = cpu;
> > +
> > +       return 0;
> > +}
> > +
> > +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> > +{
> > +       struct riscv_iommu_pmu *iommu_pmu;
> > +       unsigned int target_cpu;
> > +
> > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > +
> > +       if (cpu != iommu_pmu->on_cpu)
> > +               return 0;
> > +
> > +       iommu_pmu->on_cpu = -1;
> > +
> > +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> > +       if (target_cpu >= nr_cpu_ids)
> > +               return 0;
> > +
> > +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> > +       iommu_pmu->on_cpu = target_cpu;
> > +
> > +       return 0;
> > +}
> > +
> > +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> > +{
> > +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> > +       int idx;
> > +
> > +       if (!ovf)
> > +               return IRQ_NONE;
> > +
> > +       riscv_iommu_pmu_stop_all(pmu);
> > +
> > +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> > +               struct perf_event *event = pmu->events[idx];
> > +
> > +               if (WARN_ON_ONCE(!event))
> > +                       continue;
> > +
> > +               riscv_iommu_pmu_update(event);
> > +               riscv_iommu_pmu_set_period(event);
> > +       }
> > +
> > +       riscv_iommu_pmu_start_all(pmu);
> > +
> > +       return IRQ_HANDLED;
> > +}
> > +
> > +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> > +{
> > +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> > +       irqreturn_t ret;
> > +
> > +       /* Check whether this interrupt is for PMU */
> > +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> > +               return IRQ_NONE;
> > +
> > +       /* Process PMU IRQ */
> > +       ret = riscv_iommu_pmu_handle_irq(pmu);
> > +
> > +       /* Clear performance monitoring interrupt pending bit */
> > +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> > +
> > +       return ret;
> > +}
> > +
> > +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> > +{
> > +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> > +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> > +
> > +       return iommu->irqs[vec];
> > +}
> > +
> > +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> > +                                      struct riscv_iommu_pmu *pmu)
> > +{
> > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > +
> > +       /*
> > +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> > +        * other threaded IRQs by other queues.
> > +        */
> > +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> > +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> > +}
> > +
> > +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> > +                                    struct riscv_iommu_pmu *pmu)
> > +{
> > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > +
> > +       free_irq(irq, pmu);
> > +}
> > +
> > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > +                                const struct auxiliary_device_id *id)
> > +{
> > +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > +       struct riscv_iommu_pmu *iommu_pmu;
> > +       void __iomem *addr;
> > +       char *name;
> > +       int ret;
> > +
> > +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > +       if (!iommu_pmu)
> > +               return -ENOMEM;
> > +
> > +       iommu_pmu->reg = iommu_dev->reg;
> > +
> > +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > +       iommu_pmu->num_counters = hweight32(readl(addr));
> > +
> > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
>     lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
>
> > +       iommu_pmu->cycle_cntr_mask = readq(addr);
> iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
>
> > +
> > +       /* Assume the width of all event counters are the same */
> > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
>     lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
>
> > +       iommu_pmu->event_cntr_mask = readq(addr);
>  iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
>
> > +
> > +       iommu_pmu->pmu = (struct pmu) {
> > +               .module         = THIS_MODULE,
> > +               .parent         = &auxdev->dev,
> > +               .task_ctx_nr    = perf_invalid_context,
> > +               .event_init     = riscv_iommu_pmu_event_init,
> > +               .add            = riscv_iommu_pmu_add,
> > +               .del            = riscv_iommu_pmu_del,
> > +               .start          = riscv_iommu_pmu_start,
> > +               .stop           = riscv_iommu_pmu_stop,
> > +               .read           = riscv_iommu_pmu_read,
> > +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> > +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > +       };
> > +
> > +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> > +
> > +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> > +       if (!name) {
> > +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> > +                       dev_name(iommu_dev->dev));
> > +               return -ENOMEM;
> > +       }
> > +
> > +       /* Bind all events to the same cpu context to avoid race enabling */
> > +       iommu_pmu->on_cpu = raw_smp_processor_id();
> > +
> > +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > +       if (ret) {
> > +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > +               return ret;
> > +       }
> > +
> > +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> > +       if (ret) {
> > +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > +               goto err_cpuhp_remove;
> > +       }
> > +
> > +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > +       if (ret) {
> > +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > +               goto err_free_irq;
> > +       }
> > +
> > +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> > +                name, iommu_pmu->num_counters);
> > +
> > +       return 0;
> > +
> > +err_free_irq:
> > +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> > +err_cpuhp_remove:
> > +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > +       return ret;
> > +}
> > +
> > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > +       { .name = "iommu.pmu" },
> > +       {}
> > +};
> > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > +
> > +static struct auxiliary_driver iommu_pmu_driver = {
> > +       .probe          = riscv_iommu_pmu_probe,
> > +       .id_table       = riscv_iommu_pmu_id_table,
> > +};
> > +
> > +static int __init riscv_iommu_pmu_init(void)
> > +{
> > +       int ret;
> > +
> > +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > +                                             "perf/riscv/iommu:online",
> > +                                             riscv_iommu_pmu_online_cpu,
> > +                                             riscv_iommu_pmu_offline_cpu);
> > +       if (cpuhp_state < 0)
> > +               return cpuhp_state;
> > +
> > +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> > +       if (ret)
> > +               cpuhp_remove_multi_state(cpuhp_state);
> > +
> > +       return ret;
> > +}
> > +module_init(riscv_iommu_pmu_init);
> > +
> > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > +MODULE_LICENSE("GPL");
> > --
> > 2.43.7
> >
>
> --
> Best Regards
>  Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zhanpeng Zhang]

Hi Guo, Li,

On 6/30/26 9:17 PM, Guo Ren wrote:
> Hi Zong Li,
>
> On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
>>
>> Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
>> device driver created by the parent RISC-V IOMMU driver.
>>
>> The RISC-V IOMMU PMU separates the cycle counter from the event counters.
>> The cycle counter is not associated with iohpmevt0, so a software-defined
>> cycle event is required for the perf subsystem.
>>
>> The number and width of the counters are hardware-implemented and must
>> be detected at runtime.
>>
>> The performance monitor provides counters with filtering support to
>> collect events for specific device ID/process ID, or GSCID/PSCID.
>>
>> PMU-related definitions are moved into the perf driver, where they are
>> used exclusively.
>>
>> Suggested-by: David Laight <david.laight.linux@gmail.com>
>> Suggested-by: Guo Ren <guoren@kernel.org>
>> Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
>> Signed-off-by: Zong Li <zong.li@sifive.com>
>> ---
>>   drivers/iommu/riscv/iommu-bits.h |  61 ---
>>   drivers/perf/Kconfig             |  12 +
>>   drivers/perf/Makefile            |   1 +
>>   drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
>>   4 files changed, 716 insertions(+), 61 deletions(-)
>>   create mode 100644 drivers/perf/riscv_iommu_pmu.c
>>
>> diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
>> index f2ef9bd3cde9..6b5de913a032 100644
>> --- a/drivers/iommu/riscv/iommu-bits.h
>> +++ b/drivers/iommu/riscv/iommu-bits.h
>> @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
>>   #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
>>   #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
>>
>> -/* 5.19 Performance monitoring counter overflow status (32bits) */
>> -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
>> -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
>> -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
>> -
>> -/* 5.20 Performance monitoring counter inhibits (32bits) */
>> -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
>> -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
>> -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
>> -
>> -/* 5.21 Performance monitoring cycles counter (64bits) */
>> -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
>> -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
>> -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
>> -
>> -/* 5.22 Performance monitoring event counters (31 * 64bits) */
>> -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
>> -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
>> -
>> -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
>> -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
>> -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
>> -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
>> -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
>> -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
>> -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
>> -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
>> -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
>> -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
>> -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
>> -
>> -/* Number of defined performance-monitoring event selectors */
>> -#define RISCV_IOMMU_IOHPMEVT_CNT       31
>> -
>> -/**
>> - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
>> - *
>> - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
>> - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
>> - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
>> - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
>> - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
>> - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
>> - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
>> - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
>> - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
>> - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
>> - */
>> -enum riscv_iommu_hpmevent_id {
>> -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
>> -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
>> -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
>> -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
>> -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
>> -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
>> -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
>> -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
>> -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
>> -       RISCV_IOMMU_HPMEVENT_MAX        = 9
>> -};
>> -
>>   /* 5.24 Translation request IOVA (64bits) */
>>   #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
>>   #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
>> diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
>> index 245e7bb763b9..8cce6c2ea626 100644
>> --- a/drivers/perf/Kconfig
>> +++ b/drivers/perf/Kconfig
>> @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
>>            full perf feature support i.e. counter overflow, privilege mode
>>            filtering, counter configuration.
>>
>> +config RISCV_IOMMU_PMU
>> +       depends on RISCV || COMPILE_TEST
>> +       depends on RISCV_IOMMU
>> +       bool "RISC-V IOMMU Hardware Performance Monitor"
>> +       default y
>> +       help
>> +         Say Y if you want to use the RISC-V IOMMU performance monitor
>> +         implementation. The performance monitor is an optional hardware
>> +         feature, and whether it is actually enabled depends on IOMMU
>> +         hardware support. If the underlying hardware does not implement
>> +         the PMU, this option will have no effect.
>> +
>>   config STARFIVE_STARLINK_PMU
>>          depends on ARCH_STARFIVE || COMPILE_TEST
>>          depends on 64BIT
>> diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
>> index eb8a022dad9a..90c75f3c0ac1 100644
>> --- a/drivers/perf/Makefile
>> +++ b/drivers/perf/Makefile
>> @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
>>   obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
>>   obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
>>   obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
>> +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
>>   obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
>>   obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
>>   obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
>> diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
>> new file mode 100644
>> index 000000000000..2e856289ddc9
>> --- /dev/null
>> +++ b/drivers/perf/riscv_iommu_pmu.c
>> @@ -0,0 +1,703 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * Copyright (C) 2026 SiFive
>> + *
>> + * Authors
>> + *     Zong Li <zong.li@sifive.com>
>> + */
>> +
>> +#include <linux/auxiliary_bus.h>
>> +#include <linux/io-64-nonatomic-hi-lo.h>
>> +#include <linux/perf_event.h>
>> +
>> +#include "../iommu/riscv/iommu.h"
>> +
>> +/* 5.19 Performance monitoring counter overflow status (32bits) */
>> +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
>> +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
>> +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
>> +
>> +/* 5.20 Performance monitoring counter inhibits (32bits) */
>> +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
>> +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
>> +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
>> +
>> +/* 5.21 Performance monitoring cycles counter (64bits) */
>> +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
>> +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
>> +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
>> +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
>> +
>> +/* 5.22 Performance monitoring event counters (31 * 64bits) */
>> +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
>> +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
>> +
>> +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
>> +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
>> +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
>> +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
>> +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
>> +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
>> +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
>> +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
>> +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
>> +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
>> +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
>> +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
>> +
>> +/* The total number of counters is 31 event counters plus 1 cycle counter */
>> +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
>> +
>> +static int cpuhp_state;
>> +
>> +/**
>> + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
>> + *
>> + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
>> + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
>> + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
>> + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
>> + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
>> + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
>> + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
>> + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
>> + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
>> + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
>> + *
>> + * The specification does not define an event ID for counting the
>> + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
>> + * Event ID 0 is an invalid event and does not overlap with any valid
>> + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
>> + * event is not actually written into any register, it serves solely
>> + * as an identifier.
>> + */
>> +enum riscv_iommu_hpmevent_id {
>> +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
>> +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
>> +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
>> +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
>> +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
>> +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
>> +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
>> +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
>> +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
>> +       RISCV_IOMMU_HPMEVENT_MAX        = 9
>> +};
>> +
>> +struct riscv_iommu_pmu {
>> +       struct pmu pmu;
>> +       struct hlist_node node;
>> +       void __iomem *reg;
>> +       unsigned int on_cpu;
>> +       int num_counters;
>> +       u64 cycle_cntr_mask;
>> +       u64 event_cntr_mask;
>> +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
>> +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
>> +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
>> +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
>> +};
>> +
>> +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
>> +
>> +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
>> +       static inline u32 get_##_name(struct perf_event *event)         \
>> +       {                                                               \
>> +               return FIELD_GET(_mask, event->attr.config);            \
>> +       }                                                               \
>> +
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
>> +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
>> +
>> +/* Formats */
>> +PMU_FORMAT_ATTR(event,            "config:0-14");
>> +PMU_FORMAT_ATTR(partial_matching, "config:15");
>> +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
>> +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
>> +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
>> +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
>> +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
>> +
>> +static struct attribute *riscv_iommu_pmu_formats[] = {
>> +       &format_attr_event.attr,
>> +       &format_attr_partial_matching.attr,
>> +       &format_attr_pid_pscid.attr,
>> +       &format_attr_did_gscid.attr,
>> +       &format_attr_filter_pid_pscid.attr,
>> +       &format_attr_filter_did_gscid.attr,
>> +       &format_attr_filter_id_type.attr,
>> +       NULL,
>> +};
>> +
>> +static const struct attribute_group riscv_iommu_pmu_format_group = {
>> +       .name = "format",
>> +       .attrs = riscv_iommu_pmu_formats,
>> +};
>> +
>> +/* Events */
>> +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
>> +                                         struct device_attribute *attr,
>> +                                         char *page)
>> +{
>> +       struct perf_pmu_events_attr *pmu_attr;
>> +
>> +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
>> +
>> +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
>> +}
>> +
>> +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
>> +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
>> +
>> +static struct attribute *riscv_iommu_pmu_events[] = {
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
>> +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
>> +       NULL,
>> +};
>> +
>> +static const struct attribute_group riscv_iommu_pmu_events_group = {
>> +       .name = "events",
>> +       .attrs = riscv_iommu_pmu_events,
>> +};
>> +
>> +/* cpumask */
>> +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
>> +                                       struct device_attribute *attr,
>> +                                       char *buf)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
>> +
>> +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
>> +}
>> +
>> +static struct device_attribute riscv_iommu_cpumask_attr =
>> +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
>> +
>> +static struct attribute *riscv_iommu_cpumask_attrs[] = {
>> +       &riscv_iommu_cpumask_attr.attr,
>> +       NULL
>> +};
>> +
>> +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
>> +       .attrs = riscv_iommu_cpumask_attrs,
>> +};
>> +
>> +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
>> +       &riscv_iommu_pmu_cpumask_group,
>> +       &riscv_iommu_pmu_format_group,
>> +       &riscv_iommu_pmu_events_group,
>> +       NULL,
>> +};
>> +
>> +/* PMU Operations */
>> +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
>> +                                       u64 value)
>> +{
>> +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
>> +
>> +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> #include <include/linux/io-64-nonatomic-lo-hi.h>
> lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
>
> For better compatibility, it is recommended to use writel and readl
> instead of readq and writeq. Sorry for overlooking this last time —
> could you please make the change again?
> In the future, your driver could support RV32 directly, as the RISC-V
> IOMMU specification also supports RV32; it's just that the current
> RISC-V IOMMU driver's Kconfig depends on 64BIT.
>

I think the RV32 point is secondary here. The important part is that the
lo_hi helpers keep the actual MMIO transactions 32-bit wide.

The real point is to avoid building the driver around assumptions about
8-byte IOMMU register accesses, whose single-copy atomicity is
explicitly UNSPECIFIED by the spec. Using 32-bit MMIO transactions is
therefore the more compatible access pattern for RISC-V IOMMU registers.

>> +}
>> +
>> +/*
>> + * As stated in the RISC-V IOMMU Specification, Chapter 6:
>> + * Whether an 8 byte access to an IOMMU register is single-copy atomic
>> + * is UNSPECIFIED, and such an access may appear, internally to the
>> + * IOMMU, as if two separate 4 byte accesses - first to the high half
>> + * and second to the low half - were performed
>> + *
>> + * To make sure the driver works correctly on different hardware,
>> + * the software will always use two 4-byte access for the counter.
>> + *
>> + * This function implements the hi-lo-hi pattern to detect and handle
>> + * wraparound during the read operation:
>> + *   1. Read high half (hi)
>> + *   2. Read low half (lo)
>> + *   3. Check if low half wrapped or high half changed:
>> + *      - If lo <= lo_prev: possible wraparound occurred
>> + *      - If hi != hi_prev: high half changed during read
>> + *   4. If wraparound detected, re-read high half and assume low half is 0
>> + *   5. Update previous values for next read
>> + */
>> +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
>> +{
>> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
>> +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
>> +       u32 hi, lo;
>> +
>> +       hi = readl(addr + 4);
>> +       lo = readl(addr);
>> +
>> +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
>> +               u32 hi_tmp = readl(addr + 4);
>> +
>> +               /*
>> +                * If hi changes, then hi+1:0 must have happened while
>> +                * the code was running so it is a safe return value
>> +                */
>> +               if (hi_tmp != hi) {
>> +                       hi = hi_tmp;
>> +                       lo = 0;
>> +               }
>> +               pmu->lo_prev[idx] = ~0u;
>> +               pmu->hi_prev[idx] = hi;
>> +       }
>> +       pmu->lo_prev[idx] = lo;
>> +
>> +       value = (((u64)hi << 32) | lo) & counter_mask;
>> +
>> +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
>> +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
>> +}
> Yes! Thanks for your implementation.
>
>> +
>> +static bool is_cycle_event(u64 event)
>> +{
>> +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
>> +}
>> +
>> +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
>> +                                     u64 value)
>> +{
>> +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
>> +       if (is_cycle_event(value))
>> +               return;
>> +
>> +       /* Event counter start from idx 1 */
>> +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
>> +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
>         lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
>                pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
>
>> +}
>> +
>> +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
>> +{
>> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
>> +       u32 value = readl(addr);
>> +
>> +       writel(value & ~BIT(idx), addr);
>> +}
>> +
>> +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
>> +{
>> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
>> +       u32 value = readl(addr);
>> +
>> +       writel(value | BIT(idx), addr);
>> +}
>> +
>> +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
>> +{
>> +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
>> +       u32 used_cntr = 0;
>> +
>> +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
>> +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
>> +
>> +       writel(~used_cntr, addr);
>> +}
>> +
>> +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
>> +{
>> +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
>> +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
>> +}
>> +
>> +/* PMU APIs */
>> +static void riscv_iommu_pmu_set_period(struct perf_event *event)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
>> +       u64 period;
>> +
>> +       /*
>> +        * Limit the maximum period to prevent the counter value
>> +        * from overtaking the one we are about to program.
>> +        * In effect we are reducing max_period to account for
>> +        * interrupt latency (and we are being very conservative).
>> +        */
>> +       period = counter_mask >> 1;
>> +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
>> +       local64_set(&hwc->prev_count, period);
>> +}
>> +
>> +static int riscv_iommu_pmu_event_init(struct perf_event *event)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       struct perf_event *sibling;
>> +       int total_event_counters = pmu->num_counters - 1;
>> +       int counters = 0;
>> +
>> +       if (event->attr.type != event->pmu->type)
>> +               return -ENOENT;
>> +
>> +       if (hwc->sample_period)
>> +               return -EOPNOTSUPP;
>> +
>> +       if (event->cpu < 0)
>> +               return -EOPNOTSUPP;
>> +
>> +       event->cpu = pmu->on_cpu;
>> +
>> +       hwc->idx = -1;
>> +       hwc->config = event->attr.config;
>> +
>> +       if (event->group_leader == event)
>> +               return 0;
>> +
>> +       if (is_cycle_event(get_event(event->group_leader)))
>> +               if (++counters > total_event_counters)
>> +                       return -EINVAL;
>> +
>> +       for_each_sibling_event(sibling, event->group_leader) {
>> +               if (is_cycle_event(get_event(sibling)))
>> +                       continue;
>> +
>> +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
>> +                       return -EINVAL;
>> +
>> +               if (++counters > total_event_counters)
>> +                       return -EINVAL;
>> +       }
>> +
>> +       return 0;
>> +}
>> +
>> +static void riscv_iommu_pmu_update(struct perf_event *event)
>> +{
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       u64 delta, prev, now;
>> +       u32 idx = hwc->idx;
>> +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
>> +
>> +       do {
>> +               prev = local64_read(&hwc->prev_count);
>> +               now = riscv_iommu_pmu_get_counter(pmu, idx);
>> +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
>> +
>> +       delta = (now - prev) & counter_mask;
>> +       local64_add(delta, &event->count);
>> +}
>> +
>> +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +
>> +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
>> +               return;
>> +
>> +       if (flags & PERF_EF_RELOAD)
>> +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
>> +
>> +       hwc->state = 0;
>> +       riscv_iommu_pmu_set_period(event);
>> +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
>> +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
>> +
>> +       perf_event_update_userpage(event);
>> +}
>> +
>> +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       int idx = hwc->idx;
>> +
>> +       if (hwc->state & PERF_HES_STOPPED)
>> +               return;
>> +
>> +       riscv_iommu_pmu_disable_counter(pmu, idx);
>> +
>> +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
>> +               riscv_iommu_pmu_update(event);
>> +
>> +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
>> +}
>> +
>> +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       unsigned int num_counters = pmu->num_counters;
>> +       int idx;
>> +
>> +       /* Reserve index zero for iohpmcycles */
>> +       if (is_cycle_event(get_event(event)))
>> +               idx = 0;
>> +       else
>> +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
>> +
>> +       /* All event counters or cycle counter are in use */
>> +       if (idx == num_counters || pmu->events[idx])
>> +               return -EAGAIN;
>> +
>> +       set_bit(idx, pmu->used_counters);
>> +
>> +       pmu->events[idx] = event;
>> +       hwc->idx = idx;
>> +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
>> +       local64_set(&hwc->prev_count, 0);
>> +
>> +       if (flags & PERF_EF_START)
>> +               riscv_iommu_pmu_start(event, flags);
>> +
>> +       /* Propagate changes to the userspace mapping. */
>> +       perf_event_update_userpage(event);
>> +
>> +       return 0;
>> +}
>> +
>> +static void riscv_iommu_pmu_read(struct perf_event *event)
>> +{
>> +       riscv_iommu_pmu_update(event);
>> +}
>> +
>> +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
>> +{
>> +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
>> +       struct hw_perf_event *hwc = &event->hw;
>> +       int idx = hwc->idx;
>> +
>> +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
>> +       pmu->events[idx] = NULL;
>> +       clear_bit(idx, pmu->used_counters);
>> +
>> +       perf_event_update_userpage(event);
>> +}
>> +
>> +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
>> +{
>> +       struct riscv_iommu_pmu *iommu_pmu;
>> +
>> +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
>> +
>> +       if (iommu_pmu->on_cpu == -1)
>> +               iommu_pmu->on_cpu = cpu;
>> +
>> +       return 0;
>> +}
>> +
>> +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
>> +{
>> +       struct riscv_iommu_pmu *iommu_pmu;
>> +       unsigned int target_cpu;
>> +
>> +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
>> +
>> +       if (cpu != iommu_pmu->on_cpu)
>> +               return 0;
>> +
>> +       iommu_pmu->on_cpu = -1;
>> +
>> +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
>> +       if (target_cpu >= nr_cpu_ids)
>> +               return 0;
>> +
>> +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
>> +       iommu_pmu->on_cpu = target_cpu;
>> +
>> +       return 0;
>> +}
>> +
>> +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
>> +{
>> +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
>> +       int idx;
>> +
>> +       if (!ovf)
>> +               return IRQ_NONE;
>> +
>> +       riscv_iommu_pmu_stop_all(pmu);
>> +
>> +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
>> +               struct perf_event *event = pmu->events[idx];
>> +
>> +               if (WARN_ON_ONCE(!event))
>> +                       continue;
>> +
>> +               riscv_iommu_pmu_update(event);
>> +               riscv_iommu_pmu_set_period(event);
>> +       }
>> +
>> +       riscv_iommu_pmu_start_all(pmu);
>> +
>> +       return IRQ_HANDLED;
>> +}
>> +
>> +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
>> +{
>> +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
>> +       irqreturn_t ret;
>> +
>> +       /* Check whether this interrupt is for PMU */
>> +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
>> +               return IRQ_NONE;
>> +
>> +       /* Process PMU IRQ */
>> +       ret = riscv_iommu_pmu_handle_irq(pmu);
>> +
>> +       /* Clear performance monitoring interrupt pending bit */
>> +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
>> +
>> +       return ret;
>> +}
>> +
>> +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
>> +{
>> +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
>> +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
>> +
>> +       return iommu->irqs[vec];
>> +}
>> +
>> +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
>> +                                      struct riscv_iommu_pmu *pmu)
>> +{
>> +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
>> +
>> +       /*
>> +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
>> +        * other threaded IRQs by other queues.
>> +        */
>> +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
>> +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
>> +}
>> +
>> +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
>> +                                    struct riscv_iommu_pmu *pmu)
>> +{
>> +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
>> +
>> +       free_irq(irq, pmu);
>> +}
>> +
>> +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
>> +                                const struct auxiliary_device_id *id)
>> +{
>> +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
>> +       struct riscv_iommu_pmu *iommu_pmu;
>> +       void __iomem *addr;
>> +       char *name;
>> +       int ret;
>> +
>> +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
>> +       if (!iommu_pmu)
>> +               return -ENOMEM;
>> +
>> +       iommu_pmu->reg = iommu_dev->reg;
>> +
>> +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
>> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
>> +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
>> +       iommu_pmu->num_counters = hweight32(readl(addr));
>> +
>> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
>> +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
>      lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
>
>> +       iommu_pmu->cycle_cntr_mask = readq(addr);
> iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
>
>> +
>> +       /* Assume the width of all event counters are the same */
>> +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
>> +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
>      lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
>
>> +       iommu_pmu->event_cntr_mask = readq(addr);
>   iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
>
>> +
>> +       iommu_pmu->pmu = (struct pmu) {
>> +               .module         = THIS_MODULE,
>> +               .parent         = &auxdev->dev,
>> +               .task_ctx_nr    = perf_invalid_context,
>> +               .event_init     = riscv_iommu_pmu_event_init,
>> +               .add            = riscv_iommu_pmu_add,
>> +               .del            = riscv_iommu_pmu_del,
>> +               .start          = riscv_iommu_pmu_start,
>> +               .stop           = riscv_iommu_pmu_stop,
>> +               .read           = riscv_iommu_pmu_read,
>> +               .attr_groups    = riscv_iommu_pmu_attr_grps,
>> +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
>> +       };
>> +
>> +       auxiliary_set_drvdata(auxdev, iommu_pmu);
>> +
>> +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
>> +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
>> +       if (!name) {
>> +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
>> +                       dev_name(iommu_dev->dev));
>> +               return -ENOMEM;
>> +       }
>> +
>> +       /* Bind all events to the same cpu context to avoid race enabling */
>> +       iommu_pmu->on_cpu = raw_smp_processor_id();
>> +
>> +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
>> +       if (ret) {
>> +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
>> +               return ret;
>> +       }
>> +
>> +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
>> +       if (ret) {
>> +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
>> +               goto err_cpuhp_remove;
>> +       }
>> +
>> +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
>> +       if (ret) {
>> +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
>> +               goto err_free_irq;
>> +       }
>> +
>> +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
>> +                name, iommu_pmu->num_counters);
>> +
>> +       return 0;
>> +
>> +err_free_irq:
>> +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
>> +err_cpuhp_remove:
>> +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
>> +       return ret;
>> +}
>> +
>> +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
>> +       { .name = "iommu.pmu" },
>> +       {}
>> +};
>> +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
>> +
>> +static struct auxiliary_driver iommu_pmu_driver = {
>> +       .probe          = riscv_iommu_pmu_probe,
>> +       .id_table       = riscv_iommu_pmu_id_table,
>> +};
>> +
>> +static int __init riscv_iommu_pmu_init(void)
>> +{
>> +       int ret;
>> +
>> +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
>> +                                             "perf/riscv/iommu:online",
>> +                                             riscv_iommu_pmu_online_cpu,
>> +                                             riscv_iommu_pmu_offline_cpu);
>> +       if (cpuhp_state < 0)
>> +               return cpuhp_state;
>> +
>> +       ret = auxiliary_driver_register(&iommu_pmu_driver);
>> +       if (ret)
>> +               cpuhp_remove_multi_state(cpuhp_state);
>> +
>> +       return ret;
>> +}
>> +module_init(riscv_iommu_pmu_init);
>> +
>> +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
>> +MODULE_LICENSE("GPL");
>> --
>> 2.43.7
>>
>

Thanks,
Zhanpeng

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
>
> On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> >
> > Hi Zong Li,
> >
> > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > >
> > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > device driver created by the parent RISC-V IOMMU driver.
> > >
> > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > cycle event is required for the perf subsystem.
> > >
> > > The number and width of the counters are hardware-implemented and must
> > > be detected at runtime.
> > >
> > > The performance monitor provides counters with filtering support to
> > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > >
> > > PMU-related definitions are moved into the perf driver, where they are
> > > used exclusively.
> > >
> > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > ---
> > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > >  drivers/perf/Kconfig             |  12 +
> > >  drivers/perf/Makefile            |   1 +
> > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > >
> > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > index f2ef9bd3cde9..6b5de913a032 100644
> > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > >
> > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > -
> > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > -
> > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > -
> > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > -
> > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > -
> > > -/* Number of defined performance-monitoring event selectors */
> > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > -
> > > -/**
> > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > - *
> > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > - */
> > > -enum riscv_iommu_hpmevent_id {
> > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > -};
> > > -
> > >  /* 5.24 Translation request IOVA (64bits) */
> > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > index 245e7bb763b9..8cce6c2ea626 100644
> > > --- a/drivers/perf/Kconfig
> > > +++ b/drivers/perf/Kconfig
> > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > >           full perf feature support i.e. counter overflow, privilege mode
> > >           filtering, counter configuration.
> > >
> > > +config RISCV_IOMMU_PMU
> > > +       depends on RISCV || COMPILE_TEST
> > > +       depends on RISCV_IOMMU
> > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > +       default y
> > > +       help
> > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > +         implementation. The performance monitor is an optional hardware
> > > +         feature, and whether it is actually enabled depends on IOMMU
> > > +         hardware support. If the underlying hardware does not implement
> > > +         the PMU, this option will have no effect.
> > > +
> > >  config STARFIVE_STARLINK_PMU
> > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > >         depends on 64BIT
> > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > --- a/drivers/perf/Makefile
> > > +++ b/drivers/perf/Makefile
> > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > new file mode 100644
> > > index 000000000000..2e856289ddc9
> > > --- /dev/null
> > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > @@ -0,0 +1,703 @@
> > > +// SPDX-License-Identifier: GPL-2.0-only
> > > +/*
> > > + * Copyright (C) 2026 SiFive
> > > + *
> > > + * Authors
> > > + *     Zong Li <zong.li@sifive.com>
> > > + */
> > > +
> > > +#include <linux/auxiliary_bus.h>
> > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > +#include <linux/perf_event.h>
> > > +
> > > +#include "../iommu/riscv/iommu.h"
> > > +
> > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > +
> > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > +
> > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > +
> > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > +
> > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > +
> > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > +
> > > +static int cpuhp_state;
> > > +
> > > +/**
> > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > + *
> > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > + *
> > > + * The specification does not define an event ID for counting the
> > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > + * event is not actually written into any register, it serves solely
> > > + * as an identifier.
> > > + */
> > > +enum riscv_iommu_hpmevent_id {
> > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > +};
> > > +
> > > +struct riscv_iommu_pmu {
> > > +       struct pmu pmu;
> > > +       struct hlist_node node;
> > > +       void __iomem *reg;
> > > +       unsigned int on_cpu;
> > > +       int num_counters;
> > > +       u64 cycle_cntr_mask;
> > > +       u64 event_cntr_mask;
> > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > +};
> > > +
> > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > +
> > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > +       {                                                               \
> > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > +       }                                                               \
> > > +
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > +
> > > +/* Formats */
> > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > +
> > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > +       &format_attr_event.attr,
> > > +       &format_attr_partial_matching.attr,
> > > +       &format_attr_pid_pscid.attr,
> > > +       &format_attr_did_gscid.attr,
> > > +       &format_attr_filter_pid_pscid.attr,
> > > +       &format_attr_filter_did_gscid.attr,
> > > +       &format_attr_filter_id_type.attr,
> > > +       NULL,
> > > +};
> > > +
> > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > +       .name = "format",
> > > +       .attrs = riscv_iommu_pmu_formats,
> > > +};
> > > +
> > > +/* Events */
> > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > +                                         struct device_attribute *attr,
> > > +                                         char *page)
> > > +{
> > > +       struct perf_pmu_events_attr *pmu_attr;
> > > +
> > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > +
> > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > +}
> > > +
> > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > +
> > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > +       NULL,
> > > +};
> > > +
> > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > +       .name = "events",
> > > +       .attrs = riscv_iommu_pmu_events,
> > > +};
> > > +
> > > +/* cpumask */
> > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > +                                       struct device_attribute *attr,
> > > +                                       char *buf)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > +
> > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > +}
> > > +
> > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > +
> > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > +       &riscv_iommu_cpumask_attr.attr,
> > > +       NULL
> > > +};
> > > +
> > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > +};
> > > +
> > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > +       &riscv_iommu_pmu_cpumask_group,
> > > +       &riscv_iommu_pmu_format_group,
> > > +       &riscv_iommu_pmu_events_group,
> > > +       NULL,
> > > +};
> > > +
> > > +/* PMU Operations */
> > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > +                                       u64 value)
> > > +{
> > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > +
> > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> >
> > For better compatibility, it is recommended to use writel and readl
> > instead of readq and writeq. Sorry for overlooking this last time —
> > could you please make the change again?
> > In the future, your driver could support RV32 directly, as the RISC-V
> > IOMMU specification also supports RV32; it's just that the current
> > RISC-V IOMMU driver's Kconfig depends on 64BIT.
>
> Hi Guo,
> Thank you for your review. In my implementation for RV32, readq and
> writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> However, as you mentioned before: 64-bit access might be undefined on
> some hardware. To be safe and compatible with all hardware, we should
> use 32-bit access instead of 64-bit access.
> I will modify them in the next version
>
> I'd like to make sure I didn't miss anything important. My original
> code uses the "hi-lo" order. May I ask if there is a specific reason
> why you prefer the "lo-hi" order?
> Thanks
>

Hi all,
I have been thinking more about the IOMMU register access behavior and
would like to share my thoughts regarding the lo-hi vs. hi-lo access
order.

It seems to me that the correct 32-bit write order is strictly tied to
the underlying hardware implementation. From the software side, we
have no way of knowing whether a specific vendor designed their
hardware with a low-part trigger or a high-part trigger. Therefore,
hardcoding a fixed 32-bit write sequence could result in compatibility
issues across different hardware designs.

Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
this might imply that the hardware itself will handle the split into
two 4-byte accesses and internally manage its own trigger logic.
Because of this, for RV64 systems, software might simply use standard
8-byte writes (e.g., writeq). Delegating the access handling to the
hardware is likely the best way to maximize compatibility across all
vendor implementations. For the counter reading strategy (hi-lo-hi
pattern), I completely agree with using the hi-lo-hi pattern when
reading the PMU counters. Since this is a read operation and does not
alter hardware state.

For RV32 support, we inevitably face the 32-bit access splitting
issue. For the RV32 case, maybe we can assume the hi-lo order as the
default fallback, because the IOMMU specification uses this sequence
as example: "an access may appear, internally to the IOMMU, as if two
separate 4 byte accesses - first to the high half and second to the
low half - were performed.". Or do you prefer to use the same
assumption (hi-lo pattern) in RV64 as well?

Please let me know what you think about this.

Thanks


> >
> > > +}
> > > +
> > > +/*
> > > + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> > > + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> > > + * is UNSPECIFIED, and such an access may appear, internally to the
> > > + * IOMMU, as if two separate 4 byte accesses - first to the high half
> > > + * and second to the low half - were performed
> > > + *
> > > + * To make sure the driver works correctly on different hardware,
> > > + * the software will always use two 4-byte access for the counter.
> > > + *
> > > + * This function implements the hi-lo-hi pattern to detect and handle
> > > + * wraparound during the read operation:
> > > + *   1. Read high half (hi)
> > > + *   2. Read low half (lo)
> > > + *   3. Check if low half wrapped or high half changed:
> > > + *      - If lo <= lo_prev: possible wraparound occurred
> > > + *      - If hi != hi_prev: high half changed during read
> > > + *   4. If wraparound detected, re-read high half and assume low half is 0
> > > + *   5. Update previous values for next read
> > > + */
> > > +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > +{
> > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> > > +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > +       u32 hi, lo;
> > > +
> > > +       hi = readl(addr + 4);
> > > +       lo = readl(addr);
> > > +
> > > +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> > > +               u32 hi_tmp = readl(addr + 4);
> > > +
> > > +               /*
> > > +                * If hi changes, then hi+1:0 must have happened while
> > > +                * the code was running so it is a safe return value
> > > +                */
> > > +               if (hi_tmp != hi) {
> > > +                       hi = hi_tmp;
> > > +                       lo = 0;
> > > +               }
> > > +               pmu->lo_prev[idx] = ~0u;
> > > +               pmu->hi_prev[idx] = hi;
> > > +       }
> > > +       pmu->lo_prev[idx] = lo;
> > > +
> > > +       value = (((u64)hi << 32) | lo) & counter_mask;
> > > +
> > > +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> > > +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> > > +}
> > Yes! Thanks for your implementation.
> >
> > > +
> > > +static bool is_cycle_event(u64 event)
> > > +{
> > > +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> > > +                                     u64 value)
> > > +{
> > > +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> > > +       if (is_cycle_event(value))
> > > +               return;
> > > +
> > > +       /* Event counter start from idx 1 */
> > > +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> >        lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> >               pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> >
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > +{
> > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > +       u32 value = readl(addr);
> > > +
> > > +       writel(value & ~BIT(idx), addr);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > +{
> > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > +       u32 value = readl(addr);
> > > +
> > > +       writel(value | BIT(idx), addr);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> > > +{
> > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > +       u32 used_cntr = 0;
> > > +
> > > +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> > > +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> > > +
> > > +       writel(~used_cntr, addr);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> > > +{
> > > +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> > > +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> > > +}
> > > +
> > > +/* PMU APIs */
> > > +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > +       u64 period;
> > > +
> > > +       /*
> > > +        * Limit the maximum period to prevent the counter value
> > > +        * from overtaking the one we are about to program.
> > > +        * In effect we are reducing max_period to account for
> > > +        * interrupt latency (and we are being very conservative).
> > > +        */
> > > +       period = counter_mask >> 1;
> > > +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> > > +       local64_set(&hwc->prev_count, period);
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       struct perf_event *sibling;
> > > +       int total_event_counters = pmu->num_counters - 1;
> > > +       int counters = 0;
> > > +
> > > +       if (event->attr.type != event->pmu->type)
> > > +               return -ENOENT;
> > > +
> > > +       if (hwc->sample_period)
> > > +               return -EOPNOTSUPP;
> > > +
> > > +       if (event->cpu < 0)
> > > +               return -EOPNOTSUPP;
> > > +
> > > +       event->cpu = pmu->on_cpu;
> > > +
> > > +       hwc->idx = -1;
> > > +       hwc->config = event->attr.config;
> > > +
> > > +       if (event->group_leader == event)
> > > +               return 0;
> > > +
> > > +       if (is_cycle_event(get_event(event->group_leader)))
> > > +               if (++counters > total_event_counters)
> > > +                       return -EINVAL;
> > > +
> > > +       for_each_sibling_event(sibling, event->group_leader) {
> > > +               if (is_cycle_event(get_event(sibling)))
> > > +                       continue;
> > > +
> > > +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> > > +                       return -EINVAL;
> > > +
> > > +               if (++counters > total_event_counters)
> > > +                       return -EINVAL;
> > > +       }
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_update(struct perf_event *event)
> > > +{
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       u64 delta, prev, now;
> > > +       u32 idx = hwc->idx;
> > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > +
> > > +       do {
> > > +               prev = local64_read(&hwc->prev_count);
> > > +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> > > +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> > > +
> > > +       delta = (now - prev) & counter_mask;
> > > +       local64_add(delta, &event->count);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +
> > > +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> > > +               return;
> > > +
> > > +       if (flags & PERF_EF_RELOAD)
> > > +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> > > +
> > > +       hwc->state = 0;
> > > +       riscv_iommu_pmu_set_period(event);
> > > +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> > > +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> > > +
> > > +       perf_event_update_userpage(event);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       int idx = hwc->idx;
> > > +
> > > +       if (hwc->state & PERF_HES_STOPPED)
> > > +               return;
> > > +
> > > +       riscv_iommu_pmu_disable_counter(pmu, idx);
> > > +
> > > +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> > > +               riscv_iommu_pmu_update(event);
> > > +
> > > +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       unsigned int num_counters = pmu->num_counters;
> > > +       int idx;
> > > +
> > > +       /* Reserve index zero for iohpmcycles */
> > > +       if (is_cycle_event(get_event(event)))
> > > +               idx = 0;
> > > +       else
> > > +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> > > +
> > > +       /* All event counters or cycle counter are in use */
> > > +       if (idx == num_counters || pmu->events[idx])
> > > +               return -EAGAIN;
> > > +
> > > +       set_bit(idx, pmu->used_counters);
> > > +
> > > +       pmu->events[idx] = event;
> > > +       hwc->idx = idx;
> > > +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > +       local64_set(&hwc->prev_count, 0);
> > > +
> > > +       if (flags & PERF_EF_START)
> > > +               riscv_iommu_pmu_start(event, flags);
> > > +
> > > +       /* Propagate changes to the userspace mapping. */
> > > +       perf_event_update_userpage(event);
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_read(struct perf_event *event)
> > > +{
> > > +       riscv_iommu_pmu_update(event);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > +       struct hw_perf_event *hwc = &event->hw;
> > > +       int idx = hwc->idx;
> > > +
> > > +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> > > +       pmu->events[idx] = NULL;
> > > +       clear_bit(idx, pmu->used_counters);
> > > +
> > > +       perf_event_update_userpage(event);
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> > > +{
> > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > +
> > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > +
> > > +       if (iommu_pmu->on_cpu == -1)
> > > +               iommu_pmu->on_cpu = cpu;
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> > > +{
> > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > +       unsigned int target_cpu;
> > > +
> > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > +
> > > +       if (cpu != iommu_pmu->on_cpu)
> > > +               return 0;
> > > +
> > > +       iommu_pmu->on_cpu = -1;
> > > +
> > > +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> > > +       if (target_cpu >= nr_cpu_ids)
> > > +               return 0;
> > > +
> > > +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> > > +       iommu_pmu->on_cpu = target_cpu;
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> > > +{
> > > +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> > > +       int idx;
> > > +
> > > +       if (!ovf)
> > > +               return IRQ_NONE;
> > > +
> > > +       riscv_iommu_pmu_stop_all(pmu);
> > > +
> > > +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> > > +               struct perf_event *event = pmu->events[idx];
> > > +
> > > +               if (WARN_ON_ONCE(!event))
> > > +                       continue;
> > > +
> > > +               riscv_iommu_pmu_update(event);
> > > +               riscv_iommu_pmu_set_period(event);
> > > +       }
> > > +
> > > +       riscv_iommu_pmu_start_all(pmu);
> > > +
> > > +       return IRQ_HANDLED;
> > > +}
> > > +
> > > +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> > > +{
> > > +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> > > +       irqreturn_t ret;
> > > +
> > > +       /* Check whether this interrupt is for PMU */
> > > +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> > > +               return IRQ_NONE;
> > > +
> > > +       /* Process PMU IRQ */
> > > +       ret = riscv_iommu_pmu_handle_irq(pmu);
> > > +
> > > +       /* Clear performance monitoring interrupt pending bit */
> > > +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> > > +
> > > +       return ret;
> > > +}
> > > +
> > > +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> > > +{
> > > +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> > > +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> > > +
> > > +       return iommu->irqs[vec];
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> > > +                                      struct riscv_iommu_pmu *pmu)
> > > +{
> > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > +
> > > +       /*
> > > +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> > > +        * other threaded IRQs by other queues.
> > > +        */
> > > +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> > > +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> > > +}
> > > +
> > > +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> > > +                                    struct riscv_iommu_pmu *pmu)
> > > +{
> > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > +
> > > +       free_irq(irq, pmu);
> > > +}
> > > +
> > > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > > +                                const struct auxiliary_device_id *id)
> > > +{
> > > +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > +       void __iomem *addr;
> > > +       char *name;
> > > +       int ret;
> > > +
> > > +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > > +       if (!iommu_pmu)
> > > +               return -ENOMEM;
> > > +
> > > +       iommu_pmu->reg = iommu_dev->reg;
> > > +
> > > +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > > +       iommu_pmu->num_counters = hweight32(readl(addr));
> > > +
> > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > > +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> >     lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> >
> > > +       iommu_pmu->cycle_cntr_mask = readq(addr);
> > iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
> >
> > > +
> > > +       /* Assume the width of all event counters are the same */
> > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > > +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> >     lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> >
> > > +       iommu_pmu->event_cntr_mask = readq(addr);
> >  iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
> >
> > > +
> > > +       iommu_pmu->pmu = (struct pmu) {
> > > +               .module         = THIS_MODULE,
> > > +               .parent         = &auxdev->dev,
> > > +               .task_ctx_nr    = perf_invalid_context,
> > > +               .event_init     = riscv_iommu_pmu_event_init,
> > > +               .add            = riscv_iommu_pmu_add,
> > > +               .del            = riscv_iommu_pmu_del,
> > > +               .start          = riscv_iommu_pmu_start,
> > > +               .stop           = riscv_iommu_pmu_stop,
> > > +               .read           = riscv_iommu_pmu_read,
> > > +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> > > +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > > +       };
> > > +
> > > +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> > > +
> > > +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > > +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> > > +       if (!name) {
> > > +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> > > +                       dev_name(iommu_dev->dev));
> > > +               return -ENOMEM;
> > > +       }
> > > +
> > > +       /* Bind all events to the same cpu context to avoid race enabling */
> > > +       iommu_pmu->on_cpu = raw_smp_processor_id();
> > > +
> > > +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > +       if (ret) {
> > > +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > > +               return ret;
> > > +       }
> > > +
> > > +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> > > +       if (ret) {
> > > +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > > +               goto err_cpuhp_remove;
> > > +       }
> > > +
> > > +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > > +       if (ret) {
> > > +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > > +               goto err_free_irq;
> > > +       }
> > > +
> > > +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> > > +                name, iommu_pmu->num_counters);
> > > +
> > > +       return 0;
> > > +
> > > +err_free_irq:
> > > +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> > > +err_cpuhp_remove:
> > > +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > +       return ret;
> > > +}
> > > +
> > > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > > +       { .name = "iommu.pmu" },
> > > +       {}
> > > +};
> > > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > > +
> > > +static struct auxiliary_driver iommu_pmu_driver = {
> > > +       .probe          = riscv_iommu_pmu_probe,
> > > +       .id_table       = riscv_iommu_pmu_id_table,
> > > +};
> > > +
> > > +static int __init riscv_iommu_pmu_init(void)
> > > +{
> > > +       int ret;
> > > +
> > > +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > > +                                             "perf/riscv/iommu:online",
> > > +                                             riscv_iommu_pmu_online_cpu,
> > > +                                             riscv_iommu_pmu_offline_cpu);
> > > +       if (cpuhp_state < 0)
> > > +               return cpuhp_state;
> > > +
> > > +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> > > +       if (ret)
> > > +               cpuhp_remove_multi_state(cpuhp_state);
> > > +
> > > +       return ret;
> > > +}
> > > +module_init(riscv_iommu_pmu_init);
> > > +
> > > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > > +MODULE_LICENSE("GPL");
> > > --
> > > 2.43.7
> > >
> >
> > --
> > Best Regards
> >  Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Guo Ren]

On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
>
> On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> >
> > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > >
> > > Hi Zong Li,
> > >
> > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > >
> > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > device driver created by the parent RISC-V IOMMU driver.
> > > >
> > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > cycle event is required for the perf subsystem.
> > > >
> > > > The number and width of the counters are hardware-implemented and must
> > > > be detected at runtime.
> > > >
> > > > The performance monitor provides counters with filtering support to
> > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > >
> > > > PMU-related definitions are moved into the perf driver, where they are
> > > > used exclusively.
> > > >
> > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > ---
> > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > >  drivers/perf/Kconfig             |  12 +
> > > >  drivers/perf/Makefile            |   1 +
> > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > >
> > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > >
> > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > -
> > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > -
> > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > -
> > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > -
> > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > -
> > > > -/* Number of defined performance-monitoring event selectors */
> > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > -
> > > > -/**
> > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > - *
> > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > - */
> > > > -enum riscv_iommu_hpmevent_id {
> > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > -};
> > > > -
> > > >  /* 5.24 Translation request IOVA (64bits) */
> > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > --- a/drivers/perf/Kconfig
> > > > +++ b/drivers/perf/Kconfig
> > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > >           full perf feature support i.e. counter overflow, privilege mode
> > > >           filtering, counter configuration.
> > > >
> > > > +config RISCV_IOMMU_PMU
> > > > +       depends on RISCV || COMPILE_TEST
> > > > +       depends on RISCV_IOMMU
> > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > +       default y
> > > > +       help
> > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > +         implementation. The performance monitor is an optional hardware
> > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > +         hardware support. If the underlying hardware does not implement
> > > > +         the PMU, this option will have no effect.
> > > > +
> > > >  config STARFIVE_STARLINK_PMU
> > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > >         depends on 64BIT
> > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > --- a/drivers/perf/Makefile
> > > > +++ b/drivers/perf/Makefile
> > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > new file mode 100644
> > > > index 000000000000..2e856289ddc9
> > > > --- /dev/null
> > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > @@ -0,0 +1,703 @@
> > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > +/*
> > > > + * Copyright (C) 2026 SiFive
> > > > + *
> > > > + * Authors
> > > > + *     Zong Li <zong.li@sifive.com>
> > > > + */
> > > > +
> > > > +#include <linux/auxiliary_bus.h>
> > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > +#include <linux/perf_event.h>
> > > > +
> > > > +#include "../iommu/riscv/iommu.h"
> > > > +
> > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > +
> > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > +
> > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > +
> > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > +
> > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > +
> > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > +
> > > > +static int cpuhp_state;
> > > > +
> > > > +/**
> > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > + *
> > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > + *
> > > > + * The specification does not define an event ID for counting the
> > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > + * event is not actually written into any register, it serves solely
> > > > + * as an identifier.
> > > > + */
> > > > +enum riscv_iommu_hpmevent_id {
> > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > +};
> > > > +
> > > > +struct riscv_iommu_pmu {
> > > > +       struct pmu pmu;
> > > > +       struct hlist_node node;
> > > > +       void __iomem *reg;
> > > > +       unsigned int on_cpu;
> > > > +       int num_counters;
> > > > +       u64 cycle_cntr_mask;
> > > > +       u64 event_cntr_mask;
> > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +};
> > > > +
> > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > +
> > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > +       {                                                               \
> > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > +       }                                                               \
> > > > +
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > +
> > > > +/* Formats */
> > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > +
> > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > +       &format_attr_event.attr,
> > > > +       &format_attr_partial_matching.attr,
> > > > +       &format_attr_pid_pscid.attr,
> > > > +       &format_attr_did_gscid.attr,
> > > > +       &format_attr_filter_pid_pscid.attr,
> > > > +       &format_attr_filter_did_gscid.attr,
> > > > +       &format_attr_filter_id_type.attr,
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > +       .name = "format",
> > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > +};
> > > > +
> > > > +/* Events */
> > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > +                                         struct device_attribute *attr,
> > > > +                                         char *page)
> > > > +{
> > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > +
> > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > +
> > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > +}
> > > > +
> > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > +
> > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > +       .name = "events",
> > > > +       .attrs = riscv_iommu_pmu_events,
> > > > +};
> > > > +
> > > > +/* cpumask */
> > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > +                                       struct device_attribute *attr,
> > > > +                                       char *buf)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > +
> > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > +}
> > > > +
> > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > +
> > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > +       NULL
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > +};
> > > > +
> > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > +       &riscv_iommu_pmu_format_group,
> > > > +       &riscv_iommu_pmu_events_group,
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +/* PMU Operations */
> > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > +                                       u64 value)
> > > > +{
> > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > +
> > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > >
> > > For better compatibility, it is recommended to use writel and readl
> > > instead of readq and writeq. Sorry for overlooking this last time —
> > > could you please make the change again?
> > > In the future, your driver could support RV32 directly, as the RISC-V
> > > IOMMU specification also supports RV32; it's just that the current
> > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> >
> > Hi Guo,
> > Thank you for your review. In my implementation for RV32, readq and
> > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > However, as you mentioned before: 64-bit access might be undefined on
> > some hardware. To be safe and compatible with all hardware, we should
> > use 32-bit access instead of 64-bit access.
> > I will modify them in the next version
> >
> > I'd like to make sure I didn't miss anything important. My original
> > code uses the "hi-lo" order. May I ask if there is a specific reason
> > why you prefer the "lo-hi" order?
> > Thanks
> >
>
> Hi all,
> I have been thinking more about the IOMMU register access behavior and
> would like to share my thoughts regarding the lo-hi vs. hi-lo access
> order.
>
> It seems to me that the correct 32-bit write order is strictly tied to
> the underlying hardware implementation. From the software side, we
> have no way of knowing whether a specific vendor designed their
> hardware with a low-part trigger or a high-part trigger. Therefore,
> hardcoding a fixed 32-bit write sequence could result in compatibility
> issues across different hardware designs.
>
> Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> this might imply that the hardware itself will handle the split into
> two 4-byte accesses and internally manage its own trigger logic.
> Because of this, for RV64 systems, software might simply use standard
> 8-byte writes (e.g., writeq). Delegating the access handling to the
> hardware is likely the best way to maximize compatibility across all
> vendor implementations. For the counter reading strategy (hi-lo-hi
> pattern), I completely agree with using the hi-lo-hi pattern when
> reading the PMU counters. Since this is a read operation and does not
> alter hardware state.
>
> For RV32 support, we inevitably face the 32-bit access splitting
> issue. For the RV32 case, maybe we can assume the hi-lo order as the
> default fallback, because the IOMMU specification uses this sequence
> as example: "an access may appear, internally to the IOMMU, as if two
> separate 4 byte accesses - first to the high half and second to the
> low half - were performed.". Or do you prefer to use the same
> assumption (hi-lo pattern) in RV64 as well?
>
> Please let me know what you think about this.
>
> Thanks

You're right, RV32 is not a valid argument here — my wording was
incorrect. Thanks for pointing that out.
Also, I have no strong preference for lo-hi; that was just a careless
copy-paste on my part. I agree hi-lo is the right choice.

Thanks again for your great work.

-- 
Best Regards
 Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

On Tue, Jul 7, 2026 at 10:58 AM Guo Ren <guoren@kernel.org> wrote:
>
> On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
> >
> > On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> > >
> > > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > > >
> > > > Hi Zong Li,
> > > >
> > > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > > >
> > > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > > device driver created by the parent RISC-V IOMMU driver.
> > > > >
> > > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > > cycle event is required for the perf subsystem.
> > > > >
> > > > > The number and width of the counters are hardware-implemented and must
> > > > > be detected at runtime.
> > > > >
> > > > > The performance monitor provides counters with filtering support to
> > > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > > >
> > > > > PMU-related definitions are moved into the perf driver, where they are
> > > > > used exclusively.
> > > > >
> > > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > > ---
> > > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > > >  drivers/perf/Kconfig             |  12 +
> > > > >  drivers/perf/Makefile            |   1 +
> > > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > > >
> > > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > > >
> > > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > -
> > > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > > -
> > > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > -
> > > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > > -
> > > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > -
> > > > > -/* Number of defined performance-monitoring event selectors */
> > > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > > -
> > > > > -/**
> > > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > - *
> > > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > - */
> > > > > -enum riscv_iommu_hpmevent_id {
> > > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > -};
> > > > > -
> > > > >  /* 5.24 Translation request IOVA (64bits) */
> > > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > > --- a/drivers/perf/Kconfig
> > > > > +++ b/drivers/perf/Kconfig
> > > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > > >           full perf feature support i.e. counter overflow, privilege mode
> > > > >           filtering, counter configuration.
> > > > >
> > > > > +config RISCV_IOMMU_PMU
> > > > > +       depends on RISCV || COMPILE_TEST
> > > > > +       depends on RISCV_IOMMU
> > > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > > +       default y
> > > > > +       help
> > > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > > +         implementation. The performance monitor is an optional hardware
> > > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > > +         hardware support. If the underlying hardware does not implement
> > > > > +         the PMU, this option will have no effect.
> > > > > +
> > > > >  config STARFIVE_STARLINK_PMU
> > > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > > >         depends on 64BIT
> > > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > > --- a/drivers/perf/Makefile
> > > > > +++ b/drivers/perf/Makefile
> > > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > > new file mode 100644
> > > > > index 000000000000..2e856289ddc9
> > > > > --- /dev/null
> > > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > > @@ -0,0 +1,703 @@
> > > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > > +/*
> > > > > + * Copyright (C) 2026 SiFive
> > > > > + *
> > > > > + * Authors
> > > > > + *     Zong Li <zong.li@sifive.com>
> > > > > + */
> > > > > +
> > > > > +#include <linux/auxiliary_bus.h>
> > > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > > +#include <linux/perf_event.h>
> > > > > +
> > > > > +#include "../iommu/riscv/iommu.h"
> > > > > +
> > > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > +
> > > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > > +
> > > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > > +
> > > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > > +
> > > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > > +
> > > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > > +
> > > > > +static int cpuhp_state;
> > > > > +
> > > > > +/**
> > > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > + *
> > > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > + *
> > > > > + * The specification does not define an event ID for counting the
> > > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > > + * event is not actually written into any register, it serves solely
> > > > > + * as an identifier.
> > > > > + */
> > > > > +enum riscv_iommu_hpmevent_id {
> > > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > +};
> > > > > +
> > > > > +struct riscv_iommu_pmu {
> > > > > +       struct pmu pmu;
> > > > > +       struct hlist_node node;
> > > > > +       void __iomem *reg;
> > > > > +       unsigned int on_cpu;
> > > > > +       int num_counters;
> > > > > +       u64 cycle_cntr_mask;
> > > > > +       u64 event_cntr_mask;
> > > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +};
> > > > > +
> > > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > > +
> > > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > > +       {                                                               \
> > > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > > +       }                                                               \
> > > > > +
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > > +
> > > > > +/* Formats */
> > > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > > +
> > > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > > +       &format_attr_event.attr,
> > > > > +       &format_attr_partial_matching.attr,
> > > > > +       &format_attr_pid_pscid.attr,
> > > > > +       &format_attr_did_gscid.attr,
> > > > > +       &format_attr_filter_pid_pscid.attr,
> > > > > +       &format_attr_filter_did_gscid.attr,
> > > > > +       &format_attr_filter_id_type.attr,
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > > +       .name = "format",
> > > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > > +};
> > > > > +
> > > > > +/* Events */
> > > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > > +                                         struct device_attribute *attr,
> > > > > +                                         char *page)
> > > > > +{
> > > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > > +
> > > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > > +
> > > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > > +}
> > > > > +
> > > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > > +
> > > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > > +       .name = "events",
> > > > > +       .attrs = riscv_iommu_pmu_events,
> > > > > +};
> > > > > +
> > > > > +/* cpumask */
> > > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > > +                                       struct device_attribute *attr,
> > > > > +                                       char *buf)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > > +
> > > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > > +}
> > > > > +
> > > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > > +
> > > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > > +       NULL
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > > +       &riscv_iommu_pmu_format_group,
> > > > > +       &riscv_iommu_pmu_events_group,
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +/* PMU Operations */
> > > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > +                                       u64 value)
> > > > > +{
> > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > +
> > > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > >
> > > > For better compatibility, it is recommended to use writel and readl
> > > > instead of readq and writeq. Sorry for overlooking this last time —
> > > > could you please make the change again?
> > > > In the future, your driver could support RV32 directly, as the RISC-V
> > > > IOMMU specification also supports RV32; it's just that the current
> > > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> > >
> > > Hi Guo,
> > > Thank you for your review. In my implementation for RV32, readq and
> > > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > > However, as you mentioned before: 64-bit access might be undefined on
> > > some hardware. To be safe and compatible with all hardware, we should
> > > use 32-bit access instead of 64-bit access.
> > > I will modify them in the next version
> > >
> > > I'd like to make sure I didn't miss anything important. My original
> > > code uses the "hi-lo" order. May I ask if there is a specific reason
> > > why you prefer the "lo-hi" order?
> > > Thanks
> > >
> >
> > Hi all,
> > I have been thinking more about the IOMMU register access behavior and
> > would like to share my thoughts regarding the lo-hi vs. hi-lo access
> > order.
> >
> > It seems to me that the correct 32-bit write order is strictly tied to
> > the underlying hardware implementation. From the software side, we
> > have no way of knowing whether a specific vendor designed their
> > hardware with a low-part trigger or a high-part trigger. Therefore,
> > hardcoding a fixed 32-bit write sequence could result in compatibility
> > issues across different hardware designs.
> >
> > Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> > this might imply that the hardware itself will handle the split into
> > two 4-byte accesses and internally manage its own trigger logic.
> > Because of this, for RV64 systems, software might simply use standard
> > 8-byte writes (e.g., writeq). Delegating the access handling to the
> > hardware is likely the best way to maximize compatibility across all
> > vendor implementations. For the counter reading strategy (hi-lo-hi
> > pattern), I completely agree with using the hi-lo-hi pattern when
> > reading the PMU counters. Since this is a read operation and does not
> > alter hardware state.
> >
> > For RV32 support, we inevitably face the 32-bit access splitting
> > issue. For the RV32 case, maybe we can assume the hi-lo order as the
> > default fallback, because the IOMMU specification uses this sequence
> > as example: "an access may appear, internally to the IOMMU, as if two
> > separate 4 byte accesses - first to the high half and second to the
> > low half - were performed.". Or do you prefer to use the same
> > assumption (hi-lo pattern) in RV64 as well?
> >
> > Please let me know what you think about this.
> >
> > Thanks
>
> You're right, RV32 is not a valid argument here — my wording was
> incorrect. Thanks for pointing that out.
> Also, I have no strong preference for lo-hi; that was just a careless
> copy-paste on my part. I agree hi-lo is the right choice.
>

May I confirm if you still expect to use the hi-lo pattern to 'write'
to 64-bit registers in RV64? Or would you agree that we should let the
hardware handle this behavior, since software couldn't know the high
part/low part trigger behavior?

> Thanks again for your great work.
>
> --
> Best Regards
>  Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Guo Ren]

On Tue, Jul 7, 2026 at 11:49 AM Zong Li <zong.li@sifive.com> wrote:
>
> On Tue, Jul 7, 2026 at 10:58 AM Guo Ren <guoren@kernel.org> wrote:
> >
> > On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
> > >
> > > On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> > > >
> > > > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > > > >
> > > > > Hi Zong Li,
> > > > >
> > > > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > > > >
> > > > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > > > device driver created by the parent RISC-V IOMMU driver.
> > > > > >
> > > > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > > > cycle event is required for the perf subsystem.
> > > > > >
> > > > > > The number and width of the counters are hardware-implemented and must
> > > > > > be detected at runtime.
> > > > > >
> > > > > > The performance monitor provides counters with filtering support to
> > > > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > > > >
> > > > > > PMU-related definitions are moved into the perf driver, where they are
> > > > > > used exclusively.
> > > > > >
> > > > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > > > ---
> > > > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > > > >  drivers/perf/Kconfig             |  12 +
> > > > > >  drivers/perf/Makefile            |   1 +
> > > > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > > > >
> > > > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > > > >
> > > > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > > -
> > > > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > > > -
> > > > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > > -
> > > > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > > > -
> > > > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > > -
> > > > > > -/* Number of defined performance-monitoring event selectors */
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > > > -
> > > > > > -/**
> > > > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > > - *
> > > > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > > - */
> > > > > > -enum riscv_iommu_hpmevent_id {
> > > > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > > -};
> > > > > > -
> > > > > >  /* 5.24 Translation request IOVA (64bits) */
> > > > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > > > --- a/drivers/perf/Kconfig
> > > > > > +++ b/drivers/perf/Kconfig
> > > > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > > > >           full perf feature support i.e. counter overflow, privilege mode
> > > > > >           filtering, counter configuration.
> > > > > >
> > > > > > +config RISCV_IOMMU_PMU
> > > > > > +       depends on RISCV || COMPILE_TEST
> > > > > > +       depends on RISCV_IOMMU
> > > > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > > > +       default y
> > > > > > +       help
> > > > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > > > +         implementation. The performance monitor is an optional hardware
> > > > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > > > +         hardware support. If the underlying hardware does not implement
> > > > > > +         the PMU, this option will have no effect.
> > > > > > +
> > > > > >  config STARFIVE_STARLINK_PMU
> > > > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > > > >         depends on 64BIT
> > > > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > > > --- a/drivers/perf/Makefile
> > > > > > +++ b/drivers/perf/Makefile
> > > > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > > > new file mode 100644
> > > > > > index 000000000000..2e856289ddc9
> > > > > > --- /dev/null
> > > > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > > > @@ -0,0 +1,703 @@
> > > > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > > > +/*
> > > > > > + * Copyright (C) 2026 SiFive
> > > > > > + *
> > > > > > + * Authors
> > > > > > + *     Zong Li <zong.li@sifive.com>
> > > > > > + */
> > > > > > +
> > > > > > +#include <linux/auxiliary_bus.h>
> > > > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > > > +#include <linux/perf_event.h>
> > > > > > +
> > > > > > +#include "../iommu/riscv/iommu.h"
> > > > > > +
> > > > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > > +
> > > > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > > > +
> > > > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > > > +
> > > > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > > > +
> > > > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > > > +
> > > > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > > > +
> > > > > > +static int cpuhp_state;
> > > > > > +
> > > > > > +/**
> > > > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > > + *
> > > > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > > + *
> > > > > > + * The specification does not define an event ID for counting the
> > > > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > > > + * event is not actually written into any register, it serves solely
> > > > > > + * as an identifier.
> > > > > > + */
> > > > > > +enum riscv_iommu_hpmevent_id {
> > > > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > > +};
> > > > > > +
> > > > > > +struct riscv_iommu_pmu {
> > > > > > +       struct pmu pmu;
> > > > > > +       struct hlist_node node;
> > > > > > +       void __iomem *reg;
> > > > > > +       unsigned int on_cpu;
> > > > > > +       int num_counters;
> > > > > > +       u64 cycle_cntr_mask;
> > > > > > +       u64 event_cntr_mask;
> > > > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +};
> > > > > > +
> > > > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > > > +
> > > > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > > > +       {                                                               \
> > > > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > > > +       }                                                               \
> > > > > > +
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > > > +
> > > > > > +/* Formats */
> > > > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > > > +       &format_attr_event.attr,
> > > > > > +       &format_attr_partial_matching.attr,
> > > > > > +       &format_attr_pid_pscid.attr,
> > > > > > +       &format_attr_did_gscid.attr,
> > > > > > +       &format_attr_filter_pid_pscid.attr,
> > > > > > +       &format_attr_filter_did_gscid.attr,
> > > > > > +       &format_attr_filter_id_type.attr,
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > > > +       .name = "format",
> > > > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > > > +};
> > > > > > +
> > > > > > +/* Events */
> > > > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > > > +                                         struct device_attribute *attr,
> > > > > > +                                         char *page)
> > > > > > +{
> > > > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > > > +
> > > > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > > > +
> > > > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > > > +}
> > > > > > +
> > > > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > > > +       .name = "events",
> > > > > > +       .attrs = riscv_iommu_pmu_events,
> > > > > > +};
> > > > > > +
> > > > > > +/* cpumask */
> > > > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > > > +                                       struct device_attribute *attr,
> > > > > > +                                       char *buf)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > > > +
> > > > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > > > +}
> > > > > > +
> > > > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > > > +       NULL
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > > > +       &riscv_iommu_pmu_format_group,
> > > > > > +       &riscv_iommu_pmu_events_group,
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +/* PMU Operations */
> > > > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > > +                                       u64 value)
> > > > > > +{
> > > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > > +
> > > > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > >
> > > > > For better compatibility, it is recommended to use writel and readl
> > > > > instead of readq and writeq. Sorry for overlooking this last time —
> > > > > could you please make the change again?
> > > > > In the future, your driver could support RV32 directly, as the RISC-V
> > > > > IOMMU specification also supports RV32; it's just that the current
> > > > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> > > >
> > > > Hi Guo,
> > > > Thank you for your review. In my implementation for RV32, readq and
> > > > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > > > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > > > However, as you mentioned before: 64-bit access might be undefined on
> > > > some hardware. To be safe and compatible with all hardware, we should
> > > > use 32-bit access instead of 64-bit access.
> > > > I will modify them in the next version
> > > >
> > > > I'd like to make sure I didn't miss anything important. My original
> > > > code uses the "hi-lo" order. May I ask if there is a specific reason
> > > > why you prefer the "lo-hi" order?
> > > > Thanks
> > > >
> > >
> > > Hi all,
> > > I have been thinking more about the IOMMU register access behavior and
> > > would like to share my thoughts regarding the lo-hi vs. hi-lo access
> > > order.
> > >
> > > It seems to me that the correct 32-bit write order is strictly tied to
> > > the underlying hardware implementation. From the software side, we
> > > have no way of knowing whether a specific vendor designed their
> > > hardware with a low-part trigger or a high-part trigger. Therefore,
> > > hardcoding a fixed 32-bit write sequence could result in compatibility
> > > issues across different hardware designs.
> > >
> > > Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> > > this might imply that the hardware itself will handle the split into
> > > two 4-byte accesses and internally manage its own trigger logic.
> > > Because of this, for RV64 systems, software might simply use standard
> > > 8-byte writes (e.g., writeq). Delegating the access handling to the
> > > hardware is likely the best way to maximize compatibility across all
> > > vendor implementations. For the counter reading strategy (hi-lo-hi
> > > pattern), I completely agree with using the hi-lo-hi pattern when
> > > reading the PMU counters. Since this is a read operation and does not
> > > alter hardware state.
> > >
> > > For RV32 support, we inevitably face the 32-bit access splitting
> > > issue. For the RV32 case, maybe we can assume the hi-lo order as the
> > > default fallback, because the IOMMU specification uses this sequence
> > > as example: "an access may appear, internally to the IOMMU, as if two
> > > separate 4 byte accesses - first to the high half and second to the
> > > low half - were performed.". Or do you prefer to use the same
> > > assumption (hi-lo pattern) in RV64 as well?
> > >
> > > Please let me know what you think about this.
> > >
> > > Thanks
> >
> > You're right, RV32 is not a valid argument here — my wording was
> > incorrect. Thanks for pointing that out.
> > Also, I have no strong preference for lo-hi; that was just a careless
> > copy-paste on my part. I agree hi-lo is the right choice.
> >
>
> May I confirm if you still expect to use the hi-lo pattern to 'write'
> to 64-bit registers in RV64? Or would you agree that we should let the
> hardware handle this behavior, since software couldn't know the high
> part/low part trigger behavior?

1. I expect to use the hi-lo pattern to 'write' to 64-bit registers in
RV64. That is hi_lo_writeq or hi_lo_readq.
2. I don't agree to let the hardware handle this behavior; 64-bit MMIO
access is controversial for RISC-V IOMMU.

-- 
Best Regards
 Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Guo Ren]

On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
>
> On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> >
> > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > >
> > > Hi Zong Li,
> > >
> > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > >
> > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > device driver created by the parent RISC-V IOMMU driver.
> > > >
> > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > cycle event is required for the perf subsystem.
> > > >
> > > > The number and width of the counters are hardware-implemented and must
> > > > be detected at runtime.
> > > >
> > > > The performance monitor provides counters with filtering support to
> > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > >
> > > > PMU-related definitions are moved into the perf driver, where they are
> > > > used exclusively.
> > > >
> > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > ---
> > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > >  drivers/perf/Kconfig             |  12 +
> > > >  drivers/perf/Makefile            |   1 +
> > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > >
> > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > >
> > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > -
> > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > -
> > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > -
> > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > -
> > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > -
> > > > -/* Number of defined performance-monitoring event selectors */
> > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > -
> > > > -/**
> > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > - *
> > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > - */
> > > > -enum riscv_iommu_hpmevent_id {
> > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > -};
> > > > -
> > > >  /* 5.24 Translation request IOVA (64bits) */
> > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > --- a/drivers/perf/Kconfig
> > > > +++ b/drivers/perf/Kconfig
> > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > >           full perf feature support i.e. counter overflow, privilege mode
> > > >           filtering, counter configuration.
> > > >
> > > > +config RISCV_IOMMU_PMU
> > > > +       depends on RISCV || COMPILE_TEST
> > > > +       depends on RISCV_IOMMU
> > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > +       default y
> > > > +       help
> > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > +         implementation. The performance monitor is an optional hardware
> > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > +         hardware support. If the underlying hardware does not implement
> > > > +         the PMU, this option will have no effect.
> > > > +
> > > >  config STARFIVE_STARLINK_PMU
> > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > >         depends on 64BIT
> > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > --- a/drivers/perf/Makefile
> > > > +++ b/drivers/perf/Makefile
> > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > new file mode 100644
> > > > index 000000000000..2e856289ddc9
> > > > --- /dev/null
> > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > @@ -0,0 +1,703 @@
> > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > +/*
> > > > + * Copyright (C) 2026 SiFive
> > > > + *
> > > > + * Authors
> > > > + *     Zong Li <zong.li@sifive.com>
> > > > + */
> > > > +
> > > > +#include <linux/auxiliary_bus.h>
> > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > +#include <linux/perf_event.h>
> > > > +
> > > > +#include "../iommu/riscv/iommu.h"
> > > > +
> > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > +
> > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > +
> > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > +
> > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > +
> > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > +
> > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > +
> > > > +static int cpuhp_state;
> > > > +
> > > > +/**
> > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > + *
> > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > + *
> > > > + * The specification does not define an event ID for counting the
> > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > + * event is not actually written into any register, it serves solely
> > > > + * as an identifier.
> > > > + */
> > > > +enum riscv_iommu_hpmevent_id {
> > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > +};
> > > > +
> > > > +struct riscv_iommu_pmu {
> > > > +       struct pmu pmu;
> > > > +       struct hlist_node node;
> > > > +       void __iomem *reg;
> > > > +       unsigned int on_cpu;
> > > > +       int num_counters;
> > > > +       u64 cycle_cntr_mask;
> > > > +       u64 event_cntr_mask;
> > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > +};
> > > > +
> > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > +
> > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > +       {                                                               \
> > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > +       }                                                               \
> > > > +
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > +
> > > > +/* Formats */
> > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > +
> > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > +       &format_attr_event.attr,
> > > > +       &format_attr_partial_matching.attr,
> > > > +       &format_attr_pid_pscid.attr,
> > > > +       &format_attr_did_gscid.attr,
> > > > +       &format_attr_filter_pid_pscid.attr,
> > > > +       &format_attr_filter_did_gscid.attr,
> > > > +       &format_attr_filter_id_type.attr,
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > +       .name = "format",
> > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > +};
> > > > +
> > > > +/* Events */
> > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > +                                         struct device_attribute *attr,
> > > > +                                         char *page)
> > > > +{
> > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > +
> > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > +
> > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > +}
> > > > +
> > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > +
> > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > +       .name = "events",
> > > > +       .attrs = riscv_iommu_pmu_events,
> > > > +};
> > > > +
> > > > +/* cpumask */
> > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > +                                       struct device_attribute *attr,
> > > > +                                       char *buf)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > +
> > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > +}
> > > > +
> > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > +
> > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > +       NULL
> > > > +};
> > > > +
> > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > +};
> > > > +
> > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > +       &riscv_iommu_pmu_format_group,
> > > > +       &riscv_iommu_pmu_events_group,
> > > > +       NULL,
> > > > +};
> > > > +
> > > > +/* PMU Operations */
> > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > +                                       u64 value)
> > > > +{
> > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > +
> > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > >
> > > For better compatibility, it is recommended to use writel and readl
> > > instead of readq and writeq. Sorry for overlooking this last time —
> > > could you please make the change again?
> > > In the future, your driver could support RV32 directly, as the RISC-V
> > > IOMMU specification also supports RV32; it's just that the current
> > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> >
> > Hi Guo,
> > Thank you for your review. In my implementation for RV32, readq and
> > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > However, as you mentioned before: 64-bit access might be undefined on
> > some hardware. To be safe and compatible with all hardware, we should
> > use 32-bit access instead of 64-bit access.
> > I will modify them in the next version
> >
> > I'd like to make sure I didn't miss anything important. My original
> > code uses the "hi-lo" order. May I ask if there is a specific reason
> > why you prefer the "lo-hi" order?
> > Thanks
> >
>
> Hi all,
> I have been thinking more about the IOMMU register access behavior and
> would like to share my thoughts regarding the lo-hi vs. hi-lo access
> order.
>
> It seems to me that the correct 32-bit write order is strictly tied to
> the underlying hardware implementation. From the software side, we
> have no way of knowing whether a specific vendor designed their
> hardware with a low-part trigger or a high-part trigger. Therefore,
> hardcoding a fixed 32-bit write sequence could result in compatibility
> issues across different hardware designs.
>
> Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> this might imply that the hardware itself will handle the split into
> two 4-byte accesses and internally manage its own trigger logic.
> Because of this, for RV64 systems, software might simply use standard
> 8-byte writes (e.g., writeq). Delegating the access handling to the
> hardware is likely the best way to maximize compatibility across all
> vendor implementations. For the counter reading strategy (hi-lo-hi
> pattern), I completely agree with using the hi-lo-hi pattern when
> reading the PMU counters. Since this is a read operation and does not
> alter hardware state.
Apologies for overlooking this point. Using hi_lo_writeq/readq
explicitly enforces access ordering to prevent hardware reordering,
while maintaining RV32 compatibility. This ensures consistent 64-bit
MMIO register access semantics across both RV32 and RV64. Falling back
to 32-bit MMIO access remains the best approach, as it avoids any
ambiguity or controversy.

>
> For RV32 support, we inevitably face the 32-bit access splitting
> issue. For the RV32 case, maybe we can assume the hi-lo order as the
> default fallback, because the IOMMU specification uses this sequence
> as example: "an access may appear, internally to the IOMMU, as if two
> separate 4 byte accesses - first to the high half and second to the
> low half - were performed.". Or do you prefer to use the same
> assumption (hi-lo pattern) in RV64 as well?
>
> Please let me know what you think about this.
>
> Thanks
>
>
> > >
> > > > +}
> > > > +
> > > > +/*
> > > > + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> > > > + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> > > > + * is UNSPECIFIED, and such an access may appear, internally to the
> > > > + * IOMMU, as if two separate 4 byte accesses - first to the high half
> > > > + * and second to the low half - were performed
> > > > + *
> > > > + * To make sure the driver works correctly on different hardware,
> > > > + * the software will always use two 4-byte access for the counter.
> > > > + *
> > > > + * This function implements the hi-lo-hi pattern to detect and handle
> > > > + * wraparound during the read operation:
> > > > + *   1. Read high half (hi)
> > > > + *   2. Read low half (lo)
> > > > + *   3. Check if low half wrapped or high half changed:
> > > > + *      - If lo <= lo_prev: possible wraparound occurred
> > > > + *      - If hi != hi_prev: high half changed during read
> > > > + *   4. If wraparound detected, re-read high half and assume low half is 0
> > > > + *   5. Update previous values for next read
> > > > + */
> > > > +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > +{
> > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> > > > +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > +       u32 hi, lo;
> > > > +
> > > > +       hi = readl(addr + 4);
> > > > +       lo = readl(addr);
> > > > +
> > > > +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> > > > +               u32 hi_tmp = readl(addr + 4);
> > > > +
> > > > +               /*
> > > > +                * If hi changes, then hi+1:0 must have happened while
> > > > +                * the code was running so it is a safe return value
> > > > +                */
> > > > +               if (hi_tmp != hi) {
> > > > +                       hi = hi_tmp;
> > > > +                       lo = 0;
> > > > +               }
> > > > +               pmu->lo_prev[idx] = ~0u;
> > > > +               pmu->hi_prev[idx] = hi;
> > > > +       }
> > > > +       pmu->lo_prev[idx] = lo;
> > > > +
> > > > +       value = (((u64)hi << 32) | lo) & counter_mask;
> > > > +
> > > > +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> > > > +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> > > > +}
> > > Yes! Thanks for your implementation.
> > >
> > > > +
> > > > +static bool is_cycle_event(u64 event)
> > > > +{
> > > > +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > +                                     u64 value)
> > > > +{
> > > > +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> > > > +       if (is_cycle_event(value))
> > > > +               return;
> > > > +
> > > > +       /* Event counter start from idx 1 */
> > > > +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > > +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > >        lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > >               pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > >
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > +{
> > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > +       u32 value = readl(addr);
> > > > +
> > > > +       writel(value & ~BIT(idx), addr);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > +{
> > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > +       u32 value = readl(addr);
> > > > +
> > > > +       writel(value | BIT(idx), addr);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> > > > +{
> > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > +       u32 used_cntr = 0;
> > > > +
> > > > +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> > > > +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> > > > +
> > > > +       writel(~used_cntr, addr);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> > > > +{
> > > > +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> > > > +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> > > > +}
> > > > +
> > > > +/* PMU APIs */
> > > > +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > +       u64 period;
> > > > +
> > > > +       /*
> > > > +        * Limit the maximum period to prevent the counter value
> > > > +        * from overtaking the one we are about to program.
> > > > +        * In effect we are reducing max_period to account for
> > > > +        * interrupt latency (and we are being very conservative).
> > > > +        */
> > > > +       period = counter_mask >> 1;
> > > > +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> > > > +       local64_set(&hwc->prev_count, period);
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       struct perf_event *sibling;
> > > > +       int total_event_counters = pmu->num_counters - 1;
> > > > +       int counters = 0;
> > > > +
> > > > +       if (event->attr.type != event->pmu->type)
> > > > +               return -ENOENT;
> > > > +
> > > > +       if (hwc->sample_period)
> > > > +               return -EOPNOTSUPP;
> > > > +
> > > > +       if (event->cpu < 0)
> > > > +               return -EOPNOTSUPP;
> > > > +
> > > > +       event->cpu = pmu->on_cpu;
> > > > +
> > > > +       hwc->idx = -1;
> > > > +       hwc->config = event->attr.config;
> > > > +
> > > > +       if (event->group_leader == event)
> > > > +               return 0;
> > > > +
> > > > +       if (is_cycle_event(get_event(event->group_leader)))
> > > > +               if (++counters > total_event_counters)
> > > > +                       return -EINVAL;
> > > > +
> > > > +       for_each_sibling_event(sibling, event->group_leader) {
> > > > +               if (is_cycle_event(get_event(sibling)))
> > > > +                       continue;
> > > > +
> > > > +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> > > > +                       return -EINVAL;
> > > > +
> > > > +               if (++counters > total_event_counters)
> > > > +                       return -EINVAL;
> > > > +       }
> > > > +
> > > > +       return 0;
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_update(struct perf_event *event)
> > > > +{
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       u64 delta, prev, now;
> > > > +       u32 idx = hwc->idx;
> > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > +
> > > > +       do {
> > > > +               prev = local64_read(&hwc->prev_count);
> > > > +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> > > > +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> > > > +
> > > > +       delta = (now - prev) & counter_mask;
> > > > +       local64_add(delta, &event->count);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +
> > > > +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> > > > +               return;
> > > > +
> > > > +       if (flags & PERF_EF_RELOAD)
> > > > +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> > > > +
> > > > +       hwc->state = 0;
> > > > +       riscv_iommu_pmu_set_period(event);
> > > > +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> > > > +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> > > > +
> > > > +       perf_event_update_userpage(event);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       int idx = hwc->idx;
> > > > +
> > > > +       if (hwc->state & PERF_HES_STOPPED)
> > > > +               return;
> > > > +
> > > > +       riscv_iommu_pmu_disable_counter(pmu, idx);
> > > > +
> > > > +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> > > > +               riscv_iommu_pmu_update(event);
> > > > +
> > > > +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       unsigned int num_counters = pmu->num_counters;
> > > > +       int idx;
> > > > +
> > > > +       /* Reserve index zero for iohpmcycles */
> > > > +       if (is_cycle_event(get_event(event)))
> > > > +               idx = 0;
> > > > +       else
> > > > +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> > > > +
> > > > +       /* All event counters or cycle counter are in use */
> > > > +       if (idx == num_counters || pmu->events[idx])
> > > > +               return -EAGAIN;
> > > > +
> > > > +       set_bit(idx, pmu->used_counters);
> > > > +
> > > > +       pmu->events[idx] = event;
> > > > +       hwc->idx = idx;
> > > > +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > +       local64_set(&hwc->prev_count, 0);
> > > > +
> > > > +       if (flags & PERF_EF_START)
> > > > +               riscv_iommu_pmu_start(event, flags);
> > > > +
> > > > +       /* Propagate changes to the userspace mapping. */
> > > > +       perf_event_update_userpage(event);
> > > > +
> > > > +       return 0;
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_read(struct perf_event *event)
> > > > +{
> > > > +       riscv_iommu_pmu_update(event);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > +       int idx = hwc->idx;
> > > > +
> > > > +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> > > > +       pmu->events[idx] = NULL;
> > > > +       clear_bit(idx, pmu->used_counters);
> > > > +
> > > > +       perf_event_update_userpage(event);
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> > > > +{
> > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > +
> > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > +
> > > > +       if (iommu_pmu->on_cpu == -1)
> > > > +               iommu_pmu->on_cpu = cpu;
> > > > +
> > > > +       return 0;
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> > > > +{
> > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > +       unsigned int target_cpu;
> > > > +
> > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > +
> > > > +       if (cpu != iommu_pmu->on_cpu)
> > > > +               return 0;
> > > > +
> > > > +       iommu_pmu->on_cpu = -1;
> > > > +
> > > > +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> > > > +       if (target_cpu >= nr_cpu_ids)
> > > > +               return 0;
> > > > +
> > > > +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> > > > +       iommu_pmu->on_cpu = target_cpu;
> > > > +
> > > > +       return 0;
> > > > +}
> > > > +
> > > > +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> > > > +{
> > > > +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> > > > +       int idx;
> > > > +
> > > > +       if (!ovf)
> > > > +               return IRQ_NONE;
> > > > +
> > > > +       riscv_iommu_pmu_stop_all(pmu);
> > > > +
> > > > +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> > > > +               struct perf_event *event = pmu->events[idx];
> > > > +
> > > > +               if (WARN_ON_ONCE(!event))
> > > > +                       continue;
> > > > +
> > > > +               riscv_iommu_pmu_update(event);
> > > > +               riscv_iommu_pmu_set_period(event);
> > > > +       }
> > > > +
> > > > +       riscv_iommu_pmu_start_all(pmu);
> > > > +
> > > > +       return IRQ_HANDLED;
> > > > +}
> > > > +
> > > > +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> > > > +{
> > > > +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> > > > +       irqreturn_t ret;
> > > > +
> > > > +       /* Check whether this interrupt is for PMU */
> > > > +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> > > > +               return IRQ_NONE;
> > > > +
> > > > +       /* Process PMU IRQ */
> > > > +       ret = riscv_iommu_pmu_handle_irq(pmu);
> > > > +
> > > > +       /* Clear performance monitoring interrupt pending bit */
> > > > +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> > > > +
> > > > +       return ret;
> > > > +}
> > > > +
> > > > +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> > > > +{
> > > > +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> > > > +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> > > > +
> > > > +       return iommu->irqs[vec];
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> > > > +                                      struct riscv_iommu_pmu *pmu)
> > > > +{
> > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > +
> > > > +       /*
> > > > +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> > > > +        * other threaded IRQs by other queues.
> > > > +        */
> > > > +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> > > > +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> > > > +}
> > > > +
> > > > +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> > > > +                                    struct riscv_iommu_pmu *pmu)
> > > > +{
> > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > +
> > > > +       free_irq(irq, pmu);
> > > > +}
> > > > +
> > > > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > > > +                                const struct auxiliary_device_id *id)
> > > > +{
> > > > +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > +       void __iomem *addr;
> > > > +       char *name;
> > > > +       int ret;
> > > > +
> > > > +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > > > +       if (!iommu_pmu)
> > > > +               return -ENOMEM;
> > > > +
> > > > +       iommu_pmu->reg = iommu_dev->reg;
> > > > +
> > > > +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > > > +       iommu_pmu->num_counters = hweight32(readl(addr));
> > > > +
> > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > > > +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > >
> > > > +       iommu_pmu->cycle_cntr_mask = readq(addr);
> > > iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
> > >
> > > > +
> > > > +       /* Assume the width of all event counters are the same */
> > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > > > +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > >
> > > > +       iommu_pmu->event_cntr_mask = readq(addr);
> > >  iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
> > >
> > > > +
> > > > +       iommu_pmu->pmu = (struct pmu) {
> > > > +               .module         = THIS_MODULE,
> > > > +               .parent         = &auxdev->dev,
> > > > +               .task_ctx_nr    = perf_invalid_context,
> > > > +               .event_init     = riscv_iommu_pmu_event_init,
> > > > +               .add            = riscv_iommu_pmu_add,
> > > > +               .del            = riscv_iommu_pmu_del,
> > > > +               .start          = riscv_iommu_pmu_start,
> > > > +               .stop           = riscv_iommu_pmu_stop,
> > > > +               .read           = riscv_iommu_pmu_read,
> > > > +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> > > > +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > > > +       };
> > > > +
> > > > +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> > > > +
> > > > +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > > > +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> > > > +       if (!name) {
> > > > +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> > > > +                       dev_name(iommu_dev->dev));
> > > > +               return -ENOMEM;
> > > > +       }
> > > > +
> > > > +       /* Bind all events to the same cpu context to avoid race enabling */
> > > > +       iommu_pmu->on_cpu = raw_smp_processor_id();
> > > > +
> > > > +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > +       if (ret) {
> > > > +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > > > +               return ret;
> > > > +       }
> > > > +
> > > > +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> > > > +       if (ret) {
> > > > +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > > > +               goto err_cpuhp_remove;
> > > > +       }
> > > > +
> > > > +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > > > +       if (ret) {
> > > > +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > > > +               goto err_free_irq;
> > > > +       }
> > > > +
> > > > +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> > > > +                name, iommu_pmu->num_counters);
> > > > +
> > > > +       return 0;
> > > > +
> > > > +err_free_irq:
> > > > +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> > > > +err_cpuhp_remove:
> > > > +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > +       return ret;
> > > > +}
> > > > +
> > > > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > > > +       { .name = "iommu.pmu" },
> > > > +       {}
> > > > +};
> > > > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > > > +
> > > > +static struct auxiliary_driver iommu_pmu_driver = {
> > > > +       .probe          = riscv_iommu_pmu_probe,
> > > > +       .id_table       = riscv_iommu_pmu_id_table,
> > > > +};
> > > > +
> > > > +static int __init riscv_iommu_pmu_init(void)
> > > > +{
> > > > +       int ret;
> > > > +
> > > > +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > > > +                                             "perf/riscv/iommu:online",
> > > > +                                             riscv_iommu_pmu_online_cpu,
> > > > +                                             riscv_iommu_pmu_offline_cpu);
> > > > +       if (cpuhp_state < 0)
> > > > +               return cpuhp_state;
> > > > +
> > > > +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> > > > +       if (ret)
> > > > +               cpuhp_remove_multi_state(cpuhp_state);
> > > > +
> > > > +       return ret;
> > > > +}
> > > > +module_init(riscv_iommu_pmu_init);
> > > > +
> > > > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > > > +MODULE_LICENSE("GPL");
> > > > --
> > > > 2.43.7
> > > >
> > >
> > > --
> > > Best Regards
> > >  Guo Ren



-- 
Best Regards
 Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Zong Li]

On Tue, Jul 7, 2026 at 2:03 PM Guo Ren <guoren@kernel.org> wrote:
>
> On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
> >
> > On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> > >
> > > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > > >
> > > > Hi Zong Li,
> > > >
> > > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > > >
> > > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > > device driver created by the parent RISC-V IOMMU driver.
> > > > >
> > > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > > cycle event is required for the perf subsystem.
> > > > >
> > > > > The number and width of the counters are hardware-implemented and must
> > > > > be detected at runtime.
> > > > >
> > > > > The performance monitor provides counters with filtering support to
> > > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > > >
> > > > > PMU-related definitions are moved into the perf driver, where they are
> > > > > used exclusively.
> > > > >
> > > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > > ---
> > > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > > >  drivers/perf/Kconfig             |  12 +
> > > > >  drivers/perf/Makefile            |   1 +
> > > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > > >
> > > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > > >
> > > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > -
> > > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > > -
> > > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > -
> > > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > > -
> > > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > -
> > > > > -/* Number of defined performance-monitoring event selectors */
> > > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > > -
> > > > > -/**
> > > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > - *
> > > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > - */
> > > > > -enum riscv_iommu_hpmevent_id {
> > > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > -};
> > > > > -
> > > > >  /* 5.24 Translation request IOVA (64bits) */
> > > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > > --- a/drivers/perf/Kconfig
> > > > > +++ b/drivers/perf/Kconfig
> > > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > > >           full perf feature support i.e. counter overflow, privilege mode
> > > > >           filtering, counter configuration.
> > > > >
> > > > > +config RISCV_IOMMU_PMU
> > > > > +       depends on RISCV || COMPILE_TEST
> > > > > +       depends on RISCV_IOMMU
> > > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > > +       default y
> > > > > +       help
> > > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > > +         implementation. The performance monitor is an optional hardware
> > > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > > +         hardware support. If the underlying hardware does not implement
> > > > > +         the PMU, this option will have no effect.
> > > > > +
> > > > >  config STARFIVE_STARLINK_PMU
> > > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > > >         depends on 64BIT
> > > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > > --- a/drivers/perf/Makefile
> > > > > +++ b/drivers/perf/Makefile
> > > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > > new file mode 100644
> > > > > index 000000000000..2e856289ddc9
> > > > > --- /dev/null
> > > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > > @@ -0,0 +1,703 @@
> > > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > > +/*
> > > > > + * Copyright (C) 2026 SiFive
> > > > > + *
> > > > > + * Authors
> > > > > + *     Zong Li <zong.li@sifive.com>
> > > > > + */
> > > > > +
> > > > > +#include <linux/auxiliary_bus.h>
> > > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > > +#include <linux/perf_event.h>
> > > > > +
> > > > > +#include "../iommu/riscv/iommu.h"
> > > > > +
> > > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > +
> > > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > > +
> > > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > > +
> > > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > > +
> > > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > > +
> > > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > > +
> > > > > +static int cpuhp_state;
> > > > > +
> > > > > +/**
> > > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > + *
> > > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > + *
> > > > > + * The specification does not define an event ID for counting the
> > > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > > + * event is not actually written into any register, it serves solely
> > > > > + * as an identifier.
> > > > > + */
> > > > > +enum riscv_iommu_hpmevent_id {
> > > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > +};
> > > > > +
> > > > > +struct riscv_iommu_pmu {
> > > > > +       struct pmu pmu;
> > > > > +       struct hlist_node node;
> > > > > +       void __iomem *reg;
> > > > > +       unsigned int on_cpu;
> > > > > +       int num_counters;
> > > > > +       u64 cycle_cntr_mask;
> > > > > +       u64 event_cntr_mask;
> > > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > +};
> > > > > +
> > > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > > +
> > > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > > +       {                                                               \
> > > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > > +       }                                                               \
> > > > > +
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > > +
> > > > > +/* Formats */
> > > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > > +
> > > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > > +       &format_attr_event.attr,
> > > > > +       &format_attr_partial_matching.attr,
> > > > > +       &format_attr_pid_pscid.attr,
> > > > > +       &format_attr_did_gscid.attr,
> > > > > +       &format_attr_filter_pid_pscid.attr,
> > > > > +       &format_attr_filter_did_gscid.attr,
> > > > > +       &format_attr_filter_id_type.attr,
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > > +       .name = "format",
> > > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > > +};
> > > > > +
> > > > > +/* Events */
> > > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > > +                                         struct device_attribute *attr,
> > > > > +                                         char *page)
> > > > > +{
> > > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > > +
> > > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > > +
> > > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > > +}
> > > > > +
> > > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > > +
> > > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > > +       .name = "events",
> > > > > +       .attrs = riscv_iommu_pmu_events,
> > > > > +};
> > > > > +
> > > > > +/* cpumask */
> > > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > > +                                       struct device_attribute *attr,
> > > > > +                                       char *buf)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > > +
> > > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > > +}
> > > > > +
> > > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > > +
> > > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > > +       NULL
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > > +};
> > > > > +
> > > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > > +       &riscv_iommu_pmu_format_group,
> > > > > +       &riscv_iommu_pmu_events_group,
> > > > > +       NULL,
> > > > > +};
> > > > > +
> > > > > +/* PMU Operations */
> > > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > +                                       u64 value)
> > > > > +{
> > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > +
> > > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > >
> > > > For better compatibility, it is recommended to use writel and readl
> > > > instead of readq and writeq. Sorry for overlooking this last time —
> > > > could you please make the change again?
> > > > In the future, your driver could support RV32 directly, as the RISC-V
> > > > IOMMU specification also supports RV32; it's just that the current
> > > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> > >
> > > Hi Guo,
> > > Thank you for your review. In my implementation for RV32, readq and
> > > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > > However, as you mentioned before: 64-bit access might be undefined on
> > > some hardware. To be safe and compatible with all hardware, we should
> > > use 32-bit access instead of 64-bit access.
> > > I will modify them in the next version
> > >
> > > I'd like to make sure I didn't miss anything important. My original
> > > code uses the "hi-lo" order. May I ask if there is a specific reason
> > > why you prefer the "lo-hi" order?
> > > Thanks
> > >
> >
> > Hi all,
> > I have been thinking more about the IOMMU register access behavior and
> > would like to share my thoughts regarding the lo-hi vs. hi-lo access
> > order.
> >
> > It seems to me that the correct 32-bit write order is strictly tied to
> > the underlying hardware implementation. From the software side, we
> > have no way of knowing whether a specific vendor designed their
> > hardware with a low-part trigger or a high-part trigger. Therefore,
> > hardcoding a fixed 32-bit write sequence could result in compatibility
> > issues across different hardware designs.
> >
> > Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> > this might imply that the hardware itself will handle the split into
> > two 4-byte accesses and internally manage its own trigger logic.
> > Because of this, for RV64 systems, software might simply use standard
> > 8-byte writes (e.g., writeq). Delegating the access handling to the
> > hardware is likely the best way to maximize compatibility across all
> > vendor implementations. For the counter reading strategy (hi-lo-hi
> > pattern), I completely agree with using the hi-lo-hi pattern when
> > reading the PMU counters. Since this is a read operation and does not
> > alter hardware state.
> Apologies for overlooking this point. Using hi_lo_writeq/readq
> explicitly enforces access ordering to prevent hardware reordering,
> while maintaining RV32 compatibility. This ensures consistent 64-bit
> MMIO register access semantics across both RV32 and RV64. Falling back
> to 32-bit MMIO access remains the best approach, as it avoids any
> ambiguity or controversy.
>

I was originally a bit concerned that if our writing order is
different from the hardware design, it might cause some risks.
For example, if we write to the 64-bit register in a high-low order,
but the hardware is triggered by the high part, the hardware might
start working with the "new 32-bit high part" and the "old 32-bit low
part" before we write the new low part.
However, I checked the current driver code, and it seems we might not
need to worry about this for now. This is because we only write to
counter/event registers when the counter is inhibited.
Therefore, I think we can safely use the high-low order to rewrite the driver.
Thank you very much for your discussion.

> >
> > For RV32 support, we inevitably face the 32-bit access splitting
> > issue. For the RV32 case, maybe we can assume the hi-lo order as the
> > default fallback, because the IOMMU specification uses this sequence
> > as example: "an access may appear, internally to the IOMMU, as if two
> > separate 4 byte accesses - first to the high half and second to the
> > low half - were performed.". Or do you prefer to use the same
> > assumption (hi-lo pattern) in RV64 as well?
> >
> > Please let me know what you think about this.
> >
> > Thanks
> >
> >
> > > >
> > > > > +}
> > > > > +
> > > > > +/*
> > > > > + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> > > > > + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> > > > > + * is UNSPECIFIED, and such an access may appear, internally to the
> > > > > + * IOMMU, as if two separate 4 byte accesses - first to the high half
> > > > > + * and second to the low half - were performed
> > > > > + *
> > > > > + * To make sure the driver works correctly on different hardware,
> > > > > + * the software will always use two 4-byte access for the counter.
> > > > > + *
> > > > > + * This function implements the hi-lo-hi pattern to detect and handle
> > > > > + * wraparound during the read operation:
> > > > > + *   1. Read high half (hi)
> > > > > + *   2. Read low half (lo)
> > > > > + *   3. Check if low half wrapped or high half changed:
> > > > > + *      - If lo <= lo_prev: possible wraparound occurred
> > > > > + *      - If hi != hi_prev: high half changed during read
> > > > > + *   4. If wraparound detected, re-read high half and assume low half is 0
> > > > > + *   5. Update previous values for next read
> > > > > + */
> > > > > +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > +{
> > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> > > > > +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > +       u32 hi, lo;
> > > > > +
> > > > > +       hi = readl(addr + 4);
> > > > > +       lo = readl(addr);
> > > > > +
> > > > > +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> > > > > +               u32 hi_tmp = readl(addr + 4);
> > > > > +
> > > > > +               /*
> > > > > +                * If hi changes, then hi+1:0 must have happened while
> > > > > +                * the code was running so it is a safe return value
> > > > > +                */
> > > > > +               if (hi_tmp != hi) {
> > > > > +                       hi = hi_tmp;
> > > > > +                       lo = 0;
> > > > > +               }
> > > > > +               pmu->lo_prev[idx] = ~0u;
> > > > > +               pmu->hi_prev[idx] = hi;
> > > > > +       }
> > > > > +       pmu->lo_prev[idx] = lo;
> > > > > +
> > > > > +       value = (((u64)hi << 32) | lo) & counter_mask;
> > > > > +
> > > > > +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> > > > > +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> > > > > +}
> > > > Yes! Thanks for your implementation.
> > > >
> > > > > +
> > > > > +static bool is_cycle_event(u64 event)
> > > > > +{
> > > > > +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > +                                     u64 value)
> > > > > +{
> > > > > +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> > > > > +       if (is_cycle_event(value))
> > > > > +               return;
> > > > > +
> > > > > +       /* Event counter start from idx 1 */
> > > > > +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > > > +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > > >        lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > >               pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > > >
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > +{
> > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > +       u32 value = readl(addr);
> > > > > +
> > > > > +       writel(value & ~BIT(idx), addr);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > +{
> > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > +       u32 value = readl(addr);
> > > > > +
> > > > > +       writel(value | BIT(idx), addr);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> > > > > +{
> > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > +       u32 used_cntr = 0;
> > > > > +
> > > > > +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> > > > > +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> > > > > +
> > > > > +       writel(~used_cntr, addr);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> > > > > +{
> > > > > +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> > > > > +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> > > > > +}
> > > > > +
> > > > > +/* PMU APIs */
> > > > > +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > +       u64 period;
> > > > > +
> > > > > +       /*
> > > > > +        * Limit the maximum period to prevent the counter value
> > > > > +        * from overtaking the one we are about to program.
> > > > > +        * In effect we are reducing max_period to account for
> > > > > +        * interrupt latency (and we are being very conservative).
> > > > > +        */
> > > > > +       period = counter_mask >> 1;
> > > > > +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> > > > > +       local64_set(&hwc->prev_count, period);
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       struct perf_event *sibling;
> > > > > +       int total_event_counters = pmu->num_counters - 1;
> > > > > +       int counters = 0;
> > > > > +
> > > > > +       if (event->attr.type != event->pmu->type)
> > > > > +               return -ENOENT;
> > > > > +
> > > > > +       if (hwc->sample_period)
> > > > > +               return -EOPNOTSUPP;
> > > > > +
> > > > > +       if (event->cpu < 0)
> > > > > +               return -EOPNOTSUPP;
> > > > > +
> > > > > +       event->cpu = pmu->on_cpu;
> > > > > +
> > > > > +       hwc->idx = -1;
> > > > > +       hwc->config = event->attr.config;
> > > > > +
> > > > > +       if (event->group_leader == event)
> > > > > +               return 0;
> > > > > +
> > > > > +       if (is_cycle_event(get_event(event->group_leader)))
> > > > > +               if (++counters > total_event_counters)
> > > > > +                       return -EINVAL;
> > > > > +
> > > > > +       for_each_sibling_event(sibling, event->group_leader) {
> > > > > +               if (is_cycle_event(get_event(sibling)))
> > > > > +                       continue;
> > > > > +
> > > > > +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> > > > > +                       return -EINVAL;
> > > > > +
> > > > > +               if (++counters > total_event_counters)
> > > > > +                       return -EINVAL;
> > > > > +       }
> > > > > +
> > > > > +       return 0;
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_update(struct perf_event *event)
> > > > > +{
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       u64 delta, prev, now;
> > > > > +       u32 idx = hwc->idx;
> > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > +
> > > > > +       do {
> > > > > +               prev = local64_read(&hwc->prev_count);
> > > > > +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> > > > > +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> > > > > +
> > > > > +       delta = (now - prev) & counter_mask;
> > > > > +       local64_add(delta, &event->count);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +
> > > > > +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> > > > > +               return;
> > > > > +
> > > > > +       if (flags & PERF_EF_RELOAD)
> > > > > +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> > > > > +
> > > > > +       hwc->state = 0;
> > > > > +       riscv_iommu_pmu_set_period(event);
> > > > > +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> > > > > +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> > > > > +
> > > > > +       perf_event_update_userpage(event);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       int idx = hwc->idx;
> > > > > +
> > > > > +       if (hwc->state & PERF_HES_STOPPED)
> > > > > +               return;
> > > > > +
> > > > > +       riscv_iommu_pmu_disable_counter(pmu, idx);
> > > > > +
> > > > > +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> > > > > +               riscv_iommu_pmu_update(event);
> > > > > +
> > > > > +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       unsigned int num_counters = pmu->num_counters;
> > > > > +       int idx;
> > > > > +
> > > > > +       /* Reserve index zero for iohpmcycles */
> > > > > +       if (is_cycle_event(get_event(event)))
> > > > > +               idx = 0;
> > > > > +       else
> > > > > +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> > > > > +
> > > > > +       /* All event counters or cycle counter are in use */
> > > > > +       if (idx == num_counters || pmu->events[idx])
> > > > > +               return -EAGAIN;
> > > > > +
> > > > > +       set_bit(idx, pmu->used_counters);
> > > > > +
> > > > > +       pmu->events[idx] = event;
> > > > > +       hwc->idx = idx;
> > > > > +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > > +       local64_set(&hwc->prev_count, 0);
> > > > > +
> > > > > +       if (flags & PERF_EF_START)
> > > > > +               riscv_iommu_pmu_start(event, flags);
> > > > > +
> > > > > +       /* Propagate changes to the userspace mapping. */
> > > > > +       perf_event_update_userpage(event);
> > > > > +
> > > > > +       return 0;
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_read(struct perf_event *event)
> > > > > +{
> > > > > +       riscv_iommu_pmu_update(event);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > +       int idx = hwc->idx;
> > > > > +
> > > > > +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> > > > > +       pmu->events[idx] = NULL;
> > > > > +       clear_bit(idx, pmu->used_counters);
> > > > > +
> > > > > +       perf_event_update_userpage(event);
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > +
> > > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > > +
> > > > > +       if (iommu_pmu->on_cpu == -1)
> > > > > +               iommu_pmu->on_cpu = cpu;
> > > > > +
> > > > > +       return 0;
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > +       unsigned int target_cpu;
> > > > > +
> > > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > > +
> > > > > +       if (cpu != iommu_pmu->on_cpu)
> > > > > +               return 0;
> > > > > +
> > > > > +       iommu_pmu->on_cpu = -1;
> > > > > +
> > > > > +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> > > > > +       if (target_cpu >= nr_cpu_ids)
> > > > > +               return 0;
> > > > > +
> > > > > +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> > > > > +       iommu_pmu->on_cpu = target_cpu;
> > > > > +
> > > > > +       return 0;
> > > > > +}
> > > > > +
> > > > > +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> > > > > +{
> > > > > +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> > > > > +       int idx;
> > > > > +
> > > > > +       if (!ovf)
> > > > > +               return IRQ_NONE;
> > > > > +
> > > > > +       riscv_iommu_pmu_stop_all(pmu);
> > > > > +
> > > > > +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> > > > > +               struct perf_event *event = pmu->events[idx];
> > > > > +
> > > > > +               if (WARN_ON_ONCE(!event))
> > > > > +                       continue;
> > > > > +
> > > > > +               riscv_iommu_pmu_update(event);
> > > > > +               riscv_iommu_pmu_set_period(event);
> > > > > +       }
> > > > > +
> > > > > +       riscv_iommu_pmu_start_all(pmu);
> > > > > +
> > > > > +       return IRQ_HANDLED;
> > > > > +}
> > > > > +
> > > > > +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> > > > > +{
> > > > > +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> > > > > +       irqreturn_t ret;
> > > > > +
> > > > > +       /* Check whether this interrupt is for PMU */
> > > > > +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> > > > > +               return IRQ_NONE;
> > > > > +
> > > > > +       /* Process PMU IRQ */
> > > > > +       ret = riscv_iommu_pmu_handle_irq(pmu);
> > > > > +
> > > > > +       /* Clear performance monitoring interrupt pending bit */
> > > > > +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> > > > > +
> > > > > +       return ret;
> > > > > +}
> > > > > +
> > > > > +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> > > > > +{
> > > > > +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> > > > > +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> > > > > +
> > > > > +       return iommu->irqs[vec];
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> > > > > +                                      struct riscv_iommu_pmu *pmu)
> > > > > +{
> > > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > > +
> > > > > +       /*
> > > > > +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> > > > > +        * other threaded IRQs by other queues.
> > > > > +        */
> > > > > +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> > > > > +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> > > > > +}
> > > > > +
> > > > > +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> > > > > +                                    struct riscv_iommu_pmu *pmu)
> > > > > +{
> > > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > > +
> > > > > +       free_irq(irq, pmu);
> > > > > +}
> > > > > +
> > > > > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > > > > +                                const struct auxiliary_device_id *id)
> > > > > +{
> > > > > +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > +       void __iomem *addr;
> > > > > +       char *name;
> > > > > +       int ret;
> > > > > +
> > > > > +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > > > > +       if (!iommu_pmu)
> > > > > +               return -ENOMEM;
> > > > > +
> > > > > +       iommu_pmu->reg = iommu_dev->reg;
> > > > > +
> > > > > +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > > > > +       iommu_pmu->num_counters = hweight32(readl(addr));
> > > > > +
> > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > > > > +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > > >
> > > > > +       iommu_pmu->cycle_cntr_mask = readq(addr);
> > > > iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
> > > >
> > > > > +
> > > > > +       /* Assume the width of all event counters are the same */
> > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > > > > +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > > >
> > > > > +       iommu_pmu->event_cntr_mask = readq(addr);
> > > >  iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
> > > >
> > > > > +
> > > > > +       iommu_pmu->pmu = (struct pmu) {
> > > > > +               .module         = THIS_MODULE,
> > > > > +               .parent         = &auxdev->dev,
> > > > > +               .task_ctx_nr    = perf_invalid_context,
> > > > > +               .event_init     = riscv_iommu_pmu_event_init,
> > > > > +               .add            = riscv_iommu_pmu_add,
> > > > > +               .del            = riscv_iommu_pmu_del,
> > > > > +               .start          = riscv_iommu_pmu_start,
> > > > > +               .stop           = riscv_iommu_pmu_stop,
> > > > > +               .read           = riscv_iommu_pmu_read,
> > > > > +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> > > > > +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > > > > +       };
> > > > > +
> > > > > +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> > > > > +
> > > > > +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > > > > +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> > > > > +       if (!name) {
> > > > > +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> > > > > +                       dev_name(iommu_dev->dev));
> > > > > +               return -ENOMEM;
> > > > > +       }
> > > > > +
> > > > > +       /* Bind all events to the same cpu context to avoid race enabling */
> > > > > +       iommu_pmu->on_cpu = raw_smp_processor_id();
> > > > > +
> > > > > +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > > +       if (ret) {
> > > > > +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > > > > +               return ret;
> > > > > +       }
> > > > > +
> > > > > +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> > > > > +       if (ret) {
> > > > > +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > > > > +               goto err_cpuhp_remove;
> > > > > +       }
> > > > > +
> > > > > +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > > > > +       if (ret) {
> > > > > +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > > > > +               goto err_free_irq;
> > > > > +       }
> > > > > +
> > > > > +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> > > > > +                name, iommu_pmu->num_counters);
> > > > > +
> > > > > +       return 0;
> > > > > +
> > > > > +err_free_irq:
> > > > > +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> > > > > +err_cpuhp_remove:
> > > > > +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > > +       return ret;
> > > > > +}
> > > > > +
> > > > > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > > > > +       { .name = "iommu.pmu" },
> > > > > +       {}
> > > > > +};
> > > > > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > > > > +
> > > > > +static struct auxiliary_driver iommu_pmu_driver = {
> > > > > +       .probe          = riscv_iommu_pmu_probe,
> > > > > +       .id_table       = riscv_iommu_pmu_id_table,
> > > > > +};
> > > > > +
> > > > > +static int __init riscv_iommu_pmu_init(void)
> > > > > +{
> > > > > +       int ret;
> > > > > +
> > > > > +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > > > > +                                             "perf/riscv/iommu:online",
> > > > > +                                             riscv_iommu_pmu_online_cpu,
> > > > > +                                             riscv_iommu_pmu_offline_cpu);
> > > > > +       if (cpuhp_state < 0)
> > > > > +               return cpuhp_state;
> > > > > +
> > > > > +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> > > > > +       if (ret)
> > > > > +               cpuhp_remove_multi_state(cpuhp_state);
> > > > > +
> > > > > +       return ret;
> > > > > +}
> > > > > +module_init(riscv_iommu_pmu_init);
> > > > > +
> > > > > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > > > > +MODULE_LICENSE("GPL");
> > > > > --
> > > > > 2.43.7
> > > > >
> > > >
> > > > --
> > > > Best Regards
> > > >  Guo Ren
>
>
>
> --
> Best Regards
>  Guo Ren

Re: [PATCH v3 1/2] drivers/perf: riscv-iommu: add risc-v iommu pmu driver

[Guo Ren]

On Tue, Jul 7, 2026 at 2:34 PM Zong Li <zong.li@sifive.com> wrote:
>
> On Tue, Jul 7, 2026 at 2:03 PM Guo Ren <guoren@kernel.org> wrote:
> >
> > On Tue, Jul 7, 2026 at 10:01 AM Zong Li <zong.li@sifive.com> wrote:
> > >
> > > On Fri, Jul 3, 2026 at 9:36 AM Zong Li <zong.li@sifive.com> wrote:
> > > >
> > > > On Tue, Jun 30, 2026 at 9:17 PM Guo Ren <guoren@kernel.org> wrote:
> > > > >
> > > > > Hi Zong Li,
> > > > >
> > > > > On Tue, Jun 30, 2026 at 4:38 PM Zong Li <zong.li@sifive.com> wrote:
> > > > > >
> > > > > > Add a new driver to support the RISC-V IOMMU PMU. This is an auxiliary
> > > > > > device driver created by the parent RISC-V IOMMU driver.
> > > > > >
> > > > > > The RISC-V IOMMU PMU separates the cycle counter from the event counters.
> > > > > > The cycle counter is not associated with iohpmevt0, so a software-defined
> > > > > > cycle event is required for the perf subsystem.
> > > > > >
> > > > > > The number and width of the counters are hardware-implemented and must
> > > > > > be detected at runtime.
> > > > > >
> > > > > > The performance monitor provides counters with filtering support to
> > > > > > collect events for specific device ID/process ID, or GSCID/PSCID.
> > > > > >
> > > > > > PMU-related definitions are moved into the perf driver, where they are
> > > > > > used exclusively.
> > > > > >
> > > > > > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > > > > > Suggested-by: Guo Ren <guoren@kernel.org>
> > > > > > Link: https://lore.kernel.org/linux-riscv/20260618143634.7f3dd6c5@pumpkin/
> > > > > > Signed-off-by: Zong Li <zong.li@sifive.com>
> > > > > > ---
> > > > > >  drivers/iommu/riscv/iommu-bits.h |  61 ---
> > > > > >  drivers/perf/Kconfig             |  12 +
> > > > > >  drivers/perf/Makefile            |   1 +
> > > > > >  drivers/perf/riscv_iommu_pmu.c   | 703 +++++++++++++++++++++++++++++++
> > > > > >  4 files changed, 716 insertions(+), 61 deletions(-)
> > > > > >  create mode 100644 drivers/perf/riscv_iommu_pmu.c
> > > > > >
> > > > > > diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
> > > > > > index f2ef9bd3cde9..6b5de913a032 100644
> > > > > > --- a/drivers/iommu/riscv/iommu-bits.h
> > > > > > +++ b/drivers/iommu/riscv/iommu-bits.h
> > > > > > @@ -192,67 +192,6 @@ enum riscv_iommu_ddtp_modes {
> > > > > >  #define RISCV_IOMMU_IPSR_PMIP          BIT(RISCV_IOMMU_INTR_PM)
> > > > > >  #define RISCV_IOMMU_IPSR_PIP           BIT(RISCV_IOMMU_INTR_PQ)
> > > > > >
> > > > > > -/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > > -#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > > -#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > > -
> > > > > > -/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > > -#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > > -#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 1)
> > > > > > -
> > > > > > -/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCYCLES     0x0060
> > > > > > -#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > > -#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > > -
> > > > > > -/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > > -#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCTR_BASE + ((_n) * 0x8))
> > > > > > -
> > > > > > -/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > > -#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > > -#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > > -
> > > > > > -/* Number of defined performance-monitoring event selectors */
> > > > > > -#define RISCV_IOMMU_IOHPMEVT_CNT       31
> > > > > > -
> > > > > > -/**
> > > > > > - * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > > - *
> > > > > > - * @RISCV_IOMMU_HPMEVENT_INVALID: Invalid event, do not count
> > > > > > - * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > > - * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > > - * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > > - * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > > - */
> > > > > > -enum riscv_iommu_hpmevent_id {
> > > > > > -       RISCV_IOMMU_HPMEVENT_INVALID    = 0,
> > > > > > -       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > > -       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > > -       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > > -       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > > -       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > > -       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > > -       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > > -       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > > -       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > > -};
> > > > > > -
> > > > > >  /* 5.24 Translation request IOVA (64bits) */
> > > > > >  #define RISCV_IOMMU_REG_TR_REQ_IOVA     0x0258
> > > > > >  #define RISCV_IOMMU_TR_REQ_IOVA_VPN    GENMASK_ULL(63, 12)
> > > > > > diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
> > > > > > index 245e7bb763b9..8cce6c2ea626 100644
> > > > > > --- a/drivers/perf/Kconfig
> > > > > > +++ b/drivers/perf/Kconfig
> > > > > > @@ -105,6 +105,18 @@ config RISCV_PMU_SBI
> > > > > >           full perf feature support i.e. counter overflow, privilege mode
> > > > > >           filtering, counter configuration.
> > > > > >
> > > > > > +config RISCV_IOMMU_PMU
> > > > > > +       depends on RISCV || COMPILE_TEST
> > > > > > +       depends on RISCV_IOMMU
> > > > > > +       bool "RISC-V IOMMU Hardware Performance Monitor"
> > > > > > +       default y
> > > > > > +       help
> > > > > > +         Say Y if you want to use the RISC-V IOMMU performance monitor
> > > > > > +         implementation. The performance monitor is an optional hardware
> > > > > > +         feature, and whether it is actually enabled depends on IOMMU
> > > > > > +         hardware support. If the underlying hardware does not implement
> > > > > > +         the PMU, this option will have no effect.
> > > > > > +
> > > > > >  config STARFIVE_STARLINK_PMU
> > > > > >         depends on ARCH_STARFIVE || COMPILE_TEST
> > > > > >         depends on 64BIT
> > > > > > diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
> > > > > > index eb8a022dad9a..90c75f3c0ac1 100644
> > > > > > --- a/drivers/perf/Makefile
> > > > > > +++ b/drivers/perf/Makefile
> > > > > > @@ -20,6 +20,7 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
> > > > > >  obj-$(CONFIG_RISCV_PMU) += riscv_pmu.o
> > > > > >  obj-$(CONFIG_RISCV_PMU_LEGACY) += riscv_pmu_legacy.o
> > > > > >  obj-$(CONFIG_RISCV_PMU_SBI) += riscv_pmu_sbi.o
> > > > > > +obj-$(CONFIG_RISCV_IOMMU_PMU) += riscv_iommu_pmu.o
> > > > > >  obj-$(CONFIG_STARFIVE_STARLINK_PMU) += starfive_starlink_pmu.o
> > > > > >  obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
> > > > > >  obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
> > > > > > diff --git a/drivers/perf/riscv_iommu_pmu.c b/drivers/perf/riscv_iommu_pmu.c
> > > > > > new file mode 100644
> > > > > > index 000000000000..2e856289ddc9
> > > > > > --- /dev/null
> > > > > > +++ b/drivers/perf/riscv_iommu_pmu.c
> > > > > > @@ -0,0 +1,703 @@
> > > > > > +// SPDX-License-Identifier: GPL-2.0-only
> > > > > > +/*
> > > > > > + * Copyright (C) 2026 SiFive
> > > > > > + *
> > > > > > + * Authors
> > > > > > + *     Zong Li <zong.li@sifive.com>
> > > > > > + */
> > > > > > +
> > > > > > +#include <linux/auxiliary_bus.h>
> > > > > > +#include <linux/io-64-nonatomic-hi-lo.h>
> > > > > > +#include <linux/perf_event.h>
> > > > > > +
> > > > > > +#include "../iommu/riscv/iommu.h"
> > > > > > +
> > > > > > +/* 5.19 Performance monitoring counter overflow status (32bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOCOUNTOVF     0x0058
> > > > > > +#define RISCV_IOMMU_IOCOUNTOVF_CY      BIT(0)
> > > > > > +#define RISCV_IOMMU_IOCOUNTOVF_HPM     GENMASK_ULL(31, 1)
> > > > > > +
> > > > > > +/* 5.20 Performance monitoring counter inhibits (32bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOCOUNTINH     0x005C
> > > > > > +#define RISCV_IOMMU_IOCOUNTINH_CY      BIT(0)
> > > > > > +#define RISCV_IOMMU_IOCOUNTINH_HPM     GENMASK(31, 0)
> > > > > > +
> > > > > > +/* 5.21 Performance monitoring cycles counter (64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCYCLES    0x0060
> > > > > > +#define RISCV_IOMMU_IOHPMCYCLES_COUNTER        GENMASK_ULL(62, 0)
> > > > > > +#define RISCV_IOMMU_IOHPMCYCLES_OF     BIT_ULL(63)
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCTR(_n)   (RISCV_IOMMU_REG_IOHPMCYCLES + ((_n) * 0x8))
> > > > > > +
> > > > > > +/* 5.22 Performance monitoring event counters (31 * 64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMCTR_BASE  0x0068
> > > > > > +#define RISCV_IOMMU_IOHPMCTR_COUNTER   GENMASK_ULL(63, 0)
> > > > > > +
> > > > > > +/* 5.23 Performance monitoring event selectors (31 * 64bits) */
> > > > > > +#define RISCV_IOMMU_REG_IOHPMEVT_BASE  0x0160
> > > > > > +#define RISCV_IOMMU_REG_IOHPMEVT(_n)   (RISCV_IOMMU_REG_IOHPMEVT_BASE + ((_n) * 0x8))
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENTID   GENMASK_ULL(14, 0)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DMASK     BIT_ULL(15)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_PID_PSCID GENMASK_ULL(35, 16)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DID_GSCID GENMASK_ULL(59, 36)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_PV_PSCV   BIT_ULL(60)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_DV_GSCV   BIT_ULL(61)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_IDT       BIT_ULL(62)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_OF                BIT_ULL(63)
> > > > > > +#define RISCV_IOMMU_IOHPMEVT_EVENT     GENMASK_ULL(62, 0)
> > > > > > +
> > > > > > +/* The total number of counters is 31 event counters plus 1 cycle counter */
> > > > > > +#define RISCV_IOMMU_HPM_COUNTER_NUM    32
> > > > > > +
> > > > > > +static int cpuhp_state;
> > > > > > +
> > > > > > +/**
> > > > > > + * enum riscv_iommu_hpmevent_id - Performance-monitoring event identifier
> > > > > > + *
> > > > > > + * @RISCV_IOMMU_HPMEVENT_CYCLE: Clock cycle counter
> > > > > > + * @RISCV_IOMMU_HPMEVENT_URQ: Untranslated requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_TRQ: Translated requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_ATS_RQ: ATS translation requests
> > > > > > + * @RISCV_IOMMU_HPMEVENT_TLB_MISS: TLB misses
> > > > > > + * @RISCV_IOMMU_HPMEVENT_DD_WALK: Device directory walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_PD_WALK: Process directory walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_S_VS_WALKS: First-stage page table walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_G_WALKS: Second-stage page table walks
> > > > > > + * @RISCV_IOMMU_HPMEVENT_MAX: Value to denote maximum Event IDs
> > > > > > + *
> > > > > > + * The specification does not define an event ID for counting the
> > > > > > + * number of clock cycles, meaning there is no associated 'iohpmevt0'.
> > > > > > + * Event ID 0 is an invalid event and does not overlap with any valid
> > > > > > + * event ID. Let's repurpose ID 0 as the cycle for perf, the cycle
> > > > > > + * event is not actually written into any register, it serves solely
> > > > > > + * as an identifier.
> > > > > > + */
> > > > > > +enum riscv_iommu_hpmevent_id {
> > > > > > +       RISCV_IOMMU_HPMEVENT_CYCLE      = 0,
> > > > > > +       RISCV_IOMMU_HPMEVENT_URQ        = 1,
> > > > > > +       RISCV_IOMMU_HPMEVENT_TRQ        = 2,
> > > > > > +       RISCV_IOMMU_HPMEVENT_ATS_RQ     = 3,
> > > > > > +       RISCV_IOMMU_HPMEVENT_TLB_MISS   = 4,
> > > > > > +       RISCV_IOMMU_HPMEVENT_DD_WALK    = 5,
> > > > > > +       RISCV_IOMMU_HPMEVENT_PD_WALK    = 6,
> > > > > > +       RISCV_IOMMU_HPMEVENT_S_VS_WALKS = 7,
> > > > > > +       RISCV_IOMMU_HPMEVENT_G_WALKS    = 8,
> > > > > > +       RISCV_IOMMU_HPMEVENT_MAX        = 9
> > > > > > +};
> > > > > > +
> > > > > > +struct riscv_iommu_pmu {
> > > > > > +       struct pmu pmu;
> > > > > > +       struct hlist_node node;
> > > > > > +       void __iomem *reg;
> > > > > > +       unsigned int on_cpu;
> > > > > > +       int num_counters;
> > > > > > +       u64 cycle_cntr_mask;
> > > > > > +       u64 event_cntr_mask;
> > > > > > +       struct perf_event *events[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +       DECLARE_BITMAP(used_counters, RISCV_IOMMU_HPM_COUNTER_NUM);
> > > > > > +       u32 hi_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +       u32 lo_prev[RISCV_IOMMU_HPM_COUNTER_NUM];
> > > > > > +};
> > > > > > +
> > > > > > +#define to_riscv_iommu_pmu(p) (container_of(p, struct riscv_iommu_pmu, pmu))
> > > > > > +
> > > > > > +#define RISCV_IOMMU_PMU_ATTR_EXTRACTOR(_name, _mask)                   \
> > > > > > +       static inline u32 get_##_name(struct perf_event *event)         \
> > > > > > +       {                                                               \
> > > > > > +               return FIELD_GET(_mask, event->attr.config);            \
> > > > > > +       }                                                               \
> > > > > > +
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(event, RISCV_IOMMU_IOHPMEVT_EVENTID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(partial_matching, RISCV_IOMMU_IOHPMEVT_DMASK);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(pid_pscid, RISCV_IOMMU_IOHPMEVT_PID_PSCID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(did_gscid, RISCV_IOMMU_IOHPMEVT_DID_GSCID);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_pid_pscid, RISCV_IOMMU_IOHPMEVT_PV_PSCV);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_did_gscid, RISCV_IOMMU_IOHPMEVT_DV_GSCV);
> > > > > > +RISCV_IOMMU_PMU_ATTR_EXTRACTOR(filter_id_type, RISCV_IOMMU_IOHPMEVT_IDT);
> > > > > > +
> > > > > > +/* Formats */
> > > > > > +PMU_FORMAT_ATTR(event,            "config:0-14");
> > > > > > +PMU_FORMAT_ATTR(partial_matching, "config:15");
> > > > > > +PMU_FORMAT_ATTR(pid_pscid,        "config:16-35");
> > > > > > +PMU_FORMAT_ATTR(did_gscid,        "config:36-59");
> > > > > > +PMU_FORMAT_ATTR(filter_pid_pscid, "config:60");
> > > > > > +PMU_FORMAT_ATTR(filter_did_gscid, "config:61");
> > > > > > +PMU_FORMAT_ATTR(filter_id_type,   "config:62");
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_pmu_formats[] = {
> > > > > > +       &format_attr_event.attr,
> > > > > > +       &format_attr_partial_matching.attr,
> > > > > > +       &format_attr_pid_pscid.attr,
> > > > > > +       &format_attr_did_gscid.attr,
> > > > > > +       &format_attr_filter_pid_pscid.attr,
> > > > > > +       &format_attr_filter_did_gscid.attr,
> > > > > > +       &format_attr_filter_id_type.attr,
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_format_group = {
> > > > > > +       .name = "format",
> > > > > > +       .attrs = riscv_iommu_pmu_formats,
> > > > > > +};
> > > > > > +
> > > > > > +/* Events */
> > > > > > +static ssize_t riscv_iommu_pmu_event_show(struct device *dev,
> > > > > > +                                         struct device_attribute *attr,
> > > > > > +                                         char *page)
> > > > > > +{
> > > > > > +       struct perf_pmu_events_attr *pmu_attr;
> > > > > > +
> > > > > > +       pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
> > > > > > +
> > > > > > +       return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
> > > > > > +}
> > > > > > +
> > > > > > +#define RISCV_IOMMU_PMU_EVENT_ATTR(name, id)                   \
> > > > > > +       PMU_EVENT_ATTR_ID(name, riscv_iommu_pmu_event_show, id)
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_pmu_events[] = {
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(cycle, RISCV_IOMMU_HPMEVENT_CYCLE),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(untranslated_req, RISCV_IOMMU_HPMEVENT_URQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(translated_req, RISCV_IOMMU_HPMEVENT_TRQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ats_trans_req, RISCV_IOMMU_HPMEVENT_ATS_RQ),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(tlb_miss, RISCV_IOMMU_HPMEVENT_TLB_MISS),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(ddt_walks, RISCV_IOMMU_HPMEVENT_DD_WALK),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(pdt_walks, RISCV_IOMMU_HPMEVENT_PD_WALK),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(s_vs_pt_walks, RISCV_IOMMU_HPMEVENT_S_VS_WALKS),
> > > > > > +       RISCV_IOMMU_PMU_EVENT_ATTR(g_pt_walks, RISCV_IOMMU_HPMEVENT_G_WALKS),
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_events_group = {
> > > > > > +       .name = "events",
> > > > > > +       .attrs = riscv_iommu_pmu_events,
> > > > > > +};
> > > > > > +
> > > > > > +/* cpumask */
> > > > > > +static ssize_t riscv_iommu_cpumask_show(struct device *dev,
> > > > > > +                                       struct device_attribute *attr,
> > > > > > +                                       char *buf)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(dev_get_drvdata(dev));
> > > > > > +
> > > > > > +       return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->on_cpu));
> > > > > > +}
> > > > > > +
> > > > > > +static struct device_attribute riscv_iommu_cpumask_attr =
> > > > > > +       __ATTR(cpumask, 0444, riscv_iommu_cpumask_show, NULL);
> > > > > > +
> > > > > > +static struct attribute *riscv_iommu_cpumask_attrs[] = {
> > > > > > +       &riscv_iommu_cpumask_attr.attr,
> > > > > > +       NULL
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group riscv_iommu_pmu_cpumask_group = {
> > > > > > +       .attrs = riscv_iommu_cpumask_attrs,
> > > > > > +};
> > > > > > +
> > > > > > +static const struct attribute_group *riscv_iommu_pmu_attr_grps[] = {
> > > > > > +       &riscv_iommu_pmu_cpumask_group,
> > > > > > +       &riscv_iommu_pmu_format_group,
> > > > > > +       &riscv_iommu_pmu_events_group,
> > > > > > +       NULL,
> > > > > > +};
> > > > > > +
> > > > > > +/* PMU Operations */
> > > > > > +static void riscv_iommu_pmu_set_counter(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > > +                                       u64 value)
> > > > > > +{
> > > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > > +
> > > > > > +       writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > > #include <include/linux/io-64-nonatomic-lo-hi.h>
> > > > > lo_hi_writeq(value & counter_mask, pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx));
> > > > >
> > > > > For better compatibility, it is recommended to use writel and readl
> > > > > instead of readq and writeq. Sorry for overlooking this last time —
> > > > > could you please make the change again?
> > > > > In the future, your driver could support RV32 directly, as the RISC-V
> > > > > IOMMU specification also supports RV32; it's just that the current
> > > > > RISC-V IOMMU driver's Kconfig depends on 64BIT.
> > > >
> > > > Hi Guo,
> > > > Thank you for your review. In my implementation for RV32, readq and
> > > > writeq are replaced by hi_lo_readq and hi_lo_writeq (from
> > > > io-64-nonatomic-hi-lo.h). So, supporting RV32 should be fine.
> > > > However, as you mentioned before: 64-bit access might be undefined on
> > > > some hardware. To be safe and compatible with all hardware, we should
> > > > use 32-bit access instead of 64-bit access.
> > > > I will modify them in the next version
> > > >
> > > > I'd like to make sure I didn't miss anything important. My original
> > > > code uses the "hi-lo" order. May I ask if there is a specific reason
> > > > why you prefer the "lo-hi" order?
> > > > Thanks
> > > >
> > >
> > > Hi all,
> > > I have been thinking more about the IOMMU register access behavior and
> > > would like to share my thoughts regarding the lo-hi vs. hi-lo access
> > > order.
> > >
> > > It seems to me that the correct 32-bit write order is strictly tied to
> > > the underlying hardware implementation. From the software side, we
> > > have no way of knowing whether a specific vendor designed their
> > > hardware with a low-part trigger or a high-part trigger. Therefore,
> > > hardcoding a fixed 32-bit write sequence could result in compatibility
> > > issues across different hardware designs.
> > >
> > > Revisiting the spec regarding the UNSPECIFIED 8-byte atomic access,
> > > this might imply that the hardware itself will handle the split into
> > > two 4-byte accesses and internally manage its own trigger logic.
> > > Because of this, for RV64 systems, software might simply use standard
> > > 8-byte writes (e.g., writeq). Delegating the access handling to the
> > > hardware is likely the best way to maximize compatibility across all
> > > vendor implementations. For the counter reading strategy (hi-lo-hi
> > > pattern), I completely agree with using the hi-lo-hi pattern when
> > > reading the PMU counters. Since this is a read operation and does not
> > > alter hardware state.
> > Apologies for overlooking this point. Using hi_lo_writeq/readq
> > explicitly enforces access ordering to prevent hardware reordering,
> > while maintaining RV32 compatibility. This ensures consistent 64-bit
> > MMIO register access semantics across both RV32 and RV64. Falling back
> > to 32-bit MMIO access remains the best approach, as it avoids any
> > ambiguity or controversy.
> >
>
> I was originally a bit concerned that if our writing order is
> different from the hardware design, it might cause some risks.
> For example, if we write to the 64-bit register in a high-low order,
> but the hardware is triggered by the high part, the hardware might
> start working with the "new 32-bit high part" and the "old 32-bit low
> part" before we write the new low part.
> However, I checked the current driver code, and it seems we might not
> need to worry about this for now. This is because we only write to
> counter/event registers when the counter is inhibited.
Yes, you give out the key point.

> Therefore, I think we can safely use the high-low order to rewrite the driver.
Great, I'm waiting for your next version.

> Thank you very much for your discussion.
>
> > >
> > > For RV32 support, we inevitably face the 32-bit access splitting
> > > issue. For the RV32 case, maybe we can assume the hi-lo order as the
> > > default fallback, because the IOMMU specification uses this sequence
> > > as example: "an access may appear, internally to the IOMMU, as if two
> > > separate 4 byte accesses - first to the high half and second to the
> > > low half - were performed.". Or do you prefer to use the same
> > > assumption (hi-lo pattern) in RV64 as well?
> > >
> > > Please let me know what you think about this.
> > >
> > > Thanks
> > >
> > >
> > > > >
> > > > > > +}
> > > > > > +
> > > > > > +/*
> > > > > > + * As stated in the RISC-V IOMMU Specification, Chapter 6:
> > > > > > + * Whether an 8 byte access to an IOMMU register is single-copy atomic
> > > > > > + * is UNSPECIFIED, and such an access may appear, internally to the
> > > > > > + * IOMMU, as if two separate 4 byte accesses - first to the high half
> > > > > > + * and second to the low half - were performed
> > > > > > + *
> > > > > > + * To make sure the driver works correctly on different hardware,
> > > > > > + * the software will always use two 4-byte access for the counter.
> > > > > > + *
> > > > > > + * This function implements the hi-lo-hi pattern to detect and handle
> > > > > > + * wraparound during the read operation:
> > > > > > + *   1. Read high half (hi)
> > > > > > + *   2. Read low half (lo)
> > > > > > + *   3. Check if low half wrapped or high half changed:
> > > > > > + *      - If lo <= lo_prev: possible wraparound occurred
> > > > > > + *      - If hi != hi_prev: high half changed during read
> > > > > > + *   4. If wraparound detected, re-read high half and assume low half is 0
> > > > > > + *   5. Update previous values for next read
> > > > > > + */
> > > > > > +static u64 riscv_iommu_pmu_get_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > > +{
> > > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOHPMCTR(idx);
> > > > > > +       u64 value, counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > > +       u32 hi, lo;
> > > > > > +
> > > > > > +       hi = readl(addr + 4);
> > > > > > +       lo = readl(addr);
> > > > > > +
> > > > > > +       if (lo <= pmu->lo_prev[idx] || hi != pmu->hi_prev[idx]) {
> > > > > > +               u32 hi_tmp = readl(addr + 4);
> > > > > > +
> > > > > > +               /*
> > > > > > +                * If hi changes, then hi+1:0 must have happened while
> > > > > > +                * the code was running so it is a safe return value
> > > > > > +                */
> > > > > > +               if (hi_tmp != hi) {
> > > > > > +                       hi = hi_tmp;
> > > > > > +                       lo = 0;
> > > > > > +               }
> > > > > > +               pmu->lo_prev[idx] = ~0u;
> > > > > > +               pmu->hi_prev[idx] = hi;
> > > > > > +       }
> > > > > > +       pmu->lo_prev[idx] = lo;
> > > > > > +
> > > > > > +       value = (((u64)hi << 32) | lo) & counter_mask;
> > > > > > +
> > > > > > +       /* The bit 63 of cycle counter (i.e., idx == 0) is OF bit */
> > > > > > +       return idx ? value : (value & ~RISCV_IOMMU_IOHPMCYCLES_OF);
> > > > > > +}
> > > > > Yes! Thanks for your implementation.
> > > > >
> > > > > > +
> > > > > > +static bool is_cycle_event(u64 event)
> > > > > > +{
> > > > > > +       return event == RISCV_IOMMU_HPMEVENT_CYCLE;
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_set_event(struct riscv_iommu_pmu *pmu, u32 idx,
> > > > > > +                                     u64 value)
> > > > > > +{
> > > > > > +       /* There is no associtated IOHPMEVT0 for IOHPMCYCLES */
> > > > > > +       if (is_cycle_event(value))
> > > > > > +               return;
> > > > > > +
> > > > > > +       /* Event counter start from idx 1 */
> > > > > > +       writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > > > > +              pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > > > >        lo_hi_writeq(FIELD_GET(RISCV_IOMMU_IOHPMEVT_EVENT, value),
> > > > >               pmu->reg + RISCV_IOMMU_REG_IOHPMEVT(idx - 1));
> > > > >
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_enable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > > +{
> > > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > > +       u32 value = readl(addr);
> > > > > > +
> > > > > > +       writel(value & ~BIT(idx), addr);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_disable_counter(struct riscv_iommu_pmu *pmu, u32 idx)
> > > > > > +{
> > > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > > +       u32 value = readl(addr);
> > > > > > +
> > > > > > +       writel(value | BIT(idx), addr);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_start_all(struct riscv_iommu_pmu *pmu)
> > > > > > +{
> > > > > > +       void __iomem *addr = pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > > +       u32 used_cntr = 0;
> > > > > > +
> > > > > > +       /* The performance-monitoring counter inhibits is a 32-bit WARL register */
> > > > > > +       bitmap_to_arr32(&used_cntr, pmu->used_counters, pmu->num_counters);
> > > > > > +
> > > > > > +       writel(~used_cntr, addr);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_stop_all(struct riscv_iommu_pmu *pmu)
> > > > > > +{
> > > > > > +       writel(GENMASK_ULL(pmu->num_counters - 1, 0),
> > > > > > +              pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH);
> > > > > > +}
> > > > > > +
> > > > > > +/* PMU APIs */
> > > > > > +static void riscv_iommu_pmu_set_period(struct perf_event *event)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       u64 counter_mask = hwc->idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > > +       u64 period;
> > > > > > +
> > > > > > +       /*
> > > > > > +        * Limit the maximum period to prevent the counter value
> > > > > > +        * from overtaking the one we are about to program.
> > > > > > +        * In effect we are reducing max_period to account for
> > > > > > +        * interrupt latency (and we are being very conservative).
> > > > > > +        */
> > > > > > +       period = counter_mask >> 1;
> > > > > > +       riscv_iommu_pmu_set_counter(pmu, hwc->idx, period);
> > > > > > +       local64_set(&hwc->prev_count, period);
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_event_init(struct perf_event *event)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       struct perf_event *sibling;
> > > > > > +       int total_event_counters = pmu->num_counters - 1;
> > > > > > +       int counters = 0;
> > > > > > +
> > > > > > +       if (event->attr.type != event->pmu->type)
> > > > > > +               return -ENOENT;
> > > > > > +
> > > > > > +       if (hwc->sample_period)
> > > > > > +               return -EOPNOTSUPP;
> > > > > > +
> > > > > > +       if (event->cpu < 0)
> > > > > > +               return -EOPNOTSUPP;
> > > > > > +
> > > > > > +       event->cpu = pmu->on_cpu;
> > > > > > +
> > > > > > +       hwc->idx = -1;
> > > > > > +       hwc->config = event->attr.config;
> > > > > > +
> > > > > > +       if (event->group_leader == event)
> > > > > > +               return 0;
> > > > > > +
> > > > > > +       if (is_cycle_event(get_event(event->group_leader)))
> > > > > > +               if (++counters > total_event_counters)
> > > > > > +                       return -EINVAL;
> > > > > > +
> > > > > > +       for_each_sibling_event(sibling, event->group_leader) {
> > > > > > +               if (is_cycle_event(get_event(sibling)))
> > > > > > +                       continue;
> > > > > > +
> > > > > > +               if (sibling->pmu != event->pmu && !is_software_event(sibling))
> > > > > > +                       return -EINVAL;
> > > > > > +
> > > > > > +               if (++counters > total_event_counters)
> > > > > > +                       return -EINVAL;
> > > > > > +       }
> > > > > > +
> > > > > > +       return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_update(struct perf_event *event)
> > > > > > +{
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       u64 delta, prev, now;
> > > > > > +       u32 idx = hwc->idx;
> > > > > > +       u64 counter_mask = idx ? pmu->event_cntr_mask : pmu->cycle_cntr_mask;
> > > > > > +
> > > > > > +       do {
> > > > > > +               prev = local64_read(&hwc->prev_count);
> > > > > > +               now = riscv_iommu_pmu_get_counter(pmu, idx);
> > > > > > +       } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
> > > > > > +
> > > > > > +       delta = (now - prev) & counter_mask;
> > > > > > +       local64_add(delta, &event->count);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_start(struct perf_event *event, int flags)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +
> > > > > > +       if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
> > > > > > +               return;
> > > > > > +
> > > > > > +       if (flags & PERF_EF_RELOAD)
> > > > > > +               WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
> > > > > > +
> > > > > > +       hwc->state = 0;
> > > > > > +       riscv_iommu_pmu_set_period(event);
> > > > > > +       riscv_iommu_pmu_set_event(pmu, hwc->idx, hwc->config);
> > > > > > +       riscv_iommu_pmu_enable_counter(pmu, hwc->idx);
> > > > > > +
> > > > > > +       perf_event_update_userpage(event);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_stop(struct perf_event *event, int flags)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       int idx = hwc->idx;
> > > > > > +
> > > > > > +       if (hwc->state & PERF_HES_STOPPED)
> > > > > > +               return;
> > > > > > +
> > > > > > +       riscv_iommu_pmu_disable_counter(pmu, idx);
> > > > > > +
> > > > > > +       if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE))
> > > > > > +               riscv_iommu_pmu_update(event);
> > > > > > +
> > > > > > +       hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_add(struct perf_event *event, int flags)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       unsigned int num_counters = pmu->num_counters;
> > > > > > +       int idx;
> > > > > > +
> > > > > > +       /* Reserve index zero for iohpmcycles */
> > > > > > +       if (is_cycle_event(get_event(event)))
> > > > > > +               idx = 0;
> > > > > > +       else
> > > > > > +               idx = find_next_zero_bit(pmu->used_counters, num_counters, 1);
> > > > > > +
> > > > > > +       /* All event counters or cycle counter are in use */
> > > > > > +       if (idx == num_counters || pmu->events[idx])
> > > > > > +               return -EAGAIN;
> > > > > > +
> > > > > > +       set_bit(idx, pmu->used_counters);
> > > > > > +
> > > > > > +       pmu->events[idx] = event;
> > > > > > +       hwc->idx = idx;
> > > > > > +       hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
> > > > > > +       local64_set(&hwc->prev_count, 0);
> > > > > > +
> > > > > > +       if (flags & PERF_EF_START)
> > > > > > +               riscv_iommu_pmu_start(event, flags);
> > > > > > +
> > > > > > +       /* Propagate changes to the userspace mapping. */
> > > > > > +       perf_event_update_userpage(event);
> > > > > > +
> > > > > > +       return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_read(struct perf_event *event)
> > > > > > +{
> > > > > > +       riscv_iommu_pmu_update(event);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_del(struct perf_event *event, int flags)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = to_riscv_iommu_pmu(event->pmu);
> > > > > > +       struct hw_perf_event *hwc = &event->hw;
> > > > > > +       int idx = hwc->idx;
> > > > > > +
> > > > > > +       riscv_iommu_pmu_stop(event, PERF_EF_UPDATE);
> > > > > > +       pmu->events[idx] = NULL;
> > > > > > +       clear_bit(idx, pmu->used_counters);
> > > > > > +
> > > > > > +       perf_event_update_userpage(event);
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > > +
> > > > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > > > +
> > > > > > +       if (iommu_pmu->on_cpu == -1)
> > > > > > +               iommu_pmu->on_cpu = cpu;
> > > > > > +
> > > > > > +       return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > > +       unsigned int target_cpu;
> > > > > > +
> > > > > > +       iommu_pmu = hlist_entry_safe(node, struct riscv_iommu_pmu, node);
> > > > > > +
> > > > > > +       if (cpu != iommu_pmu->on_cpu)
> > > > > > +               return 0;
> > > > > > +
> > > > > > +       iommu_pmu->on_cpu = -1;
> > > > > > +
> > > > > > +       target_cpu = cpumask_any_but(cpu_online_mask, cpu);
> > > > > > +       if (target_cpu >= nr_cpu_ids)
> > > > > > +               return 0;
> > > > > > +
> > > > > > +       perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target_cpu);
> > > > > > +       iommu_pmu->on_cpu = target_cpu;
> > > > > > +
> > > > > > +       return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static irqreturn_t riscv_iommu_pmu_handle_irq(struct riscv_iommu_pmu *pmu)
> > > > > > +{
> > > > > > +       u32 ovf = readl(pmu->reg + RISCV_IOMMU_REG_IOCOUNTOVF);
> > > > > > +       int idx;
> > > > > > +
> > > > > > +       if (!ovf)
> > > > > > +               return IRQ_NONE;
> > > > > > +
> > > > > > +       riscv_iommu_pmu_stop_all(pmu);
> > > > > > +
> > > > > > +       for_each_set_bit(idx, (unsigned long *)&ovf, pmu->num_counters) {
> > > > > > +               struct perf_event *event = pmu->events[idx];
> > > > > > +
> > > > > > +               if (WARN_ON_ONCE(!event))
> > > > > > +                       continue;
> > > > > > +
> > > > > > +               riscv_iommu_pmu_update(event);
> > > > > > +               riscv_iommu_pmu_set_period(event);
> > > > > > +       }
> > > > > > +
> > > > > > +       riscv_iommu_pmu_start_all(pmu);
> > > > > > +
> > > > > > +       return IRQ_HANDLED;
> > > > > > +}
> > > > > > +
> > > > > > +static irqreturn_t riscv_iommu_pmu_irq_handler(int irq, void *dev_id)
> > > > > > +{
> > > > > > +       struct riscv_iommu_pmu *pmu = (struct riscv_iommu_pmu *)dev_id;
> > > > > > +       irqreturn_t ret;
> > > > > > +
> > > > > > +       /* Check whether this interrupt is for PMU */
> > > > > > +       if (!(readl_relaxed(pmu->reg + RISCV_IOMMU_REG_IPSR) & RISCV_IOMMU_IPSR_PMIP))
> > > > > > +               return IRQ_NONE;
> > > > > > +
> > > > > > +       /* Process PMU IRQ */
> > > > > > +       ret = riscv_iommu_pmu_handle_irq(pmu);
> > > > > > +
> > > > > > +       /* Clear performance monitoring interrupt pending bit */
> > > > > > +       writel_relaxed(RISCV_IOMMU_IPSR_PMIP, pmu->reg + RISCV_IOMMU_REG_IPSR);
> > > > > > +
> > > > > > +       return ret;
> > > > > > +}
> > > > > > +
> > > > > > +static unsigned int riscv_iommu_pmu_get_irq_num(struct riscv_iommu_device *iommu)
> > > > > > +{
> > > > > > +       /* Reuse ICVEC.CIV mask for all interrupt vectors mapping */
> > > > > > +       int vec = (iommu->icvec >> (RISCV_IOMMU_IPSR_PMIP * 4)) & RISCV_IOMMU_ICVEC_CIV;
> > > > > > +
> > > > > > +       return iommu->irqs[vec];
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_request_irq(struct riscv_iommu_device *iommu,
> > > > > > +                                      struct riscv_iommu_pmu *pmu)
> > > > > > +{
> > > > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > > > +
> > > > > > +       /*
> > > > > > +        * Set the IRQF_ONESHOT flag because this IRQ can be shared with
> > > > > > +        * other threaded IRQs by other queues.
> > > > > > +        */
> > > > > > +       return devm_request_irq(iommu->dev, irq, riscv_iommu_pmu_irq_handler,
> > > > > > +                               IRQF_ONESHOT | IRQF_SHARED, dev_name(iommu->dev), pmu);
> > > > > > +}
> > > > > > +
> > > > > > +static void riscv_iommu_pmu_free_irq(struct riscv_iommu_device *iommu,
> > > > > > +                                    struct riscv_iommu_pmu *pmu)
> > > > > > +{
> > > > > > +       unsigned int irq = riscv_iommu_pmu_get_irq_num(iommu);
> > > > > > +
> > > > > > +       free_irq(irq, pmu);
> > > > > > +}
> > > > > > +
> > > > > > +static int riscv_iommu_pmu_probe(struct auxiliary_device *auxdev,
> > > > > > +                                const struct auxiliary_device_id *id)
> > > > > > +{
> > > > > > +       struct  riscv_iommu_device *iommu_dev = dev_get_platdata(&auxdev->dev);
> > > > > > +       struct riscv_iommu_pmu *iommu_pmu;
> > > > > > +       void __iomem *addr;
> > > > > > +       char *name;
> > > > > > +       int ret;
> > > > > > +
> > > > > > +       iommu_pmu = devm_kzalloc(&auxdev->dev, sizeof(*iommu_pmu), GFP_KERNEL);
> > > > > > +       if (!iommu_pmu)
> > > > > > +               return -ENOMEM;
> > > > > > +
> > > > > > +       iommu_pmu->reg = iommu_dev->reg;
> > > > > > +
> > > > > > +       /* Counter number and width are hardware-implemented. Detect them by write 1s */
> > > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOCOUNTINH;
> > > > > > +       writel(RISCV_IOMMU_IOCOUNTINH_HPM, addr);
> > > > > > +       iommu_pmu->num_counters = hweight32(readl(addr));
> > > > > > +
> > > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCYCLES;
> > > > > > +       writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > > > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCYCLES_COUNTER, addr);
> > > > >
> > > > > > +       iommu_pmu->cycle_cntr_mask = readq(addr);
> > > > > iommu_pmu->cycle_cntr_mask = lo_hi_readq(addr);
> > > > >
> > > > > > +
> > > > > > +       /* Assume the width of all event counters are the same */
> > > > > > +       addr = iommu_pmu->reg + RISCV_IOMMU_REG_IOHPMCTR_BASE;
> > > > > > +       writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > > > >     lo_hi_writeq(RISCV_IOMMU_IOHPMCTR_COUNTER, addr);
> > > > >
> > > > > > +       iommu_pmu->event_cntr_mask = readq(addr);
> > > > >  iommu_pmu->event_cntr_mask = lo_hi_readq(addr);
> > > > >
> > > > > > +
> > > > > > +       iommu_pmu->pmu = (struct pmu) {
> > > > > > +               .module         = THIS_MODULE,
> > > > > > +               .parent         = &auxdev->dev,
> > > > > > +               .task_ctx_nr    = perf_invalid_context,
> > > > > > +               .event_init     = riscv_iommu_pmu_event_init,
> > > > > > +               .add            = riscv_iommu_pmu_add,
> > > > > > +               .del            = riscv_iommu_pmu_del,
> > > > > > +               .start          = riscv_iommu_pmu_start,
> > > > > > +               .stop           = riscv_iommu_pmu_stop,
> > > > > > +               .read           = riscv_iommu_pmu_read,
> > > > > > +               .attr_groups    = riscv_iommu_pmu_attr_grps,
> > > > > > +               .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
> > > > > > +       };
> > > > > > +
> > > > > > +       auxiliary_set_drvdata(auxdev, iommu_pmu);
> > > > > > +
> > > > > > +       name = devm_kasprintf(&auxdev->dev, GFP_KERNEL,
> > > > > > +                             "riscv_iommu_pmu_%s", dev_name(iommu_dev->dev));
> > > > > > +       if (!name) {
> > > > > > +               dev_err(&auxdev->dev, "Failed to create name riscv_iommu_pmu_%s\n",
> > > > > > +                       dev_name(iommu_dev->dev));
> > > > > > +               return -ENOMEM;
> > > > > > +       }
> > > > > > +
> > > > > > +       /* Bind all events to the same cpu context to avoid race enabling */
> > > > > > +       iommu_pmu->on_cpu = raw_smp_processor_id();
> > > > > > +
> > > > > > +       ret = cpuhp_state_add_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > > > +       if (ret) {
> > > > > > +               dev_err(&auxdev->dev, "Failed to register hotplug %s: %d\n", name, ret);
> > > > > > +               return ret;
> > > > > > +       }
> > > > > > +
> > > > > > +       ret = riscv_iommu_pmu_request_irq(iommu_dev, iommu_pmu);
> > > > > > +       if (ret) {
> > > > > > +               dev_err(&auxdev->dev, "Failed to request irq %s: %d\n", name, ret);
> > > > > > +               goto err_cpuhp_remove;
> > > > > > +       }
> > > > > > +
> > > > > > +       ret = perf_pmu_register(&iommu_pmu->pmu, name, -1);
> > > > > > +       if (ret) {
> > > > > > +               dev_err(&auxdev->dev, "Failed to registe %s: %d\n", name, ret);
> > > > > > +               goto err_free_irq;
> > > > > > +       }
> > > > > > +
> > > > > > +       dev_info(&auxdev->dev, "%s: Registered with %d counters\n",
> > > > > > +                name, iommu_pmu->num_counters);
> > > > > > +
> > > > > > +       return 0;
> > > > > > +
> > > > > > +err_free_irq:
> > > > > > +       riscv_iommu_pmu_free_irq(iommu_dev, iommu_pmu);
> > > > > > +err_cpuhp_remove:
> > > > > > +       cpuhp_state_remove_instance_nocalls(cpuhp_state, &iommu_pmu->node);
> > > > > > +       return ret;
> > > > > > +}
> > > > > > +
> > > > > > +static const struct auxiliary_device_id riscv_iommu_pmu_id_table[] = {
> > > > > > +       { .name = "iommu.pmu" },
> > > > > > +       {}
> > > > > > +};
> > > > > > +MODULE_DEVICE_TABLE(auxiliary, riscv_iommu_pmu_id_table);
> > > > > > +
> > > > > > +static struct auxiliary_driver iommu_pmu_driver = {
> > > > > > +       .probe          = riscv_iommu_pmu_probe,
> > > > > > +       .id_table       = riscv_iommu_pmu_id_table,
> > > > > > +};
> > > > > > +
> > > > > > +static int __init riscv_iommu_pmu_init(void)
> > > > > > +{
> > > > > > +       int ret;
> > > > > > +
> > > > > > +       cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
> > > > > > +                                             "perf/riscv/iommu:online",
> > > > > > +                                             riscv_iommu_pmu_online_cpu,
> > > > > > +                                             riscv_iommu_pmu_offline_cpu);
> > > > > > +       if (cpuhp_state < 0)
> > > > > > +               return cpuhp_state;
> > > > > > +
> > > > > > +       ret = auxiliary_driver_register(&iommu_pmu_driver);
> > > > > > +       if (ret)
> > > > > > +               cpuhp_remove_multi_state(cpuhp_state);
> > > > > > +
> > > > > > +       return ret;
> > > > > > +}
> > > > > > +module_init(riscv_iommu_pmu_init);
> > > > > > +
> > > > > > +MODULE_DESCRIPTION("RISC-V IOMMU PMU");
> > > > > > +MODULE_LICENSE("GPL");
> > > > > > --
> > > > > > 2.43.7
> > > > > >
> > > > >
> > > > > --
> > > > > Best Regards
> > > > >  Guo Ren
> >
> >
> >
> > --
> > Best Regards
> >  Guo Ren



-- 
Best Regards
 Guo Ren