[PATCH v5 3/5] selftests/resctrl: Add support for MBM and MBA tests on AMD

[Swapnil Sapkal <swapnil.sapkal@amd.com>]

From: Babu Moger <babu.moger@amd.com>

The MBM and MBA tests validate the resctrl memory bandwidth numbers
against the memory controller performance counters. On Intel these are
the iMC (Integrated Memory Controller) counters and the read event and
umask are discovered from the PMU's sysfs event files.

Commit 25e56847821f ("perf/x86/amd/uncore: Add memory controller
support") added support for the UMC (Unified Memory Controller) perf
events on AMD. Unlike Intel, the UMC PMU does not expose the CAS command
events via sysfs, so program the read event directly:
Number of CAS commands sent for reads: EventCode = 0x0a, umask = 0x1.

Keep all the vendor detection contained in enumerate_mcs() and dispatch
to a vendor specific routine to initialize the counter configuration.
Only reading the counter's perf "type" is common to both vendors, so
factor that out into a shared helper.

On AMD each memory controller (UMC) is exposed as its own PMU
(amd_umc_<n>) that is tied to a specific NUMA node, whereas on Intel
opening an iMC event on the benchmark CPU always resolves to the local
memory controllers. A UMC PMU can only be opened from a CPU that belongs
to the same node and returns -ENODEV otherwise. Since the benchmark runs
on a single CPU, open the counters that are local to that CPU and skip
the UMCs on remote nodes. This both avoids the perf_event_open() failure
and matches the local traffic accounted by mbm_local_bytes.

Assisted-by: Claude:claude-opus-4.8
Signed-off-by: Babu Moger <babu.moger@amd.com>
Co-developed-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
Signed-off-by: Swapnil Sapkal <swapnil.sapkal@amd.com>
---

    v4->v5:
    - Rebased onto kselftest/next and reworked for the linked-list counter
      management: read_from_amd_mc_dir() allocates a counter and adds it to
      the list, and the remote-node skipping iterates the list.
    - Vendor detection and dispatch now live in enumerate_mcs().

    v3->v4:
    - Contained all vendor detection in the enumeration routine and
      dispatched to per-vendor routines, factoring the common counter 'type'
      read into a shared helper. (Reinette)
    - Adapted to the upstream read-only bandwidth refactor.
    - Handled the AMD per-node UMC PMU topology: open only the UMCs local to
      the benchmark CPU and skip remote-node UMCs that would otherwise fail
      perf_event_open() with -ENODEV on multi-node systems.
    - Improved the perf_event_open() error reporting (print the errno with
      strerror()).

 tools/testing/selftests/resctrl/resctrl_val.c | 135 ++++++++++++++----
 1 file changed, 111 insertions(+), 24 deletions(-)

diff --git a/tools/testing/selftests/resctrl/resctrl_val.c b/tools/testing/selftests/resctrl/resctrl_val.c
index fdbbe2155cea..38cf58c41be1 100644
--- a/tools/testing/selftests/resctrl/resctrl_val.c
+++ b/tools/testing/selftests/resctrl/resctrl_val.c
@@ -11,6 +11,7 @@
 #include "resctrl.h"
 
 #define UNCORE_IMC		"uncore_imc_"
+#define AMD_UMC			"amd_umc_"
 #define READ_FILE_NAME		"cas_count_read"
 #define DYN_PMU_PATH		"/sys/bus/event_source/devices"
 #define SCALE			0.00006103515625
@@ -94,12 +95,8 @@ static int open_perf_read_event(int cpu_no, struct mc_counter_config *mc_counter
 	mc_counter->fd = perf_event_open(&mc_counter->pe, -1, cpu_no, -1,
 					  PERF_FLAG_FD_CLOEXEC);
 
-	if (mc_counter->fd == -1) {
-		fprintf(stderr, "Error opening leader %llx\n",
-			mc_counter->pe.config);
-
-		return -1;
-	}
+	if (mc_counter->fd == -1)
+		return -errno;
 
 	return 0;
 }
@@ -183,16 +180,14 @@ static int parse_mc_read_bw_events(char *mc_dir, unsigned int type)
 	return ret;
 }
 
-/* Get type and config of an MC counter's read event. */
-static int read_from_mc_dir(char *mc_dir)
+/* Get the perf "type" of a memory controller counter. Shared by all vendors. */
+static int get_mc_counter_type(const char *mc_dir, unsigned int *type)
 {
 	char mc_counter_type[PATH_MAX];
-	unsigned int type;
 	int path_len;
 	FILE *fp;
 	int ret;
 
-	/* Get type of MC counter */
 	path_len = snprintf(mc_counter_type, sizeof(mc_counter_type),
 			    "%s%s", mc_dir, "type");
 	if (path_len >= sizeof(mc_counter_type)) {
@@ -206,12 +201,29 @@ static int read_from_mc_dir(char *mc_dir)
 
 		return -1;
 	}
-	ret = fscanf(fp, "%u", &type);
+	ret = fscanf(fp, "%u", type);
 	fclose(fp);
 	if (ret <= 0) {
 		ksft_perror("Could not get MC type");
 		return -1;
 	}
+
+	return 0;
+}
+
+/*
+ * Intel: get type and config of an iMC counter's read event. The read event
+ * and umask are discovered from the PMU's sysfs event files.
+ */
+static int read_from_intel_mc_dir(char *mc_dir)
+{
+	unsigned int type;
+	int ret;
+
+	ret = get_mc_counter_type(mc_dir, &type);
+	if (ret)
+		return ret;
+
 	ret = parse_mc_read_bw_events(mc_dir, type);
 	if (ret) {
 		ksft_print_msg("Unable to parse bandwidth event and umask\n");
@@ -222,10 +234,42 @@ static int read_from_mc_dir(char *mc_dir)
 }
 
 /*
- * Discover the properties of the available MC (Memory Controller)
- * counters in support of needed performance measurement via perf. For each MC
- * counter get it's type and config. Also obtain each counter's event and umask
- * for the memory read events that will be measured.
+ * AMD: get type and config of a UMC counter's read event. The UMC PMU does
+ * not expose the CAS command events via sysfs, so program them directly:
+ * Number of CAS commands sent for reads: EventCode = 0x0a, umask = 0x1
+ */
+static int read_from_amd_mc_dir(char *mc_dir)
+{
+	struct mc_counter_config *mc_counter;
+	unsigned int type;
+	int ret;
+
+	ret = get_mc_counter_type(mc_dir, &type);
+	if (ret)
+		return ret;
+
+	mc_counter = calloc(1, sizeof(*mc_counter));
+	if (!mc_counter) {
+		ksft_perror("Unable to allocate memory for MC counters");
+		return -1;
+	}
+
+	mc_counter->type = type;
+	mc_counter->event = 0xa;
+	mc_counter->umask = 0x1;
+	list_add(&mc_counter->entry, &mc_counters_list);
+
+	return 0;
+}
+
+/*
+ * Discover the properties of the available MC (Memory Controller) counters.
+ * On Intel the memory controller is called Integrated Memory Controller (iMC)
+ * and on AMD it is called Unified Memory Controller (UMC). Discover the
+ * properties of the available counters in support of needed performance
+ * measurement via perf. For each MC counter get it's type and config. Also
+ * obtain each counter's event and umask for the memory read events that will
+ * be measured.
  *
  * Enumerate all these details into a linked list of structures.
  *
@@ -233,6 +277,7 @@ static int read_from_mc_dir(char *mc_dir)
  */
 static int enumerate_mcs(void)
 {
+	int (*read_mc_dir)(char *mc_dir);
 	char mc_dir[512], *temp;
 	struct dirent *ep;
 	char *sysfs_name;
@@ -241,6 +286,10 @@ static int enumerate_mcs(void)
 
 	if (get_vendor() == ARCH_INTEL) {
 		sysfs_name = UNCORE_IMC;
+		read_mc_dir = read_from_intel_mc_dir;
+	} else if (get_vendor() == ARCH_AMD) {
+		sysfs_name = AMD_UMC;
+		read_mc_dir = read_from_amd_mc_dir;
 	} else {
 		ksft_print_msg("Unsupported vendor\n");
 		return -1;
@@ -254,8 +303,9 @@ static int enumerate_mcs(void)
 				continue;
 
 			/*
-			 * mc counters are named as "uncore_imc_<n>", hence
-			 * increment the pointer to point to <n>.
+			 * MC counters are named as "uncore_imc_<n>" on Intel
+			 * and "amd_umc_<n>" on AMD, hence increment the pointer
+			 * to point to <n>.
 			 */
 			temp = temp + strlen(sysfs_name);
 
@@ -267,7 +317,7 @@ static int enumerate_mcs(void)
 			if (temp[0] >= '0' && temp[0] <= '9') {
 				sprintf(mc_dir, "%s/%s/", DYN_PMU_PATH,
 					ep->d_name);
-				ret = read_from_mc_dir(mc_dir);
+				ret = read_mc_dir(mc_dir);
 				if (ret) {
 					closedir(dp);
 
@@ -277,7 +327,9 @@ static int enumerate_mcs(void)
 		}
 		closedir(dp);
 		if (list_empty(&mc_counters_list)) {
-			ksft_print_msg("Unable to find MC counters\n");
+			ksft_print_msg("Unable to find memory controller counters\n");
+			if (get_vendor() == ARCH_AMD)
+				ksft_print_msg("Try loading amd_uncore module\n");
 
 			return -1;
 		}
@@ -335,15 +387,40 @@ static void perf_close_mc_read_mem_bw(void)
 static int perf_open_mc_read_mem_bw(int cpu_no)
 {
 	struct mc_counter_config *mc_counter;
-	int ret;
+	int ret, opened = 0;
 
 	list_for_each_entry(mc_counter, &mc_counters_list, entry)
 		mc_counter->fd = -1;
 
 	list_for_each_entry(mc_counter, &mc_counters_list, entry) {
 		ret = open_perf_read_event(cpu_no, mc_counter);
-		if (ret)
-			goto close_fds;
+		if (!ret) {
+			opened++;
+			continue;
+		}
+
+		/*
+		 * On AMD each memory controller (UMC) has its own PMU that is
+		 * tied to a specific NUMA node, unlike Intel where opening an
+		 * iMC event on the benchmark CPU always resolves to the local
+		 * memory controllers. A UMC PMU can only be opened from a CPU
+		 * that belongs to the same node and returns -ENODEV otherwise.
+		 * The benchmark runs on a single CPU, so skip the UMCs that
+		 * belong to remote nodes and measure only the memory
+		 * controllers local to the benchmark. This matches what
+		 * mbm_local_bytes accounts for on the resctrl side.
+		 */
+		if (ret == -ENODEV && get_vendor() == ARCH_AMD)
+			continue;
+
+		ksft_print_msg("Error opening memory bandwidth performance counter: %s\n",
+			       strerror(-ret));
+		goto close_fds;
+	}
+
+	if (!opened) {
+		ksft_print_msg("Could not open any memory controller counters\n");
+		goto close_fds;
 	}
 
 	return 0;
@@ -363,14 +440,20 @@ static void do_mc_read_mem_bw_test(void)
 {
 	struct mc_counter_config *mc_counter;
 
-	list_for_each_entry(mc_counter, &mc_counters_list, entry)
+	list_for_each_entry(mc_counter, &mc_counters_list, entry) {
+		if (mc_counter->fd == -1)
+			continue;
 		read_mem_bw_ioctl_perf_event_ioc_reset_enable(mc_counter);
+	}
 
 	sleep(1);
 
 	/* Stop counters after a second to get results. */
-	list_for_each_entry(mc_counter, &mc_counters_list, entry)
+	list_for_each_entry(mc_counter, &mc_counters_list, entry) {
+		if (mc_counter->fd == -1)
+			continue;
 		read_mem_bw_ioctl_perf_event_ioc_disable(mc_counter);
+	}
 }
 
 /*
@@ -395,6 +478,10 @@ static int get_read_mem_bw_mc(float *bw_mc)
 	list_for_each_entry(r, &mc_counters_list, entry) {
 		struct membw_read_format measurement;
 
+		/* Skip memory controllers that were not opened (remote node). */
+		if (r->fd == -1)
+			continue;
+
 		if (read(r->fd, &measurement, sizeof(measurement)) == -1) {
 			ksft_perror("Couldn't get read bandwidth through MC");
 			return -1;
-- 
2.43.0