[RFC PATCH] gcc-plugins: Add a plugin to ease debugging of memory ordering bugs

[RFC PATCH] gcc-plugins: Add a plugin to ease debugging of memory ordering bugs

[Gautam Menghani]

On CPUs that support out of order execution, barriers are used to make
sure that if a cpu updates a variable, all other cpus see the correct
value. However, sometimes developers forget to put the barriers in the
right place and this ends up causing bugs.

Introduce a new plugin that can insert barriers after every load/store
to memory to make it easy to verify if a given bug is due to a missing
barrier.

Testing:
To test this plugin on IBM Power systems, I reverted the commit
bb9b52bd51dc ("KVM: PPC: Book3S HV: Add missing barriers to XIVE code and document them").

I was able to see lockups in the KVM guest with the commit reverted. I
then used the barriers plugin to insert barriers, and noticed that the
lockups were no longer observed in the KVM guest.

Assisted-by: LLM IBM-Bob
Signed-off-by: Gautam Menghani <gautam@linux.ibm.com>
---
 scripts/Makefile.gcc-plugins          |   2 +
 scripts/gcc-plugins/Kconfig           |   6 ++
 scripts/gcc-plugins/barriers_plugin.c | 134 ++++++++++++++++++++++++++
 3 files changed, 142 insertions(+)
 create mode 100644 scripts/gcc-plugins/barriers_plugin.c

diff --git a/scripts/Makefile.gcc-plugins b/scripts/Makefile.gcc-plugins
index b0e1423b09c2..cefc1cdad8df 100644
--- a/scripts/Makefile.gcc-plugins
+++ b/scripts/Makefile.gcc-plugins
@@ -8,6 +8,8 @@ ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
 endif
 export DISABLE_LATENT_ENTROPY_PLUGIN
 
+gcc-plugin-$(CONFIG_GCC_PLUGIN_BARRIERS)	+= barriers_plugin.so
+
 # All the plugin CFLAGS are collected here in case a build target needs to
 # filter them out of the KBUILD_CFLAGS.
 GCC_PLUGINS_CFLAGS := $(strip $(addprefix -fplugin=$(objtree)/scripts/gcc-plugins/, $(gcc-plugin-y)) $(gcc-plugin-cflags-y)) -DGCC_PLUGINS
diff --git a/scripts/gcc-plugins/Kconfig b/scripts/gcc-plugins/Kconfig
index 6b34ba19358d..d73e00fbb732 100644
--- a/scripts/gcc-plugins/Kconfig
+++ b/scripts/gcc-plugins/Kconfig
@@ -36,4 +36,10 @@ config GCC_PLUGIN_LATENT_ENTROPY
 	   * https://grsecurity.net/
 	   * https://pax.grsecurity.net/
 
+config GCC_PLUGIN_BARRIERS
+	bool "Insert barriers after every load/store"
+	help
+	  By saying Y here barriers will be inserted after every load/store
+	  instruction.
+
 endif
diff --git a/scripts/gcc-plugins/barriers_plugin.c b/scripts/gcc-plugins/barriers_plugin.c
new file mode 100644
index 000000000000..54ee8b12e77b
--- /dev/null
+++ b/scripts/gcc-plugins/barriers_plugin.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "gcc-common.h"
+
+#if defined(__powerpc__)
+# define BARRIER_ASM "sync"
+#else
+# define BARRIER_ASM ""  /* compiler barrier only: asm volatile("" ::: "memory") */
+#endif
+
+__visible int plugin_is_GPL_compatible;
+
+static struct plugin_info barriers_plugin_info = {
+	.version = PLUGIN_VERSION,
+	.help = "Insert barriers after every load/store instruction\n"
+};
+
+/*
+ * Return true if this GIMPLE statement performs a non-stack memory access.
+ */
+static bool is_mem_based(tree expr)
+{
+	switch (TREE_CODE(expr)) {
+	case MEM_REF:
+	case INDIRECT_REF:
+	case TARGET_MEM_REF:
+		return true;
+	case COMPONENT_REF:
+		return is_mem_based(TREE_OPERAND(expr, 0));
+	default:
+		return false;
+	}
+}
+
+static bool gimple_has_mem_access(gimple stmt)
+{
+	if (gimple_code(stmt) == GIMPLE_CALL)
+		return true;
+
+	if (gimple_code(stmt) != GIMPLE_ASSIGN)
+		return false;
+
+	if (is_mem_based(gimple_assign_lhs(stmt)))
+		return true;
+
+	if (is_mem_based(gimple_assign_rhs1(stmt)))
+		return true;
+
+	return false;
+}
+
+/*
+ * Build a GIMPLE statement for "asm volatile(<barrier_instn> ::: "memory");".
+ */
+static gasm *build_mb_asm(void)
+{
+	vec <tree, va_gc> *clobbers = NULL;
+	gasm *stmt;
+	tree clob = build_tree_list(NULL_TREE, build_const_char_string(7, "memory"));
+
+	vec_safe_push(clobbers, clob);
+	stmt = gimple_build_asm_vec(BARRIER_ASM, NULL, NULL, clobbers, NULL);
+	gimple_asm_set_volatile(stmt, true);
+	return stmt;
+}
+
+static unsigned int insert_barriers_execute(void)
+{
+	basic_block bb;
+	unsigned int i;
+	gimple_stmt_iterator *gsi_p;
+	gimple stmt;
+	gasm *barrier;
+
+	FOR_EACH_BB_FN(bb, cfun) {
+		/*
+		 * Collect the iterator positions of all memory-accessing
+		 * statements first, then insert. Inserting while iterating
+		 * would visit the newly-added asm statements on the next step.
+		 */
+		auto_vec <gimple_stmt_iterator> to_instrument;
+
+		for (gimple_stmt_iterator gsi = gsi_start_bb(bb);
+		     !gsi_end_p(gsi); gsi_next(&gsi)) {
+			stmt = gsi_stmt(gsi);
+			if (!is_gimple_debug(stmt) && gimple_has_mem_access(stmt))
+				to_instrument.safe_push(gsi);
+		}
+
+		FOR_EACH_VEC_ELT(to_instrument, i, gsi_p) {
+			barrier = build_mb_asm();
+			gsi_insert_after(gsi_p, barrier, GSI_CONTINUE_LINKING);
+			update_stmt(barrier);
+		}
+	}
+	return 0;
+}
+
+#define PASS_NAME insert_barriers
+#define NO_GATE
+#define PROPERTIES_REQUIRED  PROP_gimple_any
+
+#include "gcc-generate-gimple-pass.h"
+
+__visible int plugin_init(struct plugin_name_args *plugin_info,
+			  struct plugin_gcc_version *version)
+{
+	const char * const plugin_name = plugin_info->base_name;
+	const int argc = plugin_info->argc;
+	const struct plugin_argument * const argv = plugin_info->argv;
+	int i = 0;
+
+	/*
+	 * Run before "ssa", the entry to the main GIMPLE optimization pipeline.
+	 * At this point no memory-access reordering has occurred yet, so the
+	 * inserted barrier statements are seen by
+	 * all subsequent passes and act as both compiler and hardware barriers,
+	 * exactly like the kernel's smp_mb() macro.
+	 */
+	PASS_INFO(insert_barriers, "ssa", 1, PASS_POS_INSERT_BEFORE);
+
+	if (!plugin_default_version_check(version, &gcc_version)) {
+		error(G_("incompatible gcc/plugin versions"));
+		return 1;
+	}
+
+	/* Give the information about the plugin */
+	register_callback(plugin_name, PLUGIN_INFO, NULL,
+						&barriers_plugin_info);
+
+	register_callback(plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL,
+					&insert_barriers_pass_info);
+
+	return 0;
+}
\ No newline at end of file
-- 
2.55.0