[PATCH 0/5] MIPS: Select the R10000 LL/SC workaround at boot

[PATCH 0/5] MIPS: Select the R10000 LL/SC workaround at boot

[Imre Kaloz]

Kernels for IP27, IP28 and IP30 select CONFIG_WAR_R10000_LLSC, which
retries ll/sc sequences with a branch-likely that only R10000s before
revision 3.0 need. On an R12000 Octane, dropping it makes fork, pipe,
futex and file creation 1-4% faster.

This series keeps it built in and decides at boot. Unless an affected
CPU is present, sc branches become beqz, modules are patched at load,
and tlbex and the BPF JIT emit beqz.

Patch 3 conflicts in context with the IP35 series' "MIPS: SGI-IP27:
give each CPU its own topology package id"; it resolves in one line.

Patch 5 depends on patch 2, so I'd like the whole series to go through
the MIPS tree, with an Acked-by from the BPF maintainers on patch 5.

Tested on an R12000 Octane, and the BPF JIT with test_bpf on QEMU
64-bit Malta. No R10000 before revision 3.0 was available.

Imre Kaloz (5):
  MIPS: Use SC_BEQZ in the futex and sysmips ll/sc loops
  MIPS: Select the R10000 LL/SC workaround at boot
  MIPS: SGI-IP27: Report all CPUs to the R10000 LL/SC workaround
  MIPS: tlbex: Follow the boot-time R10000 LL/SC choice
  bpf, mips: Follow the boot-time R10000 LL/SC choice

 arch/mips/Kconfig                 |   5 +-
 arch/mips/include/asm/asm.h       |   7 +-
 arch/mips/include/asm/futex.h     |   5 +-
 arch/mips/include/asm/r10k-llsc.h |  29 ++++++++
 arch/mips/kernel/Makefile         |   1 +
 arch/mips/kernel/module.c         |   5 ++
 arch/mips/kernel/r10k-llsc.c      | 111 ++++++++++++++++++++++++++++++
 arch/mips/kernel/setup.c          |   2 +
 arch/mips/kernel/smp.c            |   2 +
 arch/mips/kernel/syscall.c        |   2 +-
 arch/mips/kernel/vmlinux.lds.S    |   8 +++
 arch/mips/mm/tlbex.c              |   3 +-
 arch/mips/net/bpf_jit_comp.c      |   5 +-
 arch/mips/net/bpf_jit_comp.h      |  12 ++--
 arch/mips/net/bpf_jit_comp32.c    |   1 +
 arch/mips/net/bpf_jit_comp64.c    |   5 +-
 arch/mips/sgi-ip27/ip27-smp.c     |   2 +
 17 files changed, 188 insertions(+), 17 deletions(-)
 create mode 100644 arch/mips/include/asm/r10k-llsc.h
 create mode 100644 arch/mips/kernel/r10k-llsc.c


base-commit: 24a19b953e2a3d0ea5a8c4951a0ab8f32720c907
-- 
2.47.3

[PATCH 1/5] MIPS: Use SC_BEQZ in the futex and sysmips ll/sc loops

[Imre Kaloz]

The R10000 variants of the futex operations and of
sysmips(MIPS_ATOMIC_SET) open-code the beqzl that SC_BEQZ expands to
under CONFIG_WAR_R10000_LLSC. Use SC_BEQZ, so they follow any change
to it.

No functional change.

Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---

Notes:
    checkpatch warns about whitespace before the quoted newlines: the
    replaced lines keep the column alignment of the surrounding asm, as the
    beqzl lines they replace did.

 arch/mips/include/asm/futex.h | 5 +++--
 arch/mips/kernel/syscall.c    | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h
index 05832eb240fa..9bfae351b82e 100644
--- a/arch/mips/include/asm/futex.h
+++ b/arch/mips/include/asm/futex.h
@@ -12,6 +12,7 @@
 
 #include <linux/futex.h>
 #include <linux/uaccess.h>
+#include <asm/asm.h>
 #include <asm/asm-eva.h>
 #include <asm/barrier.h>
 #include <asm/compiler.h>
@@ -35,7 +36,7 @@
 		"	" insn	"				\n"	\
 		"	.set	arch=r4000			\n"	\
 		"2:	sc	$1, %2				\n"	\
-		"	beqzl	$1, 1b				\n"	\
+		"	" __stringify(SC_BEQZ) "	$1, 1b		\n"	\
 		__stringify(__WEAK_LLSC_MB) "			\n"	\
 		"3:						\n"	\
 		"	.insn					\n"	\
@@ -151,7 +152,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 		"	move	$1, %z5					\n"
 		"	.set	arch=r4000				\n"
 		"2:	sc	$1, %2					\n"
-		"	beqzl	$1, 1b					\n"
+		"	" __stringify(SC_BEQZ) "	$1, 1b			\n"
 		__stringify(__WEAK_LLSC_MB) "				\n"
 		"3:							\n"
 		"	.insn						\n"
diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c
index 1bfc34a2e5b3..cdc03322839f 100644
--- a/arch/mips/kernel/syscall.c
+++ b/arch/mips/kernel/syscall.c
@@ -115,7 +115,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new)
 		"1:	ll	%[old], (%[addr])			\n"
 		"	move	%[tmp], %[new]				\n"
 		"2:	sc	%[tmp], (%[addr])			\n"
-		"	beqzl	%[tmp], 1b				\n"
+		"	" __stringify(SC_BEQZ) "	%[tmp], 1b		\n"
 		"3:							\n"
 		"	.insn						\n"
 		"	.section .fixup,\"ax\"				\n"
-- 
2.47.3

[PATCH 2/5] MIPS: Select the R10000 LL/SC workaround at boot

[Imre Kaloz]

Only R10000s before revision 3.0 need the branch-likely sc retry of
CONFIG_WAR_R10000_LLSC, which the whole family mispredicts.

Patch the SC_BEQZ sites to beqz at boot and module load unless a CPU
needs it; once patched, do not start such a CPU.

Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---

Notes:
    checkpatch reports an ERROR (spaces around ':') and a do-while WARNING
    for SC_BEQZ. Its body is assembler text pasted into inline asm through
    __stringify(), not C: '9999:' is a GAS local label and ';' separates
    assembler statements. The new files fall under the MIPS entry in
    MAINTAINERS (F: arch/mips/).

 arch/mips/Kconfig                 |   5 +-
 arch/mips/include/asm/asm.h       |   7 +-
 arch/mips/include/asm/r10k-llsc.h |  29 ++++++++
 arch/mips/kernel/Makefile         |   1 +
 arch/mips/kernel/module.c         |   5 ++
 arch/mips/kernel/r10k-llsc.c      | 111 ++++++++++++++++++++++++++++++
 arch/mips/kernel/setup.c          |   2 +
 arch/mips/kernel/smp.c            |   2 +
 arch/mips/kernel/vmlinux.lds.S    |   8 +++
 9 files changed, 166 insertions(+), 4 deletions(-)
 create mode 100644 arch/mips/include/asm/r10k-llsc.h
 create mode 100644 arch/mips/kernel/r10k-llsc.c

diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index d7c67cebe065..f27d189922fe 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -2540,8 +2540,9 @@ config WAR_TX49XX_ICACHE_INDEX_INV
 config WAR_ICACHE_REFILLS
 	bool
 
-# On the R10000 up to version 2.6 (not sure about 2.7) there is a bug that
-# may cause ll / sc and lld / scd sequences to execute non-atomically.
+# R10000 CPUs before revision 3.0 may execute ll/sc and lld/scd sequences
+# non-atomically. Platforms that may have such a CPU select this, and the
+# workaround is dropped at boot when no such CPU is present.
 config WAR_R10000_LLSC
 	bool
 
diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h
index 0ed19ffed076..e69d5fb6c1a7 100644
--- a/arch/mips/include/asm/asm.h
+++ b/arch/mips/include/asm/asm.h
@@ -335,10 +335,13 @@ symbol		=	value
 /*
  * Using a branch-likely instruction to check the result of an sc instruction
  * works around a bug present in R10000 CPUs prior to revision 3.0 that could
- * cause ll-sc sequences to execute non-atomically.
+ * cause ll-sc sequences to execute non-atomically. The address of each such
+ * branch is recorded so that it can be turned into a beqz at boot when no
+ * affected CPU is present.
  */
 #ifdef CONFIG_WAR_R10000_LLSC
-# define SC_BEQZ	beqzl
+# define SC_BEQZ	.pushsection __r10k_llsc, "a"; PTR_WD 9999f;	\
+			.popsection; 9999: beqzl
 #elif !defined(CONFIG_CC_HAS_BROKEN_INLINE_COMPAT_BRANCH) && MIPS_ISA_REV >= 6
 # define SC_BEQZ	beqzc
 #else
diff --git a/arch/mips/include/asm/r10k-llsc.h b/arch/mips/include/asm/r10k-llsc.h
new file mode 100644
index 000000000000..0f0e9b3b03a0
--- /dev/null
+++ b/arch/mips/include/asm/r10k-llsc.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_R10K_LLSC_H
+#define _ASM_R10K_LLSC_H
+
+#include <linux/types.h>
+
+#ifdef CONFIG_WAR_R10000_LLSC
+extern bool r10k_llsc_war;
+extern const unsigned long __start___r10k_llsc[], __stop___r10k_llsc[];
+
+void r10k_llsc_note_cpu(unsigned int prid);
+void r10k_llsc_init(void);
+void r10k_llsc_check_secondary(void);
+unsigned int r10k_llsc_patch(const unsigned long *start,
+			     const unsigned long *end);
+#else
+#define r10k_llsc_war	false
+
+static inline void r10k_llsc_note_cpu(unsigned int prid) { }
+static inline void r10k_llsc_init(void) { }
+static inline void r10k_llsc_check_secondary(void) { }
+static inline unsigned int r10k_llsc_patch(const unsigned long *start,
+					   const unsigned long *end)
+{
+	return 0;
+}
+#endif
+
+#endif /* _ASM_R10K_LLSC_H */
diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile
index 95a1e674fd67..cd992852c31d 100644
--- a/arch/mips/kernel/Makefile
+++ b/arch/mips/kernel/Makefile
@@ -52,6 +52,7 @@ obj-$(CONFIG_CPU_R2300_FPU)	+= r2300_fpu.o
 obj-$(CONFIG_CPU_R4K_FPU)	+= r4k_fpu.o
 
 obj-$(CONFIG_SMP)		+= smp.o
+obj-$(CONFIG_WAR_R10000_LLSC)	+= r10k-llsc.o
 obj-$(CONFIG_SMP_UP)		+= smp-up.o
 obj-$(CONFIG_CPU_BMIPS)		+= smp-bmips.o bmips_vec.o bmips_5xxx_init.o
 
diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c
index 37dea772fd78..e47d0d530790 100644
--- a/arch/mips/kernel/module.c
+++ b/arch/mips/kernel/module.c
@@ -20,6 +20,7 @@
 #include <linux/spinlock.h>
 #include <linux/jump_label.h>
 #include <asm/jump_label.h>
+#include <asm/r10k-llsc.h>
 
 struct mips_hi16 {
 	struct mips_hi16 *next;
@@ -423,6 +424,10 @@ int module_finalize(const Elf_Ehdr *hdr,
 
 	INIT_LIST_HEAD(&me->arch.dbe_list);
 	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
+		if (IS_ENABLED(CONFIG_WAR_R10000_LLSC) &&
+		    !strcmp("__r10k_llsc", secstrings + s->sh_name))
+			r10k_llsc_patch((void *)s->sh_addr,
+					(void *)s->sh_addr + s->sh_size);
 		if (strcmp("__dbe_table", secstrings + s->sh_name) != 0)
 			continue;
 		me->arch.dbe_start = (void *)s->sh_addr;
diff --git a/arch/mips/kernel/r10k-llsc.c b/arch/mips/kernel/r10k-llsc.c
new file mode 100644
index 000000000000..32fd79716bf0
--- /dev/null
+++ b/arch/mips/kernel/r10k-llsc.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * R10000 CPUs before revision 3.0 may execute an ll/sc sequence
+ * non-atomically, and those up to revision 2.6 may hang in one, unless the
+ * branch retrying the sequence is a branch-likely. A kernel that may run on
+ * such a CPU is built with branch-likely retry loops, and each of them is
+ * recorded in the __r10k_llsc table. The R10000 family always predicts a
+ * branch-likely taken, so every successful sc then costs a mispredict; the
+ * recorded branches are turned into plain beqz at boot unless an affected CPU
+ * is present.
+ */
+#define pr_fmt(fmt) "r10k-llsc: " fmt
+
+#include <linux/bug.h>
+#include <linux/cache.h>
+#include <linux/cacheflush.h>
+#include <linux/init.h>
+#include <linux/printk.h>
+#include <linux/processor.h>
+#include <linux/smp.h>
+
+#include <asm/cpu.h>
+#include <asm/cpu-info.h>
+#include <asm/inst.h>
+#include <asm/r10k-llsc.h>
+
+/* Until r10k_llsc_init() has run, every recorded branch is a branch-likely. */
+bool r10k_llsc_war __ro_after_init = true;
+
+static unsigned int r10k_llsc_prid;
+
+static bool r10k_llsc_affected(unsigned int prid)
+{
+	return (prid & (PRID_COMP_MASK | PRID_IMP_MASK)) ==
+	       (PRID_COMP_LEGACY | PRID_IMP_R10000) &&
+	       (prid & PRID_REV_MASK) < PRID_REV_ENCODE_44(3, 0);
+}
+
+/*
+ * A platform whose CPUs may differ reports each CPU it will start here before
+ * r10k_llsc_init(), so that the choice covers the whole machine.
+ */
+void r10k_llsc_note_cpu(unsigned int prid)
+{
+	if (!r10k_llsc_prid && r10k_llsc_affected(prid))
+		r10k_llsc_prid = prid;
+}
+
+unsigned int r10k_llsc_patch(const unsigned long *start,
+			     const unsigned long *end)
+{
+	const unsigned long *p;
+	unsigned int n = 0;
+
+	if (r10k_llsc_war)
+		return 0;
+
+	for (p = start; p < end; p++) {
+		union mips_instruction *insn = (void *)*p;
+
+		/* Only a nop in the delay slot behaves alike under both. */
+		if (WARN_ONCE(insn[0].i_format.opcode != beql_op ||
+			      insn[0].i_format.rt || insn[1].word,
+			      "unexpected sc retry branch at %pS\n", insn))
+			continue;
+
+		insn[0].i_format.opcode = beq_op;
+		flush_icache_range((unsigned long)insn,
+				   (unsigned long)(insn + 1));
+		n++;
+	}
+
+	return n;
+}
+
+void __init r10k_llsc_init(void)
+{
+	unsigned int prid, n;
+
+	r10k_llsc_note_cpu(current_cpu_data.processor_id);
+	prid = r10k_llsc_prid;
+
+	if (prid) {
+		pr_info("enabled for R10000 revision %u.%u\n",
+			(prid >> 4) & 0xf, prid & 0xf);
+		return;
+	}
+
+	r10k_llsc_war = false;
+	n = r10k_llsc_patch(__start___r10k_llsc, __stop___r10k_llsc);
+	pr_info("not needed, %u of %lu sc retry branches patched\n", n,
+		__stop___r10k_llsc - __start___r10k_llsc);
+}
+
+/*
+ * Once the workaround has been dropped, any retry branch a CPU runs may be
+ * a plain beqz, so an affected CPU found only now is kept out of the
+ * system.
+ */
+void r10k_llsc_check_secondary(void)
+{
+	unsigned int prid = current_cpu_data.processor_id;
+
+	if (r10k_llsc_war || !r10k_llsc_affected(prid))
+		return;
+
+	pr_crit("CPU%u: R10000 revision %u.%u needs the ll/sc workaround, not starting it\n",
+		smp_processor_id(), (prid >> 4) & 0xf, prid & 0xf);
+	for (;;)
+		cpu_relax();
+}
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 1ae6d0c0e1d6..8f5f8e473a8a 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -46,6 +46,7 @@
 #include <asm/smp-ops.h>
 #include <asm/mips-cps.h>
 #include <asm/prom.h>
+#include <asm/r10k-llsc.h>
 #include <asm/fw/fw.h>
 
 #ifdef CONFIG_MIPS_ELF_APPENDED_DTB
@@ -783,6 +784,7 @@ void __init setup_arch(char **cmdline_p)
 	prefill_possible_map();
 
 	cpu_cache_init();
+	r10k_llsc_init();
 	pagetable_init();
 
 	memblock_dump_all();
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 0f28b4a62e72..8d32c252117a 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -33,6 +33,7 @@
 #include <asm/processor.h>
 #include <asm/idle.h>
 #include <asm/r4k-timer.h>
+#include <asm/r10k-llsc.h>
 #include <asm/mips-cps.h>
 #include <asm/mmu_context.h>
 #include <asm/time.h>
@@ -360,6 +361,7 @@ asmlinkage void start_secondary(void)
 	unsigned int cpu = raw_smp_processor_id();
 
 	cpu_probe();
+	r10k_llsc_check_secondary();
 	per_cpu_trap_init(false);
 	rcutree_report_cpu_starting(cpu);
 	mips_clockevent_init();
diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index 579b2cc1995a..e7c4103037bb 100644
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -128,6 +128,14 @@ SECTIONS
 		__mips_machines_end = .;
 	}
 
+#ifdef CONFIG_WAR_R10000_LLSC
+	__r10k_llsc : AT(ADDR(__r10k_llsc) - LOAD_OFFSET) {
+		__start___r10k_llsc = .;
+		KEEP(*(__r10k_llsc))
+		__stop___r10k_llsc = .;
+	}
+#endif
+
 	/* .exit.text is discarded at runtime, not link time, to deal with
 	 * references from .rodata
 	 */
-- 
2.47.3

[PATCH 3/5] MIPS: SGI-IP27: Report all CPUs to the R10000 LL/SC workaround

[Imre Kaloz]

Pass the PRId of every enabled CPU in KLCONFIG to r10k_llsc_note_cpu(),
so that an affected CPU on any node keeps the workaround.

Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
 arch/mips/sgi-ip27/ip27-smp.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/mips/sgi-ip27/ip27-smp.c b/arch/mips/sgi-ip27/ip27-smp.c
index 62733e049570..6a5f5903a443 100644
--- a/arch/mips/sgi-ip27/ip27-smp.c
+++ b/arch/mips/sgi-ip27/ip27-smp.c
@@ -15,6 +15,7 @@
 #include <asm/page.h>
 #include <asm/processor.h>
 #include <asm/ptrace.h>
+#include <asm/r10k-llsc.h>
 #include <asm/sn/agent.h>
 #include <asm/sn/arch.h>
 #include <asm/sn/gda.h>
@@ -49,6 +50,7 @@ static int node_scan_cpus(nasid_t nasid, int highest)
 				cputoslice(cpus_found) = acpu->cpu_info.physid;
 				sn_cpu_info[cpus_found].p_speed =
 							acpu->cpu_speed;
+				r10k_llsc_note_cpu(acpu->cpu_prid);
 				cpus_found++;
 			}
 			acpu = (klcpu_t *)find_component(brd, (klinfo_t *)acpu,
-- 
2.47.3

[PATCH 4/5] MIPS: tlbex: Follow the boot-time R10000 LL/SC choice

[Imre Kaloz]

The TLB handlers are generated after the boot-time R10000 LL/SC
choice. Emit the branch-likely in their ll/sc loop only when the
workaround is in use.

Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
 arch/mips/mm/tlbex.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 69ea54bdc0c3..32ce99dc90a0 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -35,6 +35,7 @@
 #include <asm/mipsregs.h>
 #include <asm/mmu_context.h>
 #include <asm/regdef.h>
+#include <asm/r10k-llsc.h>
 #include <asm/uasm.h>
 #include <asm/setup.h>
 #include <asm/tlbex.h>
@@ -95,7 +96,7 @@ static inline int __maybe_unused bcm1250_m3_war(void)
 
 static inline int __maybe_unused r10000_llsc_war(void)
 {
-	return IS_ENABLED(CONFIG_WAR_R10000_LLSC);
+	return r10k_llsc_war;
 }
 
 static int use_bbit_insns(void)
-- 
2.47.3

[PATCH 5/5] bpf, mips: Follow the boot-time R10000 LL/SC choice

[Imre Kaloz]

Programs are JITed after the R10000 LL/SC workaround has been selected
at boot. Emit the branch-likely in the atomic and cmpxchg ll/sc loops
only when the workaround is in use.

Signed-off-by: Imre Kaloz <kaloz@kernel.org>
---
 arch/mips/net/bpf_jit_comp.c   |  5 +++--
 arch/mips/net/bpf_jit_comp.h   | 12 +++++++-----
 arch/mips/net/bpf_jit_comp32.c |  1 +
 arch/mips/net/bpf_jit_comp64.c |  5 +++--
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index 6ee4abe6a1f7..11e50193e205 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -112,6 +112,7 @@
 #include <asm/cacheflush.h>
 #include <asm/cpu-features.h>
 #include <asm/isa-rev.h>
+#include <asm/r10k-llsc.h>
 #include <asm/uasm.h>
 
 #include "bpf_jit_comp.h"
@@ -432,7 +433,7 @@ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code)
 		break;
 	}
 	emit(ctx, sc, MIPS_R_T8, off, dst);
-	emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset);
+	emit_llsc_beqz(ctx, MIPS_R_T8, -16 - LLSC_offset);
 	emit(ctx, nop); /* Delay slot */
 
 	if (code & BPF_FETCH) {
@@ -449,7 +450,7 @@ void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off)
 	emit(ctx, bne, MIPS_R_T9, res, 12);
 	emit(ctx, move, MIPS_R_T8, src);     /* Delay slot */
 	emit(ctx, sc, MIPS_R_T8, off, dst);
-	emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset);
+	emit_llsc_beqz(ctx, MIPS_R_T8, -20 - LLSC_offset);
 	emit(ctx, move, res, MIPS_R_T9);     /* Delay slot */
 	clobber_reg(ctx, res);
 }
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
index a37fe20818eb..3129691bccde 100644
--- a/arch/mips/net/bpf_jit_comp.h
+++ b/arch/mips/net/bpf_jit_comp.h
@@ -98,11 +98,13 @@ do {								\
 #define emit(...) __emit(__VA_ARGS__)
 
 /* Workaround for R10000 ll/sc errata */
-#ifdef CONFIG_WAR_R10000_LLSC
-#define LLSC_beqz	beqzl
-#else
-#define LLSC_beqz	beqz
-#endif
+static inline void emit_llsc_beqz(struct jit_context *ctx, u8 rs, s16 off)
+{
+	if (r10k_llsc_war)
+		emit(ctx, beqzl, rs, off);
+	else
+		emit(ctx, beqz, rs, off);
+}
 
 /* Workaround for Loongson-3 ll/sc errata */
 #ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c
index 15a2a153dc87..07aece9edc8e 100644
--- a/arch/mips/net/bpf_jit_comp32.c
+++ b/arch/mips/net/bpf_jit_comp32.c
@@ -18,6 +18,7 @@
 #include <linux/bpf.h>
 #include <asm/cpu-features.h>
 #include <asm/isa-rev.h>
+#include <asm/r10k-llsc.h>
 #include <asm/uasm.h>
 
 #include "bpf_jit_comp.h"
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index 6681ccac9dd9..fd10c8b0ece8 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -17,6 +17,7 @@
 #include <linux/bpf.h>
 #include <asm/cpu-features.h>
 #include <asm/isa-rev.h>
+#include <asm/r10k-llsc.h>
 #include <asm/uasm.h>
 
 #include "bpf_jit_comp.h"
@@ -407,7 +408,7 @@ static void emit_atomic_r64(struct jit_context *ctx,
 		break;
 	}
 	emit(ctx, scd, t2, off, dst);
-	emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
+	emit_llsc_beqz(ctx, t2, -16 - LLSC_offset);
 	emit(ctx, nop); /* Delay slot */
 
 	if (code & BPF_FETCH) {
@@ -428,7 +429,7 @@ static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
 	emit(ctx, bne, t1, r0, 12);
 	emit(ctx, move, t2, src);      /* Delay slot */
 	emit(ctx, scd, t2, off, dst);
-	emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
+	emit_llsc_beqz(ctx, t2, -20 - LLSC_offset);
 	emit(ctx, move, r0, t1);       /* Delay slot */
 
 	clobber_reg(ctx, r0);
-- 
2.47.3