[PATCH v1 0/5] Support trampoline for LoongArch

[PATCH v1 0/5] Support trampoline for LoongArch

[Chenghao Duan]

v1: Support trampoline for LoongArch. The following feature tests
have been completed:
1. fentry
2. fexit
3. fmod_ret

TODO: The support for the struct_ops feature will be provided in
subsequent patches.

Chenghao Duan (4):
  LoongArch: BPF: The operation commands needed to add a trampoline
  LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch
  LoongArch: BPF: Add bpf trampoline support for Loongarch
  LoongArch: BPF: Update the code to rename validate_code to
    validate_ctx.

George Guo (1):
  LoongArch: Support fixmap

 arch/loongarch/include/asm/fixmap.h |   2 +
 arch/loongarch/include/asm/inst.h   |  19 ++
 arch/loongarch/kernel/inst.c        |  85 +++++
 arch/loongarch/kernel/setup.c       |   1 +
 arch/loongarch/mm/init.c            | 111 ++++++-
 arch/loongarch/net/bpf_jit.c        | 497 +++++++++++++++++++++++++++-
 arch/loongarch/net/bpf_jit.h        |   6 +
 7 files changed, 714 insertions(+), 7 deletions(-)

Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>

-- 
2.25.1

[PATCH v1 1/5] LoongArch: Support fixmap

[Chenghao Duan]

From: George Guo <guodongtai@kylinos.cn>

Support fixmap for Loongarch.

In the functions patch_map and patch_unmap, __set_fixmap are used.
Therefore, remove the __init identifier from these functions.

Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
Signed-off-by: George Guo <guodongtai@kylinos.cn>
---
 arch/loongarch/include/asm/fixmap.h |   2 +
 arch/loongarch/kernel/setup.c       |   1 +
 arch/loongarch/mm/init.c            | 111 ++++++++++++++++++++++++++--
 3 files changed, 108 insertions(+), 6 deletions(-)

diff --git a/arch/loongarch/include/asm/fixmap.h b/arch/loongarch/include/asm/fixmap.h
index d2e55ae55..b579ad2be 100644
--- a/arch/loongarch/include/asm/fixmap.h
+++ b/arch/loongarch/include/asm/fixmap.h
@@ -13,6 +13,7 @@
 enum fixed_addresses {
 	FIX_HOLE,
 	FIX_EARLYCON_MEM_BASE,
+	FIX_TEXT_POKE0,
 	__end_of_fixed_addresses
 };
 
@@ -20,6 +21,7 @@ enum fixed_addresses {
 #define FIXADDR_START	(FIXADDR_TOP - FIXADDR_SIZE)
 #define FIXMAP_PAGE_IO	PAGE_KERNEL_SUC
 
+void __init early_fixmap_init(void);
 extern void __set_fixmap(enum fixed_addresses idx,
 			 phys_addr_t phys, pgprot_t flags);
 
diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
index 56934fe58..368786952 100644
--- a/arch/loongarch/kernel/setup.c
+++ b/arch/loongarch/kernel/setup.c
@@ -597,6 +597,7 @@ void __init setup_arch(char **cmdline_p)
 	memblock_init();
 	pagetable_init();
 	bootcmdline_init(cmdline_p);
+	early_fixmap_init();
 	parse_early_param();
 	reserve_initrd_mem();
 
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 188b52bbb..68abb7bad 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -36,6 +36,109 @@
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 
+#define SPAN_NR_ENTRIES(vstart, vend, shift) \
+	((((vend) - 1) >> (shift)) - ((vstart) >> (shift)) + 1)
+
+#define NR_BM_PTE_TABLES \
+	SPAN_NR_ENTRIES(FIXADDR_START, FIXADDR_TOP, PMD_SHIFT)
+#define NR_BM_PMD_TABLES \
+	SPAN_NR_ENTRIES(FIXADDR_START, FIXADDR_TOP, PUD_SHIFT)
+
+static_assert(NR_BM_PMD_TABLES == 1);
+
+#define __BM_TABLE_IDX(addr, shift) \
+	(((addr) >> (shift)) - (FIXADDR_START >> (shift)))
+
+#define BM_PTE_TABLE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
+
+static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss;
+static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
+static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+
+static inline pte_t *fixmap_pte(unsigned long addr)
+{
+	return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)];
+}
+
+static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
+{
+	pmd_t pmd = READ_ONCE(*pmdp);
+	pte_t *ptep;
+
+	if (!pmd_present(pmd)) {
+		ptep = bm_pte[BM_PTE_TABLE_IDX(addr)];
+		pmd_populate_kernel(&init_mm, pmdp, ptep);
+		kernel_pte_init(ptep);
+	}
+}
+
+static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr,
+					 unsigned long end)
+{
+	unsigned long next;
+	pud_t pud = READ_ONCE(*pudp);
+	pmd_t *pmdp;
+
+	if (pud_none(pud))
+		pud_populate(&init_mm, pudp, bm_pmd);
+
+	pmdp = pmd_offset(pudp, addr);
+
+#ifndef __PAGETABLE_PMD_FOLDED
+	pmd_init(pmdp);
+#endif
+	do {
+		next = pmd_addr_end(addr, end);
+		early_fixmap_init_pte(pmdp, addr);
+	} while (pmdp++, addr = next, addr != end);
+}
+
+static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
+					 unsigned long end)
+{
+	p4d_t p4d = READ_ONCE(*p4dp);
+	pud_t *pudp;
+
+#ifndef __PAGETABLE_PUD_FOLDED
+	if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) &&
+	    p4d_phys(p4d) != __pa_symbol(bm_pud)) {
+		/*
+		 * We only end up here if the kernel mapping and the fixmap
+		 * share the top level pgd entry, which should only happen on
+		 * 16k/4 levels configurations.
+		 */
+		BUG_ON(!IS_ENABLED(CONFIG_PAGE_SIZE_16KB));
+	}
+#endif
+
+	if (p4d_none(p4d))
+		p4d_populate(&init_mm, p4dp, bm_pud);
+
+	pudp = pud_offset(p4dp, addr);
+
+#ifndef __PAGETABLE_PUD_FOLDED
+	pud_init(pudp);
+#endif
+	early_fixmap_init_pmd(pudp, addr, end);
+}
+
+/*
+ * The p*d_populate functions call virt_to_phys implicitly so they can't be used
+ * directly on kernel symbols (bm_p*d). This function is called too early to use
+ * lm_alias so __p*d_populate functions must be used to populate with the
+ * physical address from __pa_symbol.
+ */
+void __init early_fixmap_init(void)
+{
+	unsigned long addr = FIXADDR_START;
+	unsigned long end = FIXADDR_TOP;
+
+	pgd_t *pgdp = pgd_offset_k(addr);
+	p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+	early_fixmap_init_pud(p4dp, addr, end);
+}
+
 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
 EXPORT_SYMBOL(empty_zero_page);
 
@@ -209,7 +312,7 @@ pte_t * __init populate_kernel_pte(unsigned long addr)
 	return pte_offset_kernel(pmd, addr);
 }
 
-void __init __set_fixmap(enum fixed_addresses idx,
+void __set_fixmap(enum fixed_addresses idx,
 			       phys_addr_t phys, pgprot_t flags)
 {
 	unsigned long addr = __fix_to_virt(idx);
@@ -217,11 +320,7 @@ void __init __set_fixmap(enum fixed_addresses idx,
 
 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
 
-	ptep = populate_kernel_pte(addr);
-	if (!pte_none(ptep_get(ptep))) {
-		pte_ERROR(*ptep);
-		return;
-	}
+	ptep = fixmap_pte(addr);
 
 	if (pgprot_val(flags))
 		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
-- 
2.25.1

[PATCH v1 2/5] LoongArch: BPF: The operation commands needed to add a trampoline

[Chenghao Duan]

Add instrctions:
emit_NOP
emit_BREAK

Add branch jump function:
larch_insn_gen_beq
larch_insn_gen_bne

Add instruction copy function: larch_insn_text_copy
The implementation of larch_insn_text_copy uses the fixmap
FIX_TEXT_POKE0.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Youling Tang <tangyouling@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/include/asm/inst.h | 19 +++++++
 arch/loongarch/kernel/inst.c      | 85 +++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)

diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
index 3089785ca..dd6e07781 100644
--- a/arch/loongarch/include/asm/inst.h
+++ b/arch/loongarch/include/asm/inst.h
@@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs);
 int larch_insn_read(void *addr, u32 *insnp);
 int larch_insn_write(void *addr, u32 insn);
 int larch_insn_patch_text(void *addr, u32 insn);
+int larch_insn_text_copy(void *dst, void *src, size_t len);
 
 u32 larch_insn_gen_nop(void);
 u32 larch_insn_gen_b(unsigned long pc, unsigned long dest);
@@ -511,6 +512,8 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
 u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
+u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
+u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
 
 static inline bool signed_imm_check(long val, unsigned int bit)
 {
@@ -778,6 +781,22 @@ static inline void emit_##NAME(union loongarch_instruction *insn,	\
 
 DEF_EMIT_REG3SA2_FORMAT(alsld, alsld_op)
 
+#define DEF_EMIT_NOP(NAME)						\
+static inline void emit_##NAME(union loongarch_instruction *insn)	\
+{									\
+	insn->word = INSN_NOP;						\
+}
+
+DEF_EMIT_NOP(NOP)
+
+#define DEF_EMIT_BREAK(NAME)						\
+static inline void emit_##NAME(union loongarch_instruction *insn)	\
+{									\
+	insn->word = INSN_BREAK;					\
+}
+
+DEF_EMIT_BREAK(BREAK)
+
 struct pt_regs;
 
 void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc);
diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c
index 14d7d700b..a47dc3575 100644
--- a/arch/loongarch/kernel/inst.c
+++ b/arch/loongarch/kernel/inst.c
@@ -10,6 +10,33 @@
 
 static DEFINE_RAW_SPINLOCK(patch_lock);
 
+static bool is_image_text(unsigned long addr)
+{
+	return core_kernel_text(addr);
+}
+
+static void  *patch_map(void *addr, int fixmap)
+{
+	unsigned long uintaddr = (uintptr_t)addr;
+	bool image = is_image_text(uintaddr);
+	struct page *page;
+	phys_addr_t phys;
+
+	if (image)
+		phys = __pa_symbol(addr);
+	else {
+		page = vmalloc_to_page(addr);
+	phys = page_to_phys(page) + offset_in_page(addr);
+	}
+
+	return (void *)set_fixmap_offset(fixmap, phys);
+}
+
+static void patch_unmap(int fixmap)
+{
+	clear_fixmap(fixmap);
+}
+
 void simu_pc(struct pt_regs *regs, union loongarch_instruction insn)
 {
 	unsigned long pc = regs->csr_era;
@@ -218,6 +245,36 @@ int larch_insn_patch_text(void *addr, u32 insn)
 	return ret;
 }
 
+int larch_insn_text_copy(void *dst, void *src, size_t len)
+{
+	unsigned long flags;
+	size_t wlen = 0;
+	size_t size;
+	void *waddr;
+	void *ptr;
+	int ret = 0;
+
+	raw_spin_lock_irqsave(&patch_lock, flags);
+	while (wlen < len) {
+		ptr = dst + wlen;
+		size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr),
+			     len - wlen);
+
+		waddr = patch_map(ptr, FIX_TEXT_POKE0);
+		ret = copy_to_kernel_nofault(waddr, src + wlen, size);
+		patch_unmap(FIX_TEXT_POKE0);
+
+		if (ret) {
+			pr_err("%s: operation failed\n", __func__);
+			break;
+		}
+		wlen += size;
+	}
+	raw_spin_unlock_irqrestore(&patch_lock, flags);
+
+	return ret;
+}
+
 u32 larch_insn_gen_nop(void)
 {
 	return INSN_NOP;
@@ -336,3 +393,31 @@ u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
 
 	return insn.word;
 }
+
+u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
+{
+	union loongarch_instruction insn;
+
+	if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
+		pr_warn("The generated beq instruction is out of range.\n");
+		return INSN_BREAK;
+	}
+
+	emit_beq(&insn, rd, rj, imm >> 2);
+
+	return insn.word;
+}
+
+u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
+{
+	union loongarch_instruction insn;
+
+	if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
+		pr_warn("The generated bne instruction is out of range.\n");
+		return INSN_BREAK;
+	}
+
+	emit_bne(&insn, rj, rd, imm >> 2);
+
+	return insn.word;
+}
-- 
2.25.1

[PATCH v1 0/5] Support trampoline for LoongArch

[Chenghao Duan]

v1: Support trampoline for LoongArch. The following feature tests
have been completed:
1. fentry
2. fexit
3. fmod_ret

TODO: The support for the struct_ops feature will be provided in
subsequent patches.

Chenghao Duan (4):
  LoongArch: BPF: The operation commands needed to add a trampoline
  LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch
  LoongArch: BPF: Add bpf trampoline support for Loongarch
  LoongArch: BPF: Update the code to rename validate_code to
    validate_ctx.

George Guo (1):
  LoongArch: Support fixmap

 arch/loongarch/include/asm/fixmap.h |   2 +
 arch/loongarch/include/asm/inst.h   |  19 ++
 arch/loongarch/kernel/inst.c        |  85 +++++
 arch/loongarch/kernel/setup.c       |   1 +
 arch/loongarch/mm/init.c            | 111 ++++++-
 arch/loongarch/net/bpf_jit.c        | 497 +++++++++++++++++++++++++++-
 arch/loongarch/net/bpf_jit.h        |   6 +
 7 files changed, 714 insertions(+), 7 deletions(-)

Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>

-- 
2.25.1

[PATCH v1 1/5] LoongArch: Support fixmap

[Chenghao Duan]

From: George Guo <guodongtai@kylinos.cn>

Support fixmap for Loongarch.

In the functions patch_map and patch_unmap, __set_fixmap are used.
Therefore, remove the __init identifier from these functions.

Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
Signed-off-by: George Guo <guodongtai@kylinos.cn>
---
 arch/loongarch/include/asm/fixmap.h |   2 +
 arch/loongarch/kernel/setup.c       |   1 +
 arch/loongarch/mm/init.c            | 111 ++++++++++++++++++++++++++--
 3 files changed, 108 insertions(+), 6 deletions(-)

diff --git a/arch/loongarch/include/asm/fixmap.h b/arch/loongarch/include/asm/fixmap.h
index d2e55ae55..b579ad2be 100644
--- a/arch/loongarch/include/asm/fixmap.h
+++ b/arch/loongarch/include/asm/fixmap.h
@@ -13,6 +13,7 @@
 enum fixed_addresses {
 	FIX_HOLE,
 	FIX_EARLYCON_MEM_BASE,
+	FIX_TEXT_POKE0,
 	__end_of_fixed_addresses
 };
 
@@ -20,6 +21,7 @@ enum fixed_addresses {
 #define FIXADDR_START	(FIXADDR_TOP - FIXADDR_SIZE)
 #define FIXMAP_PAGE_IO	PAGE_KERNEL_SUC
 
+void __init early_fixmap_init(void);
 extern void __set_fixmap(enum fixed_addresses idx,
 			 phys_addr_t phys, pgprot_t flags);
 
diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
index 56934fe58..368786952 100644
--- a/arch/loongarch/kernel/setup.c
+++ b/arch/loongarch/kernel/setup.c
@@ -597,6 +597,7 @@ void __init setup_arch(char **cmdline_p)
 	memblock_init();
 	pagetable_init();
 	bootcmdline_init(cmdline_p);
+	early_fixmap_init();
 	parse_early_param();
 	reserve_initrd_mem();
 
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 188b52bbb..68abb7bad 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -36,6 +36,109 @@
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 
+#define SPAN_NR_ENTRIES(vstart, vend, shift) \
+	((((vend) - 1) >> (shift)) - ((vstart) >> (shift)) + 1)
+
+#define NR_BM_PTE_TABLES \
+	SPAN_NR_ENTRIES(FIXADDR_START, FIXADDR_TOP, PMD_SHIFT)
+#define NR_BM_PMD_TABLES \
+	SPAN_NR_ENTRIES(FIXADDR_START, FIXADDR_TOP, PUD_SHIFT)
+
+static_assert(NR_BM_PMD_TABLES == 1);
+
+#define __BM_TABLE_IDX(addr, shift) \
+	(((addr) >> (shift)) - (FIXADDR_START >> (shift)))
+
+#define BM_PTE_TABLE_IDX(addr)	__BM_TABLE_IDX(addr, PMD_SHIFT)
+
+static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss;
+static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused;
+static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
+
+static inline pte_t *fixmap_pte(unsigned long addr)
+{
+	return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)];
+}
+
+static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
+{
+	pmd_t pmd = READ_ONCE(*pmdp);
+	pte_t *ptep;
+
+	if (!pmd_present(pmd)) {
+		ptep = bm_pte[BM_PTE_TABLE_IDX(addr)];
+		pmd_populate_kernel(&init_mm, pmdp, ptep);
+		kernel_pte_init(ptep);
+	}
+}
+
+static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr,
+					 unsigned long end)
+{
+	unsigned long next;
+	pud_t pud = READ_ONCE(*pudp);
+	pmd_t *pmdp;
+
+	if (pud_none(pud))
+		pud_populate(&init_mm, pudp, bm_pmd);
+
+	pmdp = pmd_offset(pudp, addr);
+
+#ifndef __PAGETABLE_PMD_FOLDED
+	pmd_init(pmdp);
+#endif
+	do {
+		next = pmd_addr_end(addr, end);
+		early_fixmap_init_pte(pmdp, addr);
+	} while (pmdp++, addr = next, addr != end);
+}
+
+static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
+					 unsigned long end)
+{
+	p4d_t p4d = READ_ONCE(*p4dp);
+	pud_t *pudp;
+
+#ifndef __PAGETABLE_PUD_FOLDED
+	if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) &&
+	    p4d_phys(p4d) != __pa_symbol(bm_pud)) {
+		/*
+		 * We only end up here if the kernel mapping and the fixmap
+		 * share the top level pgd entry, which should only happen on
+		 * 16k/4 levels configurations.
+		 */
+		BUG_ON(!IS_ENABLED(CONFIG_PAGE_SIZE_16KB));
+	}
+#endif
+
+	if (p4d_none(p4d))
+		p4d_populate(&init_mm, p4dp, bm_pud);
+
+	pudp = pud_offset(p4dp, addr);
+
+#ifndef __PAGETABLE_PUD_FOLDED
+	pud_init(pudp);
+#endif
+	early_fixmap_init_pmd(pudp, addr, end);
+}
+
+/*
+ * The p*d_populate functions call virt_to_phys implicitly so they can't be used
+ * directly on kernel symbols (bm_p*d). This function is called too early to use
+ * lm_alias so __p*d_populate functions must be used to populate with the
+ * physical address from __pa_symbol.
+ */
+void __init early_fixmap_init(void)
+{
+	unsigned long addr = FIXADDR_START;
+	unsigned long end = FIXADDR_TOP;
+
+	pgd_t *pgdp = pgd_offset_k(addr);
+	p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+	early_fixmap_init_pud(p4dp, addr, end);
+}
+
 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
 EXPORT_SYMBOL(empty_zero_page);
 
@@ -209,7 +312,7 @@ pte_t * __init populate_kernel_pte(unsigned long addr)
 	return pte_offset_kernel(pmd, addr);
 }
 
-void __init __set_fixmap(enum fixed_addresses idx,
+void __set_fixmap(enum fixed_addresses idx,
 			       phys_addr_t phys, pgprot_t flags)
 {
 	unsigned long addr = __fix_to_virt(idx);
@@ -217,11 +320,7 @@ void __init __set_fixmap(enum fixed_addresses idx,
 
 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
 
-	ptep = populate_kernel_pte(addr);
-	if (!pte_none(ptep_get(ptep))) {
-		pte_ERROR(*ptep);
-		return;
-	}
+	ptep = fixmap_pte(addr);
 
 	if (pgprot_val(flags))
 		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
-- 
2.25.1

[PATCH v1 2/5] LoongArch: BPF: The operation commands needed to add a trampoline

[Chenghao Duan]

Add instrctions:
emit_NOP
emit_BREAK

Add branch jump function:
larch_insn_gen_beq
larch_insn_gen_bne

Add instruction copy function: larch_insn_text_copy
The implementation of larch_insn_text_copy uses the fixmap
FIX_TEXT_POKE0.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Youling Tang <tangyouling@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/include/asm/inst.h | 19 +++++++
 arch/loongarch/kernel/inst.c      | 85 +++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)

diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
index 3089785ca..dd6e07781 100644
--- a/arch/loongarch/include/asm/inst.h
+++ b/arch/loongarch/include/asm/inst.h
@@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs);
 int larch_insn_read(void *addr, u32 *insnp);
 int larch_insn_write(void *addr, u32 insn);
 int larch_insn_patch_text(void *addr, u32 insn);
+int larch_insn_text_copy(void *dst, void *src, size_t len);
 
 u32 larch_insn_gen_nop(void);
 u32 larch_insn_gen_b(unsigned long pc, unsigned long dest);
@@ -511,6 +512,8 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
 u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
 u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
+u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
+u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
 
 static inline bool signed_imm_check(long val, unsigned int bit)
 {
@@ -778,6 +781,22 @@ static inline void emit_##NAME(union loongarch_instruction *insn,	\
 
 DEF_EMIT_REG3SA2_FORMAT(alsld, alsld_op)
 
+#define DEF_EMIT_NOP(NAME)						\
+static inline void emit_##NAME(union loongarch_instruction *insn)	\
+{									\
+	insn->word = INSN_NOP;						\
+}
+
+DEF_EMIT_NOP(NOP)
+
+#define DEF_EMIT_BREAK(NAME)						\
+static inline void emit_##NAME(union loongarch_instruction *insn)	\
+{									\
+	insn->word = INSN_BREAK;					\
+}
+
+DEF_EMIT_BREAK(BREAK)
+
 struct pt_regs;
 
 void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc);
diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c
index 14d7d700b..a47dc3575 100644
--- a/arch/loongarch/kernel/inst.c
+++ b/arch/loongarch/kernel/inst.c
@@ -10,6 +10,33 @@
 
 static DEFINE_RAW_SPINLOCK(patch_lock);
 
+static bool is_image_text(unsigned long addr)
+{
+	return core_kernel_text(addr);
+}
+
+static void  *patch_map(void *addr, int fixmap)
+{
+	unsigned long uintaddr = (uintptr_t)addr;
+	bool image = is_image_text(uintaddr);
+	struct page *page;
+	phys_addr_t phys;
+
+	if (image)
+		phys = __pa_symbol(addr);
+	else {
+		page = vmalloc_to_page(addr);
+	phys = page_to_phys(page) + offset_in_page(addr);
+	}
+
+	return (void *)set_fixmap_offset(fixmap, phys);
+}
+
+static void patch_unmap(int fixmap)
+{
+	clear_fixmap(fixmap);
+}
+
 void simu_pc(struct pt_regs *regs, union loongarch_instruction insn)
 {
 	unsigned long pc = regs->csr_era;
@@ -218,6 +245,36 @@ int larch_insn_patch_text(void *addr, u32 insn)
 	return ret;
 }
 
+int larch_insn_text_copy(void *dst, void *src, size_t len)
+{
+	unsigned long flags;
+	size_t wlen = 0;
+	size_t size;
+	void *waddr;
+	void *ptr;
+	int ret = 0;
+
+	raw_spin_lock_irqsave(&patch_lock, flags);
+	while (wlen < len) {
+		ptr = dst + wlen;
+		size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr),
+			     len - wlen);
+
+		waddr = patch_map(ptr, FIX_TEXT_POKE0);
+		ret = copy_to_kernel_nofault(waddr, src + wlen, size);
+		patch_unmap(FIX_TEXT_POKE0);
+
+		if (ret) {
+			pr_err("%s: operation failed\n", __func__);
+			break;
+		}
+		wlen += size;
+	}
+	raw_spin_unlock_irqrestore(&patch_lock, flags);
+
+	return ret;
+}
+
 u32 larch_insn_gen_nop(void)
 {
 	return INSN_NOP;
@@ -336,3 +393,31 @@ u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
 
 	return insn.word;
 }
+
+u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
+{
+	union loongarch_instruction insn;
+
+	if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
+		pr_warn("The generated beq instruction is out of range.\n");
+		return INSN_BREAK;
+	}
+
+	emit_beq(&insn, rd, rj, imm >> 2);
+
+	return insn.word;
+}
+
+u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
+{
+	union loongarch_instruction insn;
+
+	if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
+		pr_warn("The generated bne instruction is out of range.\n");
+		return INSN_BREAK;
+	}
+
+	emit_bne(&insn, rj, rd, imm >> 2);
+
+	return insn.word;
+}
-- 
2.25.1

Re: [PATCH v1 2/5] LoongArch: BPF: The operation commands needed to add a trampoline

[Hengqi Chen]

On Wed, Jun 11, 2025 at 1:37 PM Chenghao Duan <duanchenghao@kylinos.cn> wrote:
>
> Add instrctions:
> emit_NOP
> emit_BREAK
>
> Add branch jump function:
> larch_insn_gen_beq
> larch_insn_gen_bne
>
> Add instruction copy function: larch_insn_text_copy
> The implementation of larch_insn_text_copy uses the fixmap
> FIX_TEXT_POKE0.
>
> Signed-off-by: George Guo <guodongtai@kylinos.cn>
> Signed-off-by: Youling Tang <tangyouling@kylinos.cn>
> Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
> ---
>  arch/loongarch/include/asm/inst.h | 19 +++++++
>  arch/loongarch/kernel/inst.c      | 85 +++++++++++++++++++++++++++++++
>  2 files changed, 104 insertions(+)
>
> diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
> index 3089785ca..dd6e07781 100644
> --- a/arch/loongarch/include/asm/inst.h
> +++ b/arch/loongarch/include/asm/inst.h
> @@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs);
>  int larch_insn_read(void *addr, u32 *insnp);
>  int larch_insn_write(void *addr, u32 insn);
>  int larch_insn_patch_text(void *addr, u32 insn);
> +int larch_insn_text_copy(void *dst, void *src, size_t len);
>
>  u32 larch_insn_gen_nop(void);
>  u32 larch_insn_gen_b(unsigned long pc, unsigned long dest);
> @@ -511,6 +512,8 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
>  u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
>  u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
>  u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
> +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
> +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
>
>  static inline bool signed_imm_check(long val, unsigned int bit)
>  {
> @@ -778,6 +781,22 @@ static inline void emit_##NAME(union loongarch_instruction *insn,  \
>
>  DEF_EMIT_REG3SA2_FORMAT(alsld, alsld_op)
>
> +#define DEF_EMIT_NOP(NAME)                                             \
> +static inline void emit_##NAME(union loongarch_instruction *insn)      \
> +{                                                                      \
> +       insn->word = INSN_NOP;                                          \
> +}
> +
> +DEF_EMIT_NOP(NOP)
> +

We have the following helpers in bpf_jit.h already:
  * emit_nop
  * emit_cond_jmp
  * emit_uncond_jmp

These can be used directly.

> +#define DEF_EMIT_BREAK(NAME)                                           \
> +static inline void emit_##NAME(union loongarch_instruction *insn)      \
> +{                                                                      \
> +       insn->word = INSN_BREAK;                                        \
> +}
> +
> +DEF_EMIT_BREAK(BREAK)
> +
>  struct pt_regs;
>
>  void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int *pc);
> diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c
> index 14d7d700b..a47dc3575 100644
> --- a/arch/loongarch/kernel/inst.c
> +++ b/arch/loongarch/kernel/inst.c
> @@ -10,6 +10,33 @@
>
>  static DEFINE_RAW_SPINLOCK(patch_lock);
>
> +static bool is_image_text(unsigned long addr)
> +{
> +       return core_kernel_text(addr);
> +}
> +
> +static void  *patch_map(void *addr, int fixmap)

                    ^ extra space here

> +{
> +       unsigned long uintaddr = (uintptr_t)addr;
> +       bool image = is_image_text(uintaddr);
> +       struct page *page;
> +       phys_addr_t phys;
> +
> +       if (image)
> +               phys = __pa_symbol(addr);
> +       else {
> +               page = vmalloc_to_page(addr);
> +       phys = page_to_phys(page) + offset_in_page(addr);

weird indentation, please fix.

> +       }
> +
> +       return (void *)set_fixmap_offset(fixmap, phys);
> +}
> +
> +static void patch_unmap(int fixmap)
> +{
> +       clear_fixmap(fixmap);
> +}
> +
>  void simu_pc(struct pt_regs *regs, union loongarch_instruction insn)
>  {
>         unsigned long pc = regs->csr_era;
> @@ -218,6 +245,36 @@ int larch_insn_patch_text(void *addr, u32 insn)
>         return ret;
>  }
>
> +int larch_insn_text_copy(void *dst, void *src, size_t len)
> +{
> +       unsigned long flags;
> +       size_t wlen = 0;
> +       size_t size;
> +       void *waddr;
> +       void *ptr;
> +       int ret = 0;
> +
> +       raw_spin_lock_irqsave(&patch_lock, flags);
> +       while (wlen < len) {
> +               ptr = dst + wlen;
> +               size = min_t(size_t, PAGE_SIZE - offset_in_page(ptr),
> +                            len - wlen);
> +
> +               waddr = patch_map(ptr, FIX_TEXT_POKE0);
> +               ret = copy_to_kernel_nofault(waddr, src + wlen, size);
> +               patch_unmap(FIX_TEXT_POKE0);
> +
> +               if (ret) {
> +                       pr_err("%s: operation failed\n", __func__);
> +                       break;
> +               }
> +               wlen += size;
> +       }
> +       raw_spin_unlock_irqrestore(&patch_lock, flags);
> +
> +       return ret;
> +}
> +
>  u32 larch_insn_gen_nop(void)
>  {
>         return INSN_NOP;
> @@ -336,3 +393,31 @@ u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
>
>         return insn.word;
>  }
> +
> +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
> +{
> +       union loongarch_instruction insn;
> +
> +       if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
> +               pr_warn("The generated beq instruction is out of range.\n");
> +               return INSN_BREAK;
> +       }
> +
> +       emit_beq(&insn, rd, rj, imm >> 2);
> +
> +       return insn.word;
> +}
> +
> +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
> +{
> +       union loongarch_instruction insn;
> +
> +       if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
> +               pr_warn("The generated bne instruction is out of range.\n");
> +               return INSN_BREAK;
> +       }
> +
> +       emit_bne(&insn, rj, rd, imm >> 2);
> +
> +       return insn.word;
> +}
> --
> 2.25.1
>
>

[PATCH v1 3/5] LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch

[Chenghao Duan]

Implement the bpf_arch_text_poke function for the LoongArch
architecture. On LoongArch, since symbol addresses in the direct mapping
region cannot be reached via relative jump instructions from the paged
mapping region, we use the move_imm+jirl instruction pair as absolute
jump instructions. These require 2-5 instructions, so we reserve 5 NOP
instructions in the program as placeholders for function jumps.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 64 ++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index ea357a3ed..0e31d4d66 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -4,6 +4,7 @@
  *
  * Copyright (C) 2022 Loongson Technology Corporation Limited
  */
+#include <linux/memory.h>
 #include "bpf_jit.h"
 
 #define REG_TCC		LOONGARCH_GPR_A6
@@ -1351,3 +1352,66 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 {
 	return true;
 }
+
+static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 ip, u64 target)
+{
+	s64 offset = (s64)(target - ip);
+
+	if (offset && (offset >= -SZ_128M && offset < SZ_128M)) {
+		emit_insn(ctx, bl, offset >> 2);
+	} else {
+		move_imm(ctx, LOONGARCH_GPR_T1, target, false);
+		emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
+	}
+
+	return 0;
+}
+
+static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
+{
+	s64 off = 0;
+	struct jit_ctx ctx;
+
+	ctx.idx = 0;
+	ctx.image = (union loongarch_instruction *)insns;
+
+	if (!target) {
+		emit_insn((&ctx), NOP);
+		emit_insn((&ctx), NOP);
+		return 0;
+	}
+
+	off = (s64)(target - ip);
+	return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO,
+					(unsigned long)ip, (unsigned long)target);
+}
+
+int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
+			void *old_addr, void *new_addr)
+{
+	u32 old_insns[5] = {[0 ... 4] = INSN_NOP};
+	u32 new_insns[5] = {[0 ... 4] = INSN_NOP};
+	bool is_call = poke_type == BPF_MOD_CALL;
+	int ret;
+
+	if (!is_kernel_text((unsigned long)ip) &&
+		!is_bpf_text_address((unsigned long)ip))
+		return -ENOTSUPP;
+
+	ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call);
+	if (ret)
+		return ret;
+
+	if (memcmp(ip, old_insns, 5 * 4))
+		return -EFAULT;
+
+	ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call);
+	if (ret)
+		return ret;
+
+	mutex_lock(&text_mutex);
+	if (memcmp(ip, new_insns, 5 * 4))
+		ret = larch_insn_text_copy(ip, new_insns, 5 * 4);
+	mutex_unlock(&text_mutex);
+	return ret;
+}
-- 
2.25.1

Re: [PATCH v1 3/5] LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch

[kernel test robot]

Hi Chenghao,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/net]
[also build test WARNING on bpf-next/master bpf/master linus/master v6.16-rc1 next-20250611]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Chenghao-Duan/LoongArch-Support-fixmap/20250611-133843
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git net
patch link:    https://lore.kernel.org/r/20250611053625.352091-4-duanchenghao%40kylinos.cn
patch subject: [PATCH v1 3/5] LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch
config: loongarch-randconfig-002-20250612 (https://download.01.org/0day-ci/archive/20250612/202506120407.Dqc4kJYr-lkp@intel.com/config)
compiler: loongarch64-linux-gcc (GCC) 12.4.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250612/202506120407.Dqc4kJYr-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202506120407.Dqc4kJYr-lkp@intel.com/

All warnings (new ones prefixed by >>):

   arch/loongarch/net/bpf_jit.c: In function 'gen_jump_or_nops':
>> arch/loongarch/net/bpf_jit.c:1380:13: warning: variable 'off' set but not used [-Wunused-but-set-variable]
    1380 |         s64 off = 0;
         |             ^~~


vim +/off +1380 arch/loongarch/net/bpf_jit.c

  1377	
  1378	static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
  1379	{
> 1380		s64 off = 0;
  1381		struct jit_ctx ctx;
  1382	
  1383		ctx.idx = 0;
  1384		ctx.image = (union loongarch_instruction *)insns;
  1385	
  1386		if (!target) {
  1387			emit_insn((&ctx), NOP);
  1388			emit_insn((&ctx), NOP);
  1389			return 0;
  1390		}
  1391	
  1392		off = (s64)(target - ip);
  1393		return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO,
  1394						(unsigned long)ip, (unsigned long)target);
  1395	}
  1396	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

[PATCH v1 4/5] LoongArch: BPF: Add bpf trampoline support for Loongarch

[Chenghao Duan]

BPF trampoline is the critical infrastructure of the BPF subsystem, acting
as a mediator between kernel functions and BPF programs. Numerous important
features, such as using BPF program for zero overhead kernel introspection,
rely on this key component. We can't wait to support bpf trampoline on
Loongarch.

The related tests have passed, Including the following technical points:
1. fentry
2. fmod_ret
3. fexit

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 423 +++++++++++++++++++++++++++++++++++
 arch/loongarch/net/bpf_jit.h |   6 +
 2 files changed, 429 insertions(+)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 0e31d4d66..cdcb73f93 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -7,6 +7,10 @@
 #include <linux/memory.h>
 #include "bpf_jit.h"
 
+#define LOONGARCH_MAX_REG_ARGS 8
+#define LOONGARCH_FENTRY_NINSNS 2
+#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
+
 #define REG_TCC		LOONGARCH_GPR_A6
 #define TCC_SAVED	LOONGARCH_GPR_S5
 
@@ -1386,6 +1390,20 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
 					(unsigned long)ip, (unsigned long)target);
 }
 
+static int emit_call(struct jit_ctx *ctx, u64 addr)
+{
+	u64 ip;
+	s64 off = 0;
+
+	if (addr && ctx->image && ctx->ro_image) {
+		ip = (u64)(ctx->image + ctx->idx);
+		off = addr - ip;
+	}
+
+	return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, ip, addr);
+}
+
+
 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
 			void *old_addr, void *new_addr)
 {
@@ -1415,3 +1433,408 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
 	mutex_unlock(&text_mutex);
 	return ret;
 }
+
+static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
+{
+	int i = 0;
+
+	for (i = 0; i < nargs; i++) {
+		emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
+		args_off -= 8;
+	}
+}
+
+static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
+{
+	int i = 0;
+
+	for (i = 0; i < nargs; i++) {
+		emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
+		args_off -= 8;
+	}
+}
+
+static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
+			   int args_off, int retval_off,
+			   int run_ctx_off, bool save_ret)
+{
+	int ret;
+	u32 *branch;
+	struct bpf_prog *p = l->link.prog;
+	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
+
+	if (l->cookie) {
+		move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
+		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
+	} else {
+		emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP,
+				-run_ctx_off + cookie_off);
+	}
+
+	/* arg1: prog */
+	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
+	/* arg2: &run_ctx */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
+	ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
+	if (ret)
+		return ret;
+
+	/* store prog start time */
+	move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
+
+	/* if (__bpf_prog_enter(prog) == 0)
+	 *      goto skip_exec_of_prog;
+	 *
+	 */
+	branch = (u32 *)ctx->image + ctx->idx;
+	/* NOP reserved for conditional jump */
+	emit_insn(ctx, NOP);
+
+	/* arg1: &args_off */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
+	if (!p->jited)
+		move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
+	ret = emit_call(ctx, (const u64)p->bpf_func);
+	if (ret)
+		return ret;
+
+	if (save_ret) {
+		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+	}
+
+	/* update branch with beqz */
+	if (ctx->image) {
+		int offset = (void *)(u32 *)(&ctx->image[ctx->idx]) - (void *)branch;
+		*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
+	}
+
+	/* arg1: prog */
+	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
+	/* arg2: prog start time */
+	move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
+	/* arg3: &run_ctx */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
+	ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
+
+	return ret;
+}
+
+static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
+			       int args_off, int retval_off, int run_ctx_off, u32 **branches)
+{
+	int i;
+
+	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
+	for (i = 0; i < tl->nr_links; i++) {
+		invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off,
+			run_ctx_off, true);
+		emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
+		branches[i] = (u32 *)ctx->image + ctx->idx;
+		emit_insn(ctx, NOP);
+	}
+}
+
+void *bpf_arch_text_copy(void *dst, void *src, size_t len)
+{
+	if (larch_insn_text_copy(dst, src, len))
+		return ERR_PTR(-EINVAL);
+
+	return dst;
+}
+
+u64 bpf_jit_alloc_exec_limit(void)
+{
+	return VMALLOC_END - VMALLOC_START;
+}
+
+void *arch_alloc_bpf_trampoline(unsigned int size)
+{
+	void *image;
+
+	if (WARN_ON_ONCE(size > PAGE_SIZE))
+		return NULL;
+	image = bpf_jit_alloc_exec(PAGE_SIZE);
+	if (image) {
+		jit_fill_hole(image, size);
+		set_vm_flush_reset_perms(image);
+	}
+
+	return image;
+}
+
+void arch_free_bpf_trampoline(void *image, unsigned int size)
+{
+	WARN_ON_ONCE(size > PAGE_SIZE);
+	bpf_jit_free_exec(image);
+}
+
+int arch_protect_bpf_trampoline(void *image, unsigned int size)
+{
+	return 0;
+}
+
+int bpf_arch_text_invalidate(void *dst, size_t len)
+{
+	jit_fill_hole(dst, len);
+	return 0;
+}
+
+static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+					 const struct btf_func_model *m,
+					 struct bpf_tramp_links *tlinks,
+					 void *func_addr, u32 flags)
+{
+	int i;
+	int stack_size = 0, nargs = 0;
+	int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off;
+	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
+	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
+	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
+	int ret, save_ret;
+	void *orig_call = func_addr;
+	u32 **branches = NULL;
+
+	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
+		return -ENOTSUPP;
+
+	/*
+	 * FP + 8       [ RA to parent func ] return address to parent
+	 *                    function
+	 * FP + 0       [ FP of parent func ] frame pointer of parent
+	 *                    function
+	 * FP - 8       [ T0 to traced func ] return address of traced
+	 *                    function
+	 * FP - 16      [ FP of traced func ] frame pointer of traced
+	 *                    function
+	 *
+	 * FP - retval_off  [ return value      ] BPF_TRAMP_F_CALL_ORIG or
+	 *                    BPF_TRAMP_F_RET_FENTRY_RET
+	 *                  [ argN              ]
+	 *                  [ ...               ]
+	 * FP - args_off    [ arg1              ]
+	 *
+	 * FP - nargs_off   [ regs count        ]
+	 *
+	 * FP - ip_off      [ traced func   ] BPF_TRAMP_F_IP_ARG
+	 *
+	 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
+	 *
+	 * FP - sreg_off    [ callee saved reg  ]
+	 *
+	 */
+
+	if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
+	    return -ENOTSUPP;
+
+	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
+		return -ENOTSUPP;
+
+	stack_size = 0;
+
+	/* room of trampoline frame to store return address and frame pointer */
+	stack_size += 16;
+
+	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
+	if (save_ret) {
+		/* Save BPF R0 and A0 */
+		stack_size += 16;
+		retval_off = stack_size;
+	}
+
+	/* room of trampoline frame to store args */
+	nargs = m->nr_args;
+	stack_size += nargs * 8;
+	args_off = stack_size;
+	/* room of trampoline frame to store args number */
+	stack_size += 8;
+	nargs_off = stack_size;
+
+	/* room of trampoline frame to store ip address */
+	if (flags & BPF_TRAMP_F_IP_ARG) {
+		stack_size += 8;
+		ip_off = stack_size;
+	}
+
+	/* room of trampoline frame to store struct bpf_tramp_run_ctx */
+	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
+	run_ctx_off = stack_size;
+
+	stack_size += 8;
+	sreg_off = stack_size;
+
+	stack_size = round_up(stack_size, 16);
+
+	/* For the trampoline called from function entry */
+	/* RA and FP for parent function*/
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
+	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
+	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
+	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
+
+	/* RA and FP for traced function*/
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
+	emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
+	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
+	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
+
+	/* callee saved register S1 to pass start time */
+	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
+
+	/* store ip address of the traced function */
+	if (flags & BPF_TRAMP_F_IP_ARG) {
+		move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
+		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
+	}
+
+	/* store nargs number*/
+	move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
+	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
+
+	store_args(ctx, nargs, args_off);
+
+	/* To traced function */
+	orig_call += LOONGARCH_FENTRY_NBYTES;
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
+		ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < fentry->nr_links; i++) {
+		ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off,
+			run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET);
+		if (ret)
+			return ret;
+	}
+
+	if (fmod_ret->nr_links) {
+		branches  = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
+		if (!branches)
+			return -ENOMEM;
+
+		invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off,
+			run_ctx_off, branches);
+	}
+
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		restore_args(ctx, m->nr_args, args_off);
+		ret = emit_call(ctx, (const u64)orig_call);
+		if (ret)
+			goto out;
+		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+		im->ip_after_call = ctx->ro_image + ctx->idx;
+		/* Reserve space for the move_imm + jirl instruction */
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+	}
+
+	for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
+		int offset = (u32 *)&ctx->image[ctx->idx] - branches[i];
+		*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
+	}
+
+	for (i = 0; i < fexit->nr_links; i++) {
+		ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off,
+				run_ctx_off, false);
+		if (ret)
+			goto out;
+	}
+
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		im->ip_epilogue = ctx->ro_image + ctx->idx;
+		move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
+		ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
+		if (ret)
+			goto out;
+	}
+
+	if (flags & BPF_TRAMP_F_RESTORE_REGS)
+		restore_args(ctx, m->nr_args, args_off);
+
+	if (save_ret) {
+		emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+	}
+
+	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
+
+	/* trampoline called from function entry */
+	emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
+	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
+
+	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
+	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
+
+	if (flags & BPF_TRAMP_F_SKIP_FRAME)
+		/* return to parent function */
+		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
+	else
+		/* return to traced function */
+		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
+
+	ret = ctx->idx;
+out:
+	kfree(branches);
+
+	return ret;
+}
+
+int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
+				void *ro_image_end, const struct btf_func_model *m,
+				u32 flags, struct bpf_tramp_links *tlinks,
+				void *func_addr)
+{
+	int ret;
+	void *image, *tmp;
+	u32 size = ro_image_end - ro_image;
+
+	image = kvmalloc(size, GFP_KERNEL);
+	if (!image)
+		return -ENOMEM;
+
+	struct jit_ctx ctx = {
+		.image = (union loongarch_instruction *)image,
+		.ro_image = (union loongarch_instruction *)ro_image,
+		.idx = 0,
+	};
+
+	jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
+	ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
+	if (ret > 0 && validate_code(&ctx) < 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	tmp = bpf_arch_text_copy(ro_image, image, size);
+	if (IS_ERR(tmp)) {
+		ret = PTR_ERR(tmp);
+		goto out;
+	}
+
+	bpf_flush_icache(ro_image, ro_image_end);
+out:
+	kvfree(image);
+	return ret < 0 ? ret : size;
+}
+
+int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
+			     struct bpf_tramp_links *tlinks, void *func_addr)
+{
+	struct bpf_tramp_image im;
+	struct jit_ctx ctx;
+	int ret;
+
+	ctx.image = NULL;
+	ctx.idx = 0;
+
+	ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
+	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
+}
diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h
index 68586338e..51a12b809 100644
--- a/arch/loongarch/net/bpf_jit.h
+++ b/arch/loongarch/net/bpf_jit.h
@@ -18,6 +18,7 @@ struct jit_ctx {
 	u32 *offset;
 	int num_exentries;
 	union loongarch_instruction *image;
+	union loongarch_instruction *ro_image;
 	u32 stack_size;
 };
 
@@ -303,3 +304,8 @@ static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch
 
 	return -EINVAL;
 }
+
+static inline void bpf_flush_icache(void *start, void *end)
+{
+	flush_icache_range((unsigned long)start, (unsigned long)end);
+}
-- 
2.25.1

[PATCH v1 5/5] LoongArch: BPF: Update the code to rename validate_code to validate_ctx.

[Chenghao Duan]

Update the code to rename validate_code to validate_ctx.
validate_code is used to check the validity of code.
validate_ctx is used to check both code validity and table entry
correctness.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index cdcb73f93..ee38a9824 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1177,6 +1177,14 @@ static int validate_code(struct jit_ctx *ctx)
 			return -1;
 	}
 
+	return 0;
+}
+
+static int validate_ctx(struct jit_ctx *ctx)
+{
+	if (validate_code(ctx))
+		return -1;
+
 	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
 		return -1;
 
@@ -1285,7 +1293,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	build_epilogue(&ctx);
 
 	/* 3. Extra pass to validate JITed code */
-	if (validate_code(&ctx)) {
+	if (validate_ctx(&ctx)) {
 		bpf_jit_binary_free(header);
 		prog = orig_prog;
 		goto out_offset;
-- 
2.25.1

[PATCH v1 3/5] LoongArch: BPF: Add bpf_arch_text_poke support for Loongarch

[Chenghao Duan]

Implement the bpf_arch_text_poke function for the LoongArch
architecture. On LoongArch, since symbol addresses in the direct mapping
region cannot be reached via relative jump instructions from the paged
mapping region, we use the move_imm+jirl instruction pair as absolute
jump instructions. These require 2-5 instructions, so we reserve 5 NOP
instructions in the program as placeholders for function jumps.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 64 ++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index ea357a3ed..0e31d4d66 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -4,6 +4,7 @@
  *
  * Copyright (C) 2022 Loongson Technology Corporation Limited
  */
+#include <linux/memory.h>
 #include "bpf_jit.h"
 
 #define REG_TCC		LOONGARCH_GPR_A6
@@ -1351,3 +1352,66 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 {
 	return true;
 }
+
+static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 ip, u64 target)
+{
+	s64 offset = (s64)(target - ip);
+
+	if (offset && (offset >= -SZ_128M && offset < SZ_128M)) {
+		emit_insn(ctx, bl, offset >> 2);
+	} else {
+		move_imm(ctx, LOONGARCH_GPR_T1, target, false);
+		emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
+	}
+
+	return 0;
+}
+
+static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
+{
+	s64 off = 0;
+	struct jit_ctx ctx;
+
+	ctx.idx = 0;
+	ctx.image = (union loongarch_instruction *)insns;
+
+	if (!target) {
+		emit_insn((&ctx), NOP);
+		emit_insn((&ctx), NOP);
+		return 0;
+	}
+
+	off = (s64)(target - ip);
+	return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO,
+					(unsigned long)ip, (unsigned long)target);
+}
+
+int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
+			void *old_addr, void *new_addr)
+{
+	u32 old_insns[5] = {[0 ... 4] = INSN_NOP};
+	u32 new_insns[5] = {[0 ... 4] = INSN_NOP};
+	bool is_call = poke_type == BPF_MOD_CALL;
+	int ret;
+
+	if (!is_kernel_text((unsigned long)ip) &&
+		!is_bpf_text_address((unsigned long)ip))
+		return -ENOTSUPP;
+
+	ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call);
+	if (ret)
+		return ret;
+
+	if (memcmp(ip, old_insns, 5 * 4))
+		return -EFAULT;
+
+	ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call);
+	if (ret)
+		return ret;
+
+	mutex_lock(&text_mutex);
+	if (memcmp(ip, new_insns, 5 * 4))
+		ret = larch_insn_text_copy(ip, new_insns, 5 * 4);
+	mutex_unlock(&text_mutex);
+	return ret;
+}
-- 
2.25.1

[PATCH v1 4/5] LoongArch: BPF: Add bpf trampoline support for Loongarch

[Chenghao Duan]

BPF trampoline is the critical infrastructure of the BPF subsystem, acting
as a mediator between kernel functions and BPF programs. Numerous important
features, such as using BPF program for zero overhead kernel introspection,
rely on this key component. We can't wait to support bpf trampoline on
Loongarch.

The related tests have passed, Including the following technical points:
1. fentry
2. fmod_ret
3. fexit

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 423 +++++++++++++++++++++++++++++++++++
 arch/loongarch/net/bpf_jit.h |   6 +
 2 files changed, 429 insertions(+)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 0e31d4d66..cdcb73f93 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -7,6 +7,10 @@
 #include <linux/memory.h>
 #include "bpf_jit.h"
 
+#define LOONGARCH_MAX_REG_ARGS 8
+#define LOONGARCH_FENTRY_NINSNS 2
+#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
+
 #define REG_TCC		LOONGARCH_GPR_A6
 #define TCC_SAVED	LOONGARCH_GPR_S5
 
@@ -1386,6 +1390,20 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
 					(unsigned long)ip, (unsigned long)target);
 }
 
+static int emit_call(struct jit_ctx *ctx, u64 addr)
+{
+	u64 ip;
+	s64 off = 0;
+
+	if (addr && ctx->image && ctx->ro_image) {
+		ip = (u64)(ctx->image + ctx->idx);
+		off = addr - ip;
+	}
+
+	return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, ip, addr);
+}
+
+
 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
 			void *old_addr, void *new_addr)
 {
@@ -1415,3 +1433,408 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
 	mutex_unlock(&text_mutex);
 	return ret;
 }
+
+static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
+{
+	int i = 0;
+
+	for (i = 0; i < nargs; i++) {
+		emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
+		args_off -= 8;
+	}
+}
+
+static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
+{
+	int i = 0;
+
+	for (i = 0; i < nargs; i++) {
+		emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
+		args_off -= 8;
+	}
+}
+
+static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
+			   int args_off, int retval_off,
+			   int run_ctx_off, bool save_ret)
+{
+	int ret;
+	u32 *branch;
+	struct bpf_prog *p = l->link.prog;
+	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
+
+	if (l->cookie) {
+		move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
+		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
+	} else {
+		emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP,
+				-run_ctx_off + cookie_off);
+	}
+
+	/* arg1: prog */
+	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
+	/* arg2: &run_ctx */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
+	ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
+	if (ret)
+		return ret;
+
+	/* store prog start time */
+	move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
+
+	/* if (__bpf_prog_enter(prog) == 0)
+	 *      goto skip_exec_of_prog;
+	 *
+	 */
+	branch = (u32 *)ctx->image + ctx->idx;
+	/* NOP reserved for conditional jump */
+	emit_insn(ctx, NOP);
+
+	/* arg1: &args_off */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
+	if (!p->jited)
+		move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
+	ret = emit_call(ctx, (const u64)p->bpf_func);
+	if (ret)
+		return ret;
+
+	if (save_ret) {
+		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+	}
+
+	/* update branch with beqz */
+	if (ctx->image) {
+		int offset = (void *)(u32 *)(&ctx->image[ctx->idx]) - (void *)branch;
+		*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
+	}
+
+	/* arg1: prog */
+	move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
+	/* arg2: prog start time */
+	move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
+	/* arg3: &run_ctx */
+	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
+	ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
+
+	return ret;
+}
+
+static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
+			       int args_off, int retval_off, int run_ctx_off, u32 **branches)
+{
+	int i;
+
+	emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
+	for (i = 0; i < tl->nr_links; i++) {
+		invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off,
+			run_ctx_off, true);
+		emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
+		branches[i] = (u32 *)ctx->image + ctx->idx;
+		emit_insn(ctx, NOP);
+	}
+}
+
+void *bpf_arch_text_copy(void *dst, void *src, size_t len)
+{
+	if (larch_insn_text_copy(dst, src, len))
+		return ERR_PTR(-EINVAL);
+
+	return dst;
+}
+
+u64 bpf_jit_alloc_exec_limit(void)
+{
+	return VMALLOC_END - VMALLOC_START;
+}
+
+void *arch_alloc_bpf_trampoline(unsigned int size)
+{
+	void *image;
+
+	if (WARN_ON_ONCE(size > PAGE_SIZE))
+		return NULL;
+	image = bpf_jit_alloc_exec(PAGE_SIZE);
+	if (image) {
+		jit_fill_hole(image, size);
+		set_vm_flush_reset_perms(image);
+	}
+
+	return image;
+}
+
+void arch_free_bpf_trampoline(void *image, unsigned int size)
+{
+	WARN_ON_ONCE(size > PAGE_SIZE);
+	bpf_jit_free_exec(image);
+}
+
+int arch_protect_bpf_trampoline(void *image, unsigned int size)
+{
+	return 0;
+}
+
+int bpf_arch_text_invalidate(void *dst, size_t len)
+{
+	jit_fill_hole(dst, len);
+	return 0;
+}
+
+static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+					 const struct btf_func_model *m,
+					 struct bpf_tramp_links *tlinks,
+					 void *func_addr, u32 flags)
+{
+	int i;
+	int stack_size = 0, nargs = 0;
+	int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off;
+	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
+	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
+	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
+	int ret, save_ret;
+	void *orig_call = func_addr;
+	u32 **branches = NULL;
+
+	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
+		return -ENOTSUPP;
+
+	/*
+	 * FP + 8       [ RA to parent func ] return address to parent
+	 *                    function
+	 * FP + 0       [ FP of parent func ] frame pointer of parent
+	 *                    function
+	 * FP - 8       [ T0 to traced func ] return address of traced
+	 *                    function
+	 * FP - 16      [ FP of traced func ] frame pointer of traced
+	 *                    function
+	 *
+	 * FP - retval_off  [ return value      ] BPF_TRAMP_F_CALL_ORIG or
+	 *                    BPF_TRAMP_F_RET_FENTRY_RET
+	 *                  [ argN              ]
+	 *                  [ ...               ]
+	 * FP - args_off    [ arg1              ]
+	 *
+	 * FP - nargs_off   [ regs count        ]
+	 *
+	 * FP - ip_off      [ traced func   ] BPF_TRAMP_F_IP_ARG
+	 *
+	 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
+	 *
+	 * FP - sreg_off    [ callee saved reg  ]
+	 *
+	 */
+
+	if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
+	    return -ENOTSUPP;
+
+	if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
+		return -ENOTSUPP;
+
+	stack_size = 0;
+
+	/* room of trampoline frame to store return address and frame pointer */
+	stack_size += 16;
+
+	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
+	if (save_ret) {
+		/* Save BPF R0 and A0 */
+		stack_size += 16;
+		retval_off = stack_size;
+	}
+
+	/* room of trampoline frame to store args */
+	nargs = m->nr_args;
+	stack_size += nargs * 8;
+	args_off = stack_size;
+	/* room of trampoline frame to store args number */
+	stack_size += 8;
+	nargs_off = stack_size;
+
+	/* room of trampoline frame to store ip address */
+	if (flags & BPF_TRAMP_F_IP_ARG) {
+		stack_size += 8;
+		ip_off = stack_size;
+	}
+
+	/* room of trampoline frame to store struct bpf_tramp_run_ctx */
+	stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
+	run_ctx_off = stack_size;
+
+	stack_size += 8;
+	sreg_off = stack_size;
+
+	stack_size = round_up(stack_size, 16);
+
+	/* For the trampoline called from function entry */
+	/* RA and FP for parent function*/
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
+	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
+	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
+	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
+
+	/* RA and FP for traced function*/
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
+	emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
+	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
+	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
+
+	/* callee saved register S1 to pass start time */
+	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
+
+	/* store ip address of the traced function */
+	if (flags & BPF_TRAMP_F_IP_ARG) {
+		move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
+		emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
+	}
+
+	/* store nargs number*/
+	move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
+	emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
+
+	store_args(ctx, nargs, args_off);
+
+	/* To traced function */
+	orig_call += LOONGARCH_FENTRY_NBYTES;
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
+		ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < fentry->nr_links; i++) {
+		ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off,
+			run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET);
+		if (ret)
+			return ret;
+	}
+
+	if (fmod_ret->nr_links) {
+		branches  = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
+		if (!branches)
+			return -ENOMEM;
+
+		invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off,
+			run_ctx_off, branches);
+	}
+
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		restore_args(ctx, m->nr_args, args_off);
+		ret = emit_call(ctx, (const u64)orig_call);
+		if (ret)
+			goto out;
+		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+		im->ip_after_call = ctx->ro_image + ctx->idx;
+		/* Reserve space for the move_imm + jirl instruction */
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+		emit_insn(ctx, NOP);
+	}
+
+	for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
+		int offset = (u32 *)&ctx->image[ctx->idx] - branches[i];
+		*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
+	}
+
+	for (i = 0; i < fexit->nr_links; i++) {
+		ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off,
+				run_ctx_off, false);
+		if (ret)
+			goto out;
+	}
+
+	if (flags & BPF_TRAMP_F_CALL_ORIG) {
+		im->ip_epilogue = ctx->ro_image + ctx->idx;
+		move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
+		ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
+		if (ret)
+			goto out;
+	}
+
+	if (flags & BPF_TRAMP_F_RESTORE_REGS)
+		restore_args(ctx, m->nr_args, args_off);
+
+	if (save_ret) {
+		emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
+		emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
+	}
+
+	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
+
+	/* trampoline called from function entry */
+	emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
+	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
+
+	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
+	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
+	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
+
+	if (flags & BPF_TRAMP_F_SKIP_FRAME)
+		/* return to parent function */
+		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
+	else
+		/* return to traced function */
+		emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
+
+	ret = ctx->idx;
+out:
+	kfree(branches);
+
+	return ret;
+}
+
+int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
+				void *ro_image_end, const struct btf_func_model *m,
+				u32 flags, struct bpf_tramp_links *tlinks,
+				void *func_addr)
+{
+	int ret;
+	void *image, *tmp;
+	u32 size = ro_image_end - ro_image;
+
+	image = kvmalloc(size, GFP_KERNEL);
+	if (!image)
+		return -ENOMEM;
+
+	struct jit_ctx ctx = {
+		.image = (union loongarch_instruction *)image,
+		.ro_image = (union loongarch_instruction *)ro_image,
+		.idx = 0,
+	};
+
+	jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
+	ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
+	if (ret > 0 && validate_code(&ctx) < 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	tmp = bpf_arch_text_copy(ro_image, image, size);
+	if (IS_ERR(tmp)) {
+		ret = PTR_ERR(tmp);
+		goto out;
+	}
+
+	bpf_flush_icache(ro_image, ro_image_end);
+out:
+	kvfree(image);
+	return ret < 0 ? ret : size;
+}
+
+int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
+			     struct bpf_tramp_links *tlinks, void *func_addr)
+{
+	struct bpf_tramp_image im;
+	struct jit_ctx ctx;
+	int ret;
+
+	ctx.image = NULL;
+	ctx.idx = 0;
+
+	ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
+	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
+}
diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h
index 68586338e..51a12b809 100644
--- a/arch/loongarch/net/bpf_jit.h
+++ b/arch/loongarch/net/bpf_jit.h
@@ -18,6 +18,7 @@ struct jit_ctx {
 	u32 *offset;
 	int num_exentries;
 	union loongarch_instruction *image;
+	union loongarch_instruction *ro_image;
 	u32 stack_size;
 };
 
@@ -303,3 +304,8 @@ static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch
 
 	return -EINVAL;
 }
+
+static inline void bpf_flush_icache(void *start, void *end)
+{
+	flush_icache_range((unsigned long)start, (unsigned long)end);
+}
-- 
2.25.1

[PATCH v1 5/5] LoongArch: BPF: Update the code to rename validate_code to validate_ctx.

[Chenghao Duan]

Update the code to rename validate_code to validate_ctx.
validate_code is used to check the validity of code.
validate_ctx is used to check both code validity and table entry
correctness.

Signed-off-by: George Guo <guodongtai@kylinos.cn>
Signed-off-by: Chenghao Duan <duanchenghao@kylinos.cn>
---
 arch/loongarch/net/bpf_jit.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index cdcb73f93..ee38a9824 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1177,6 +1177,14 @@ static int validate_code(struct jit_ctx *ctx)
 			return -1;
 	}
 
+	return 0;
+}
+
+static int validate_ctx(struct jit_ctx *ctx)
+{
+	if (validate_code(ctx))
+		return -1;
+
 	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
 		return -1;
 
@@ -1285,7 +1293,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	build_epilogue(&ctx);
 
 	/* 3. Extra pass to validate JITed code */
-	if (validate_code(&ctx)) {
+	if (validate_ctx(&ctx)) {
 		bpf_jit_binary_free(header);
 		prog = orig_prog;
 		goto out_offset;
-- 
2.25.1