mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Nylon Chen <nylon.chen@sifive.com>
To: Paul Walmsley <pjw@kernel.org>,
	Palmer Dabbelt <palmer@dabbelt.com>,
	 Albert Ou <aou@eecs.berkeley.edu>,
	Alexandre Ghiti <alex@ghiti.fr>,  Shuah Khan <shuah@kernel.org>
Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org,
	 linux-kselftest@vger.kernel.org,
	Nylon Chen <nylon.chen@sifive.com>
Subject: [PATCH 1/2] riscv: traps: Add support for vector misalignment handler
Date: Mon, 16 Nov 2026 23:38:37 -0800	[thread overview]
Message-ID: <20261116-riscv-vector-misaligned-v1-1-6c4d0ca905b7@sifive.com> (raw)
In-Reply-To: <20261116-riscv-vector-misaligned-v1-0-6c4d0ca905b7@sifive.com>

Implement misaligned load/store emulation for RISC-V vector instructions.
This adds support for handling misaligned memory accesses in vector
operations, which may occur when the effective address is not naturally
aligned for the element size.

The implementation includes:
- Vector load emulator (misaligned_v_ld_emulator)
- Vector store emulator (misaligned_v_st_emulator)
- Support for various vector instruction formats:
  - Unit-stride and strided load/store
  - Indexed load/store
  - Segment load/store
  - Whole register load/store
  - Fault-only-first loads

Both user mode and kernel mode traps are emulated. In user mode, the
vector register file has already been saved to current->thread.vstate
by kernel_vector_begin(), so register contents are accessed there. In
kernel mode the registers are still live in hardware for the duration
of the trap (IRQs stay disabled, nothing else can touch them), so
get_vreg()/set_vreg() instead use vstart together with a
tail-undisturbed vsetvli to read or modify only the target element in
place, the same technique OpenSBI's M-mode handler uses. This avoids
depending on CONFIG_RISCV_ISA_V_PREEMPTIVE or any per-task kernel
vector state allocation.

Signed-off-by: Nylon Chen <nylon.chen@sifive.com>
---
 arch/riscv/include/asm/vector.h        |   4 +
 arch/riscv/kernel/Makefile             |   1 +
 arch/riscv/kernel/traps_misaligned.c   |  39 +-
 arch/riscv/kernel/traps_v_misaligned.c | 902 +++++++++++++++++++++++++++++++++
 4 files changed, 944 insertions(+), 2 deletions(-)

diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index cd09f32e1ea3..712c38383998 100644
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -51,6 +51,8 @@ void put_cpu_vector_context(void);
 void riscv_v_thread_free(struct task_struct *tsk);
 void __init riscv_v_setup_ctx_cache(void);
 void riscv_v_thread_alloc(struct task_struct *tsk);
+int misaligned_v_ld_emulator(struct pt_regs *regs, unsigned long insn);
+int misaligned_v_st_emulator(struct pt_regs *regs, unsigned long insn);
 void __init update_regset_vector_info(unsigned long size);
 
 static inline u32 riscv_v_flags(void)
@@ -411,6 +413,8 @@ bool riscv_v_vstate_ctrl_user_allowed(void);
 
 struct pt_regs;
 
+int misaligned_v_ld_emulator(struct pt_regs *regs, unsigned long insn);
+int misaligned_v_st_emulator(struct pt_regs *regs, unsigned long insn);
 static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
 static __always_inline bool has_vector(void) { return false; }
 static __always_inline bool insn_is_vector(u32 insn_buf) { return false; }
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f54c336bf547..a618b28c21ee 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -76,6 +76,7 @@ obj-$(CONFIG_MMU) += vdso.o vdso/
 obj-$(CONFIG_RISCV_USER_CFI) += vdso_cfi/
 
 obj-$(CONFIG_RISCV_MISALIGNED)	+= traps_misaligned.o
+obj-$(CONFIG_RISCV_MISALIGNED)	+= traps_v_misaligned.o
 obj-$(CONFIG_RISCV_MISALIGNED)	+= unaligned_access_speed.o
 obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)		+= copy-unaligned.o
 obj-$(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)	+= vec-copy-unaligned.o
diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c
index 6e8ae6c66322..3a16ec91e150 100644
--- a/arch/riscv/kernel/traps_misaligned.c
+++ b/arch/riscv/kernel/traps_misaligned.c
@@ -206,15 +206,39 @@ static int handle_vector_misaligned_load(struct pt_regs *regs)
 		return 0;
 	}
 
-	/* If vector instruction we don't emulate it yet */
 	regs->epc = epc;
-	return -1;
+	return misaligned_v_ld_emulator(regs, insn);
+}
+
+static int handle_vector_misaligned_store(struct pt_regs *regs)
+{
+	unsigned long epc = regs->epc;
+	unsigned long insn;
+
+	if (get_insn(regs, epc, &insn))
+		return -1;
+
+	/* Only return 0 when in check_vector_unaligned_access_emulated */
+	if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
+		*this_cpu_ptr(&vector_misaligned_access) =
+			RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
+		regs->epc = epc + INSN_LEN(insn);
+		return 0;
+	}
+
+	regs->epc = epc;
+	return misaligned_v_st_emulator(regs, insn);
 }
 #else
 static int handle_vector_misaligned_load(struct pt_regs *regs)
 {
 	return -1;
 }
+
+static int handle_vector_misaligned_store(struct pt_regs *regs)
+{
+	return -1;
+}
 #endif
 
 static int handle_scalar_misaligned_load(struct pt_regs *regs)
@@ -445,6 +469,17 @@ int handle_misaligned_load(struct pt_regs *regs)
 
 int handle_misaligned_store(struct pt_regs *regs)
 {
+	unsigned long epc = regs->epc;
+	unsigned long insn;
+
+	if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
+		if (get_insn(regs, epc, &insn))
+			return -1;
+
+		if (insn_is_vector(insn))
+			return handle_vector_misaligned_store(regs);
+	}
+
 	if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
 		return handle_scalar_misaligned_store(regs);
 
diff --git a/arch/riscv/kernel/traps_v_misaligned.c b/arch/riscv/kernel/traps_v_misaligned.c
new file mode 100644
index 000000000000..29557a51a347
--- /dev/null
+++ b/arch/riscv/kernel/traps_v_misaligned.c
@@ -0,0 +1,902 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ *
+ * Copyright (c) 2025 SiFive Inc.
+ *
+ * Authors:
+ *   Nylon Chen <nylon.chen@sifive.com>
+ *   Zong Li <zong.li@sifive.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/perf_event.h>
+#include <linux/irq.h>
+#include <linux/stringify.h>
+#include <linux/sched/signal.h>
+
+#include <linux/processor.h>
+#include <asm/ptrace.h>
+#include <asm/csr.h>
+#include <linux/entry-common.h>
+#include <asm/hwprobe.h>
+#include <linux/cpufeature.h>
+#include <asm/vector.h>
+#include <asm/simd.h>
+#include <linux/slab.h>
+#include <asm/asm-prototypes.h>
+
+#define INSN_MASK_VECTOR_UNIT_STRIDE			0xfdf0707f
+#define INSN_MASK_VECTOR_FAULT_ONLY_FIRST		0xfdf0707f
+#define INSN_MASK_VECTOR_STRIDE				0xfc00707f
+#define INSN_MASK_VECTOR_INDEXED			0xfc00707f
+#define INSN_MASK_VECTOR_WHOLE_REG			0xfff0707f
+
+#define VSEW3(bits) ({							\
+	unsigned long __b = (unsigned long)(bits);			\
+	(__b == 16UL ? 5UL : (__b == 32UL ? 6UL : 7UL));		\
+})
+
+#define INSN_MATCH_VLUXSEG(n, bits) \
+	((((unsigned long)(n) - 1UL) << 29) | 0x04000007UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VSUXSEG(n, bits) \
+	((((unsigned long)(n) - 1UL) << 29) | 0x04000027UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VLOXSEG(n, bits) \
+	((((unsigned long)(n) - 1UL) << 29) | 0x0c000007UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VSOXSEG(n, bits) \
+	((((unsigned long)(n) - 1UL) << 29) | 0x0c000027UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VLSSEG(n, bits)  \
+	((((unsigned long)(n) - 1UL) << 29) | 0x08000007UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VSSSEG(n, bits)  \
+	((((unsigned long)(n) - 1UL) << 29) | 0x08000027UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VSSEG(n, bits)   \
+	((((unsigned long)(n) - 1UL) << 29) | 0x00004027UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VLSEG(n, bits)   \
+	((((unsigned long)(n) - 1UL) << 29) | 0x00004007UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VLSEGFF(n, bits) \
+	((((unsigned long)(n) - 1UL) << 29) | 0x01000007UL | (VSEW3(bits) << 12))
+
+#define INSN_MATCH_VLE16V		0x00005007
+#define INSN_MATCH_VLE32V		0x00006007
+#define INSN_MATCH_VLE64V		0x00007007
+#define INSN_MATCH_VSE16V		0x00005027
+#define INSN_MATCH_VSE32V		0x00006027
+#define INSN_MATCH_VSE64V		0x00007027
+#define INSN_MATCH_VLSE16V		0x08005007
+#define INSN_MATCH_VLSE32V		0x08006007
+#define INSN_MATCH_VLSE64V		0x08007007
+#define INSN_MATCH_VSSE16V		0x08005027
+#define INSN_MATCH_VSSE32V		0x08006027
+#define INSN_MATCH_VSSE64V		0x08007027
+#define INSN_MATCH_VLOXEI16V		0x0c005007
+#define INSN_MATCH_VLOXEI32V		0x0c006007
+#define INSN_MATCH_VLOXEI64V		0x0c007007
+#define INSN_MATCH_VSOXEI16V		0x0c005027
+#define INSN_MATCH_VSOXEI32V		0x0c006027
+#define INSN_MATCH_VSOXEI64V		0x0c007027
+#define INSN_MATCH_VLUXEI16V		0x04005007
+#define INSN_MATCH_VLUXEI32V		0x04006007
+#define INSN_MATCH_VLUXEI64V		0x04007007
+#define INSN_MATCH_VSUXEI16V		0x04005027
+#define INSN_MATCH_VSUXEI32V		0x04006027
+#define INSN_MATCH_VSUXEI64V		0x04007027
+#define INSN_MATCH_VLE16FFV		0x01005007
+#define INSN_MATCH_VLE32FFV		0x01006007
+#define INSN_MATCH_VLE64FFV		0x01007007
+#define INSN_MATCH_VL1RE8V		0x02800007
+#define INSN_MATCH_VL1RE16V		0x02805007
+#define INSN_MATCH_VL1RE32V		0x02806007
+#define INSN_MATCH_VL1RE64V		0x02807007
+#define INSN_MATCH_VL2RE8V		0x22800007
+#define INSN_MATCH_VL2RE16V		0x22805007
+#define INSN_MATCH_VL2RE32V		0x22806007
+#define INSN_MATCH_VL2RE64V		0x22807007
+#define INSN_MATCH_VL4RE8V		0x62800007
+#define INSN_MATCH_VL4RE16V		0x62805007
+#define INSN_MATCH_VL4RE32V		0x62806007
+#define INSN_MATCH_VL4RE64V		0x62807007
+#define INSN_MATCH_VL8RE8V		0xe2800007
+#define INSN_MATCH_VL8RE16V		0xe2805007
+#define INSN_MATCH_VL8RE32V		0xe2806007
+#define INSN_MATCH_VL8RE64V		0xe2807007
+#define INSN_MATCH_VS1RV		0x02800027
+#define INSN_MATCH_VS2RV		0x22800027
+#define INSN_MATCH_VS4RV		0x62800027
+#define INSN_MATCH_VS8RV		0xe2800027
+
+#define IS_VECTOR_INSN_MATCH(insn, match, mask)					\
+({										\
+	u32 _i = (u32)(insn);							\
+	u32 _m = (u32)(match);							\
+	u32 _k = (u32)(mask);							\
+	(((_i & _k) == (_m & _k)));						\
+})
+
+#define IS_UNIT_STRIDE_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_UNIT_STRIDE)
+
+#define IS_STRIDE_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_STRIDE)
+
+#define IS_INDEXED_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_INDEXED)
+
+#define IS_FAULT_ONLY_FIRST_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_FAULT_ONLY_FIRST)
+
+#define IS_WHOLE_REG_MATCH(insn, match) \
+	IS_VECTOR_INSN_MATCH(insn, match, INSN_MASK_VECTOR_WHOLE_REG)
+
+#define IS_UNIT_STRIDE_LOAD(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLE16V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLE32V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLE64V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(2, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(3, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(4, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(5, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(6, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(7, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(8, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(2, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(3, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(4, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(5, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(6, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(7, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(8, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(2, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(3, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(4, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(5, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(6, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(7, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VLSEG(8, 64)); \
+})
+
+#define IS_UNIT_STRIDE_STORE(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSE16V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSE32V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSE64V) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(2, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(3, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(4, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(5, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(6, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(7, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(8, 16)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(2, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(3, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(4, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(5, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(6, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(7, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(8, 32)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(2, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(3, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(4, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(5, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(6, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(7, 64)) || \
+	IS_UNIT_STRIDE_MATCH(__insn, INSN_MATCH_VSSEG(8, 64)); \
+})
+
+#define IS_STRIDE_LOAD(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSE16V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSE32V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSE64V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(2, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(3, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(4, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(5, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(6, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(7, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(8, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(2, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(3, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(4, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(5, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(6, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(7, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(8, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(2, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(3, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(4, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(5, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(6, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(7, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VLSSEG(8, 64)); \
+})
+
+#define IS_STRIDE_STORE(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSE16V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSE32V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSE64V) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(2, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(3, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(4, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(5, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(6, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(7, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(8, 16)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(2, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(3, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(4, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(5, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(6, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(7, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(8, 32)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(2, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(3, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(4, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(5, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(6, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(7, 64)) || \
+	IS_STRIDE_MATCH(__insn, INSN_MATCH_VSSSEG(8, 64)); \
+})
+
+#define IS_INDEXED_LOAD(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXEI16V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXEI32V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXEI64V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXEI16V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXEI32V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXEI64V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLUXSEG(8, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VLOXSEG(8, 64)); \
+})
+
+#define IS_INDEXED_STORE(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXEI16V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXEI32V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXEI64V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXEI16V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXEI32V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXEI64V) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSUXSEG(8, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(2, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(3, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(4, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(5, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(6, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(7, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(8, 16)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(2, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(3, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(4, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(5, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(6, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(7, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(8, 32)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(2, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(3, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(4, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(5, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(6, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(7, 64)) || \
+	IS_INDEXED_MATCH(__insn, INSN_MATCH_VSOXSEG(8, 64)); \
+})
+
+#define IS_FAULT_ONLY_FIRST_LOAD(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLE16FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLE32FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLE64FFV) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(2, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(3, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(4, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(5, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(6, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(7, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(8, 16)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(2, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(3, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(4, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(5, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(6, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(7, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(8, 32)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(2, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(3, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(4, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(5, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(6, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(7, 64)) || \
+	IS_FAULT_ONLY_FIRST_MATCH(__insn, INSN_MATCH_VLSEGFF(8, 64)); \
+})
+
+#define IS_WHOLE_REG_LOAD(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL1RE8V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL1RE16V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL1RE32V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL1RE64V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL2RE8V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL2RE16V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL2RE32V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL2RE64V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL4RE8V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL4RE16V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL4RE32V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL4RE64V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL8RE8V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL8RE16V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL8RE32V) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VL8RE64V); \
+})
+
+#define IS_WHOLE_REG_STORE(insn) ({ \
+	typeof(insn) __insn = (insn); \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VS1RV) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VS2RV) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VS4RV) || \
+	IS_WHOLE_REG_MATCH(__insn, INSN_MATCH_VS8RV); \
+})
+
+#define VM_MASK				0x1
+#define VIEW_MASK			0x3
+#define VSEW_MASK			0x3
+#define VLMUL_MASK			0x7
+#define VD_MASK				0x1f
+#define VS2_MASK			0x1f
+
+#define SH_VSEW				3
+#define SH_VIEW				12
+#define SH_VD				7
+#define SH_VS2				20
+#define SH_VM				25
+#define SH_MEW				28
+
+#define GET_VD(insn)			(((insn) >> SH_VD) & VD_MASK)
+#define GET_VS2(insn)			(((insn) >> SH_VS2) & VS2_MASK)
+#define GET_VIEW(insn)			(((insn) >> SH_VIEW) & VIEW_MASK)
+#define IS_MASKED(insn)			((((insn) >> SH_VM) & VM_MASK) == 0)
+#define GET_MEW(insn)			(((insn) >> SH_MEW) & 1)
+#define GET_VSEW(vtype)			(((vtype) >> SH_VSEW) & VSEW_MASK)
+#define GET_VLMUL(vtype)		((vtype) & VLMUL_MASK)
+#define GET_LEN(view)			(1UL << (view))
+#define GET_NF(insn)			(1 + (((insn) >> 29) & 7))
+#define GET_VEMUL(vlmul, view, vsew)	(((vlmul) + (view) - (vsew)) & 7)
+#define GET_EMUL(vemul)						\
+({								\
+	unsigned long _v = (unsigned long)(vemul);		\
+	(1UL << (_v >= 4 ? 0 : _v));				\
+})
+
+#if defined(CONFIG_64BIT)
+#define LOG_REGBYTES		3
+#else
+#define LOG_REGBYTES		2
+#endif
+
+#define SH_RS1			15
+#define SH_RS2			20
+
+#define SHIFT_RIGHT(x, y)					\
+({								\
+	long _y = (long)(y);					\
+	__typeof__(x) _x = (x);					\
+	(_y < 0) ? (_x << (unsigned long)(-_y))			\
+	: (_x >> (unsigned long)_y);				\
+})
+
+#define REG_MASK                        \
+	((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
+
+#define REG_OFFSET(insn, pos)           \
+	(SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
+
+#define REG_PTR(insn, pos, regs)                                               \
+	((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)))
+
+#define GET_RS1(insn, regs)		(*REG_PTR(insn, SH_RS1, regs))
+#define GET_RS2(insn, regs)		(*REG_PTR(insn, SH_RS2, regs))
+
+/*
+ * VLEN_MAX defines the maximum vector length supported by the emulator.
+ * The mask array size must be at least VLEN_MAX / 8 bytes to hold v0 mask.
+ * Current hardware implementations typically have VLEN of 128-1024 bits.
+ * We use 4096 bits (512 bytes) as a reasonable upper bound that covers
+ * known implementations while keeping stack usage manageable.
+ */
+#define VLEN_MAX 4096
+
+/*
+ * set_vreg - Write bytes into a vector register at a specific position.
+ *
+ * In user mode, kernel_vector_begin() has already saved the hardware
+ * vector register file into current->thread.vstate.datap, so the target
+ * bytes are modified there and restored to hardware by kernel_vector_end().
+ *
+ * In kernel mode, the vector registers are "live" in hardware for the
+ * duration of the trap (IRQs are disabled, nothing else can touch them).
+ * vstart skips elements before @pos, and the tail-undisturbed (tu) policy
+ * on vsetvli preserves elements after @pos+@size, so only [pos, pos+size)
+ * is modified without corrupting the rest of the register. This is the
+ * same technique OpenSBI's M-mode vector misaligned handler uses.
+ */
+static inline void set_vreg(ulong vlenb, ulong which, ulong pos, ulong size,
+			    const u8 *bytes, bool is_usermode)
+{
+	if (is_usermode) {
+		struct __riscv_v_ext_state *vstate = &current->thread.vstate;
+		unsigned char *vdata = vstate->datap;
+
+		if (!vdata)
+			return;
+		memcpy(vdata + which * vlenb + pos, bytes, size);
+		return;
+	}
+
+	/* Kernel mode: directly access the live hardware vector registers. */
+	pos += (which % 8) * vlenb;
+	bytes -= pos;
+
+	__asm__ __volatile__(
+		".option push\n\t"
+		".option arch, +v\n\t"
+		"vsetvli x0, %0, e8, m8, tu, ma\n\t"
+		".option pop\n\t"
+		: : "r"(pos + size));
+
+	csr_write(CSR_VSTART, pos);
+
+	switch (which / 8) {
+	case 0:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vle8.v v0, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 1:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vle8.v v8, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 2:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vle8.v v16, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 3:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vle8.v v24, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	}
+}
+
+/*
+ * get_vreg - Read bytes from a vector register at a specific position.
+ *
+ * Same technique as set_vreg but in reverse: uses vse8.v to store register
+ * contents to memory.
+ */
+static inline void get_vreg(ulong vlenb, ulong which, ulong pos, ulong size,
+			    u8 *bytes, bool is_usermode)
+{
+	if (is_usermode) {
+		struct __riscv_v_ext_state *vstate = &current->thread.vstate;
+		const unsigned char *vdata = vstate->datap;
+
+		if (!vdata) {
+			memset(bytes, 0, size);
+			return;
+		}
+		memcpy(bytes, vdata + which * vlenb + pos, size);
+		return;
+	}
+
+	/* Kernel mode: directly access the live hardware vector registers. */
+	pos += (which % 8) * vlenb;
+	bytes -= pos;
+
+	__asm__ __volatile__(
+		".option push\n\t"
+		".option arch, +v\n\t"
+		"vsetvli x0, %0, e8, m8, tu, ma\n\t"
+		".option pop\n\t"
+		: : "r"(pos + size));
+
+	csr_write(CSR_VSTART, pos);
+
+	switch (which / 8) {
+	case 0:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vse8.v v0, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 1:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vse8.v v8, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 2:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vse8.v v16, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	case 3:
+		__asm__ __volatile__(
+			".option push\n\t"
+			".option arch, +v\n\t"
+			"vse8.v v24, (%0)\n\t"
+			".option pop\n\t"
+			: : "r"(bytes) : "memory");
+		break;
+	}
+}
+
+/*
+ * vsetvl - Restore vl and vtype CSRs after get_vreg/set_vreg clobber them.
+ *
+ * get_vreg/set_vreg use vsetvli internally, which modifies vl and vtype.
+ * This must be called after kernel-mode emulation completes to restore the
+ * original values. vsetvl also implicitly resets vstart to 0, which is the
+ * correct post-instruction state per the V spec.
+ */
+static inline void vsetvl(ulong vl, ulong vtype)
+{
+	__asm__ __volatile__(
+		".option push\n\t"
+		".option arch, +v\n\t"
+		"vsetvl x0, %0, %1\n\t"
+		".option pop\n\t"
+		: : "r"(vl), "r"(vtype));
+}
+
+int misaligned_v_ld_emulator(struct pt_regs *regs, unsigned long insn)
+{
+	int ret = 0;
+	unsigned long vl, vtype, vlenb, vstart, base, stride;
+	unsigned long vd, vs2, view, vsew, vlmul, emul, vemul;
+	unsigned long len, nf;
+	bool illegal, masked;
+	bool is_usermode = user_mode(regs);
+	u8 mask[VLEN_MAX / 8];
+	u8 bytes[8 * sizeof(u64)];
+
+	/*
+	 * User mode: IRQs are enabled by do_trap_misaligned(), so we can use
+	 * kernel_vector_begin() which saves the user's vector state to datap.
+	 * We then read CSRs from the saved vstate (not hardware) because
+	 * kernel_vector_begin() -> riscv_v_vstate_save() modifies the hw CSRs.
+	 *
+	 * Kernel mode: IRQs are disabled (NMI-like context), may_use_simd()
+	 * returns false, so kernel_vector_begin() cannot be used. Instead we
+	 * enable vector directly and read CSRs from hardware; get_vreg/
+	 * set_vreg access individual register elements in place via vstart +
+	 * vsetvli(tu), without disturbing the rest of the register file.
+	 */
+	if (is_usermode) {
+		struct __riscv_v_ext_state *vstate = &current->thread.vstate;
+
+		kernel_vector_begin();
+		vl = vstate->vl;
+		vtype = vstate->vtype;
+		vstart = vstate->vstart;
+		vlenb = vstate->vlenb;
+	} else {
+		riscv_v_enable();
+		vl = csr_read(CSR_VL);
+		vtype = csr_read(CSR_VTYPE);
+		vstart = csr_read(CSR_VSTART);
+		vlenb = csr_read(CSR_VLENB);
+	}
+
+	base = GET_RS1(insn, regs);
+	stride = GET_RS2(insn, regs);
+	vd = GET_VD(insn);
+	vs2 = GET_VS2(insn);
+	view = GET_VIEW(insn);
+	vsew = GET_VSEW(vtype);
+	vlmul = GET_VLMUL(vtype);
+	illegal = GET_MEW(insn);
+	masked = IS_MASKED(insn);
+	len = GET_LEN(view);
+	nf = GET_NF(insn);
+	vemul = GET_VEMUL(vlmul, view, vsew);
+	emul = GET_EMUL(vemul);
+
+	if (IS_UNIT_STRIDE_LOAD(insn) || IS_FAULT_ONLY_FIRST_LOAD(insn)) {
+		stride = nf * len;
+	} else if (IS_WHOLE_REG_LOAD(insn)) {
+		vl = (nf * vlenb) >> view;
+		nf = 1;
+		vemul = 0;
+		emul = 1;
+		stride = nf * len;
+	} else if (IS_INDEXED_LOAD(insn)) {
+		len = 1 << vsew;
+		vemul = (vlmul + vsew - vsew) & 7;
+		emul = 1 << ((vemul & 4) ? 0 : vemul);
+		stride = nf * len;
+	}
+
+	if (illegal || vlenb > VLEN_MAX / 8) {
+		if (is_usermode)
+			force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->epc);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if (masked)
+		get_vreg(vlenb, 0, 0, vlenb, mask, is_usermode);
+
+	do {
+		if (!masked || ((mask[vstart / 8] >> (vstart % 8)) & 1)) {
+			unsigned long addr = base + vstart * stride;
+
+			if (IS_INDEXED_LOAD(insn)) {
+				unsigned long offset = 0;
+
+				get_vreg(vlenb, vs2, vstart << view, 1 << view,
+					 (u8 *)&offset, is_usermode);
+				addr = base + offset;
+			}
+
+			csr_write(CSR_VSTART, vstart);
+
+			for (unsigned long seg = 0; seg < nf; seg++) {
+				for (unsigned long i = 0; i < len; i++) {
+					u8 *src = (u8 *)(addr + seg * len + i);
+
+					if (is_usermode) {
+						if (get_user(bytes[seg * len + i],
+							     (u8 __user *)src)) {
+							if (IS_FAULT_ONLY_FIRST_LOAD(insn) &&
+							    vstart != 0) {
+								vl = vstart;
+								break;
+							}
+							ret = -EFAULT;
+							goto out;
+						}
+					} else {
+						bytes[seg * len + i] = *src;
+					}
+				}
+			}
+
+			for (unsigned long seg = 0; seg < nf; seg++)
+				set_vreg(vlenb, vd + seg * emul, vstart * len, len,
+					 &bytes[seg * len], is_usermode);
+		}
+	} while (++vstart < vl);
+
+out:
+	if (ret == 0)
+		regs->epc += 4;
+
+	if (is_usermode) {
+		kernel_vector_end();
+	} else {
+		/*
+		 * Restore vl/vtype clobbered by get_vreg/set_vreg's vsetvli.
+		 * vsetvl also resets vstart to 0 (the correct post-insn state).
+		 */
+		vsetvl(vl, vtype);
+		riscv_v_disable();
+	}
+
+	return ret;
+}
+
+int misaligned_v_st_emulator(struct pt_regs *regs, unsigned long insn)
+{
+	int ret = 0;
+	unsigned long vl, vtype, vlenb, vstart, base, stride;
+	unsigned long vd, vs2, view, vsew, vlmul, emul, vemul;
+	unsigned long len, nf;
+	bool illegal, masked;
+	bool is_usermode = user_mode(regs);
+	u8 mask[VLEN_MAX / 8];
+	u8 bytes[8 * sizeof(u64)];
+
+	/*
+	 * User mode: IRQs are enabled by do_trap_misaligned(), so we can use
+	 * kernel_vector_begin() which saves the user's vector state to datap.
+	 *
+	 * Kernel mode: IRQs are disabled, may_use_simd() returns false. Enable
+	 * vector directly and access registers via vstart + vsetvli(tu).
+	 */
+	if (is_usermode) {
+		struct __riscv_v_ext_state *vstate = &current->thread.vstate;
+
+		kernel_vector_begin();
+		vl = vstate->vl;
+		vtype = vstate->vtype;
+		vstart = vstate->vstart;
+		vlenb = vstate->vlenb;
+	} else {
+		riscv_v_enable();
+		vl = csr_read(CSR_VL);
+		vtype = csr_read(CSR_VTYPE);
+		vstart = csr_read(CSR_VSTART);
+		vlenb = csr_read(CSR_VLENB);
+	}
+
+	base = GET_RS1(insn, regs);
+	stride = GET_RS2(insn, regs);
+	vd = GET_VD(insn);
+	vs2 = GET_VS2(insn);
+	view = GET_VIEW(insn);
+	vsew = GET_VSEW(vtype);
+	vlmul = GET_VLMUL(vtype);
+	illegal = GET_MEW(insn);
+	masked = IS_MASKED(insn);
+	len = GET_LEN(view);
+	nf = GET_NF(insn);
+	vemul = GET_VEMUL(vlmul, view, vsew);
+	emul = GET_EMUL(vemul);
+
+	if (IS_UNIT_STRIDE_STORE(insn)) {
+		stride = nf * len;
+	} else if (IS_WHOLE_REG_STORE(insn)) {
+		vl = (nf * vlenb) >> view;
+		nf = 1;
+		vemul = 0;
+		emul = 1;
+		stride = nf * len;
+	} else if (IS_INDEXED_STORE(insn)) {
+		len = 1 << vsew;
+		vemul = (vlmul + vsew - vsew) & 7;
+		emul = 1 << ((vemul & 4) ? 0 : vemul);
+		stride = nf * len;
+	}
+
+	if (illegal || vlenb > VLEN_MAX / 8) {
+		if (is_usermode)
+			force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->epc);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if (masked)
+		get_vreg(vlenb, 0, 0, vlenb, mask, is_usermode);
+
+	do {
+		if (!masked || ((mask[vstart / 8] >> (vstart % 8)) & 1)) {
+			unsigned long addr = base + vstart * stride;
+
+			if (IS_INDEXED_STORE(insn)) {
+				unsigned long offset = 0;
+
+				get_vreg(vlenb, vs2, vstart << view, 1 << view,
+					 (u8 *)&offset, is_usermode);
+				addr = base + offset;
+			}
+
+			for (unsigned long seg = 0; seg < nf; seg++)
+				get_vreg(vlenb, vd + seg * emul, vstart * len, len,
+					 &bytes[seg * len], is_usermode);
+
+			csr_write(CSR_VSTART, vstart);
+			for (unsigned long seg = 0; seg < nf; seg++) {
+				for (unsigned long i = 0; i < len; i++) {
+					u8 *dst = (u8 *)(addr + seg * len + i);
+
+					if (is_usermode) {
+						if (put_user(bytes[seg * len + i],
+							     (u8 __user *)dst)) {
+							ret = -EFAULT;
+							goto out;
+						}
+					} else {
+						*dst = bytes[seg * len + i];
+					}
+				}
+			}
+		}
+	} while (++vstart < vl);
+
+out:
+	if (ret == 0)
+		regs->epc += 4;
+
+	if (is_usermode) {
+		kernel_vector_end();
+	} else {
+		/* Restore vl/vtype clobbered by get_vreg's vsetvli. */
+		vsetvl(vl, vtype);
+		riscv_v_disable();
+	}
+
+	return ret;
+}

-- 
2.56.0


  reply	other threads:[~2026-11-17  7:38 UTC|newest]

Thread overview: 3+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17  7:38 [PATCH 0/2] riscv: Add support for vector misaligned load/store Nylon Chen
2026-11-17  7:38 ` Nylon Chen [this message]
2026-11-17  7:38 ` [PATCH 2/2] selftests/riscv: Add vector misaligned load/store tests Nylon Chen

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261116-riscv-vector-misaligned-v1-1-6c4d0ca905b7@sifive.com \
    --to=nylon.chen@sifive.com \
    --cc=alex@ghiti.fr \
    --cc=aou@eecs.berkeley.edu \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-kselftest@vger.kernel.org \
    --cc=linux-riscv@lists.infradead.org \
    --cc=palmer@dabbelt.com \
    --cc=pjw@kernel.org \
    --cc=shuah@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®