* [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions
2026-08-28 15:33 [RFC PATCH 0/5] rseq: add support for RSEQ operations odion
@ 2026-08-28 15:33 ` odion
2026-08-29 22:34 ` Dmitry Vyukov
2026-08-28 15:33 ` [RFC PATCH 2/5] rseq: add per-task rseq operation state odion
` (3 subsequent siblings)
4 siblings, 1 reply; 11+ messages in thread
From: odion @ 2026-08-28 15:33 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Peter Zijlstra, Paul E. McKenney, Boqun Feng, LKML,
Thomas Gleixner, Dmitry Vyukov, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell, Olivier Dion
From: Olivier Dion <odion@efficios.com>
Introduce userspace ABI for rseq operations: a per-thread list of
operations the kernel applies on return to user space.
The main motivation for this work is to encourage TCMalloc to migrate to
RSEQ v2 [0] and use the glibc RSEQ region.
Indeed, TCMalloc relies on the behavior of RSEQ v1, that reset the
cpu_id bits in the RSEQ shared region, to invalidate a per-cpu pointer
cached in a TLS. This hack requires TCMalloc users to use a glibc
tunable to disable RSEQ registration for threads so that TCMalloc can
register its own region, overlapping the TLS cache.
Overall, this puts the users in a situation of choosing between a fast
sched_getcpu() and TCMalloc, plus some downsides such as requiring a
initial-exec model for the cache TLS in a shared-library and not being
compatible with the glibc
RSEQ operations aim at solving this by offering operations that are
executed by the kernel, on behalf of a task after it is scheduled,
before returning to userspace. Operations are per-task and registered
throught prctl. They are opt-in and only introduce overhead on tasks
that register them.
Add enum rseq_op_type and the rseq_op_node / rseq_op_reset /
rseq_op_reset_with_stride structures, the RSEQ_CS_FLAG_RSEQ_OP_*
availability/enabled flags, the RSEQ_OP_LIST_LIMIT walk bound, and the
PR_RSEQ_OP prctl with its REGISTER/UNREGISTER sub-commands.
The operation list is a circular doubly-linked list anchored by a
kernel-owned sentinel embedded in struct rseq.
[0] Documentation/userspace-api/rseq.rst (Optimized RSEQ v2)
Link: https://lore.kernel.org/lkml/20260428221058.149538293@kernel.org
Signed-off-by: Olivier Dion <odion@efficios.com>
---
include/uapi/linux/prctl.h | 12 +++++
include/uapi/linux/rseq.h | 104 ++++++++++++++++++++++++++++++++++---
2 files changed, 108 insertions(+), 8 deletions(-)
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index b6ec6f693719..4cb6356a271a 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -396,6 +396,18 @@ struct prctl_mm_map {
*/
# define PR_RSEQ_SLICE_EXT_ENABLE 0x01
+/*
+ * RSEQ operation registration.
+ *
+ * arg3 is the user address of a struct rseq_op_node embedded in one of the
+ * rseq operation structures (see uapi/linux/rseq.h). Registering the first
+ * operation enables rseq operation processing for the thread; unregistering
+ * the last one disables it.
+ */
+#define PR_RSEQ_OP 82
+# define PR_RSEQ_OP_REGISTER 1
+# define PR_RSEQ_OP_UNREGISTER 2
+
/*
* Get or set the control flow integrity (CFI) configuration for the
* current thread.
diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h
index ca6fe1f9d05e..b664d1991c48 100644
--- a/include/uapi/linux/rseq.h
+++ b/include/uapi/linux/rseq.h
@@ -13,6 +13,11 @@
#include <linux/types.h>
#include <asm/byteorder.h>
+/*
+ * Maximum number of nodes walked in the rseq operation list.
+ */
+#define RSEQ_OP_LIST_LIMIT 2048
+
enum rseq_cpu_id_state {
RSEQ_CPU_ID_UNINITIALIZED = -1,
RSEQ_CPU_ID_REGISTRATION_FAILED = -2,
@@ -33,6 +38,8 @@ enum rseq_cs_flags_bit {
/* User read only feature flags */
RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4,
RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT = 5,
+ RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT = 6,
+ RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT = 7,
};
enum rseq_cs_flags {
@@ -47,6 +54,10 @@ enum rseq_cs_flags {
(1U << RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT),
RSEQ_CS_FLAG_SLICE_EXT_ENABLED =
(1U << RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT),
+ RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE =
+ (1U << RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT),
+ RSEQ_CS_FLAG_RSEQ_OP_ENABLED =
+ (1U << RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT),
};
/*
@@ -86,6 +97,77 @@ struct rseq_slice_ctrl {
};
};
+/*
+ * enum rseq_op_type - Type of an rseq operation
+ * @RSEQ_OP_RESET: Plain reset. Uses struct rseq_op_reset.
+ * @RSEQ_OP_RESET_WITH_STRIDE_CPUID: Reset indexed by the current CPU ID.
+ * Uses struct rseq_op_reset_with_stride.
+ * @RSEQ_OP_RESET_WITH_STRIDE_MMCID: Reset indexed by the current MM CID.
+ * Uses struct rseq_op_reset_with_stride.
+ */
+enum rseq_op_type {
+ RSEQ_OP_RESET,
+ RSEQ_OP_RESET_WITH_STRIDE_CPUID,
+ RSEQ_OP_RESET_WITH_STRIDE_MMCID,
+ RSEQ_OP_NR,
+};
+
+/*
+ * struct rseq_op_node - Common header linking an rseq operation into the list
+ * @next: Address of the next node. Owned by the kernel.
+ * @prev: Address of the previous node. Owned by the kernel.
+ * @type: Operation type. See enum rseq_op_type.
+ * @reserved: Must be zero on registration.
+ *
+ * User space allocates the node, sets @type and zeroes @next, @prev and
+ * @reserved before passing it to prctl(PR_RSEQ_OP, PR_RSEQ_OP_REGISTER, node).
+ * The kernel owns @next and @prev for the lifetime of the registration and
+ * links the node into a circular doubly-linked list anchored by an internal
+ * sentinel in struct rseq. User space must not touch @next or @prev while the
+ * node is registered.
+ */
+struct rseq_op_node {
+ __u64 next;
+ __u64 prev;
+ struct {
+ __u8 type; /* enum rseq_op_type */
+ __u8 reserved[7];
+ };
+};
+
+/*
+ * struct rseq_op_reset - Reset one word to a value on return to user space
+ * @node: Operation list node.
+ * @src: Address of the source word, or 0 to reset @dst to zero.
+ * @dst: Address of the destination word.
+ * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
+ */
+struct rseq_op_reset {
+ struct rseq_op_node node;
+ __u64 src;
+ __u64 dst;
+ __u32 len;
+};
+
+/*
+ * struct rseq_op_reset_with_stride - Reset one word in a strided array
+ * @node: Operation list node.
+ * @src: Address of the source word, or 0 to reset the slot to zero.
+ * @dst: Base address of the strided destination array.
+ * @dst_stride: Stride in bytes between consecutive array slots.
+ * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
+ *
+ * The destination slot is @dst + @dst_stride * index, where index is the
+ * current CPU ID or MM CID depending on the operation type.
+ */
+struct rseq_op_reset_with_stride {
+ struct rseq_op_node node;
+ __u64 src;
+ __u64 dst;
+ __u64 dst_stride;
+ __u32 len;
+};
+
/*
* The original size and alignment of the allocation for struct rseq is
* 32 bytes.
@@ -191,15 +273,21 @@ struct rseq {
struct rseq_slice_ctrl slice_ctrl;
/*
- * Before rseq became extensible, its original size was 32 bytes even
- * though the active rseq area was only 20 bytes.
- * Exposing a 32 bytes feature size would make life needlessly painful
- * for userspace. Therefore, add a reserved byte after byte 32
- * to bump the rseq feature size from 32 to 33.
- * The next field to be added to the rseq area will be larger
- * than one byte, and will replace this reserved byte.
+ * Sentinel of the circular doubly-linked list of rseq operations
+ * registered via prctl(PR_RSEQ_OP, ...). Fully owned and maintained by
+ * the kernel: it is initialized to point to itself on registration and
+ * user space must never read or write it directly.
+ *
+ * The kernel only use next and prev from rseq_op_list. The rest of the
+ * bytes are reserved for later usage and should be zeroed.
*/
- __u8 __reserved;
+ union {
+ struct rseq_op_node rseq_op_list;
+ struct {
+ __u64 op_used[2];
+ __u64 reserved;
+ };
+ };
/*
* Flexible array member at end of structure, after last feature field.
--
2.54.0
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions
2026-08-28 15:33 ` [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions odion
@ 2026-08-29 22:34 ` Dmitry Vyukov
2026-08-29 22:47 ` Dmitry Vyukov
0 siblings, 1 reply; 11+ messages in thread
From: Dmitry Vyukov @ 2026-08-29 22:34 UTC (permalink / raw)
To: odion
Cc: Mathieu Desnoyers, Peter Zijlstra, Paul E. McKenney, Boqun Feng,
LKML, Thomas Gleixner, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell
On Fri, 28 Aug 2026 at 17:33, <odion@efficios.com> wrote:
>
> From: Olivier Dion <odion@efficios.com>
>
> Introduce userspace ABI for rseq operations: a per-thread list of
> operations the kernel applies on return to user space.
>
> The main motivation for this work is to encourage TCMalloc to migrate to
> RSEQ v2 [0] and use the glibc RSEQ region.
>
> Indeed, TCMalloc relies on the behavior of RSEQ v1, that reset the
> cpu_id bits in the RSEQ shared region, to invalidate a per-cpu pointer
> cached in a TLS. This hack requires TCMalloc users to use a glibc
> tunable to disable RSEQ registration for threads so that TCMalloc can
> register its own region, overlapping the TLS cache.
>
> Overall, this puts the users in a situation of choosing between a fast
> sched_getcpu() and TCMalloc, plus some downsides such as requiring a
> initial-exec model for the cache TLS in a shared-library and not being
> compatible with the glibc
>
> RSEQ operations aim at solving this by offering operations that are
> executed by the kernel, on behalf of a task after it is scheduled,
> before returning to userspace. Operations are per-task and registered
> throught prctl. They are opt-in and only introduce overhead on tasks
> that register them.
>
> Add enum rseq_op_type and the rseq_op_node / rseq_op_reset /
> rseq_op_reset_with_stride structures, the RSEQ_CS_FLAG_RSEQ_OP_*
> availability/enabled flags, the RSEQ_OP_LIST_LIMIT walk bound, and the
> PR_RSEQ_OP prctl with its REGISTER/UNREGISTER sub-commands.
>
> The operation list is a circular doubly-linked list anchored by a
> kernel-owned sentinel embedded in struct rseq.
>
> [0] Documentation/userspace-api/rseq.rst (Optimized RSEQ v2)
Hi Olivier,
This is very cool, thanks for working on this.
> Link: https://lore.kernel.org/lkml/20260428221058.149538293@kernel.org
> Signed-off-by: Olivier Dion <odion@efficios.com>
> ---
> include/uapi/linux/prctl.h | 12 +++++
> include/uapi/linux/rseq.h | 104 ++++++++++++++++++++++++++++++++++---
> 2 files changed, 108 insertions(+), 8 deletions(-)
>
> diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
> index b6ec6f693719..4cb6356a271a 100644
> --- a/include/uapi/linux/prctl.h
> +++ b/include/uapi/linux/prctl.h
> @@ -396,6 +396,18 @@ struct prctl_mm_map {
> */
> # define PR_RSEQ_SLICE_EXT_ENABLE 0x01
>
> +/*
> + * RSEQ operation registration.
> + *
> + * arg3 is the user address of a struct rseq_op_node embedded in one of the
> + * rseq operation structures (see uapi/linux/rseq.h). Registering the first
> + * operation enables rseq operation processing for the thread; unregistering
> + * the last one disables it.
> + */
> +#define PR_RSEQ_OP 82
> +# define PR_RSEQ_OP_REGISTER 1
> +# define PR_RSEQ_OP_UNREGISTER 2
> +
> /*
> * Get or set the control flow integrity (CFI) configuration for the
> * current thread.
> diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h
> index ca6fe1f9d05e..b664d1991c48 100644
> --- a/include/uapi/linux/rseq.h
> +++ b/include/uapi/linux/rseq.h
> @@ -13,6 +13,11 @@
> #include <linux/types.h>
> #include <asm/byteorder.h>
>
> +/*
> + * Maximum number of nodes walked in the rseq operation list.
> + */
> +#define RSEQ_OP_LIST_LIMIT 2048
Does this belong to the user header? Kernel and userspace are not
necessary built with the same headers, and we don't necessary want to
set this const in stone. I think it may be more flexible to make this
impl detail and just return ENOSPC.
> enum rseq_cpu_id_state {
> RSEQ_CPU_ID_UNINITIALIZED = -1,
> RSEQ_CPU_ID_REGISTRATION_FAILED = -2,
> @@ -33,6 +38,8 @@ enum rseq_cs_flags_bit {
> /* User read only feature flags */
> RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4,
> RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT = 5,
> + RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT = 6,
> + RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT = 7,
> };
>
> enum rseq_cs_flags {
> @@ -47,6 +54,10 @@ enum rseq_cs_flags {
> (1U << RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT),
> RSEQ_CS_FLAG_SLICE_EXT_ENABLED =
> (1U << RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT),
> + RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE =
> + (1U << RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT),
> + RSEQ_CS_FLAG_RSEQ_OP_ENABLED =
> + (1U << RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT),
> };
>
> /*
> @@ -86,6 +97,77 @@ struct rseq_slice_ctrl {
> };
> };
>
> +/*
> + * enum rseq_op_type - Type of an rseq operation
> + * @RSEQ_OP_RESET: Plain reset. Uses struct rseq_op_reset.
> + * @RSEQ_OP_RESET_WITH_STRIDE_CPUID: Reset indexed by the current CPU ID.
> + * Uses struct rseq_op_reset_with_stride.
> + * @RSEQ_OP_RESET_WITH_STRIDE_MMCID: Reset indexed by the current MM CID.
> + * Uses struct rseq_op_reset_with_stride.
Do you have use-cases for these STRIDE ops?
I can think of some, but they would require executing ops when a task
is descheduled, rather than when it's scheduled in.
> + */
> +enum rseq_op_type {
> + RSEQ_OP_RESET,
> + RSEQ_OP_RESET_WITH_STRIDE_CPUID,
> + RSEQ_OP_RESET_WITH_STRIDE_MMCID,
> + RSEQ_OP_NR,
> +};
> +
> +/*
> + * struct rseq_op_node - Common header linking an rseq operation into the list
> + * @next: Address of the next node. Owned by the kernel.
> + * @prev: Address of the previous node. Owned by the kernel.
> + * @type: Operation type. See enum rseq_op_type.
> + * @reserved: Must be zero on registration.
> + *
> + * User space allocates the node, sets @type and zeroes @next, @prev and
> + * @reserved before passing it to prctl(PR_RSEQ_OP, PR_RSEQ_OP_REGISTER, node).
> + * The kernel owns @next and @prev for the lifetime of the registration and
> + * links the node into a circular doubly-linked list anchored by an internal
> + * sentinel in struct rseq. User space must not touch @next or @prev while the
> + * node is registered.
Have you considered keeping copies of these structs in the kernel?
That (1) is more consistent with most of the other kernel APIs, (2)
will make executing ops faster since we don't need to copy_from_user
all the time, (3) will make code much simpler since we won't need to
re-verify all data, (4) will make the code more secure since
user-space won't be able to mess with these structs, (5) will simplify
the contract and documentation.
There are robust lists that keep lists in user-space, but these are
supposed to be concurrently modified by userspace. But these rseq ops
are generally not expected to be added/removed often (and the current
contract does not allow that anyway).
> + */
> +struct rseq_op_node {
> + __u64 next;
> + __u64 prev;
> + struct {
> + __u8 type; /* enum rseq_op_type */
> + __u8 reserved[7];
> + };
> +};
> +
> +/*
> + * struct rseq_op_reset - Reset one word to a value on return to user space
> + * @node: Operation list node.
> + * @src: Address of the source word, or 0 to reset @dst to zero.
Is "or 0 to reset" part implemented in the series?
It would be useful to see an op that contains a const that is written,
rather than an address. Address invovles additional indirection +
copy_from_user.
And the contract for concurrent changes of the value pointed by src
from user-space are not documented (atomicity?), w/o that having the
address is not very useful.
> + * @dst: Address of the destination word.
> + * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
> + */
> +struct rseq_op_reset {
> + struct rseq_op_node node;
> + __u64 src;
> + __u64 dst;
> + __u32 len;
> +};
> +
> +/*
> + * struct rseq_op_reset_with_stride - Reset one word in a strided array
> + * @node: Operation list node.
> + * @src: Address of the source word, or 0 to reset the slot to zero.
> + * @dst: Base address of the strided destination array.
> + * @dst_stride: Stride in bytes between consecutive array slots.
> + * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
> + *
> + * The destination slot is @dst + @dst_stride * index, where index is the
> + * current CPU ID or MM CID depending on the operation type.
> + */
> +struct rseq_op_reset_with_stride {
> + struct rseq_op_node node;
> + __u64 src;
> + __u64 dst;
> + __u64 dst_stride;
> + __u32 len;
> +};
> +
> /*
> * The original size and alignment of the allocation for struct rseq is
> * 32 bytes.
> @@ -191,15 +273,21 @@ struct rseq {
> struct rseq_slice_ctrl slice_ctrl;
>
> /*
> - * Before rseq became extensible, its original size was 32 bytes even
> - * though the active rseq area was only 20 bytes.
> - * Exposing a 32 bytes feature size would make life needlessly painful
> - * for userspace. Therefore, add a reserved byte after byte 32
> - * to bump the rseq feature size from 32 to 33.
> - * The next field to be added to the rseq area will be larger
> - * than one byte, and will replace this reserved byte.
> + * Sentinel of the circular doubly-linked list of rseq operations
> + * registered via prctl(PR_RSEQ_OP, ...). Fully owned and maintained by
> + * the kernel: it is initialized to point to itself on registration and
> + * user space must never read or write it directly.
> + *
> + * The kernel only use next and prev from rseq_op_list. The rest of the
> + * bytes are reserved for later usage and should be zeroed.
> */
Do we really need the union and the op_used/reserved? Isn't it easier
to say that everything needs to be 0'ed? What would userspace do with
op_used? It's not possible to skip initialization of these fields when
using both memset, struct initialization syntax, and relying on 0 init
of global data.
> - __u8 __reserved;
> + union {
> + struct rseq_op_node rseq_op_list;
> + struct {
> + __u64 op_used[2];
> + __u64 reserved;
> + };
> + };
>
> /*
> * Flexible array member at end of structure, after last feature field.
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions
2026-08-29 22:34 ` Dmitry Vyukov
@ 2026-08-29 22:47 ` Dmitry Vyukov
0 siblings, 0 replies; 11+ messages in thread
From: Dmitry Vyukov @ 2026-08-29 22:47 UTC (permalink / raw)
To: odion
Cc: Mathieu Desnoyers, Peter Zijlstra, Paul E. McKenney, Boqun Feng,
LKML, Thomas Gleixner, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell
On Sun, 30 Aug 2026 at 00:34, Dmitry Vyukov <dvyukov@google.com> wrote:
>
> On Fri, 28 Aug 2026 at 17:33, <odion@efficios.com> wrote:
> >
> > From: Olivier Dion <odion@efficios.com>
> >
> > Introduce userspace ABI for rseq operations: a per-thread list of
> > operations the kernel applies on return to user space.
> >
> > The main motivation for this work is to encourage TCMalloc to migrate to
> > RSEQ v2 [0] and use the glibc RSEQ region.
> >
> > Indeed, TCMalloc relies on the behavior of RSEQ v1, that reset the
> > cpu_id bits in the RSEQ shared region, to invalidate a per-cpu pointer
> > cached in a TLS. This hack requires TCMalloc users to use a glibc
> > tunable to disable RSEQ registration for threads so that TCMalloc can
> > register its own region, overlapping the TLS cache.
> >
> > Overall, this puts the users in a situation of choosing between a fast
> > sched_getcpu() and TCMalloc, plus some downsides such as requiring a
> > initial-exec model for the cache TLS in a shared-library and not being
> > compatible with the glibc
> >
> > RSEQ operations aim at solving this by offering operations that are
> > executed by the kernel, on behalf of a task after it is scheduled,
> > before returning to userspace. Operations are per-task and registered
> > throught prctl. They are opt-in and only introduce overhead on tasks
> > that register them.
> >
> > Add enum rseq_op_type and the rseq_op_node / rseq_op_reset /
> > rseq_op_reset_with_stride structures, the RSEQ_CS_FLAG_RSEQ_OP_*
> > availability/enabled flags, the RSEQ_OP_LIST_LIMIT walk bound, and the
> > PR_RSEQ_OP prctl with its REGISTER/UNREGISTER sub-commands.
> >
> > The operation list is a circular doubly-linked list anchored by a
> > kernel-owned sentinel embedded in struct rseq.
> >
> > [0] Documentation/userspace-api/rseq.rst (Optimized RSEQ v2)
>
> Hi Olivier,
>
> This is very cool, thanks for working on this.
>
> > Link: https://lore.kernel.org/lkml/20260428221058.149538293@kernel.org
> > Signed-off-by: Olivier Dion <odion@efficios.com>
> > ---
> > include/uapi/linux/prctl.h | 12 +++++
> > include/uapi/linux/rseq.h | 104 ++++++++++++++++++++++++++++++++++---
> > 2 files changed, 108 insertions(+), 8 deletions(-)
> >
> > diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
> > index b6ec6f693719..4cb6356a271a 100644
> > --- a/include/uapi/linux/prctl.h
> > +++ b/include/uapi/linux/prctl.h
> > @@ -396,6 +396,18 @@ struct prctl_mm_map {
> > */
> > # define PR_RSEQ_SLICE_EXT_ENABLE 0x01
> >
> > +/*
> > + * RSEQ operation registration.
> > + *
> > + * arg3 is the user address of a struct rseq_op_node embedded in one of the
> > + * rseq operation structures (see uapi/linux/rseq.h). Registering the first
> > + * operation enables rseq operation processing for the thread; unregistering
> > + * the last one disables it.
> > + */
> > +#define PR_RSEQ_OP 82
> > +# define PR_RSEQ_OP_REGISTER 1
> > +# define PR_RSEQ_OP_UNREGISTER 2
> > +
> > /*
> > * Get or set the control flow integrity (CFI) configuration for the
> > * current thread.
> > diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h
> > index ca6fe1f9d05e..b664d1991c48 100644
> > --- a/include/uapi/linux/rseq.h
> > +++ b/include/uapi/linux/rseq.h
> > @@ -13,6 +13,11 @@
> > #include <linux/types.h>
> > #include <asm/byteorder.h>
> >
> > +/*
> > + * Maximum number of nodes walked in the rseq operation list.
> > + */
> > +#define RSEQ_OP_LIST_LIMIT 2048
>
> Does this belong to the user header? Kernel and userspace are not
> necessary built with the same headers, and we don't necessary want to
> set this const in stone. I think it may be more flexible to make this
> impl detail and just return ENOSPC.
>
> > enum rseq_cpu_id_state {
> > RSEQ_CPU_ID_UNINITIALIZED = -1,
> > RSEQ_CPU_ID_REGISTRATION_FAILED = -2,
> > @@ -33,6 +38,8 @@ enum rseq_cs_flags_bit {
> > /* User read only feature flags */
> > RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4,
> > RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT = 5,
> > + RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT = 6,
> > + RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT = 7,
> > };
> >
> > enum rseq_cs_flags {
> > @@ -47,6 +54,10 @@ enum rseq_cs_flags {
> > (1U << RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT),
> > RSEQ_CS_FLAG_SLICE_EXT_ENABLED =
> > (1U << RSEQ_CS_FLAG_SLICE_EXT_ENABLED_BIT),
> > + RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE =
> > + (1U << RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE_BIT),
> > + RSEQ_CS_FLAG_RSEQ_OP_ENABLED =
> > + (1U << RSEQ_CS_FLAG_RSEQ_OP_ENABLED_BIT),
> > };
> >
> > /*
> > @@ -86,6 +97,77 @@ struct rseq_slice_ctrl {
> > };
> > };
> >
> > +/*
> > + * enum rseq_op_type - Type of an rseq operation
> > + * @RSEQ_OP_RESET: Plain reset. Uses struct rseq_op_reset.
> > + * @RSEQ_OP_RESET_WITH_STRIDE_CPUID: Reset indexed by the current CPU ID.
> > + * Uses struct rseq_op_reset_with_stride.
> > + * @RSEQ_OP_RESET_WITH_STRIDE_MMCID: Reset indexed by the current MM CID.
> > + * Uses struct rseq_op_reset_with_stride.
>
> Do you have use-cases for these STRIDE ops?
> I can think of some, but they would require executing ops when a task
> is descheduled, rather than when it's scheduled in.
>
> > + */
> > +enum rseq_op_type {
> > + RSEQ_OP_RESET,
> > + RSEQ_OP_RESET_WITH_STRIDE_CPUID,
> > + RSEQ_OP_RESET_WITH_STRIDE_MMCID,
> > + RSEQ_OP_NR,
> > +};
> > +
> > +/*
> > + * struct rseq_op_node - Common header linking an rseq operation into the list
> > + * @next: Address of the next node. Owned by the kernel.
> > + * @prev: Address of the previous node. Owned by the kernel.
> > + * @type: Operation type. See enum rseq_op_type.
> > + * @reserved: Must be zero on registration.
> > + *
> > + * User space allocates the node, sets @type and zeroes @next, @prev and
> > + * @reserved before passing it to prctl(PR_RSEQ_OP, PR_RSEQ_OP_REGISTER, node).
> > + * The kernel owns @next and @prev for the lifetime of the registration and
> > + * links the node into a circular doubly-linked list anchored by an internal
> > + * sentinel in struct rseq. User space must not touch @next or @prev while the
> > + * node is registered.
>
> Have you considered keeping copies of these structs in the kernel?
> That (1) is more consistent with most of the other kernel APIs, (2)
> will make executing ops faster since we don't need to copy_from_user
> all the time, (3) will make code much simpler since we won't need to
> re-verify all data, (4) will make the code more secure since
> user-space won't be able to mess with these structs, (5) will simplify
> the contract and documentation.
> There are robust lists that keep lists in user-space, but these are
> supposed to be concurrently modified by userspace. But these rseq ops
> are generally not expected to be added/removed often (and the current
> contract does not allow that anyway).
>
> > + */
> > +struct rseq_op_node {
> > + __u64 next;
> > + __u64 prev;
> > + struct {
> > + __u8 type; /* enum rseq_op_type */
> > + __u8 reserved[7];
> > + };
> > +};
> > +
> > +/*
> > + * struct rseq_op_reset - Reset one word to a value on return to user space
> > + * @node: Operation list node.
> > + * @src: Address of the source word, or 0 to reset @dst to zero.
>
> Is "or 0 to reset" part implemented in the series?
>
> It would be useful to see an op that contains a const that is written,
> rather than an address. Address invovles additional indirection +
> copy_from_user.
>
> And the contract for concurrent changes of the value pointed by src
> from user-space are not documented (atomicity?), w/o that having the
> address is not very useful.
>
>
> > + * @dst: Address of the destination word.
> > + * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
> > + */
> > +struct rseq_op_reset {
> > + struct rseq_op_node node;
> > + __u64 src;
> > + __u64 dst;
> > + __u32 len;
> > +};
> > +
> > +/*
> > + * struct rseq_op_reset_with_stride - Reset one word in a strided array
> > + * @node: Operation list node.
> > + * @src: Address of the source word, or 0 to reset the slot to zero.
> > + * @dst: Base address of the strided destination array.
> > + * @dst_stride: Stride in bytes between consecutive array slots.
> > + * @len: Word length in bytes. Must be 4 or 8 (8 is 64-bit only).
> > + *
> > + * The destination slot is @dst + @dst_stride * index, where index is the
> > + * current CPU ID or MM CID depending on the operation type.
It would be useful to explicitly clarify when is the max value for index.
I understand it's "number of CPUs". But what about online/offline
CPUs? Where/how user is expected to read the right value? If I use
CIDs and affinity mask, can I expect the index is bounded by the
affinity bits?
> > + */
> > +struct rseq_op_reset_with_stride {
> > + struct rseq_op_node node;
> > + __u64 src;
> > + __u64 dst;
> > + __u64 dst_stride;
> > + __u32 len;
> > +};
> > +
> > /*
> > * The original size and alignment of the allocation for struct rseq is
> > * 32 bytes.
> > @@ -191,15 +273,21 @@ struct rseq {
> > struct rseq_slice_ctrl slice_ctrl;
> >
> > /*
> > - * Before rseq became extensible, its original size was 32 bytes even
> > - * though the active rseq area was only 20 bytes.
> > - * Exposing a 32 bytes feature size would make life needlessly painful
> > - * for userspace. Therefore, add a reserved byte after byte 32
> > - * to bump the rseq feature size from 32 to 33.
> > - * The next field to be added to the rseq area will be larger
> > - * than one byte, and will replace this reserved byte.
> > + * Sentinel of the circular doubly-linked list of rseq operations
> > + * registered via prctl(PR_RSEQ_OP, ...). Fully owned and maintained by
> > + * the kernel: it is initialized to point to itself on registration and
> > + * user space must never read or write it directly.
> > + *
> > + * The kernel only use next and prev from rseq_op_list. The rest of the
> > + * bytes are reserved for later usage and should be zeroed.
> > */
>
> Do we really need the union and the op_used/reserved? Isn't it easier
> to say that everything needs to be 0'ed? What would userspace do with
> op_used? It's not possible to skip initialization of these fields when
> using both memset, struct initialization syntax, and relying on 0 init
> of global data.
>
> > - __u8 __reserved;
> > + union {
> > + struct rseq_op_node rseq_op_list;
> > + struct {
> > + __u64 op_used[2];
> > + __u64 reserved;
> > + };
> > + };
> >
> > /*
> > * Flexible array member at end of structure, after last feature field.
> > --
> > 2.54.0
> >
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC PATCH 2/5] rseq: add per-task rseq operation state
2026-08-28 15:33 [RFC PATCH 0/5] rseq: add support for RSEQ operations odion
2026-08-28 15:33 ` [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions odion
@ 2026-08-28 15:33 ` odion
2026-08-29 22:37 ` Dmitry Vyukov
2026-08-28 15:33 ` [RFC PATCH 3/5] rseq: apply operations on exit to user space odion
` (2 subsequent siblings)
4 siblings, 1 reply; 11+ messages in thread
From: odion @ 2026-08-28 15:33 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Peter Zijlstra, Paul E. McKenney, Boqun Feng, LKML,
Thomas Gleixner, Dmitry Vyukov, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell, Olivier Dion
From: Olivier Dion <odion@efficios.com>
Add the task-side state for rseq operations: the rseq_event::rseq_op
state indicating operation processing is enabled for the task, and the
nr_ops counter in struct rseq_data. nr_ops is edge triggered: its 0<->1
transition enables/disables rseq_event::rseq_op and the user visible
RSEQ_CS_FLAG_RSEQ_OP_ENABLED flag. This makes so that only thread that
enable RSEQ operations through prctl takes a performance hit.
Include rseq_op in the sched-switch raise condition so pending operations
force a return through the rseq exit path, and declare the rseq_op_prctl()
entry point (with a CONFIG_RSEQ=n stub).
Signed-off-by: Olivier Dion <odion@efficios.com>
---
include/linux/rseq.h | 10 +++++++++-
include/linux/rseq_types.h | 9 +++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index 7ef79b25e714..d1e33906d863 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -69,7 +69,9 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
* was via interrupt from user space. ev->has_rseq does not have
* to be evaluated here because rseq_v2() implies has_rseq.
*/
- bool raise = ev->user_irq | ev->ids_changed;
+ bool raise = (ev->user_irq |
+ ev->ids_changed |
+ ev->rseq_op);
if (raise) {
ev->sched_switch = true;
@@ -172,6 +174,8 @@ static inline unsigned int rseq_alloc_align(void)
return 1U << get_count_order(offsetof(struct rseq, end));
}
+int rseq_op_prctl(unsigned long arg2, unsigned long arg3);
+
#else /* CONFIG_RSEQ */
static inline bool rseq_v2(struct task_struct *t) { return false; }
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
@@ -182,6 +186,10 @@ static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
static inline void rseq_execve(struct task_struct *t) { }
+static inline int rseq_op_prctl(unsigned long arg2, unsigned long arg3)
+{
+ return -ENOTSUPP;
+}
#endif /* !CONFIG_RSEQ */
#ifdef CONFIG_DEBUG_RSEQ
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 85739a63e85e..059292695c34 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -23,6 +23,7 @@ struct rseq;
* exit to user
* @ids_changed: Indicator that IDs need to be updated
* @user_irq: True on interrupt entry from user mode
+ * @rseq_op: Rseq operation processing is enabled for the task
* @has_rseq: Greater than 0 if the task has a rseq pointer installed.
* Contains the RSEQ version number
* @error: Compound error code for the slow path to analyze
@@ -44,6 +45,7 @@ struct rseq_event {
u8 sched_switch;
u8 ids_changed;
u8 user_irq;
+ u8 rseq_op;
};
};
@@ -115,6 +117,7 @@ struct rseq_slice {
* @event: Storage for event management
* @ids: Storage for cached CPU ID and MM CID
* @slice: Storage for time slice extension data
+ * @nr_ops: Number of registered rseq operations
*/
struct rseq_data {
struct rseq __user *usrptr;
@@ -125,6 +128,12 @@ struct rseq_data {
#ifdef CONFIG_RSEQ_SLICE_EXTENSION
struct rseq_slice slice;
#endif
+ /*
+ * Number of rseq operations registered for the task. Edge triggered:
+ * the 0<->1 transition enables/disables rseq_event::rseq_op and the
+ * RSEQ_CS_FLAG_RSEQ_OP_ENABLED user flag.
+ */
+ u32 nr_ops;
};
#else /* CONFIG_RSEQ */
--
2.54.0
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [RFC PATCH 2/5] rseq: add per-task rseq operation state
2026-08-28 15:33 ` [RFC PATCH 2/5] rseq: add per-task rseq operation state odion
@ 2026-08-29 22:37 ` Dmitry Vyukov
0 siblings, 0 replies; 11+ messages in thread
From: Dmitry Vyukov @ 2026-08-29 22:37 UTC (permalink / raw)
To: odion
Cc: Mathieu Desnoyers, Peter Zijlstra, Paul E. McKenney, Boqun Feng,
LKML, Thomas Gleixner, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell
On Fri, 28 Aug 2026 at 17:34, <odion@efficios.com> wrote:
>
> From: Olivier Dion <odion@efficios.com>
>
> Add the task-side state for rseq operations: the rseq_event::rseq_op
> state indicating operation processing is enabled for the task, and the
> nr_ops counter in struct rseq_data. nr_ops is edge triggered: its 0<->1
> transition enables/disables rseq_event::rseq_op and the user visible
> RSEQ_CS_FLAG_RSEQ_OP_ENABLED flag. This makes so that only thread that
> enable RSEQ operations through prctl takes a performance hit.
>
> Include rseq_op in the sched-switch raise condition so pending operations
> force a return through the rseq exit path, and declare the rseq_op_prctl()
> entry point (with a CONFIG_RSEQ=n stub).
>
> Signed-off-by: Olivier Dion <odion@efficios.com>
> ---
> include/linux/rseq.h | 10 +++++++++-
> include/linux/rseq_types.h | 9 +++++++++
> 2 files changed, 18 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/rseq.h b/include/linux/rseq.h
> index 7ef79b25e714..d1e33906d863 100644
> --- a/include/linux/rseq.h
> +++ b/include/linux/rseq.h
> @@ -69,7 +69,9 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
> * was via interrupt from user space. ev->has_rseq does not have
> * to be evaluated here because rseq_v2() implies has_rseq.
> */
> - bool raise = ev->user_irq | ev->ids_changed;
> + bool raise = (ev->user_irq |
> + ev->ids_changed |
> + ev->rseq_op);
>
> if (raise) {
> ev->sched_switch = true;
> @@ -172,6 +174,8 @@ static inline unsigned int rseq_alloc_align(void)
> return 1U << get_count_order(offsetof(struct rseq, end));
> }
>
> +int rseq_op_prctl(unsigned long arg2, unsigned long arg3);
> +
> #else /* CONFIG_RSEQ */
> static inline bool rseq_v2(struct task_struct *t) { return false; }
> static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
> @@ -182,6 +186,10 @@ static inline void rseq_force_update(void) { }
> static inline void rseq_virt_userspace_exit(void) { }
> static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
> static inline void rseq_execve(struct task_struct *t) { }
> +static inline int rseq_op_prctl(unsigned long arg2, unsigned long arg3)
> +{
> + return -ENOTSUPP;
> +}
> #endif /* !CONFIG_RSEQ */
>
> #ifdef CONFIG_DEBUG_RSEQ
> diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
> index 85739a63e85e..059292695c34 100644
> --- a/include/linux/rseq_types.h
> +++ b/include/linux/rseq_types.h
> @@ -23,6 +23,7 @@ struct rseq;
> * exit to user
> * @ids_changed: Indicator that IDs need to be updated
> * @user_irq: True on interrupt entry from user mode
> + * @rseq_op: Rseq operation processing is enabled for the task
> * @has_rseq: Greater than 0 if the task has a rseq pointer installed.
> * Contains the RSEQ version number
> * @error: Compound error code for the slow path to analyze
> @@ -44,6 +45,7 @@ struct rseq_event {
> u8 sched_switch;
> u8 ids_changed;
> u8 user_irq;
> + u8 rseq_op;
> };
> };
>
> @@ -115,6 +117,7 @@ struct rseq_slice {
> * @event: Storage for event management
> * @ids: Storage for cached CPU ID and MM CID
> * @slice: Storage for time slice extension data
> + * @nr_ops: Number of registered rseq operations
> */
> struct rseq_data {
> struct rseq __user *usrptr;
> @@ -125,6 +128,12 @@ struct rseq_data {
> #ifdef CONFIG_RSEQ_SLICE_EXTENSION
> struct rseq_slice slice;
> #endif
> + /*
> + * Number of rseq operations registered for the task. Edge triggered:
> + * the 0<->1 transition enables/disables rseq_event::rseq_op and the
> + * RSEQ_CS_FLAG_RSEQ_OP_ENABLED user flag.
> + */
> + u32 nr_ops;
If we copy ops to kernel memory, it would be useful to have 1 op
embeded here (with fallback array for more ops). For now the only
known use-case for this is memory allocators, so I would assume in
most cases it will be either 0 or 1 ops registered.
> };
>
> #else /* CONFIG_RSEQ */
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC PATCH 3/5] rseq: apply operations on exit to user space
2026-08-28 15:33 [RFC PATCH 0/5] rseq: add support for RSEQ operations odion
2026-08-28 15:33 ` [RFC PATCH 1/5] rseq: uapi: add rseq operation definitions odion
2026-08-28 15:33 ` [RFC PATCH 2/5] rseq: add per-task rseq operation state odion
@ 2026-08-28 15:33 ` odion
2026-08-29 22:42 ` Dmitry Vyukov
2026-08-28 15:33 ` [RFC PATCH 4/5] rseq: register and unregister operations via prctl odion
2026-08-28 15:33 ` [RFC PATCH 5/5] selftests/rseq: add coverage for rseq operations odion
4 siblings, 1 reply; 11+ messages in thread
From: odion @ 2026-08-28 15:33 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Peter Zijlstra, Paul E. McKenney, Boqun Feng, LKML,
Thomas Gleixner, Dmitry Vyukov, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell, Olivier Dion
From: Olivier Dion <odion@efficios.com>
Add rseq_apply_ops(), which walks the task's circular operation list on
return to user space and applies each operation.
Wire the walk into rseq_exit_user_update() for both the ids-unchanged
and ids-changed paths, gated on rseq_event::rseq_op. Since operation
processing is a persistent per-task state rather than a one-shot event,
introduce rseq_clear_one_shot_events() which preserves the rseq_op bit
when clearing events on the way out.
len, src and dst are re-read from user memory and re-validated on every
application, since user space may change them after registration. A
broken ABI contract will fault the process.
Signed-off-by: Olivier Dion <odion@efficios.com>
---
include/linux/rseq_entry.h | 157 +++++++++++++++++++++++++++++++++++--
1 file changed, 152 insertions(+), 5 deletions(-)
diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
index ed9da6e41a2a..df083b0ee14a 100644
--- a/include/linux/rseq_entry.h
+++ b/include/linux/rseq_entry.h
@@ -247,6 +247,137 @@ static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, un
#define rseq_slice_clear_user(rseq, efault) do { } while (0)
#endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
+/*
+ * Validate and perform a single reset word operation: reset @dst to *@src,
+ * or to zero when @src is 0. @len, @src and @dst come from user memory and
+ * are re-validated on every application, since user space may have changed
+ * them since registration. A broken ABI contract returns false, which faults
+ * the process.
+ */
+static __always_inline bool rseq_op_reset_word(u64 src, u64 dst, u32 len,
+ void __user *node)
+{
+ if (unlikely(!IS_ALIGNED(dst, len) || (src && !IS_ALIGNED(src, len)))) {
+ pr_info_ratelimited("rseq: bad operation alignment len=%u src=%llx dst=%llx from %p\n",
+ len, src, dst, node);
+ return false;
+ }
+
+ switch (len) {
+ case 4: {
+ u32 __user *udst = (u32 __user *)dst;
+ u32 v = 0;
+
+ if (src) {
+ u32 __user *usrc = (u32 __user *)src;
+
+ scoped_user_read_access(usrc, efault)
+ unsafe_get_user(v, usrc, efault);
+ }
+ scoped_user_write_access(udst, efault)
+ unsafe_put_user(v, udst, efault);
+ return true;
+ }
+ case 8: {
+ u64 __user *udst = (u64 __user *)dst;
+ u64 v = 0;
+
+ if (src) {
+ u64 __user *usrc = (u64 __user *)src;
+
+ scoped_user_read_access(usrc, efault)
+ unsafe_get_user(v, usrc, efault);
+ }
+ scoped_user_write_access(udst, efault)
+ unsafe_put_user(v, udst, efault);
+ return true;
+ }
+ default:
+ pr_info("rseq: bad operation length=%u from %p\n", len, node);
+ return false;
+ }
+efault:
+ pr_info("rseq: fault while applying operation from %p\n", node);
+ return false;
+}
+
+static __always_inline bool rseq_apply_ops(struct task_struct *t)
+{
+ struct rseq __user *rseq = t->rseq.usrptr;
+ struct rseq_op_node __user *sentinel = &rseq->rseq_op_list;
+ struct rseq_op_node __user *node;
+ unsigned int limit = RSEQ_OP_LIST_LIMIT;
+ u64 next;
+
+ WARN_ONCE(!t->rseq.event.rseq_op, "rseq operations not enabled");
+
+ scoped_user_read_access(sentinel, efault)
+ unsafe_get_user(next, &sentinel->next, efault);
+
+ node = (struct rseq_op_node __user *)next;
+
+ while (node != sentinel && limit--) {
+ u8 type;
+
+ scoped_user_read_access(node, efault) {
+ unsafe_get_user(type, &node->type, efault);
+ unsafe_get_user(next, &node->next, efault);
+ }
+
+ switch (type) {
+ case RSEQ_OP_RESET: {
+ struct rseq_op_reset __user *op =
+ (struct rseq_op_reset __user *)node;
+ u64 src, dst;
+ u32 len;
+
+ scoped_user_read_access(op, efault) {
+ unsafe_get_user(src, &op->src, efault);
+ unsafe_get_user(dst, &op->dst, efault);
+ unsafe_get_user(len, &op->len, efault);
+ }
+
+ if (!rseq_op_reset_word(src, dst, len, node))
+ return false;
+ break;
+ }
+ case RSEQ_OP_RESET_WITH_STRIDE_CPUID: /* fall through */
+ case RSEQ_OP_RESET_WITH_STRIDE_MMCID: {
+ struct rseq_op_reset_with_stride __user *op =
+ (struct rseq_op_reset_with_stride __user *)node;
+ u64 src, dst, dst_stride;
+ u32 len, index;
+
+ scoped_user_read_access(op, efault) {
+ unsafe_get_user(src, &op->src, efault);
+ unsafe_get_user(dst, &op->dst, efault);
+ unsafe_get_user(dst_stride, &op->dst_stride, efault);
+ unsafe_get_user(len, &op->len, efault);
+ }
+ index = (type == RSEQ_OP_RESET_WITH_STRIDE_CPUID) ?
+ t->rseq.ids.cpu_id : t->rseq.ids.mm_cid;
+ dst += dst_stride * index;
+
+ if (!rseq_op_reset_word(src, dst, len, node))
+ return false;
+
+ break;
+ }
+ default:
+ pr_info("rseq: bad operation type=%u from %p\n",
+ type, node);
+ return false;
+ }
+
+ node = (struct rseq_op_node __user *)next;
+ }
+
+ return node == sentinel;
+efault:
+ pr_info("rseq: fault while walking operation list\n");
+ return false;
+}
+
bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
static __always_inline void rseq_note_user_irq_entry(void)
@@ -632,6 +763,10 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
if (unlikely(!rseq_update_user_cs(t, regs, csaddr)))
return false;
}
+
+ if (t->rseq.event.rseq_op && !rseq_apply_ops(t))
+ return false;
+
return true;
}
@@ -642,11 +777,22 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
.node_id = cpu_to_node(cpu),
};
- return rseq_update_usr(t, regs, &ids);
+ if (!rseq_update_usr(t, regs, &ids))
+ return false;
+
+ if (t->rseq.event.rseq_op && !rseq_apply_ops(t))
+ return false;
+
+ return true;
efault:
return false;
}
+static __always_inline void rseq_clear_one_shot_events(struct rseq_event *ev)
+{
+ ev->events &= (struct rseq_event){ .rseq_op = true }.events;
+}
+
static __always_inline bool __rseq_exit_to_user_mode_restart(struct pt_regs *regs)
{
struct task_struct *t = current;
@@ -674,8 +820,9 @@ static __always_inline bool __rseq_exit_to_user_mode_restart(struct pt_regs *reg
if (unlikely(!rseq_exit_user_update(regs, t)))
return true;
}
- /* Clear state so next entry starts from a clean slate */
- t->rseq.event.events = 0;
+ /* Clear one-shot events so next entry starts from a clean slate */
+ rseq_clear_one_shot_events(&t->rseq.event);
+
return false;
}
@@ -730,7 +877,7 @@ static __always_inline void rseq_syscall_exit_to_user_mode(void)
/* Needed to remove the store for the !lockdep case */
if (IS_ENABLED(CONFIG_LOCKDEP)) {
WARN_ON_ONCE(ev->sched_switch);
- ev->events = 0;
+ rseq_clear_one_shot_events(ev);
}
}
@@ -747,7 +894,7 @@ static __always_inline void rseq_irqentry_exit_to_user_mode(void)
* interrupt did not result in a schedule and therefore the
* rseq processing could not clear it.
*/
- ev->events = 0;
+ rseq_clear_one_shot_events(ev);
}
void __rseq_debug_syscall_return(struct pt_regs *regs);
--
2.54.0
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [RFC PATCH 3/5] rseq: apply operations on exit to user space
2026-08-28 15:33 ` [RFC PATCH 3/5] rseq: apply operations on exit to user space odion
@ 2026-08-29 22:42 ` Dmitry Vyukov
0 siblings, 0 replies; 11+ messages in thread
From: Dmitry Vyukov @ 2026-08-29 22:42 UTC (permalink / raw)
To: odion
Cc: Mathieu Desnoyers, Peter Zijlstra, Paul E. McKenney, Boqun Feng,
LKML, Thomas Gleixner, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell
On Fri, 28 Aug 2026 at 17:34, <odion@efficios.com> wrote:
>
> From: Olivier Dion <odion@efficios.com>
>
> Add rseq_apply_ops(), which walks the task's circular operation list on
> return to user space and applies each operation.
>
> Wire the walk into rseq_exit_user_update() for both the ids-unchanged
> and ids-changed paths, gated on rseq_event::rseq_op. Since operation
> processing is a persistent per-task state rather than a one-shot event,
> introduce rseq_clear_one_shot_events() which preserves the rseq_op bit
> when clearing events on the way out.
>
> len, src and dst are re-read from user memory and re-validated on every
> application, since user space may change them after registration. A
> broken ABI contract will fault the process.
>
> Signed-off-by: Olivier Dion <odion@efficios.com>
> ---
> include/linux/rseq_entry.h | 157 +++++++++++++++++++++++++++++++++++--
> 1 file changed, 152 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
> index ed9da6e41a2a..df083b0ee14a 100644
> --- a/include/linux/rseq_entry.h
> +++ b/include/linux/rseq_entry.h
> @@ -247,6 +247,137 @@ static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, un
> #define rseq_slice_clear_user(rseq, efault) do { } while (0)
> #endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
>
> +/*
> + * Validate and perform a single reset word operation: reset @dst to *@src,
> + * or to zero when @src is 0. @len, @src and @dst come from user memory and
> + * are re-validated on every application, since user space may have changed
> + * them since registration. A broken ABI contract returns false, which faults
> + * the process.
> + */
> +static __always_inline bool rseq_op_reset_word(u64 src, u64 dst, u32 len,
> + void __user *node)
> +{
> + if (unlikely(!IS_ALIGNED(dst, len) || (src && !IS_ALIGNED(src, len)))) {
> + pr_info_ratelimited("rseq: bad operation alignment len=%u src=%llx dst=%llx from %p\n",
> + len, src, dst, node);
> + return false;
> + }
> +
> + switch (len) {
> + case 4: {
> + u32 __user *udst = (u32 __user *)dst;
> + u32 v = 0;
> +
> + if (src) {
> + u32 __user *usrc = (u32 __user *)src;
> +
> + scoped_user_read_access(usrc, efault)
> + unsafe_get_user(v, usrc, efault);
> + }
> + scoped_user_write_access(udst, efault)
> + unsafe_put_user(v, udst, efault);
> + return true;
> + }
> + case 8: {
> + u64 __user *udst = (u64 __user *)dst;
> + u64 v = 0;
> +
> + if (src) {
> + u64 __user *usrc = (u64 __user *)src;
> +
> + scoped_user_read_access(usrc, efault)
> + unsafe_get_user(v, usrc, efault);
> + }
> + scoped_user_write_access(udst, efault)
> + unsafe_put_user(v, udst, efault);
> + return true;
> + }
> + default:
> + pr_info("rseq: bad operation length=%u from %p\n", len, node);
pr_info_ratelimited here and below.
Kernel does not generally warn on the global console about misbehaving
userspace programs (which generally do not have access to the output
anyway, as it requires root nowadays).
If you decide to do that, it should be at least rate-limited to avoid DoS.
> + return false;
> + }
> +efault:
> + pr_info("rseq: fault while applying operation from %p\n", node);
> + return false;
> +}
> +
> +static __always_inline bool rseq_apply_ops(struct task_struct *t)
> +{
> + struct rseq __user *rseq = t->rseq.usrptr;
> + struct rseq_op_node __user *sentinel = &rseq->rseq_op_list;
> + struct rseq_op_node __user *node;
> + unsigned int limit = RSEQ_OP_LIST_LIMIT;
> + u64 next;
> +
> + WARN_ONCE(!t->rseq.event.rseq_op, "rseq operations not enabled");
> +
> + scoped_user_read_access(sentinel, efault)
> + unsafe_get_user(next, &sentinel->next, efault);
> +
> + node = (struct rseq_op_node __user *)next;
> +
> + while (node != sentinel && limit--) {
> + u8 type;
> +
> + scoped_user_read_access(node, efault) {
> + unsafe_get_user(type, &node->type, efault);
> + unsafe_get_user(next, &node->next, efault);
> + }
> +
> + switch (type) {
> + case RSEQ_OP_RESET: {
> + struct rseq_op_reset __user *op =
> + (struct rseq_op_reset __user *)node;
> + u64 src, dst;
> + u32 len;
> +
> + scoped_user_read_access(op, efault) {
> + unsafe_get_user(src, &op->src, efault);
> + unsafe_get_user(dst, &op->dst, efault);
> + unsafe_get_user(len, &op->len, efault);
> + }
> +
> + if (!rseq_op_reset_word(src, dst, len, node))
> + return false;
> + break;
> + }
> + case RSEQ_OP_RESET_WITH_STRIDE_CPUID: /* fall through */
> + case RSEQ_OP_RESET_WITH_STRIDE_MMCID: {
> + struct rseq_op_reset_with_stride __user *op =
> + (struct rseq_op_reset_with_stride __user *)node;
> + u64 src, dst, dst_stride;
> + u32 len, index;
> +
> + scoped_user_read_access(op, efault) {
> + unsafe_get_user(src, &op->src, efault);
> + unsafe_get_user(dst, &op->dst, efault);
> + unsafe_get_user(dst_stride, &op->dst_stride, efault);
> + unsafe_get_user(len, &op->len, efault);
> + }
> + index = (type == RSEQ_OP_RESET_WITH_STRIDE_CPUID) ?
> + t->rseq.ids.cpu_id : t->rseq.ids.mm_cid;
> + dst += dst_stride * index;
> +
> + if (!rseq_op_reset_word(src, dst, len, node))
> + return false;
Declaration of src/dst/len and rseq_op_reset_word call potentially can
be deduplicated across these 2 cases.
> +
> + break;
> + }
> + default:
> + pr_info("rseq: bad operation type=%u from %p\n",
> + type, node);
> + return false;
> + }
> +
> + node = (struct rseq_op_node __user *)next;
> + }
> +
> + return node == sentinel;
> +efault:
> + pr_info("rseq: fault while walking operation list\n");
> + return false;
> +}
> +
> bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
>
> static __always_inline void rseq_note_user_irq_entry(void)
> @@ -632,6 +763,10 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
> if (unlikely(!rseq_update_user_cs(t, regs, csaddr)))
> return false;
> }
> +
> + if (t->rseq.event.rseq_op && !rseq_apply_ops(t))
> + return false;
> +
> return true;
> }
>
> @@ -642,11 +777,22 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
> .node_id = cpu_to_node(cpu),
> };
>
> - return rseq_update_usr(t, regs, &ids);
> + if (!rseq_update_usr(t, regs, &ids))
> + return false;
> +
> + if (t->rseq.event.rseq_op && !rseq_apply_ops(t))
> + return false;
> +
> + return true;
> efault:
> return false;
> }
>
> +static __always_inline void rseq_clear_one_shot_events(struct rseq_event *ev)
> +{
> + ev->events &= (struct rseq_event){ .rseq_op = true }.events;
> +}
> +
> static __always_inline bool __rseq_exit_to_user_mode_restart(struct pt_regs *regs)
> {
> struct task_struct *t = current;
> @@ -674,8 +820,9 @@ static __always_inline bool __rseq_exit_to_user_mode_restart(struct pt_regs *reg
> if (unlikely(!rseq_exit_user_update(regs, t)))
> return true;
> }
> - /* Clear state so next entry starts from a clean slate */
> - t->rseq.event.events = 0;
> + /* Clear one-shot events so next entry starts from a clean slate */
> + rseq_clear_one_shot_events(&t->rseq.event);
> +
> return false;
> }
>
> @@ -730,7 +877,7 @@ static __always_inline void rseq_syscall_exit_to_user_mode(void)
> /* Needed to remove the store for the !lockdep case */
> if (IS_ENABLED(CONFIG_LOCKDEP)) {
> WARN_ON_ONCE(ev->sched_switch);
> - ev->events = 0;
> + rseq_clear_one_shot_events(ev);
> }
> }
>
> @@ -747,7 +894,7 @@ static __always_inline void rseq_irqentry_exit_to_user_mode(void)
> * interrupt did not result in a schedule and therefore the
> * rseq processing could not clear it.
> */
> - ev->events = 0;
> + rseq_clear_one_shot_events(ev);
> }
>
> void __rseq_debug_syscall_return(struct pt_regs *regs);
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC PATCH 4/5] rseq: register and unregister operations via prctl
2026-08-28 15:33 [RFC PATCH 0/5] rseq: add support for RSEQ operations odion
` (2 preceding siblings ...)
2026-08-28 15:33 ` [RFC PATCH 3/5] rseq: apply operations on exit to user space odion
@ 2026-08-28 15:33 ` odion
2026-08-29 22:50 ` Dmitry Vyukov
2026-08-28 15:33 ` [RFC PATCH 5/5] selftests/rseq: add coverage for rseq operations odion
4 siblings, 1 reply; 11+ messages in thread
From: odion @ 2026-08-28 15:33 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Peter Zijlstra, Paul E. McKenney, Boqun Feng, LKML,
Thomas Gleixner, Dmitry Vyukov, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell, Olivier Dion
From: Olivier Dion <odion@efficios.com>
Implement the PR_RSEQ_OP prctl. Registration splices a user-provided
pristine node into the head of the circular operation list, and
unregistration unsplices it after validating it is properly linked
between its neighbours; the kernel owns the node's next/prev links for
the lifetime of the registration. The 0<->1 nr_ops transition reflects
the enabled state into rseq_event::rseq_op and the user visible flag via
rseq_op_update_enabled().
Initialize the operation list sentinel as an empty self-pointing circular
list on rseq registration, and reset nr_ops / rseq_op state there.
Apply pending operations on the slow path and on signal delivery, and
factor the common "clear error and force SIGSEGV" fixup into
force_fault().
Signed-off-by: Olivier Dion <odion@efficios.com>
---
kernel/rseq.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++---
kernel/sys.c | 5 ++
2 files changed, 212 insertions(+), 13 deletions(-)
diff --git a/kernel/rseq.c b/kernel/rseq.c
index e75e3a5e312c..a277c11dae99 100644
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -253,17 +253,18 @@ static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs)
static void rseq_slowpath_update_usr(struct pt_regs *regs)
{
/*
- * Preserve has_rseq and user_irq state. The generic entry code clears
- * user_irq on the way out, the non-generic entry architectures are not
- * setting user_irq.
+ * Preserve has_rseq, rseq_op and user_irq state. The generic entry
+ * code clears user_irq on the way out, the non-generic entry
+ * architectures are not setting user_irq.
*/
const struct rseq_event evt_mask = {
.has_rseq = RSEQ_HAS_RSEQ_VERSION_MASK,
+ .rseq_op = true,
.user_irq = true,
};
struct task_struct *t = current;
struct rseq_ids ids;
- bool event;
+ bool event, should_fault = false;
if (unlikely(t->flags & PF_EXITING))
return;
@@ -300,7 +301,12 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
ids.node_id = cpu_to_node(ids.cpu_id);
- if (unlikely(!rseq_update_usr(t, regs, &ids))) {
+ if (unlikely(!rseq_update_usr(t, regs, &ids)))
+ should_fault = true;
+ else if (t->rseq.event.rseq_op && unlikely(!rseq_apply_ops(t)))
+ should_fault = true;
+
+ if (should_fault) {
/*
* Clear the errors just in case this might survive magically, but
* leave the rest intact.
@@ -329,8 +335,20 @@ void __rseq_handle_slowpath(struct pt_regs *regs)
rseq_slowpath_update_usr(regs);
}
+static inline void force_fault(int sig)
+{
+ /*
+ * Clear the errors just in case this might survive magically, but leave
+ * the rest intact.
+ */
+ current->rseq.event.error = 0;
+ force_sigsegv(sig);
+}
+
void __rseq_signal_deliver(int sig, struct pt_regs *regs)
{
+ bool should_fault = false;
+
rseq_stat_inc(rseq_stats.signal);
/*
@@ -339,14 +357,13 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs)
* the interrupted context as after this point the instruction
* pointer in @regs points to the signal handler.
*/
- if (unlikely(!rseq_handle_cs(current, regs))) {
- /*
- * Clear the errors just in case this might survive
- * magically, but leave the rest intact.
- */
- current->rseq.event.error = 0;
- force_sigsegv(sig);
- }
+ if (unlikely(!rseq_handle_cs(current, regs)))
+ should_fault = true;
+ else if (current->rseq.event.rseq_op && unlikely(!rseq_apply_ops(current)))
+ should_fault = true;
+
+ if (should_fault)
+ force_fault(sig);
/*
* In legacy mode, force the update of IDs before returning to user
@@ -436,6 +453,8 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED;
}
}
+ if (version > 1)
+ rseqfl |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
scoped_user_write_access(rseq, efault) {
/*
@@ -458,8 +477,16 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
* registrations.
*/
if (version > 1) {
+ u64 sentinel = (u64)&rseq->rseq_op_list;
+
if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION))
unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
+ /*
+ * Initialize the rseq operation list sentinel as an
+ * empty circular doubly-linked list pointing to itself.
+ */
+ unsafe_put_user(sentinel, &rseq->rseq_op_list.next, efault);
+ unsafe_put_user(sentinel, &rseq->rseq_op_list.prev, efault);
}
}
@@ -474,6 +501,12 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
#ifdef CONFIG_RSEQ_SLICE_EXTENSION
current->rseq.slice.state.enabled = !!(rseqfl & RSEQ_CS_FLAG_SLICE_EXT_ENABLED);
#endif
+ /*
+ * A fresh registration starts with no operations, so operation
+ * processing is disabled until the first one is registered.
+ */
+ current->rseq.event.rseq_op = false;
+ current->rseq.nr_ops = 0;
/*
* Ensure the cpu_id_start and cpu_id fields are updated before
@@ -887,3 +920,164 @@ device_initcall(rseq_slice_init);
#else
static void rseq_slice_ext_init(struct dentry *root_dir) { }
#endif /* CONFIG_RSEQ_SLICE_EXTENSION */
+
+/*
+ * Reflect the operation enabled state into the user visible flags field.
+ * Called on the 0<->1 transition of nr_ops. The kill path is taken on fault
+ * because losing this update leaves user space and kernel state inconsistent.
+ */
+static int rseq_op_update_enabled(struct task_struct *t, bool enable)
+{
+ struct rseq __user *rseq = t->rseq.usrptr;
+ u32 rflags;
+
+ if (get_user(rflags, &rseq->flags))
+ return -EFAULT;
+
+ rflags &= ~RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
+ rflags |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
+ if (enable)
+ rflags |= RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
+
+ if (put_user(rflags, &rseq->flags))
+ return -EFAULT;
+
+ t->rseq.event.rseq_op = enable;
+ return 0;
+}
+
+/*
+ * Register @node at the head of the circular doubly-linked operation list
+ * anchored by the kernel owned sentinel in struct rseq.
+ */
+static int rseq_op_register(struct task_struct *t, struct rseq_op_node __user *node)
+{
+ struct rseq_op_node __user *sentinel = &t->rseq.usrptr->rseq_op_list;
+ struct rseq_op_node __user *first;
+ u64 next, prev, first_addr;
+ u8 type, i;
+
+ if (t->rseq.nr_ops >= RSEQ_OP_LIST_LIMIT)
+ return -ENOSPC;
+
+ if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
+ return -EINVAL;
+ if (!access_ok(node, sizeof(*node)))
+ return -EFAULT;
+
+ /*
+ * The node links are owned by the kernel. User space must present a
+ * pristine node: next, prev and the reserved bytes all zeroed, and a
+ * known operation type.
+ */
+ if (get_user(next, &node->next) || get_user(prev, &node->prev) ||
+ get_user(type, &node->type))
+ return -EFAULT;
+ if (next || prev)
+ return -EINVAL;
+ if (type >= RSEQ_OP_NR)
+ return -EINVAL;
+ for (i = 1; i < sizeof(node->reserved); i++) {
+ u8 r;
+
+ if (get_user(r, &node->reserved[i]))
+ return -EFAULT;
+ if (r)
+ return -EINVAL;
+ }
+
+ /* Splice the node in right after the sentinel. */
+ if (get_user(first_addr, &sentinel->next))
+ goto die;
+ first = (struct rseq_op_node __user *)first_addr;
+
+ if (put_user((u64)(unsigned long)first, &node->next) ||
+ put_user((u64)(unsigned long)sentinel, &node->prev) ||
+ put_user((u64)(unsigned long)node, &first->prev) ||
+ put_user((u64)(unsigned long)node, &sentinel->next))
+ goto die;
+
+ t->rseq.nr_ops += 1;
+
+ if (t->rseq.nr_ops == 1)
+ return rseq_op_update_enabled(t, true) ? -EFAULT : 0;
+ return 0;
+die:
+ force_sig(SIGSEGV);
+ return -EFAULT;
+}
+
+/*
+ * Unregister @node from the operation list. The node links are validated
+ * against its neighbours to reject bogus or double unregistration.
+ */
+static int rseq_op_unregister(struct task_struct *t, struct rseq_op_node __user *node)
+{
+ struct rseq_op_node __user *prev, *next;
+ u64 prev_addr, next_addr, tmp;
+
+ if (!t->rseq.nr_ops)
+ return -ENOENT;
+
+ if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
+ return -EINVAL;
+ if (!access_ok(node, sizeof(*node)))
+ return -EFAULT;
+
+ if (get_user(next_addr, &node->next) || get_user(prev_addr, &node->prev))
+ return -EFAULT;
+ prev = (struct rseq_op_node __user *)prev_addr;
+ next = (struct rseq_op_node __user *)next_addr;
+
+ /* A registered node always has both links set. */
+ if (!prev || !next)
+ return -EINVAL;
+
+ /* Verify the node is properly linked between its neighbours. */
+ if (get_user(tmp, &prev->next))
+ goto die;
+ if (tmp != (u64)(unsigned long)node)
+ return -EINVAL;
+ if (get_user(tmp, &next->prev))
+ goto die;
+ if (tmp != (u64)(unsigned long)node)
+ return -EINVAL;
+
+ /* Unsplice and clear the node links so it can be reused. */
+ if (put_user(next_addr, &prev->next) ||
+ put_user(prev_addr, &next->prev) ||
+ put_user(0ULL, &node->next) ||
+ put_user(0ULL, &node->prev))
+ goto die;
+
+ t->rseq.nr_ops -= 1;
+
+ if (t->rseq.nr_ops == 0)
+ return rseq_op_update_enabled(t, false) ? -EFAULT : 0;
+ return 0;
+die:
+ force_sig(SIGSEGV);
+ return -EFAULT;
+}
+
+int rseq_op_prctl(unsigned long arg2, unsigned long arg3)
+{
+ struct rseq_op_node __user *node = (struct rseq_op_node __user *)arg3;
+ struct task_struct *t = current;
+
+ if (!t->rseq.usrptr)
+ return -ENXIO;
+ if (!rseq_v2(t))
+ return -ENOTSUPP;
+ if (!node)
+ return -EINVAL;
+
+ switch (arg2) {
+ case PR_RSEQ_OP_REGISTER:
+ return rseq_op_register(t, node);
+ case PR_RSEQ_OP_UNREGISTER:
+ return rseq_op_unregister(t, node);
+ default:
+ return -EINVAL;
+ }
+}
diff --git a/kernel/sys.c b/kernel/sys.c
index df69bd71de03..494c91b03c3f 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -2889,6 +2889,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
return -EINVAL;
error = rseq_slice_extension_prctl(arg2, arg3);
break;
+ case PR_RSEQ_OP:
+ if (arg4 || arg5)
+ return -EINVAL;
+ error = rseq_op_prctl(arg2, arg3);
+ break;
case PR_GET_CFI:
if (arg2 != PR_CFI_BRANCH_LANDING_PADS)
return -EINVAL;
--
2.54.0
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [RFC PATCH 4/5] rseq: register and unregister operations via prctl
2026-08-28 15:33 ` [RFC PATCH 4/5] rseq: register and unregister operations via prctl odion
@ 2026-08-29 22:50 ` Dmitry Vyukov
0 siblings, 0 replies; 11+ messages in thread
From: Dmitry Vyukov @ 2026-08-29 22:50 UTC (permalink / raw)
To: odion
Cc: Mathieu Desnoyers, Peter Zijlstra, Paul E. McKenney, Boqun Feng,
LKML, Thomas Gleixner, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell
On Fri, 28 Aug 2026 at 17:34, <odion@efficios.com> wrote:
>
> From: Olivier Dion <odion@efficios.com>
>
> Implement the PR_RSEQ_OP prctl. Registration splices a user-provided
> pristine node into the head of the circular operation list, and
> unregistration unsplices it after validating it is properly linked
> between its neighbours; the kernel owns the node's next/prev links for
> the lifetime of the registration. The 0<->1 nr_ops transition reflects
> the enabled state into rseq_event::rseq_op and the user visible flag via
> rseq_op_update_enabled().
>
> Initialize the operation list sentinel as an empty self-pointing circular
> list on rseq registration, and reset nr_ops / rseq_op state there.
>
> Apply pending operations on the slow path and on signal delivery, and
> factor the common "clear error and force SIGSEGV" fixup into
> force_fault().
>
> Signed-off-by: Olivier Dion <odion@efficios.com>
> ---
> kernel/rseq.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++---
> kernel/sys.c | 5 ++
> 2 files changed, 212 insertions(+), 13 deletions(-)
>
> diff --git a/kernel/rseq.c b/kernel/rseq.c
> index e75e3a5e312c..a277c11dae99 100644
> --- a/kernel/rseq.c
> +++ b/kernel/rseq.c
> @@ -253,17 +253,18 @@ static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs)
> static void rseq_slowpath_update_usr(struct pt_regs *regs)
> {
> /*
> - * Preserve has_rseq and user_irq state. The generic entry code clears
> - * user_irq on the way out, the non-generic entry architectures are not
> - * setting user_irq.
> + * Preserve has_rseq, rseq_op and user_irq state. The generic entry
> + * code clears user_irq on the way out, the non-generic entry
> + * architectures are not setting user_irq.
> */
> const struct rseq_event evt_mask = {
> .has_rseq = RSEQ_HAS_RSEQ_VERSION_MASK,
> + .rseq_op = true,
> .user_irq = true,
> };
> struct task_struct *t = current;
> struct rseq_ids ids;
> - bool event;
> + bool event, should_fault = false;
>
> if (unlikely(t->flags & PF_EXITING))
> return;
> @@ -300,7 +301,12 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
>
> ids.node_id = cpu_to_node(ids.cpu_id);
>
> - if (unlikely(!rseq_update_usr(t, regs, &ids))) {
> + if (unlikely(!rseq_update_usr(t, regs, &ids)))
> + should_fault = true;
> + else if (t->rseq.event.rseq_op && unlikely(!rseq_apply_ops(t)))
> + should_fault = true;
> +
> + if (should_fault) {
> /*
> * Clear the errors just in case this might survive magically, but
> * leave the rest intact.
> @@ -329,8 +335,20 @@ void __rseq_handle_slowpath(struct pt_regs *regs)
> rseq_slowpath_update_usr(regs);
> }
>
> +static inline void force_fault(int sig)
> +{
> + /*
> + * Clear the errors just in case this might survive magically, but leave
> + * the rest intact.
> + */
> + current->rseq.event.error = 0;
> + force_sigsegv(sig);
> +}
> +
> void __rseq_signal_deliver(int sig, struct pt_regs *regs)
> {
> + bool should_fault = false;
> +
> rseq_stat_inc(rseq_stats.signal);
>
> /*
> @@ -339,14 +357,13 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs)
> * the interrupted context as after this point the instruction
> * pointer in @regs points to the signal handler.
> */
> - if (unlikely(!rseq_handle_cs(current, regs))) {
> - /*
> - * Clear the errors just in case this might survive
> - * magically, but leave the rest intact.
> - */
> - current->rseq.event.error = 0;
> - force_sigsegv(sig);
> - }
> + if (unlikely(!rseq_handle_cs(current, regs)))
> + should_fault = true;
> + else if (current->rseq.event.rseq_op && unlikely(!rseq_apply_ops(current)))
Just to make sure: membarrier delivered to a task will also trigger
rseq_apply_ops, right?
> + should_fault = true;
> +
> + if (should_fault)
> + force_fault(sig);
>
> /*
> * In legacy mode, force the update of IDs before returning to user
> @@ -436,6 +453,8 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
> rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED;
> }
> }
> + if (version > 1)
> + rseqfl |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
>
> scoped_user_write_access(rseq, efault) {
> /*
> @@ -458,8 +477,16 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
> * registrations.
> */
> if (version > 1) {
> + u64 sentinel = (u64)&rseq->rseq_op_list;
> +
> if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION))
> unsafe_put_user(0U, &rseq->slice_ctrl.all, efault);
> + /*
> + * Initialize the rseq operation list sentinel as an
> + * empty circular doubly-linked list pointing to itself.
> + */
> + unsafe_put_user(sentinel, &rseq->rseq_op_list.next, efault);
> + unsafe_put_user(sentinel, &rseq->rseq_op_list.prev, efault);
Is there always enough space to write these? Can this corrupt program memory?
> }
> }
>
> @@ -474,6 +501,12 @@ static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u3
> #ifdef CONFIG_RSEQ_SLICE_EXTENSION
> current->rseq.slice.state.enabled = !!(rseqfl & RSEQ_CS_FLAG_SLICE_EXT_ENABLED);
> #endif
> + /*
> + * A fresh registration starts with no operations, so operation
> + * processing is disabled until the first one is registered.
> + */
> + current->rseq.event.rseq_op = false;
> + current->rseq.nr_ops = 0;
>
> /*
> * Ensure the cpu_id_start and cpu_id fields are updated before
> @@ -887,3 +920,164 @@ device_initcall(rseq_slice_init);
> #else
> static void rseq_slice_ext_init(struct dentry *root_dir) { }
> #endif /* CONFIG_RSEQ_SLICE_EXTENSION */
> +
> +/*
> + * Reflect the operation enabled state into the user visible flags field.
> + * Called on the 0<->1 transition of nr_ops. The kill path is taken on fault
> + * because losing this update leaves user space and kernel state inconsistent.
> + */
> +static int rseq_op_update_enabled(struct task_struct *t, bool enable)
> +{
> + struct rseq __user *rseq = t->rseq.usrptr;
> + u32 rflags;
> +
> + if (get_user(rflags, &rseq->flags))
> + return -EFAULT;
> +
> + rflags &= ~RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
> + rflags |= RSEQ_CS_FLAG_RSEQ_OP_AVAILABLE;
> + if (enable)
> + rflags |= RSEQ_CS_FLAG_RSEQ_OP_ENABLED;
> +
> + if (put_user(rflags, &rseq->flags))
> + return -EFAULT;
> +
> + t->rseq.event.rseq_op = enable;
> + return 0;
> +}
> +
> +/*
> + * Register @node at the head of the circular doubly-linked operation list
> + * anchored by the kernel owned sentinel in struct rseq.
> + */
> +static int rseq_op_register(struct task_struct *t, struct rseq_op_node __user *node)
> +{
> + struct rseq_op_node __user *sentinel = &t->rseq.usrptr->rseq_op_list;
> + struct rseq_op_node __user *first;
> + u64 next, prev, first_addr;
> + u8 type, i;
> +
> + if (t->rseq.nr_ops >= RSEQ_OP_LIST_LIMIT)
> + return -ENOSPC;
> +
> + if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
> + return -EINVAL;
> + if (!access_ok(node, sizeof(*node)))
> + return -EFAULT;
> +
> + /*
> + * The node links are owned by the kernel. User space must present a
> + * pristine node: next, prev and the reserved bytes all zeroed, and a
> + * known operation type.
> + */
> + if (get_user(next, &node->next) || get_user(prev, &node->prev) ||
> + get_user(type, &node->type))
> + return -EFAULT;
> + if (next || prev)
> + return -EINVAL;
> + if (type >= RSEQ_OP_NR)
> + return -EINVAL;
> + for (i = 1; i < sizeof(node->reserved); i++) {
> + u8 r;
> +
> + if (get_user(r, &node->reserved[i]))
> + return -EFAULT;
> + if (r)
> + return -EINVAL;
> + }
> +
> + /* Splice the node in right after the sentinel. */
> + if (get_user(first_addr, &sentinel->next))
> + goto die;
> + first = (struct rseq_op_node __user *)first_addr;
> +
> + if (put_user((u64)(unsigned long)first, &node->next) ||
> + put_user((u64)(unsigned long)sentinel, &node->prev) ||
> + put_user((u64)(unsigned long)node, &first->prev) ||
> + put_user((u64)(unsigned long)node, &sentinel->next))
> + goto die;
> +
> + t->rseq.nr_ops += 1;
> +
> + if (t->rseq.nr_ops == 1)
> + return rseq_op_update_enabled(t, true) ? -EFAULT : 0;
> + return 0;
> +die:
> + force_sig(SIGSEGV);
> + return -EFAULT;
> +}
> +
> +/*
> + * Unregister @node from the operation list. The node links are validated
> + * against its neighbours to reject bogus or double unregistration.
> + */
> +static int rseq_op_unregister(struct task_struct *t, struct rseq_op_node __user *node)
> +{
> + struct rseq_op_node __user *prev, *next;
> + u64 prev_addr, next_addr, tmp;
> +
> + if (!t->rseq.nr_ops)
> + return -ENOENT;
> +
> + if (!IS_ALIGNED((unsigned long)node, __alignof__(struct rseq_op_node)))
> + return -EINVAL;
> + if (!access_ok(node, sizeof(*node)))
> + return -EFAULT;
> +
> + if (get_user(next_addr, &node->next) || get_user(prev_addr, &node->prev))
> + return -EFAULT;
> + prev = (struct rseq_op_node __user *)prev_addr;
> + next = (struct rseq_op_node __user *)next_addr;
> +
> + /* A registered node always has both links set. */
> + if (!prev || !next)
> + return -EINVAL;
> +
> + /* Verify the node is properly linked between its neighbours. */
> + if (get_user(tmp, &prev->next))
> + goto die;
> + if (tmp != (u64)(unsigned long)node)
> + return -EINVAL;
> + if (get_user(tmp, &next->prev))
> + goto die;
> + if (tmp != (u64)(unsigned long)node)
> + return -EINVAL;
> +
> + /* Unsplice and clear the node links so it can be reused. */
> + if (put_user(next_addr, &prev->next) ||
> + put_user(prev_addr, &next->prev) ||
> + put_user(0ULL, &node->next) ||
> + put_user(0ULL, &node->prev))
> + goto die;
> +
> + t->rseq.nr_ops -= 1;
> +
> + if (t->rseq.nr_ops == 0)
> + return rseq_op_update_enabled(t, false) ? -EFAULT : 0;
> + return 0;
> +die:
> + force_sig(SIGSEGV);
> + return -EFAULT;
> +}
> +
> +int rseq_op_prctl(unsigned long arg2, unsigned long arg3)
> +{
> + struct rseq_op_node __user *node = (struct rseq_op_node __user *)arg3;
> + struct task_struct *t = current;
> +
> + if (!t->rseq.usrptr)
> + return -ENXIO;
> + if (!rseq_v2(t))
> + return -ENOTSUPP;
> + if (!node)
> + return -EINVAL;
> +
> + switch (arg2) {
> + case PR_RSEQ_OP_REGISTER:
> + return rseq_op_register(t, node);
> + case PR_RSEQ_OP_UNREGISTER:
> + return rseq_op_unregister(t, node);
> + default:
> + return -EINVAL;
> + }
> +}
> diff --git a/kernel/sys.c b/kernel/sys.c
> index df69bd71de03..494c91b03c3f 100644
> --- a/kernel/sys.c
> +++ b/kernel/sys.c
> @@ -2889,6 +2889,11 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
> return -EINVAL;
> error = rseq_slice_extension_prctl(arg2, arg3);
> break;
> + case PR_RSEQ_OP:
> + if (arg4 || arg5)
> + return -EINVAL;
> + error = rseq_op_prctl(arg2, arg3);
> + break;
> case PR_GET_CFI:
> if (arg2 != PR_CFI_BRANCH_LANDING_PADS)
> return -EINVAL;
> --
> 2.54.0
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC PATCH 5/5] selftests/rseq: add coverage for rseq operations
2026-08-28 15:33 [RFC PATCH 0/5] rseq: add support for RSEQ operations odion
` (3 preceding siblings ...)
2026-08-28 15:33 ` [RFC PATCH 4/5] rseq: register and unregister operations via prctl odion
@ 2026-08-28 15:33 ` odion
4 siblings, 0 replies; 11+ messages in thread
From: odion @ 2026-08-28 15:33 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Peter Zijlstra, Paul E. McKenney, Boqun Feng, LKML,
Thomas Gleixner, Dmitry Vyukov, David Matlack, Marco Elver,
Sean Christopherson, Wei Liu, Florian Weimer, Mathias Stearn,
Chris Kennelly, Blake Oler, Rich Felker, Matthew Wilcox,
Greg Kroah-Hartman, Carlos O'Donell, Olivier Dion
From: Olivier Dion <odion@efficios.com>
Add selftest-side support for the rseq operation ABI and wire a new
rseq_op_test into the rseq selftest suite.
The new test covers:
- reset before entering a signal handler,
- reset on return to userspace after poll(),
- reset after scheduler-driven preemption,
- no-operation cases where nothing should be reset,
- register-time rejection of non-pristine nodes and unknown types,
- fatal cases for corrupted list links (self and transitive cycles),
- invalid destination and next pointers,
- unsupported lengths,
- unaligned destinations, and
- NULL-source clearing.
Also extend the selftest ABI/helpers with the rseq operation node
definitions and prctl-based register/unregister helpers, and add a
dedicated runner that disables glibc's implicit rseq registration.
Signed-off-by: Olivier Dion <odion@efficios.com>
---
tools/testing/selftests/rseq/.gitignore | 4 +-
tools/testing/selftests/rseq/Makefile | 4 +-
tools/testing/selftests/rseq/rseq-abi.h | 146 ++++++++++++++++++++----
tools/testing/selftests/rseq/rseq.c | 23 +++-
tools/testing/selftests/rseq/rseq.h | 75 ++++++++++++
5 files changed, 223 insertions(+), 29 deletions(-)
diff --git a/tools/testing/selftests/rseq/.gitignore b/tools/testing/selftests/rseq/.gitignore
index ec01d164c1f0..8d5f02d79b98 100644
--- a/tools/testing/selftests/rseq/.gitignore
+++ b/tools/testing/selftests/rseq/.gitignore
@@ -2,7 +2,6 @@
basic_percpu_ops_test
basic_percpu_ops_mm_cid_test
basic_test
-basic_rseq_op_test
param_test
param_test_benchmark
param_test_compare_twice
@@ -11,3 +10,6 @@ param_test_mm_cid_benchmark
param_test_mm_cid_compare_twice
syscall_errors_test
slice_test
+check_optimized
+legacy_check
+rseq_op_test
\ No newline at end of file
diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile
index 50d69e22ee7a..5e761df0e5aa 100644
--- a/tools/testing/selftests/rseq/Makefile
+++ b/tools/testing/selftests/rseq/Makefile
@@ -22,12 +22,14 @@ TEST_GEN_PROGS_EXTENDED = librseq.so \
param_test_compare_twice \
param_test_mm_cid \
param_test_mm_cid_compare_twice \
+ rseq_op_test \
syscall_errors_test \
legacy_check \
slice_test \
check_optimized
-TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh run_legacy_check.sh run_timeslice_test.sh
+TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh run_legacy_check.sh run_timeslice_test.sh \
+ run_rseq_op_test.sh
TEST_FILES := settings
diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h
index 5f4ea2152c2f..051636f84194 100644
--- a/tools/testing/selftests/rseq/rseq-abi.h
+++ b/tools/testing/selftests/rseq/rseq-abi.h
@@ -26,6 +26,10 @@ enum rseq_abi_cs_flags_bit {
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0,
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1,
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2,
+ RSEQ_ABI_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4,
+ RSEQ_ABI_CS_FLAG_SLICE_EXT_ENABLED_BIT = 5,
+ RSEQ_ABI_CS_FLAG_RSEQ_OP_AVAILABLE_BIT = 6,
+ RSEQ_ABI_CS_FLAG_RSEQ_OP_ENABLED_BIT = 7,
};
enum rseq_abi_cs_flags {
@@ -35,6 +39,14 @@ enum rseq_abi_cs_flags {
(1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT),
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE =
(1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT),
+ RSEQ_ABI_CS_FLAG_SLICE_EXT_AVAILABLE =
+ (1U << RSEQ_ABI_CS_FLAG_SLICE_EXT_AVAILABLE_BIT),
+ RSEQ_ABI_CS_FLAG_SLICE_EXT_ENABLED =
+ (1U << RSEQ_ABI_CS_FLAG_SLICE_EXT_ENABLED_BIT),
+ RSEQ_ABI_CS_FLAG_RSEQ_OP_AVAILABLE =
+ (1U << RSEQ_ABI_CS_FLAG_RSEQ_OP_AVAILABLE_BIT),
+ RSEQ_ABI_CS_FLAG_RSEQ_OP_ENABLED =
+ (1U << RSEQ_ABI_CS_FLAG_RSEQ_OP_ENABLED_BIT),
};
/*
@@ -74,6 +86,103 @@ struct rseq_abi_slice_ctrl {
};
};
+union rseq_ptr {
+ __u64 ptr64;
+
+ /*
+ * The "arch" field provides architecture accessor for
+ * the ptr field based on architecture pointer size and
+ * endianness.
+ */
+ struct {
+#ifdef __LP64__
+ __u64 ptr;
+#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN)
+ __u32 padding; /* Initialized to zero. */
+ __u32 ptr;
+#else
+ __u32 ptr;
+ __u32 padding; /* Initialized to zero. */
+#endif
+ } arch;
+};
+
+/*
+ * Maximum number of nodes walked in the rseq operation list.
+ */
+#define RSEQ_ABI_OP_LIST_LIMIT 2048
+
+/*
+ * enum rseq_abi_op_type - Type of an rseq operation
+ * @RSEQ_ABI_OP_RESET: Plain reset. Uses struct rseq_abi_op_reset.
+ * @RSEQ_ABI_OP_RESET_WITH_STRIDE_CPUID: Reset indexed by the current CPU ID.
+ * Uses struct rseq_abi_op_reset_with_stride.
+ * @RSEQ_ABI_OP_RESET_WITH_STRIDE_MMCID: Reset indexed by the current MM CID.
+ * Uses struct rseq_abi_op_reset_with_stride.
+ */
+enum rseq_abi_op_type {
+ RSEQ_ABI_OP_RESET,
+ RSEQ_ABI_OP_RESET_WITH_STRIDE_CPUID,
+ RSEQ_ABI_OP_RESET_WITH_STRIDE_MMCID,
+ RSEQ_ABI_OP_NR,
+};
+
+/*
+ * struct rseq_abi_op_node - Common header linking an rseq operation into the list
+ * @next: Address of the next node. Owned by the kernel.
+ * @prev: Address of the previous node. Owned by the kernel.
+ * @type: Operation type. See enum rseq_abi_op_type.
+ * @reserved: Must be zero on registration.
+ *
+ * User space allocates the node, sets @type and zeroes @next, @prev and
+ * @reserved before passing it to prctl(PR_RSEQ_OP, PR_RSEQ_OP_REGISTER, node).
+ * The kernel owns @next and @prev for the lifetime of the registration and
+ * links the node into a circular doubly-linked list anchored by an internal
+ * sentinel in struct rseq_abi. User space must not touch @next or @prev while
+ * the node is registered.
+ */
+struct rseq_abi_op_node {
+ __u64 next;
+ __u64 prev;
+ struct {
+ __u8 type; /* enum rseq_abi_op_type */
+ __u8 reserved[7];
+ };
+};
+
+/*
+ * struct rseq_abi_op_reset - Reset one word to a value on return to user space
+ * @node: Operation list node.
+ * @src: Address of the source word, or 0 to reset @dst to zero.
+ * @dst: Address of the destination word.
+ * @len: Word length in bytes. Must be 4 or 8.
+ */
+struct rseq_abi_op_reset {
+ struct rseq_abi_op_node node;
+ __u64 src;
+ __u64 dst;
+ __u32 len;
+};
+
+/*
+ * struct rseq_abi_op_reset_with_stride - Reset one word in a strided array
+ * @node: Operation list node.
+ * @src: Address of the source word, or 0 to reset the slot to zero.
+ * @dst: Base address of the strided destination array.
+ * @dst_stride: Stride in bytes between consecutive array slots.
+ * @len: Word length in bytes. Must be 4 or 8.
+ *
+ * The destination slot is @dst + @dst_stride * index, where index is the
+ * current CPU ID or MM CID depending on the operation type.
+ */
+struct rseq_abi_op_reset_with_stride {
+ struct rseq_abi_op_node node;
+ __u64 src;
+ __u64 dst;
+ __u64 dst_stride;
+ __u32 len;
+};
+
/*
* struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always
* contained within a single cache-line.
@@ -127,26 +236,7 @@ struct rseq_abi {
* atomicity semantics. This field should only be updated by the
* thread which registered this data structure. Aligned on 64-bit.
*/
- union {
- __u64 ptr64;
-
- /*
- * The "arch" field provides architecture accessor for
- * the ptr field based on architecture pointer size and
- * endianness.
- */
- struct {
-#ifdef __LP64__
- __u64 ptr;
-#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN)
- __u32 padding; /* Initialized to zero. */
- __u32 ptr;
-#else
- __u32 ptr;
- __u32 padding; /* Initialized to zero. */
-#endif
- } arch;
- } rseq_cs;
+ union rseq_ptr rseq_cs;
/*
* Restartable sequences flags field.
@@ -192,9 +282,21 @@ struct rseq_abi {
struct rseq_abi_slice_ctrl slice_ctrl;
/*
- * Place holder to push the size above 32 bytes.
+ * Sentinel of the circular doubly-linked list of rseq operations
+ * registered via prctl(PR_RSEQ_OP, ...). Fully owned and maintained by
+ * the kernel: it is initialized to point to itself on registration and
+ * user space must never read or write it directly.
+ *
+ * The kernel only use next and prev from rseq_op_list. The rest of the
+ * bytes are reserved for later usage and should be zeroed.
*/
- __u8 __reserved;
+ union {
+ struct rseq_abi_op_node rseq_op_list;
+ struct {
+ __u64 op_used[2];
+ __u64 reserved;
+ };
+ };
/*
* Flexible array member at end of structure, after last feature field.
diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c
index be0d0a97031e..6a5406f3353b 100644
--- a/tools/testing/selftests/rseq/rseq.c
+++ b/tools/testing/selftests/rseq/rseq.c
@@ -116,6 +116,17 @@ bool rseq_available(void)
}
}
+/* The rseq areas need to be at least 32 bytes. */
+static
+unsigned int get_rseq_min_alloc_size(void)
+{
+ unsigned int alloc_size = rseq_size;
+
+ if ((int) alloc_size < ORIG_RSEQ_ALLOC_SIZE)
+ alloc_size = ORIG_RSEQ_ALLOC_SIZE;
+ return alloc_size;
+}
+
/*
* Return the feature size supported by the kernel.
*
@@ -261,12 +272,14 @@ void rseq_init(void)
/* rseq flags are deprecated, always set to 0. */
rseq_flags = 0;
+ {
+ unsigned int rseq_kernel_feature_size = get_rseq_kernel_feature_size();
- /*
- * Set the size to 0 until at least one thread registers to mimic the
- * libc behavior.
- */
- rseq_size = 0;
+ if (rseq_kernel_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE)
+ rseq_size = rseq_kernel_feature_size;
+ else
+ rseq_size = ORIG_RSEQ_ALLOC_SIZE;
+ }
}
static __attribute__((destructor))
diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h
index c62ebb9290c0..8f65d272e7cf 100644
--- a/tools/testing/selftests/rseq/rseq.h
+++ b/tools/testing/selftests/rseq/rseq.h
@@ -18,6 +18,9 @@
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
+#include <string.h>
+#include <sys/prctl.h>
+#include <linux/kernel.h>
#include "rseq-abi.h"
#include "compiler.h"
@@ -395,4 +398,76 @@ int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode per
}
}
+/*
+ * prctl commands for the rseq operation list. The kernel owns the list: the
+ * feature is enabled implicitly by registering the first operation and
+ * disabled by unregistering the last one. There is no explicit enable knob.
+ */
+#ifndef PR_RSEQ_OP
+#define PR_RSEQ_OP 82
+#define PR_RSEQ_OP_REGISTER 1
+#define PR_RSEQ_OP_UNREGISTER 2
+#endif
+
+/*
+ * Initialize a plain reset operation. Sets the operation type, the destination
+ * and source words, the word length, and leaves the node in a pristine state
+ * (next, prev and reserved bytes zeroed) as required by the kernel at
+ * registration time.
+ */
+static inline
+void rseq_op_reset_init(struct rseq_abi_op_reset *op,
+ void *dst, void *src, size_t len)
+{
+ op->node.next = 0;
+ op->node.prev = 0;
+ op->node.type = RSEQ_ABI_OP_RESET;
+ memset(op->node.reserved, 0, sizeof(op->node.reserved));
+ op->src = (__u64)(unsigned long)src;
+ op->dst = (__u64)(unsigned long)dst;
+ op->len = len;
+}
+
+/*
+ * Initialize a strided reset operation indexed by the current CPU ID or MM CID
+ * depending on @type.
+ */
+static inline
+void rseq_op_reset_with_stride_init(struct rseq_abi_op_reset_with_stride *op,
+ enum rseq_abi_op_type type,
+ void *dst, void *src,
+ size_t dst_stride, size_t len)
+{
+ op->node.next = 0;
+ op->node.prev = 0;
+ op->node.type = type;
+ memset(op->node.reserved, 0, sizeof(op->node.reserved));
+ op->src = (__u64)(unsigned long)src;
+ op->dst = (__u64)(unsigned long)dst;
+ op->dst_stride = (__u64)dst_stride;
+ op->len = len;
+}
+
+/*
+ * Register an rseq operation node. The kernel links the pristine node into its
+ * internal list and enables operation processing when the first node is
+ * registered. Returns the prctl() return value (0 on success).
+ */
+static inline
+int rseq_op_register(struct rseq_abi_op_node *node)
+{
+ return prctl(PR_RSEQ_OP, PR_RSEQ_OP_REGISTER, (unsigned long)node, 0, 0);
+}
+
+/*
+ * Unregister an rseq operation node. The kernel unlinks the node, clears its
+ * next/prev links, and disables operation processing when the last node is
+ * unregistered. Returns the prctl() return value (0 on success).
+ */
+static inline
+int rseq_op_unregister(struct rseq_abi_op_node *node)
+{
+ return prctl(PR_RSEQ_OP, PR_RSEQ_OP_UNREGISTER, (unsigned long)node, 0, 0);
+}
+
#endif /* RSEQ_H_ */
--
2.54.0
^ permalink raw reply [flat|nested] 11+ messages in thread