[PATCH 3/4] io_uring: add io_req_{set,clear,switch}_flags()

[Caleb Sander Mateos <csander@purestorage.com>]

clang likes to set/clear flag bits using the narrowest possible or/and
instructions (e.g. an 8-bit or if all flag bits are in the same byte).
This causes store-to-load forwarding failures on subsequent wider loads
of req->flags.

Introduce helpers io_req_set_flags() and io_req_clear_flags() to set and
clear bits in req->flags. Use a volatile write to prevent clang from
emitting a narrower store to part of req->flags. The downside of
volatile writes is that consecutive req->flags updates must store to
req->flags multiple times. So provide io_req_switch_flags() as well,
which can both set and clear flags in a single volatile write.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 include/linux/io_uring.h | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h
index 505caa99c621..c70f7887ea72 100644
--- a/include/linux/io_uring.h
+++ b/include/linux/io_uring.h
@@ -5,10 +5,35 @@
 #include <linux/io_uring_types.h>
 #include <linux/sched.h>
 #include <linux/xarray.h>
 #include <uapi/linux/io_uring.h>
 
+/*
+ * clang likes to set/clear flag bits using the narrowest possible or/and
+ * instructions (e.g. an 8-bit or if all flag bits are in the same byte).
+ * This causes store-to-load forwarding failures on subsequent wider loads
+ * of req->flags.
+ * Store to req->flags using volatile writes to prevent this "optimization".
+ */
+static inline void io_req_switch_flags(struct io_kiocb *req,
+				       io_req_flags_t clear_flags,
+				       io_req_flags_t set_flags)
+{
+	WRITE_ONCE(req->flags, (req->flags | set_flags) & ~clear_flags);
+}
+
+static inline void io_req_set_flags(struct io_kiocb *req, io_req_flags_t flags)
+{
+	io_req_switch_flags(req, 0, flags);
+}
+
+static inline void io_req_clear_flags(struct io_kiocb *req,
+				      io_req_flags_t flags)
+{
+	io_req_switch_flags(req, flags, 0);
+}
+
 static inline void req_set_fail(struct io_kiocb *req)
 {
 	req->flags |= REQ_F_FAIL;
 	if (req->flags & REQ_F_CQE_SKIP) {
 		req->flags &= ~REQ_F_CQE_SKIP;
-- 
2.56.1