* Re: Linux 7.2.14
2026-10-30 12:12 Linux 7.2.14 Greg Kroah-Hartman
@ 2026-10-30 12:12 ` Greg Kroah-Hartman
0 siblings, 0 replies; 2+ messages in thread
From: Greg Kroah-Hartman @ 2026-10-30 12:12 UTC (permalink / raw)
To: linux-kernel, akpm, torvalds, stable; +Cc: lwn, jslaby, Greg Kroah-Hartman
diff --git a/Makefile b/Makefile
index dfd40298a0cf..ab29f857e9ed 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
VERSION = 7
PATCHLEVEL = 2
-SUBLEVEL = 13
+SUBLEVEL = 14
EXTRAVERSION =
NAME = Baby Opossum Posse
diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c
index 86292965f493..0e4783ff7d22 100644
--- a/crypto/asymmetric_keys/restrict.c
+++ b/crypto/asymmetric_keys/restrict.c
@@ -243,10 +243,14 @@ static int key_or_keyring_common(struct key *dest_keyring,
if (IS_ERR(key))
key = NULL;
} else if (trusted->type == &key_type_asymmetric) {
+ const struct asymmetric_key_ids *kids;
const struct asymmetric_key_id **signer_ids;
- signer_ids = (const struct asymmetric_key_id **)
- asymmetric_key_ids(trusted)->id;
+ kids = asymmetric_key_ids(trusted);
+ if (!kids)
+ return -ENOKEY;
+
+ signer_ids = (const struct asymmetric_key_id **)kids->id;
/*
* The auth_ids come from the candidate key (the
diff --git a/fs/coredump.c b/fs/coredump.c
index e68a76ff92a3..19c1cfe86472 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -508,7 +508,9 @@ static int zap_threads(struct task_struct *tsk,
int nr = -EAGAIN;
spin_lock_irq(&tsk->sighand->siglock);
- if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task) {
+ /* A freeze requested before the dump would be lost with TIF_SIGPENDING. */
+ if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task &&
+ !freezing(tsk) && !(tsk->jobctl & JOBCTL_TRAP_FREEZE)) {
/* Allow SIGKILL, see prepare_signal() */
signal->core_state = core_state;
nr = zap_process(signal, exit_code);
@@ -562,6 +564,8 @@ static void coredump_finish(bool core_dumped)
current->signal->core_state = NULL;
spin_unlock_irq(¤t->sighand->siglock);
+ /* A released thread may exit and be freed before it is woken. */
+ guard(rcu)();
while ((curr = next) != NULL) {
next = curr->next;
task = curr->task;
@@ -570,6 +574,7 @@ static void coredump_finish(bool core_dumped)
* ->task == NULL before we read ->next.
*/
smp_mb();
+ /* Any wakeup now lets the thread exit, rcu keeps it alive. */
curr->task = NULL;
wake_up_process(task);
}
@@ -577,13 +582,8 @@ static void coredump_finish(bool core_dumped)
static bool dump_interrupted(void)
{
- /*
- * SIGKILL or freezing() interrupt the coredumping. Perhaps we
- * can do try_to_freeze() and check __fatal_signal_pending(),
- * but then we need to teach dump_write() to restart and clear
- * TIF_SIGPENDING.
- */
- return fatal_signal_pending(current) || freezing(current);
+ /* Only SIGKILL and the freezers set it after zap_threads(). */
+ return task_sigpending(current);
}
static void wait_for_dump_helpers(struct file *file)
@@ -1196,6 +1196,8 @@ void vfs_coredump(const kernel_siginfo_t *siginfo)
if (coredump_wait(siginfo->si_signo, &core_state) < 0)
return;
+ /* Task work must not cut the dump short, see signal_pending(). */
+ guard(no_notify_signal)();
scoped_with_creds(cred)
do_coredump(&cn, &cprm, &argv, &argc, binfmt);
coredump_cleanup(&cn, &cprm);
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
index fb8e52574397..8515560e2a4b 100644
--- a/fs/ntfs/file.c
+++ b/fs/ntfs/file.c
@@ -620,12 +620,20 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
static vm_fault_t ntfs_filemap_page_mkwrite(struct vm_fault *vmf)
{
struct inode *inode = file_inode(vmf->vma->vm_file);
+ struct address_space *mapping = inode->i_mapping;
vm_fault_t ret;
sb_start_pagefault(inode->i_sb);
file_update_time(vmf->vma->vm_file);
+ /*
+ * Serialize against truncate/fallocate which hold the lock
+ * exclusively while invalidating pagecache and changing extents.
+ */
+ filemap_invalidate_lock_shared(mapping);
ret = iomap_page_mkwrite(vmf, &ntfs_page_mkwrite_iomap_ops, NULL);
+ filemap_invalidate_unlock_shared(mapping);
+
sb_end_pagefault(inode->i_sb);
return ret;
}
@@ -1061,7 +1069,6 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le
struct ntfs_volume *vol = ni->vol;
int err = 0;
loff_t old_size;
- bool map_locked = false;
if (mode & ~(NTFS_FALLOC_FL_SUPPORTED))
return -EOPNOTSUPP;
@@ -1090,16 +1097,12 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le
inode_lock(vi);
if (NInoCompressed(ni) || NInoEncrypted(ni)) {
- err = -EOPNOTSUPP;
- goto out;
+ inode_unlock(vi);
+ return -EOPNOTSUPP;
}
inode_dio_wait(vi);
- if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE |
- FALLOC_FL_INSERT_RANGE)) {
- filemap_invalidate_lock(vi->i_mapping);
- map_locked = true;
- }
+ filemap_invalidate_lock(vi->i_mapping);
switch (mode & FALLOC_FL_MODE_MASK) {
case FALLOC_FL_ALLOCATE_RANGE:
@@ -1124,14 +1127,13 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le
err = file_modified(file);
out:
- if (map_locked)
- filemap_invalidate_unlock(vi->i_mapping);
+ if (!err && mode == 0 && NInoNonResident(ni) &&
+ offset > old_size) {
+ truncate_pagecache(vi, old_size);
+ pagecache_isize_extended(vi, old_size, offset);
+ }
+ filemap_invalidate_unlock(vi->i_mapping);
if (!err) {
- if (mode == 0 && NInoNonResident(ni) &&
- offset > old_size) {
- truncate_pagecache(vi, old_size);
- pagecache_isize_extended(vi, old_size, offset);
- }
NInoSetFileNameDirty(ni);
inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi));
mark_inode_dirty(vi);
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 20116462cf3d..896ec1616f58 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -513,8 +513,13 @@ struct io_ring_ctx {
u32 iowq_limits[2];
struct callback_head poll_wq_task_work;
+ /* deferred ->poll_wq wake up for local task_work */
+ struct rcu_head poll_wq_rcu;
+ atomic_t poll_wq_rcu_pending;
struct list_head defer_list;
unsigned nr_drained;
+ /* drain request dispatched from ->defer_list, until it completes */
+ struct io_kiocb *drain_req;
/* protected by ->completion_lock */
unsigned nr_req_allocated;
diff --git a/include/linux/key-type.h b/include/linux/key-type.h
index bb97bd3e5af4..c4f329b6ae3f 100644
--- a/include/linux/key-type.h
+++ b/include/linux/key-type.h
@@ -161,6 +161,7 @@ struct key_type {
const void *in, const void *in2);
/* internal fields */
+ atomic64_t index_id; /* Opaque keyring index ID */
struct list_head link; /* link in types list */
struct lock_class_key lock_class; /* key->sem lock class */
} __randomize_layout;
diff --git a/include/linux/key.h b/include/linux/key.h
index 81b8f05c6898..e40b5267658f 100644
--- a/include/linux/key.h
+++ b/include/linux/key.h
@@ -107,6 +107,7 @@ struct keyring_name;
struct key_tag {
struct rcu_head rcu;
+ atomic64_t index_id; /* Opaque keyring index ID */
refcount_t usage;
bool removed; /* T when subject removed */
};
@@ -129,6 +130,8 @@ struct keyring_index_key {
struct key_type *type;
struct key_tag *domain_tag; /* Domain of operation */
const char *description;
+ u64 type_id; /* Cached index IDs */
+ u64 domain_id;
};
union key_payload {
@@ -251,6 +254,8 @@ struct key {
struct key_type *type; /* type of key */
struct key_tag *domain_tag; /* Domain of operation */
char *description;
+ u64 type_id;
+ u64 domain_id;
};
};
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5b36a3364ca0..e2f472e6ca94 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1805,7 +1805,7 @@ extern struct pid __rcu *cad_pid;
* I am cleaning dirty pages from some other bdi. */
#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
#define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */
-#define PF__HOLE__00800000 0x00800000
+#define PF_NO_NOTIFY_SIGNAL 0x00800000 /* see no_notify_signal_save() */
#define PF__HOLE__01000000 0x01000000
#define PF__HOLE__02000000 0x02000000
#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index efa1e2d6c7a6..bfb84f025b67 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -2,6 +2,7 @@
#ifndef _LINUX_SCHED_SIGNAL_H
#define _LINUX_SCHED_SIGNAL_H
+#include <linux/cleanup.h>
#include <linux/rculist.h>
#include <linux/signal.h>
#include <linux/sched.h>
@@ -384,14 +385,36 @@ static inline int task_sigpending(struct task_struct *p)
return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
}
+/* Prevent TIF_NOTIFY_SIGNAL from interrupting this task. */
+static inline unsigned int no_notify_signal_save(void)
+{
+ unsigned int flags = current->flags;
+
+ current->flags |= PF_NO_NOTIFY_SIGNAL;
+ return flags;
+}
+
+/* Restore the previous PF_NO_NOTIFY_SIGNAL state. */
+static inline void no_notify_signal_restore(unsigned int flags)
+{
+ current_restore_flags(flags, PF_NO_NOTIFY_SIGNAL);
+}
+
+DEFINE_LOCK_GUARD_0(no_notify_signal,
+ _T->flags = no_notify_signal_save(),
+ no_notify_signal_restore(_T->flags),
+ unsigned int flags)
+
static inline int signal_pending(struct task_struct *p)
{
/*
* TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same
* behavior in terms of ensuring that we break out of wait loops
- * so that notify signal callbacks can be processed.
+ * so that notify signal callbacks can be processed. Not for a task
+ * that asked not to be interrupted by it, see no_notify_signal_save().
*/
- if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)))
+ if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)) &&
+ likely(!(READ_ONCE(p->flags) & PF_NO_NOTIFY_SIGNAL)))
return 1;
return task_sigpending(p);
}
diff --git a/io_uring/cancel.c b/io_uring/cancel.c
index 4f706aff14fc..75a21ecfe69c 100644
--- a/io_uring/cancel.c
+++ b/io_uring/cancel.c
@@ -72,7 +72,16 @@ static bool io_cancel_cb(struct io_wq_work *work, void *data)
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct io_cancel_data *cd = data;
- return io_cancel_req_match(req, cd);
+ if (!io_cancel_req_match(req, cd))
+ return false;
+ /*
+ * Pending work is gone once it has matched. Work that is running may
+ * park the request where the same cancellation looks next. Don't make
+ * it skip the request as one that it has dealt with.
+ */
+ if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY))
+ req->cancel_seq_set = false;
+ return true;
}
static int io_async_cancel_one(struct io_uring_task *tctx,
@@ -117,6 +126,8 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
*/
if (!ret)
return 0;
+ if (ret == -EALREADY)
+ cd->running = true;
ret = io_poll_cancel(ctx, cd, issue_flags);
if (ret != -ENOENT)
@@ -180,6 +191,7 @@ static int __io_async_cancel(struct io_cancel_data *cd,
struct io_tctx_node *node;
int ret, nr = 0;
+ cd->running = false;
do {
ret = io_try_cancel(tctx, cd, issue_flags);
if (ret == -ENOENT)
@@ -195,6 +207,9 @@ static int __io_async_cancel(struct io_cancel_data *cd,
mutex_lock(&ctx->tctx_lock);
ret = -ENOENT;
list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
+ /* has been done above, don't count what is running there */
+ if (all && node->task->io_uring == tctx)
+ continue;
ret = io_async_cancel_one(node->task->io_uring, cd);
if (ret != -ENOENT) {
if (!all)
@@ -204,7 +219,15 @@ static int __io_async_cancel(struct io_cancel_data *cd,
}
mutex_unlock(&ctx->tctx_lock);
io_ring_submit_unlock(ctx, issue_flags);
- return all ? nr : ret;
+ if (all)
+ return nr;
+ /*
+ * The worker may be done with the work that was found running, and
+ * flagged, above. That isn't the same as not having found it.
+ */
+ if (ret == -ENOENT && cd->running)
+ ret = -EALREADY;
+ return ret;
}
int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
@@ -308,7 +331,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
if (ret != -EALREADY)
goto out;
- if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) {
+ if (sc.timeout.tv_sec != -1 || sc.timeout.tv_nsec != -1) {
struct timespec64 ts = {
.tv_sec = sc.timeout.tv_sec,
.tv_nsec = sc.timeout.tv_nsec
@@ -636,8 +659,18 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
io_uring_drop_tctx_refs(current);
/* only skip the sleep for local work this task may run */
xa_for_each(&tctx->xa, index, node) {
- if (io_local_work_pending(node->ctx) &&
- io_allowed_defer_tw_run(node->ctx))
+ if (!(node->ctx->flags & IORING_SETUP_DEFER_TASKRUN) ||
+ !io_allowed_defer_tw_run(node->ctx))
+ continue;
+ /*
+ * The wake up io_uring_try_cancel_requests() asked for
+ * may have been issued while we were still running. Ask
+ * again now that it can't be missed.
+ */
+ atomic_set(&node->ctx->cq_wait_nr, 1);
+ /* pairs with the xchg() in mpscq_push() */
+ smp_mb();
+ if (io_local_work_pending(node->ctx))
goto end_wait;
}
/*
diff --git a/io_uring/cancel.h b/io_uring/cancel.h
index 1b201a094303..e03a6fb0b7f9 100644
--- a/io_uring/cancel.h
+++ b/io_uring/cancel.h
@@ -11,6 +11,8 @@ struct io_cancel_data {
u8 opcode;
u32 flags;
int seq;
+ /* io_try_cancel() has found it being run by io-wq, and flagged it */
+ bool running;
};
int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c
index 3ae4804765b9..30027fd46045 100644
--- a/io_uring/fdinfo.c
+++ b/io_uring/fdinfo.c
@@ -154,11 +154,17 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
cq_entries = min(cq_tail - cq_head, ctx->cq_entries);
for (i = 0; i < cq_entries; i++) {
struct io_uring_cqe *cqe;
- bool cqe32 = false;
+ bool cqe32 = false, two_entries = false;
- cqe = &r->cqes[(cq_head & cq_mask)];
- if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32)
+ if (ctx->flags & IORING_SETUP_CQE32) {
+ /* all entries have twice the size */
+ cqe = &r->cqes[(cq_head & cq_mask) << 1];
cqe32 = true;
+ } else {
+ cqe = &r->cqes[(cq_head & cq_mask)];
+ two_entries = cqe->flags & IORING_CQE_F_32;
+ cqe32 = two_entries;
+ }
seq_printf(m, "%5u: user_data:%llu, res:%d, flags:%x",
cq_head & cq_mask, cqe->user_data, cqe->res,
cqe->flags);
@@ -167,7 +173,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
cqe->big_cqe[0], cqe->big_cqe[1]);
seq_printf(m, "\n");
cq_head++;
- if (cqe32) {
+ if (two_entries) {
cq_head++;
i++;
}
diff --git a/io_uring/futex.c b/io_uring/futex.c
index eaee14242a3a..9203998654ff 100644
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -152,6 +152,24 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0;
}
+/*
+ * A wait on a private futex does not block the submitter. Even a single
+ * threaded task can go on and have the mm private hash installed by clone()
+ * or prctl(), which does not pick up waiters from the global hash. Make
+ * sure the private hash exists before the first waiter is queued.
+ */
+static int io_futex_prep_private(struct io_kiocb *req)
+{
+ int ret = futex_hash_allocate_default();
+
+ if (unlikely(ret))
+ return ret;
+
+ /* inflight tracking only needed for mm private hash */
+ io_req_track_inflight(req);
+ return 0;
+}
+
int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
@@ -161,9 +179,8 @@ int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (unlikely(ret))
return ret;
- /* inflight tracking only needed for mm private hash */
if (!(iof->futex_flags & FLAGS_SHARED))
- io_req_track_inflight(req);
+ return io_futex_prep_private(req);
return 0;
}
@@ -213,10 +230,13 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return ret;
}
- /* inflight tracking only needed for mm private hash */
for (i = 0; i < iof->futex_nr; i++) {
if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) {
- io_req_track_inflight(req);
+ ret = io_futex_prep_private(req);
+ if (ret) {
+ kfree(ifd);
+ return ret;
+ }
break;
}
}
@@ -283,6 +303,8 @@ int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
*/
__set_current_state(TASK_RUNNING);
hlist_add_head(&req->hash_node, &ctx->futex_list);
+ if (io_wq_cancel_seen(req, issue_flags))
+ __io_futex_cancel(req);
} else {
iof->futexv_unqueued = 1;
if (woken != -1)
@@ -323,6 +345,8 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)
&ifd->q, NULL, NULL);
if (!ret) {
hlist_add_head(&req->hash_node, &ctx->futex_list);
+ if (io_wq_cancel_seen(req, issue_flags))
+ __io_futex_cancel(req);
io_ring_submit_unlock(ctx, issue_flags);
return IOU_ISSUE_SKIP_COMPLETE;
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index fa403ed24596..59aa1ab58492 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -55,6 +55,8 @@ struct io_worker {
struct io_wq_acct *acct;
struct io_wq_work *cur_work;
+ /* rest of the hashed chain ->cur_work was taken off the list with */
+ struct io_wq_work *next_work;
raw_spinlock_t lock;
struct completion ref_done;
@@ -466,11 +468,9 @@ static void io_wq_dec_running(struct io_worker *worker)
if (!atomic_dec_and_test(&acct->nr_running))
return;
- if (!worker->cur_work)
- return;
if (!io_acct_run_queue(acct))
return;
- if (io_wq_hash_defer(worker->cur_work, acct)) {
+ if (worker->cur_work && io_wq_hash_defer(worker->cur_work, acct)) {
raw_spin_unlock(&acct->lock);
return;
}
@@ -582,6 +582,17 @@ static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
return NULL;
}
+static void __io_assign_current_work(struct io_worker *worker,
+ struct io_wq_work *work)
+{
+ raw_spin_lock(&worker->lock);
+ worker->cur_work = work;
+ /* stays valid while in between two items of a chain */
+ if (work)
+ worker->next_work = wq_next_work(work);
+ raw_spin_unlock(&worker->lock);
+}
+
static void io_assign_current_work(struct io_worker *worker,
struct io_wq_work *work)
{
@@ -590,9 +601,7 @@ static void io_assign_current_work(struct io_worker *worker,
cond_resched();
}
- raw_spin_lock(&worker->lock);
- worker->cur_work = work;
- raw_spin_unlock(&worker->lock);
+ __io_assign_current_work(worker, work);
}
/*
@@ -623,9 +632,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
* work list, but isn't yet discoverable as the
* current work item for this worker.
*/
- raw_spin_lock(&worker->lock);
- worker->cur_work = work;
- raw_spin_unlock(&worker->lock);
+ __io_assign_current_work(worker, work);
}
raw_spin_unlock(&acct->lock);
@@ -641,21 +648,28 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
/* handle a whole dependent link */
do {
bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
- struct io_wq_work *next_hashed, *linked;
+ struct io_wq_work *next_hashed, *linked, *done = work;
unsigned int work_flags = atomic_read(&work->flags);
unsigned int hash = __io_wq_is_hashed(work_flags)
? __io_get_work_hash(work_flags)
: -1U;
+ bool last;
next_hashed = wq_next_work(work);
- if (do_kill &&
- (work_flags & IO_WQ_WORK_UNBOUND))
+ /*
+ * io_wq_exit_workers() signals us once, so that we
+ * don't sleep for ever. If that was before this work
+ * became the current one, the signal is gone.
+ */
+ if (do_kill && (work_flags & IO_WQ_WORK_UNBOUND))
atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
+ else if (do_kill)
+ set_notify_signal(current);
io_wq_submit_work(work);
io_assign_current_work(worker, NULL);
- linked = io_wq_free_work(work);
+ last = io_wq_put_work(work, &linked);
work = next_hashed;
if (!work && linked && !io_wq_is_hashed(linked)) {
work = linked;
@@ -664,6 +678,13 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
io_assign_current_work(worker, work);
if (linked)
io_wq_enqueue(wq, linked);
+ /*
+ * Not before the link is current or pending work again.
+ * This may be the last time that a task is woken up
+ * which exits and looks for requests to cancel.
+ */
+ if (last)
+ io_wq_free_work(done);
if (hash != -1U && !next_hashed) {
/* serialize hash clear with wake_up() */
@@ -682,6 +703,35 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
} while (1);
}
+/*
+ * The worker has decided to exit, and has given up its ->nr_workers slot and
+ * its place on the free list. io_wq_enqueue() may have picked it from that
+ * list, or seen it running, after it last looked at the work list. Leave
+ * ->nr_running as well and check again. Returns false if the worker has to
+ * stay.
+ */
+static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker)
+{
+ /* turns io_wq_dec_running() and the scheduler hooks into no-ops */
+ clear_bit(IO_WORKER_F_UP, &worker->flags);
+ atomic_dec(&acct->nr_running);
+ if (!io_acct_run_queue(acct))
+ return true;
+ raw_spin_unlock(&acct->lock);
+
+ raw_spin_lock(&acct->workers_lock);
+ if (acct->nr_workers >= acct->max_workers) {
+ /* a worker was created in our place, the work is its business */
+ raw_spin_unlock(&acct->workers_lock);
+ return true;
+ }
+ acct->nr_workers++;
+ raw_spin_unlock(&acct->workers_lock);
+ atomic_inc(&acct->nr_running);
+ set_bit(IO_WORKER_F_UP, &worker->flags);
+ return false;
+}
+
static int io_wq_worker(void *data)
{
struct io_worker *worker = data;
@@ -715,19 +765,41 @@ static int io_wq_worker(void *data)
* idle-exit, drop the worker as well. This is used to avoid
* keeping io-wq workers around for tasks that no longer have
* any active io_uring instances.
+ * Not as long as a request of ours for another worker is
+ * pending, though. It is accounted as a running worker, which
+ * others rely on, and io_worker_exit() would cancel it.
*/
- if ((last_timeout && (exit_mask || acct->nr_workers > 1)) ||
- test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) {
+ if (((last_timeout && (exit_mask || acct->nr_workers > 1)) ||
+ test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) &&
+ !test_bit(0, &worker->create_state)) {
acct->nr_workers--;
+ if (test_and_clear_bit(IO_WORKER_F_FREE, &worker->flags))
+ hlist_nulls_del_init_rcu(&worker->nulls_node);
raw_spin_unlock(&acct->workers_lock);
__set_current_state(TASK_RUNNING);
- break;
+ if (io_worker_retire(acct, worker))
+ break;
+ last_timeout = false;
+ continue;
}
last_timeout = false;
__io_worker_idle(acct, worker);
raw_spin_unlock(&acct->workers_lock);
if (io_run_task_work())
continue;
+ /*
+ * io_wq_enqueue() leaves new work to us for as long as we are
+ * accounted as running. Get out of ->nr_running before taking
+ * the last look at the work list, rather than when scheduling
+ * out. No need to ask for another worker then, we can do it.
+ */
+ clear_bit(IO_WORKER_F_RUNNING, &worker->flags);
+ atomic_dec(&acct->nr_running);
+ if (io_acct_run_queue(acct)) {
+ io_wq_worker_running(current);
+ io_worker_handle_work(acct, worker);
+ continue;
+ }
ret = schedule_timeout(WORKER_IDLE_TIMEOUT);
if (signal_pending(current)) {
struct ksignal ksig;
@@ -993,9 +1065,13 @@ void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable)
static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq)
{
do {
+ struct io_wq_work *linked;
+
atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
io_wq_submit_work(work);
- work = io_wq_free_work(work);
+ if (io_wq_put_work(work, &linked))
+ io_wq_free_work(work);
+ work = linked;
} while (work);
}
@@ -1103,6 +1179,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker,
static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
{
struct io_cb_cancel_data *match = data;
+ struct io_wq_work *work;
/*
* Hold the lock to avoid ->cur_work going out of scope, caller
@@ -1111,6 +1188,19 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
raw_spin_lock(&worker->lock);
if (__io_wq_worker_cancel(worker, match, worker->cur_work))
match->nr_running++;
+ /*
+ * Hashed work is taken off the work list a chain at a time, what the
+ * worker hasn't started yet can only be found here. It doesn't move on
+ * without the lock. Just mark the work, it fails once it is its turn.
+ */
+ for (work = worker->next_work; work; work = wq_next_work(work)) {
+ if (match->nr_running && !match->cancel_all)
+ break;
+ if (match->fn(work, match->data)) {
+ atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
+ match->nr_running++;
+ }
+ }
raw_spin_unlock(&worker->lock);
return match->nr_running && !match->cancel_all;
@@ -1319,6 +1409,8 @@ static bool io_task_work_match(struct callback_head *cb, void *data)
void io_wq_exit_start(struct io_wq *wq)
{
set_bit(IO_WQ_BIT_EXIT, &wq->state);
+ /* Pairs with task_work_add() in io_queue_worker_create(). */
+ smp_mb__after_atomic();
}
static void io_wq_cancel_tw_create(struct io_wq *wq)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 3d8c8a50fd31..33f0c6ab7dba 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -405,17 +405,46 @@ static void io_prep_async_link(struct io_kiocb *req)
}
}
+/* Called for every request that is issued: keep it inline */
+static __always_inline bool io_assign_file(struct io_kiocb *req,
+ const struct io_issue_def *def,
+ unsigned int issue_flags)
+{
+ if (req->file || !def->needs_file)
+ return true;
+
+ if (req->flags & REQ_F_FIXED_FILE)
+ req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
+ else
+ req->file = io_file_get_normal(req, req->cqe.fd);
+
+ return !!req->file;
+}
+
static void io_queue_iowq(struct io_kiocb *req)
{
struct io_uring_task *tctx = req->tctx;
BUG_ON(!tctx);
+ lockdep_assert_held(&req->ctx->uring_lock);
if ((current->flags & PF_KTHREAD) || !tctx->io_wq) {
io_req_task_queue_fail(req, -ECANCELED);
return;
}
+ /*
+ * The file is looked up as late as possible, as it may be the result
+ * of an earlier request of the link. This one is up now, and what kind
+ * of work it is depends on the file. If that fails, leave the error
+ * handling to the worker.
+ *
+ * Not for a splice: whatever it writes to, it may wait for its input
+ * for ever. It has to remain unbound work that isn't hashed.
+ */
+ if (req->opcode != IORING_OP_SPLICE)
+ io_assign_file(req, &io_issue_defs[req->opcode], 0);
+
/* init ->work of the whole link before punting */
io_prep_async_link(req);
@@ -454,12 +483,15 @@ unsigned io_linked_nr(struct io_kiocb *req)
return nr;
}
-static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
+static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx)
{
- bool drain_seen = false, first = true;
+ bool drain_seen = false, first = true, drain_queued = false;
lockdep_assert_held(&ctx->uring_lock);
__io_req_caches_free(ctx);
+ /* nothing outside of ->defer_list is alive */
+ if (ctx->nr_req_allocated == ctx->nr_drained)
+ ctx->drain_req = NULL;
while (!list_empty(&ctx->defer_list)) {
struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
@@ -467,14 +499,21 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
drain_seen |= de->req->flags & REQ_F_IO_DRAIN;
if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained)
- return;
+ return drain_queued;
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
+ if (de->req->flags & REQ_F_IO_DRAIN) {
+ drain_queued = true;
+ /* it runs alone, track its completion unless it's a link */
+ if (!de->req->link)
+ ctx->drain_req = de->req;
+ }
io_req_task_queue(de->req);
kfree(de);
first = false;
}
+ return drain_queued;
}
void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
@@ -738,6 +777,13 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
return false;
+ /*
+ * A single entry may be left if a 32b CQE is asked for. It must not be
+ * handed out later on if this fails, as the CQE is going to overflow.
+ */
+ ctx->cqe_cached = NULL;
+ ctx->cqe_sentinel = NULL;
+
/*
* Post dummy CQE if a 32b CQE is needed and there's only room for a
* 16b CQE before the ring wraps.
@@ -930,9 +976,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
/*
* Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
- * the submitter task context, IOPOLL protects with uring_lock.
+ * the submitter task context, IOPOLL protects with uring_lock. A drain
+ * request ends the drain under uring_lock as its CQE is posted.
*/
- if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) {
+ if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) ||
+ (req->flags & (REQ_F_REISSUE | REQ_F_IO_DRAIN))) {
defer_complete:
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
@@ -1133,11 +1181,33 @@ static void io_free_batch_list(struct io_ring_ctx *ctx,
} while (node);
}
+/*
+ * The dispatched drain request ran alone, and anything submitted after it was
+ * deferred. Once it completed, new requests may run if nothing is waiting.
+ */
+static __cold bool io_drain_req_completed(struct io_ring_ctx *ctx)
+{
+ struct io_wq_work_node *node;
+
+ __wq_list_for_each(node, &ctx->submit_state.compl_reqs) {
+ struct io_kiocb *req = container_of(node, struct io_kiocb,
+ comp_list);
+
+ /* a requeued request hasn't completed */
+ if (req == ctx->drain_req && !(req->flags & REQ_F_REISSUE)) {
+ ctx->drain_req = NULL;
+ return true;
+ }
+ }
+ return false;
+}
+
void __io_submit_flush_completions(struct io_ring_ctx *ctx)
__must_hold(&ctx->uring_lock)
{
struct io_submit_state *state = &ctx->submit_state;
struct io_wq_work_node *node;
+ bool drain_done = false;
__io_cq_lock(ctx);
__wq_list_for_each(node, &state->compl_reqs) {
@@ -1159,13 +1229,21 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
}
__io_cq_unlock_post(ctx);
+ if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) && ctx->drain_req)
+ drain_done = io_drain_req_completed(ctx);
+
if (!wq_list_empty(&state->compl_reqs)) {
io_free_batch_list(ctx, state->compl_reqs.first);
INIT_WQ_LIST(&state->compl_reqs);
}
- if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE))
- io_queue_deferred(ctx);
+ if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
+ /* a drain request started from the list keeps the drain going */
+ if (io_queue_deferred(ctx))
+ drain_done = false;
+ if (drain_done && list_empty(&ctx->defer_list))
+ ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
+ }
ctx->submit_state.cq_flush = false;
}
@@ -1358,20 +1436,6 @@ static __cold void io_drain_req(struct io_kiocb *req)
ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
}
-static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
- unsigned int issue_flags)
-{
- if (req->file || !def->needs_file)
- return true;
-
- if (req->flags & REQ_F_FIXED_FILE)
- req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
- else
- req->file = io_file_get_normal(req, req->cqe.fd);
-
- return !!req->file;
-}
-
#define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT)
static inline int __io_issue_sqe(struct io_kiocb *req,
@@ -1418,6 +1482,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
ret = __io_issue_sqe(req, issue_flags, def);
if (ret == IOU_COMPLETE) {
+ /* not all handlers take care of the internal restart codes */
+ if (unlikely(req->cqe.res <= -ERESTARTSYS))
+ req->cqe.res = io_fixup_restart_res(req->cqe.res);
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
io_req_complete_defer(req);
else
@@ -1432,6 +1499,8 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
if (req->flags & REQ_F_IOPOLL)
io_iopoll_req_issued(req, issue_flags);
}
+ if (unlikely(ret <= -ERESTARTSYS))
+ ret = io_fixup_restart_res(ret);
return ret;
}
@@ -1454,22 +1523,35 @@ int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
return ret;
}
-struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
+/*
+ * io-wq is done with @work. Returns true if it held the last reference, and
+ * hands out the next request of the link then. The caller has to make sure
+ * that cancellations can find the latter before it calls io_wq_free_work().
+ */
+bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct io_kiocb *nxt = NULL;
- if (req_ref_put_and_test_atomic(req)) {
- if (req->flags & IO_REQ_LINK_FLAGS) {
- struct io_ring_ctx *ctx = req->ctx;
+ *linked = NULL;
+ if (!req_ref_put_and_test_atomic(req))
+ return false;
- mutex_lock(&ctx->uring_lock);
- nxt = io_req_find_next(req);
- mutex_unlock(&ctx->uring_lock);
- }
- io_free_req(req);
+ if (req->flags & IO_REQ_LINK_FLAGS) {
+ struct io_ring_ctx *ctx = req->ctx;
+
+ mutex_lock(&ctx->uring_lock);
+ nxt = io_req_find_next(req);
+ mutex_unlock(&ctx->uring_lock);
}
- return nxt ? &nxt->work : NULL;
+ if (nxt)
+ *linked = &nxt->work;
+ return true;
+}
+
+void io_wq_free_work(struct io_wq_work *work)
+{
+ io_free_req(container_of(work, struct io_kiocb, work));
}
void io_wq_submit_work(struct io_wq_work *work)
@@ -1480,7 +1562,7 @@ void io_wq_submit_work(struct io_wq_work *work)
bool needs_poll = false;
int ret = 0, err = -ECANCELED;
- /* one will be dropped by io_wq_free_work() after returning to io-wq */
+ /* one will be dropped by io_wq_put_work() after returning to io-wq */
if (!(req->flags & REQ_F_REFCOUNT))
__io_req_set_refcount(req, 2);
else
@@ -1758,17 +1840,27 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
* A 128b op on a non-128b SQ requires mixed SQE support as
* well as 2 contiguous entries.
*/
- if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || *left < 2 ||
+ if (!(ctx->flags & IORING_SETUP_SQE_MIXED) ||
(unsigned)(sqe - ctx->sq_sqes) >= ctx->sq_entries - 1)
return io_init_fail_req(req, -EINVAL);
+ if (*left >= 2) {
+ /*
+ * A 128b operation on a mixed SQ uses two entries, so
+ * we have to increment the head and cached refs, and
+ * decrement what's left.
+ */
+ current->io_uring->cached_refs++;
+ (*left)--;
+ } else if (!(ctx->flags & IORING_SETUP_SQPOLL) ||
+ !__io_sqring_entries(ctx)) {
+ return io_init_fail_req(req, -EINVAL);
+ }
/*
- * A 128b operation on a mixed SQ uses two entries, so we have
- * to increment the head and cached refs, and decrement what's
- * left.
+ * else: the SQPOLL thread caps the number of entries it submits
+ * in one go, which is none of the application's business. Take
+ * the 2nd entry on top.
*/
- current->io_uring->cached_refs++;
ctx->cached_sq_head++;
- (*left)--;
}
if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
@@ -2297,6 +2389,7 @@ struct io_tctx_exit {
struct callback_head task_work;
struct completion completion;
struct io_ring_ctx *ctx;
+ bool in_cancel;
};
static __cold void io_tctx_exit_cb(struct callback_head *cb)
@@ -2311,7 +2404,8 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb)
* tctx can be NULL if the queueing of this task_work raced with
* work cancelation off the exec path.
*/
- if (tctx && !atomic_read(&tctx->in_cancel))
+ work->in_cancel = tctx && atomic_read(&tctx->in_cancel);
+ if (tctx && !work->in_cancel)
io_uring_del_tctx_node((unsigned long)work->ctx);
complete(&work->completion);
}
@@ -2405,6 +2499,14 @@ static __cold void io_ring_exit_work(struct work_struct *work)
* wait is marked as interruptible.
*/
wait_for_completion_interruptible(&exit.completion);
+ /*
+ * The task removes the node when it is done with its
+ * cancellation. Give it the time for that: it doesn't get back
+ * to it as long as it finds new task_work each time it has run
+ * the old one, and we may be one of many who send it.
+ */
+ if (exit.in_cancel)
+ schedule_timeout_idle(HZ / 20);
mutex_lock(&ctx->uring_lock);
mutex_lock(&ctx->tctx_lock);
}
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index cb736b815422..684553242c41 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -203,7 +203,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
__cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx);
void __io_submit_flush_completions(struct io_ring_ctx *ctx);
-struct io_wq_work *io_wq_free_work(struct io_wq_work *work);
+bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked);
+void io_wq_free_work(struct io_wq_work *work);
void io_wq_submit_work(struct io_wq_work *work);
void io_free_req(struct io_kiocb *req);
@@ -348,6 +349,24 @@ static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags,
req->big_cqe.extra2 = extra2;
}
+static inline ssize_t io_fixup_restart_res(ssize_t ret)
+{
+ switch (ret) {
+ case -ERESTARTSYS:
+ case -ERESTARTNOINTR:
+ case -ERESTARTNOHAND:
+ case -ERESTART_RESTARTBLOCK:
+ /*
+ * We can't just restart the syscall, since previously
+ * submitted sqes may already be in progress. Just fail
+ * this IO with EINTR.
+ */
+ return -EINTR;
+ default:
+ return ret;
+ }
+}
+
static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache,
struct io_kiocb *req)
{
@@ -410,6 +429,29 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
lockdep_assert_held(&ctx->uring_lock);
}
+/*
+ * For a handler that is run by io-wq and parks the request on a list where
+ * cancellations find it, rather than completing it. Whoever tried to cancel
+ * the request before that found it running. It has flagged the work item,
+ * has searched the list in vain, and won't come back. To be called with
+ * the lock the search takes held: if the search came first, the flag is
+ * visible. For most lists that is ->uring_lock, timeouts use ->timeout_lock.
+ */
+static inline bool __io_wq_cancel_seen(struct io_kiocb *req,
+ unsigned int issue_flags)
+{
+ return (issue_flags & IO_URING_F_IOWQ) &&
+ (atomic_read(&req->work.flags) & IO_WQ_WORK_CANCEL);
+}
+
+static inline bool io_wq_cancel_seen(struct io_kiocb *req,
+ unsigned int issue_flags)
+{
+ lockdep_assert_held(&req->ctx->uring_lock);
+
+ return __io_wq_cancel_seen(req, issue_flags);
+}
+
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
{
/* order cqe stores with ring update */
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 5065a3eff809..fd0c9c49c13e 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -203,7 +203,7 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,
if (unlikely(tail == head))
return sel;
- if (head + 1 == tail)
+ if ((__u16)(head + 1) == tail)
req->flags |= REQ_F_BL_EMPTY;
buf = io_ring_head_to_buf(br, head, bl->mask);
diff --git a/io_uring/loop.c b/io_uring/loop.c
index bbbb6ef14e6a..ca185d9c0624 100644
--- a/io_uring/loop.c
+++ b/io_uring/loop.c
@@ -9,12 +9,6 @@ static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
}
-static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait)
-{
- atomic_set(&ctx->cq_wait_nr, nr_wait);
- set_current_state(TASK_INTERRUPTIBLE);
-}
-
static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
{
__set_current_state(TASK_RUNNING);
@@ -24,7 +18,7 @@ static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
unsigned nr_wait)
{
- io_loop_wait_start(ctx, nr_wait);
+ io_cq_wait_arm(ctx, nr_wait);
if (unlikely(io_local_work_pending(ctx) ||
io_loop_nr_cqes(ctx, lp) <= 0) ||
diff --git a/io_uring/mpscq.h b/io_uring/mpscq.h
index f910526766fd..da9474497cc8 100644
--- a/io_uring/mpscq.h
+++ b/io_uring/mpscq.h
@@ -25,9 +25,10 @@
* its predecessor. In between, the tail end of the queue is not yet reachable
* from the head. mpscq_pop() detects this and returns NULL, while mpscq_empty()
* reports false. The consumer must not treat such a NULL as "queue empty" - it
- * should retry later. The window is two instructions wide, but a producer can
- * be preempted inside it, so the consumer must not spin on it while holding
- * resources the producer might need to make progress.
+ * should retry later. The window is two instructions wide and the producer
+ * can't be preempted inside it, but it can be interrupted, so the consumer must
+ * not spin on it while holding resources the producer might need to make
+ * progress.
*
* The consumer side only supports a single consumer at a time, callers must
* provide their own serialization for it. The stub node is what allows the
@@ -69,6 +70,12 @@ static inline bool mpscq_push(struct mpscq *q, struct llist_node *node)
struct llist_node *prev;
node->next = NULL;
+ /*
+ * All the consumer can do about a node that isn't linked yet is to try
+ * again. If it is what preempted us, and we don't get to run as long
+ * as it does, that never ends.
+ */
+ preempt_disable();
/*
* xchg() implies a full barrier, so the initialization of the
* entry (including ->next above) is visible before the node can
@@ -77,6 +84,7 @@ static inline bool mpscq_push(struct mpscq *q, struct llist_node *node)
*/
prev = xchg(&q->tail, node);
WRITE_ONCE(prev->next, node);
+ preempt_enable();
return prev == &q->stub;
}
diff --git a/io_uring/napi.c b/io_uring/napi.c
index bfc771445912..ca0fe6f1871b 100644
--- a/io_uring/napi.c
+++ b/io_uring/napi.c
@@ -6,7 +6,7 @@
#ifdef CONFIG_NET_RX_BUSY_POLL
/* Timeout for cleanout of stale entries. */
-#define NAPI_TIMEOUT (60 * SEC_CONVERSION)
+#define NAPI_TIMEOUT (60 * HZ)
struct io_napi_entry {
unsigned int napi_id;
@@ -378,6 +378,8 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq)
if (iowq->timeout != KTIME_MAX) {
ktime_t dt = ktime_sub(iowq->timeout, io_get_time(ctx));
+ /* an absolute timeout may have passed already */
+ dt = max_t(ktime_t, dt, 0);
iowq->napi_busy_poll_dt = min_t(u64, iowq->napi_busy_poll_dt, dt);
}
diff --git a/io_uring/net.c b/io_uring/net.c
index 37fc7baa55b2..e982ce1fa4de 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -511,6 +511,24 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
return nbufs;
}
+/*
+ * A receive that is retried after a partial transfer keeps writing to what it
+ * has mapped. io_net_kbuf_recyle() only commits what was filled so far, which
+ * takes a buffer out of circulation if it's a single one that is consumed as a
+ * whole. The tail of an incrementally consumed buffer, or the yet unused
+ * buffers of a bundle, would be up for grabs. They can't be committed in
+ * advance either, the CQE has no way to tell if they end up not being used.
+ */
+static bool io_net_kbuf_can_retry(struct io_kiocb *req,
+ struct io_buffer_list *bl,
+ struct io_async_msghdr *kmsg)
+{
+ /* not ring provided, or committed at selection time */
+ if (!(req->flags & REQ_F_BUFFERS_COMMIT))
+ return true;
+ return !(bl->flags & IOBL_INC) && iter_is_ubuf(&kmsg->msg.msg_iter);
+}
+
static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
struct io_async_msghdr *kmsg, int len)
{
@@ -1122,7 +1140,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
io_kbuf_recycle(req, sel.buf_list, issue_flags);
return IOU_RETRY;
}
- if (ret > 0 && io_net_retry(sock, flags)) {
+ if (ret > 0 && io_net_retry(sock, flags) &&
+ io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) {
sr->done_io += ret;
return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
}
@@ -1209,6 +1228,8 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
*sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
if (!sel->addr)
return -ENOBUFS;
+ /* ring provided buffers can be up to 4G, keep len an int */
+ len = min_t(size_t, len, MAX_RW_COUNT);
sr->buf = sel->addr;
sr->len = len;
map_ubuf:
@@ -1283,7 +1304,8 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
io_kbuf_recycle(req, sel.buf_list, issue_flags);
return IOU_RETRY;
}
- if (ret > 0 && io_net_retry(sock, flags)) {
+ if (ret > 0 && io_net_retry(sock, flags) &&
+ io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) {
sr->len -= ret;
sr->buf += ret;
sr->done_io += ret;
diff --git a/io_uring/openclose.c b/io_uring/openclose.c
index c71242915dad..24cf3ce2e2bc 100644
--- a/io_uring/openclose.c
+++ b/io_uring/openclose.c
@@ -375,13 +375,16 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files,
fds[1] = alloc_slot ? ret : slot - 1;
files[1] = NULL;
- io_ring_submit_unlock(ctx, issue_flags);
-
- if (!copy_to_user(p->fds, fds, sizeof(fds)))
+ /*
+ * Hang on to the lock. If this fails, the slots have to be cleared, and
+ * they must still hold what was put there.
+ */
+ if (!copy_to_user(p->fds, fds, sizeof(fds))) {
+ io_ring_submit_unlock(ctx, issue_flags);
return 0;
+ }
ret = -EFAULT;
- io_ring_submit_lock(ctx, issue_flags);
err:
if (fds[0] != -1)
io_fixed_fd_remove(ctx, fds[0]);
diff --git a/io_uring/poll.c b/io_uring/poll.c
index 5447a7c24dce..4f04bda42f7f 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -241,12 +241,14 @@ static inline void io_poll_execute(struct io_kiocb *req, int res,
*/
static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
{
+ bool again;
int v;
if (unlikely(tw.cancel))
return -ECANCELED;
do {
+ again = false;
v = atomic_read(&req->poll_refs);
if (unlikely(v != 1)) {
@@ -310,9 +312,15 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
} else {
int ret;
- /* multiple refs and HUP, ensure we loop once more */
- if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1)
- v--;
+ /*
+ * Multiple refs and HUP, ensure we loop once more. Drop
+ * down to one ref now, so the next pass ends unless a
+ * wake up came in.
+ */
+ if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) {
+ atomic_sub(v - 1, &req->poll_refs);
+ again = true;
+ }
ret = io_poll_issue(req, tw);
if (ret == IOU_COMPLETE)
@@ -330,7 +338,8 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
* Release all references, retry if someone tried to restart
* task_work while we were executing it.
*/
- } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK);
+ v &= IO_POLL_REF_MASK;
+ } while (again || atomic_cmpxchg(&req->poll_refs, v, 0) != v);
io_napi_add(req);
return IOU_POLL_NO_ACTION;
@@ -617,6 +626,18 @@ static int __io_arm_poll_handler(struct io_kiocb *req,
io_poll_add_hash(req, issue_flags);
+ /*
+ * Whoever tried to cancel the request while io-wq was running it has
+ * flagged the work item. If it searched the hash table before the
+ * request got there it found nothing, and it won't come back. Both
+ * the search and the insertion are done with ->uring_lock held, and
+ * the flag is set before the search. If the search came first,
+ * the flag is visible by now.
+ */
+ if (ipt->owning &&
+ (atomic_read(&req->work.flags) & IO_WQ_WORK_CANCEL))
+ io_poll_mark_cancelled(req);
+
if (mask && (poll->events & EPOLLET) &&
io_poll_can_finish_inline(req, ipt)) {
__io_poll_execute(req, mask, 0);
@@ -906,6 +927,15 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
ipt.pt._qproc = io_poll_queue_proc;
+ /*
+ * A poll update or a spurious wake up arms the request again. Forget
+ * the entries of the previous round like io_arm_apoll() does, a stale
+ * second entry would be taken for a third wait queue.
+ */
+ if (req_has_async_data(req))
+ io_req_async_data_free(req);
+ req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL);
+
ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);
if (ret > 0) {
io_req_set_res(req, ipt.result_mask, 0);
diff --git a/io_uring/register.c b/io_uring/register.c
index dab3eb4258df..2ce36623e25a 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -496,6 +496,17 @@ static void io_register_free_rings(struct io_ring_ctx *ctx,
IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \
IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED)
+static bool io_sqe_is_128(struct io_ring_ctx *ctx,
+ const struct io_uring_sqe *sqe)
+{
+ u8 opcode = READ_ONCE(sqe->opcode);
+
+ if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || opcode >= IORING_OP_LAST)
+ return false;
+ opcode = array_index_nospec(opcode, IORING_OP_LAST);
+ return io_issue_defs[opcode].is_128;
+}
+
static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
{
struct io_ctx_config config;
@@ -505,6 +516,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
struct io_uring_params *p = &config.p;
struct io_rings_layout *rl = &config.layout;
u32 *o_sq_array, *n_sq_array = NULL;
+ bool cqe32_tail = false;
+ unsigned int sqe128_src = 0;
int ret;
memset(&config, 0, sizeof(config));
@@ -603,13 +616,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
old_head = READ_ONCE(o.rings->sq.head);
if (tail - old_head > p->sq_entries)
goto overflow;
- for (i = old_head; i < tail; i++) {
+ for (i = old_head; i != tail; i++) {
+ bool sqe128_tail = sqe128_src;
unsigned int dst, src;
size_t sq_size;
dst = i & (p->sq_entries - 1);
src = i & (ctx->sq_entries - 1);
- if (n_sq_array) {
+ if (sqe128_tail) {
+ /* it follows the first half, whatever the array says */
+ src = sqe128_src;
+ sqe128_src = 0;
+ if (n_sq_array)
+ WRITE_ONCE(n_sq_array[dst], dst);
+ } else if (n_sq_array) {
src = READ_ONCE(o_sq_array[src]);
if (unlikely(src >= ctx->sq_entries)) {
WRITE_ONCE(n_sq_array[dst], UINT_MAX);
@@ -618,6 +638,19 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
WRITE_ONCE(n_sq_array[dst], dst);
}
+ /*
+ * A 128b SQE on a SQE_MIXED ring takes two contiguous entries
+ * and must not wrap. A smaller ring may place it on the last
+ * entry, fail the resize if that happens.
+ */
+ if (!sqe128_tail && i + 1 != tail &&
+ src < ctx->sq_entries - 1 &&
+ io_sqe_is_128(ctx, &o.sq_sqes[src])) {
+ if (dst == p->sq_entries - 1)
+ goto overflow;
+ sqe128_src = src + 1;
+ }
+
sq_size = sizeof(struct io_uring_sqe);
if (ctx->flags & IORING_SETUP_SQE128) {
dst <<= 1;
@@ -640,7 +673,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
ret = -EOVERFLOW;
goto out;
}
- for (i = old_head; i < tail; i++) {
+ for (i = old_head; i != tail; i++) {
unsigned index, dst_mask, src_mask;
size_t cq_size;
@@ -654,6 +687,21 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
src_mask = (ctx->cq_entries << 1) - 1;
dst_mask = (p->cq_entries << 1) - 1;
}
+ /*
+ * A 32b CQE on a CQE_MIXED ring takes two contiguous entries
+ * and must not wrap. A smaller ring may place it on the last
+ * entry, fail the resize if that happens.
+ */
+ if (ctx->flags & IORING_SETUP_CQE_MIXED) {
+ if (cqe32_tail) {
+ cqe32_tail = false;
+ } else if (READ_ONCE(o.rings->cqes[index & src_mask].flags) &
+ IORING_CQE_F_32) {
+ if ((index & dst_mask) == dst_mask)
+ goto overflow;
+ cqe32_tail = true;
+ }
+ }
memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size);
}
WRITE_ONCE(n.rings->cq.head, old_head);
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 45a96e6546fc..a00b89868cb3 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -624,24 +624,6 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
smp_store_release(&req->iopoll_completed, 1);
}
-static inline ssize_t io_fixup_restart_res(ssize_t ret)
-{
- switch (ret) {
- case -ERESTARTSYS:
- case -ERESTARTNOINTR:
- case -ERESTARTNOHAND:
- case -ERESTART_RESTARTBLOCK:
- /*
- * We can't just restart the syscall, since previously
- * submitted sqes may already be in progress. Just fail
- * this IO with EINTR.
- */
- return -EINTR;
- default:
- return ret;
- }
-}
-
static inline void io_rw_done(struct io_kiocb *req, ssize_t ret)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c
index 2460bd605266..2991eaec256d 100644
--- a/io_uring/sqpoll.c
+++ b/io_uring/sqpoll.c
@@ -380,6 +380,13 @@ static int io_sq_thread(void *data)
*/
smp_mb__after_atomic();
+ /*
+ * Nothing gets submitted before the ring is
+ * enabled, which comes with a wake up.
+ */
+ if (READ_ONCE(ctx->flags) & IORING_SETUP_R_DISABLED)
+ continue;
+
if (io_sqring_entries(ctx)) {
needs_sched = false;
break;
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index c4dd26cf342d..ca03cb95ad8d 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -171,6 +171,14 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
raw_spin_lock_irq(&ctx->timeout_lock);
seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
+ /*
+ * A fired timeout bumps ->cq_timeouts before its CQE is posted, which
+ * leaves the sequence behind the last flush until it is. No event has
+ * happened then, see io_timeout().
+ */
+ if ((s32)(seq - ctx->cq_last_tm_flush) < 0)
+ seq = ctx->cq_last_tm_flush;
+
list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
struct io_kiocb *req = cmd_to_io_kiocb(timeout);
u32 events_needed, events_got;
@@ -321,22 +329,29 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
-static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
- struct io_cancel_data *cd)
+static struct io_kiocb *io_timeout_find(struct io_ring_ctx *ctx,
+ struct io_cancel_data *cd)
__must_hold(&ctx->timeout_lock)
{
struct io_timeout *timeout;
- struct io_timeout_data *io;
- struct io_kiocb *req = NULL;
list_for_each_entry(timeout, &ctx->timeout_list, list) {
- struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
+ struct io_kiocb *req = cmd_to_io_kiocb(timeout);
- if (io_cancel_req_match(tmp, cd)) {
- req = tmp;
- break;
- }
+ if (io_cancel_req_match(req, cd))
+ return req;
}
+ return NULL;
+}
+
+static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
+ struct io_cancel_data *cd)
+ __must_hold(&ctx->timeout_lock)
+{
+ struct io_kiocb *req = io_timeout_find(ctx, cd);
+ struct io_timeout *timeout;
+ struct io_timeout_data *io;
+
if (!req)
return ERR_PTR(-ENOENT);
@@ -386,6 +401,9 @@ static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw)
};
ret = io_try_cancel(req->tctx, &cd, 0);
+ /* a number of requests for futex and waitid */
+ if (ret > 0)
+ ret = 0;
} else {
ret = -ECANCELED;
}
@@ -458,6 +476,8 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
io = req->async_data;
if (hrtimer_try_to_cancel(&io->timer) == -1)
return -EALREADY;
+ if (mode == HRTIMER_MODE_ABS)
+ ts = timens_ktime_to_host(io_timeout_get_clock(io), ts);
hrtimer_setup(&io->timer, io_link_timeout_fn, io_timeout_get_clock(io), mode);
hrtimer_start(&io->timer, ts, mode);
return 0;
@@ -468,15 +488,26 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
__must_hold(&ctx->timeout_lock)
{
struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };
- struct io_kiocb *req = io_timeout_extract(ctx, &cd);
+ struct io_kiocb *req = io_timeout_find(ctx, &cd);
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_timeout_data *data;
- if (IS_ERR(req))
- return PTR_ERR(req);
+ if (!req)
+ return -ENOENT;
- timeout->off = 0; /* noseq */
data = req->async_data;
+ /* multishot requests only make sense with rel values */
+ if ((data->flags & IORING_TIMEOUT_MULTISHOT) && mode == HRTIMER_MODE_ABS)
+ return -EINVAL;
+
+ req = io_timeout_extract(ctx, &cd);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+ /* for a multishot request this isn't a sequence, keep the repeats */
+ if (!(data->flags & IORING_TIMEOUT_MULTISHOT))
+ timeout->off = 0; /* noseq */
+ if (mode == HRTIMER_MODE_ABS)
+ time = timens_ktime_to_host(io_timeout_get_clock(data), time);
data->time = time;
list_add_tail(&timeout->list, &ctx->timeout_list);
@@ -509,7 +540,12 @@ int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
IORING_TIMEOUT_ABS |
IORING_TIMEOUT_IMMEDIATE_ARG))
return -EINVAL;
- ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2), tr->flags);
+ /*
+ * The clock is that of the timeout to update, an absolute time
+ * is converted to the host's view once that has been found.
+ */
+ ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2),
+ tr->flags & ~IORING_TIMEOUT_ABS);
if (ret)
return ret;
} else if (tr->flags) {
@@ -645,11 +681,19 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
struct io_ring_ctx *ctx = req->ctx;
struct io_timeout_data *data = req->async_data;
+ struct io_timeout *first;
struct list_head *entry;
- u32 tail, off = timeout->off;
+ u32 tail, base, off = timeout->off;
raw_spin_lock_irq(&ctx->timeout_lock);
+ /* the list is searched under ->timeout_lock, see io_wq_cancel_seen() */
+ if (__io_wq_cancel_seen(req, issue_flags)) {
+ raw_spin_unlock_irq(&ctx->timeout_lock);
+ io_req_task_queue_fail(req, -ECANCELED);
+ return IOU_ISSUE_SKIP_COMPLETE;
+ }
+
/*
* sqe->off holds how many events that need to occur for this
* timeout event to be satisfied. If it isn't set, then this is
@@ -661,13 +705,27 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
}
tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
- timeout->target_seq = tail + off;
- /* Update the last seq here in case io_flush_timeouts() hasn't.
- * This is safe because ->completion_lock is held, and submissions
- * and completions are never mixed in the same ->completion_lock section.
+ /*
+ * Update the last seq here in case io_flush_timeouts() hasn't, it only
+ * gets called once sequences are in use. Leave it alone if there are
+ * timeouts waiting for one. CQEs may have been posted since the last
+ * flush, and a target that has been reached must not fall behind it.
*/
- ctx->cq_last_tm_flush = tail;
+ first = list_first_entry_or_null(&ctx->timeout_list, struct io_timeout,
+ list);
+ if (!first || io_is_timeout_noseq(cmd_to_io_kiocb(first)))
+ ctx->cq_last_tm_flush = tail;
+ base = ctx->cq_last_tm_flush;
+
+ /*
+ * A fired timeout bumps ->cq_timeouts before its CQE is posted, which
+ * leaves the sequence behind the last flush until it is. Nothing else
+ * separates the two then, so count from the flush.
+ */
+ if ((s32)(tail - base) < 0)
+ tail = base;
+ timeout->target_seq = tail + off;
/*
* Insertion sort, ensuring the first entry in the list is always
@@ -679,8 +737,8 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
if (io_is_timeout_noseq(nxt))
continue;
- /* nxt.seq is behind @tail, otherwise would've been completed */
- if (off >= nextt->target_seq - tail)
+ /* no target is behind the last flush, see io_flush_timeouts() */
+ if (timeout->target_seq - base >= nextt->target_seq - base)
break;
}
add:
diff --git a/io_uring/tw.c b/io_uring/tw.c
index f9affd68dd26..0b5066f97224 100644
--- a/io_uring/tw.c
+++ b/io_uring/tw.c
@@ -148,6 +148,37 @@ static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx)
}
}
+static void io_poll_wq_wake_rcu(struct rcu_head *rcu)
+{
+ struct io_ring_ctx *ctx = container_of(rcu, struct io_ring_ctx,
+ poll_wq_rcu);
+
+ atomic_set(&ctx->poll_wq_rcu_pending, 0);
+ io_poll_wq_wake(ctx);
+ percpu_ref_put(&ctx->refs);
+}
+
+/*
+ * io_uring_poll() reports pending local work as EPOLLIN, as it takes a trip to
+ * the kernel to turn it into CQEs. Let those know who are asleep already.
+ */
+static void io_local_work_wake_poll(struct io_ring_ctx *ctx, unsigned int flags)
+{
+ if (!(flags & IOU_F_TWQ_IN_WAKE)) {
+ io_poll_wq_wake(ctx);
+ return;
+ }
+ /*
+ * Called from a wake up handler with some wait queue lock held, this
+ * one could lead back to it. Do it from a clean context.
+ */
+ if (!wq_has_sleeper(&ctx->poll_wq) ||
+ atomic_xchg(&ctx->poll_wq_rcu_pending, 1))
+ return;
+ percpu_ref_get(&ctx->refs);
+ call_rcu_hurry(&ctx->poll_wq_rcu, io_poll_wq_wake_rcu);
+}
+
void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
{
struct io_ring_ctx *ctx = req->ctx;
@@ -165,7 +196,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
/*
* The xchg() in mpscq_push() implies a full barrier, which pairs with
- * the barrier in set_current_state() on the io_cqring_wait() side. This
+ * the barrier in io_cq_wait_arm() on the io_cqring_wait() side. This
* ensures that either we see the updated ->cq_wait_nr, or waiters going
* to sleep will observe the work added to the list, which is similar to
* the wait/wake task state sync.
@@ -174,6 +205,9 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
io_ctx_mark_taskrun(ctx);
if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD)
io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE);
+ /* whoever sets the flag wakes ->poll_wq afterwards */
+ if (data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED)
+ io_local_work_wake_poll(ctx, flags);
}
/*
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 7c1337f29a5e..f2301c455de2 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -87,7 +87,7 @@ static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
io_ring_submit_lock(ctx, issue_flags);
- hlist_del(&req->hash_node);
+ hlist_del_init(&req->hash_node);
io_ring_submit_unlock(ctx, issue_flags);
}
diff --git a/io_uring/wait.c b/io_uring/wait.c
index a48523c2f356..fb1c082993a7 100644
--- a/io_uring/wait.c
+++ b/io_uring/wait.c
@@ -3,6 +3,7 @@
* Waiting for completion events
*/
#include <linux/kernel.h>
+#include <linux/sched/rt.h>
#include <linux/sched/signal.h>
#include <linux/io_uring.h>
#include <linux/time_namespace.h>
@@ -26,7 +27,8 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
*/
if (io_should_wake(iowq) || io_has_work(iowq->ctx))
return autoremove_wake_function(curr, mode, wake_flags, key);
- return -1;
+ /* keep going, a waiter further down may be content with less */
+ return 0;
}
int io_run_task_work_sig(struct io_ring_ctx *ctx)
@@ -76,9 +78,16 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
if (iowq->timeout == KTIME_MAX ||
ktime_compare(iowq->min_timeout, iowq->timeout) >= 0)
goto out_wake;
- /* work we may need to run, wake function will see if we need to wake */
- if (io_has_work(ctx))
- goto out_wake;
+ /*
+ * Work we need to run. It isn't known if that is going to post a CQE,
+ * so this isn't a timeout. Carry on with the normal one if it doesn't.
+ */
+ if (io_has_work(ctx)) {
+ iowq->min_timeout = 0;
+ iowq->cq_tail = iowq->cq_min_tail + 1;
+ wake_up_process(iowq->wq.private);
+ return HRTIMER_NORESTART;
+ }
/* got events since we started waiting, min timeout is done */
scoped_guard(rcu) {
struct io_rings *rings = io_get_rings(ctx);
@@ -112,23 +121,32 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer)
return io_cqring_timer_wakeup(timer);
}
+/* Same policy as for hrtimer sleepers, see __hrtimer_setup_sleeper(). */
+static enum hrtimer_mode io_cqring_timer_mode(void)
+{
+ if (IS_ENABLED(CONFIG_PREEMPT_RT) && rt_or_dl_task_policy(current))
+ return HRTIMER_MODE_ABS_HARD;
+ return HRTIMER_MODE_ABS;
+}
+
static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
clockid_t clock_id, ktime_t start_time)
{
+ enum hrtimer_mode mode = io_cqring_timer_mode();
ktime_t timeout;
if (iowq->min_timeout) {
timeout = ktime_add_ns(iowq->min_timeout, start_time);
hrtimer_setup_on_stack(&iowq->t, io_cqring_min_timer_wakeup, clock_id,
- HRTIMER_MODE_ABS);
+ mode);
} else {
timeout = iowq->timeout;
hrtimer_setup_on_stack(&iowq->t, io_cqring_timer_wakeup, clock_id,
- HRTIMER_MODE_ABS);
+ mode);
}
hrtimer_set_expires_range_ns(&iowq->t, timeout, 0);
- hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
+ hrtimer_start_expires(&iowq->t, mode);
if (!READ_ONCE(iowq->hit_timeout))
schedule();
@@ -257,8 +275,7 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
unsigned long check_cq;
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
- atomic_set(&ctx->cq_wait_nr, nr_wait);
- set_current_state(TASK_INTERRUPTIBLE);
+ io_cq_wait_arm(ctx, nr_wait);
} else {
prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
TASK_INTERRUPTIBLE);
@@ -291,9 +308,18 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags,
check_cq = READ_ONCE(ctx->check_cq);
if (unlikely(check_cq)) {
- /* let the caller flush overflows, retry */
- if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
+ if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) {
io_cqring_do_overflow_flush(ctx);
+ /*
+ * No room for what has overflown. It takes the
+ * application to change that, so return to it.
+ */
+ if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT,
+ &ctx->check_cq)) {
+ ret = 0;
+ break;
+ }
+ }
if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) {
ret = -EBADR;
break;
diff --git a/io_uring/wait.h b/io_uring/wait.h
index b7b9c46b1b01..eb5ae8a6fc94 100644
--- a/io_uring/wait.h
+++ b/io_uring/wait.h
@@ -14,6 +14,21 @@
*/
#define IO_CQ_WAKE_INIT (-1)
+/*
+ * Start a wait cycle of the current task. The add that uses up ->cq_wait_nr
+ * may be one that queued its entry long ago and only now gets to look at the
+ * count. As nobody else wakes us after that, it has to find us sleeping, hence
+ * the task state must be visible before the count is. The barrier pairs with
+ * the xchg() in mpscq_push(), see io_req_local_work_add().
+ */
+static inline void io_cq_wait_arm(struct io_ring_ctx *ctx, int nr_wait)
+{
+ __set_current_state(TASK_INTERRUPTIBLE);
+ atomic_set_release(&ctx->cq_wait_nr, nr_wait);
+ /* order the above against looking at the work list */
+ smp_mb();
+}
+
struct ext_arg {
size_t argsz;
struct timespec64 ts;
diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index 2b756017423f..172cf770b29b 100644
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -154,9 +154,12 @@ static bool __io_waitid_cancel(struct io_kiocb *req, bool copy_si)
/*
* Mark us canceled regardless of ownership. This will prevent a
- * potential retry from a spurious wakeup.
+ * potential retry from a spurious wakeup. Only the first cancel
+ * attempt should take a reference for the ownership handoff.
*/
- atomic_or(IO_WAITID_CANCEL_FLAG, &iw->refs);
+ if (atomic_fetch_or(IO_WAITID_CANCEL_FLAG, &iw->refs) &
+ IO_WAITID_CANCEL_FLAG)
+ return false;
/* claim ownership */
if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
@@ -265,12 +268,18 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
return 0;
list_del_init(&wait->entry);
- smp_store_release(&iw->head, NULL);
- /* cancel is in progress */
+ /*
+ * Cancel or issue is in progress. Leave ->head alone, so that whoever
+ * owns the request has to take the wait queue lock, which we hold,
+ * before it can complete and free it.
+ */
if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
return 1;
+ /* pairs with the smp_load_acquire() in io_waitid_remove_wq() */
+ smp_store_release(&iw->head, NULL);
+
req->io_task_work.func = io_waitid_cb;
__io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE);
return 1;
@@ -343,6 +352,8 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
* a waitqueue callback, or if someone cancels it.
*/
if (!io_waitid_drop_issue_ref(req)) {
+ if (io_wq_cancel_seen(req, issue_flags))
+ __io_waitid_cancel(req, true);
io_ring_submit_unlock(ctx, issue_flags);
return IOU_ISSUE_SKIP_COMPLETE;
}
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 0a9e8a58b99a..6324d9196df6 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -1474,7 +1474,7 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
cqe->flags |= IORING_CQE_F_32;
area = io_zcrx_iov_to_area(niov);
- offset = off + (net_iov_idx(niov) << ifq->niov_shift);
+ offset = off + ((u64)net_iov_idx(niov) << ifq->niov_shift);
rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1);
rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT);
rcqe->__pad = 0;
diff --git a/kernel/exit.c b/kernel/exit.c
index f7c48f1c404f..256f9ab9cbfe 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -979,10 +979,11 @@ void __noreturn do_exit(long code)
exit_sem(tsk);
exit_shm(tsk);
- exit_files(tsk);
- exit_fs(tsk);
+ /* Hang the tty up before the last close of it can clear the session. */
if (group_dead)
disassociate_ctty(1);
+ exit_files(tsk);
+ exit_fs(tsk);
exit_nsproxy_namespaces(tsk);
exit_task_work(tsk);
exit_thread(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index 854448152696..26485ba0dac7 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1658,6 +1658,7 @@ static int copy_files(u64 clone_flags, struct task_struct *tsk,
if (clone_flags & CLONE_FILES) {
atomic_inc(&oldf->count);
+ tsk->files = oldf;
return 0;
}
@@ -2181,6 +2182,8 @@ __latent_entropy struct task_struct *copy_process(
INIT_LIST_HEAD(&p->sibling);
rcu_copy_process(p);
p->vfork_done = NULL;
+ /* Set by copy_files(), exit_files() on the error path skips NULL. */
+ p->files = NULL;
spin_lock_init(&p->alloc_lock);
init_sigpending(&p->pending);
@@ -2282,7 +2285,7 @@ __latent_entropy struct task_struct *copy_process(
goto bad_fork_cleanup_semundo;
retval = copy_fs(clone_flags, p);
if (retval)
- goto bad_fork_cleanup_files;
+ goto bad_fork_cleanup_semundo;
retval = copy_sighand(clone_flags, p);
if (retval)
goto bad_fork_cleanup_fs;
@@ -2596,8 +2599,6 @@ __latent_entropy struct task_struct *copy_process(
__cleanup_sighand(p->sighand);
bad_fork_cleanup_fs:
exit_fs(p); /* blocking */
-bad_fork_cleanup_files:
- exit_files(p); /* blocking */
bad_fork_cleanup_semundo:
exit_sem(p);
bad_fork_cleanup_security:
@@ -2608,6 +2609,8 @@ __latent_entropy struct task_struct *copy_process(
perf_event_free_task(p);
bad_fork_sched_cancel_fork:
sched_cancel_fork(p);
+ /* ->release() of a file may need scx_fork_rwsem for write. */
+ exit_files(p); /* blocking */
bad_fork_cleanup_policy:
lockdep_free_task(p);
#ifdef CONFIG_NUMA
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index 4ade0f37ed54..127a5fd23e70 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -238,9 +238,10 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns)
* kernel_wait4() will also block until our children traced from the
* parent namespace are detached and become EXIT_DEAD.
*/
+ /* Task work must not busy-loop the reaper, see signal_pending(). */
+ guard(no_notify_signal)();
do {
clear_thread_flag(TIF_SIGPENDING);
- clear_thread_flag(TIF_NOTIFY_SIGNAL);
rc = kernel_wait4(-1, NULL, __WALL, NULL);
} while (rc != -ECHILD);
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index d041645d9d17..4e9822a87aab 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -1227,6 +1227,12 @@ int ptrace_request(struct task_struct *child, long request,
case PTRACE_SETSIGMASK: {
sigset_t new_set;
+ /* A user worker only ever takes SIGKILL and SIGSTOP. */
+ if (child->flags & PF_USER_WORKER) {
+ ret = -EPERM;
+ break;
+ }
+
if (addr != sizeof(sigset_t)) {
ret = -EINVAL;
break;
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 3ee16f7f22cd..c117cf172dfc 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -2292,6 +2292,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
lockdep_assert_rq_held(dst_rq);
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
+ if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
+ wait_ops_state(p, SCX_OPSS_DISPATCHING);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
list_add(&p->scx.dsq_list.node, &dst_dsq->list);
@@ -2462,8 +2464,13 @@ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
raw_spin_rq_lock(src_rq);
/* task_rq couldn't have changed if we're still the holding cpu */
- return likely(p->scx.holding_cpu == cpu) &&
- !WARN_ON_ONCE(src_rq != task_rq(p));
+ if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
+ if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
+ wait_ops_state(p, SCX_OPSS_DISPATCHING);
+ return true;
+ }
+
+ return false;
}
static bool consume_remote_task(struct rq *this_rq,
@@ -2511,6 +2518,9 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(src_rq);
+ if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
+ wait_ops_state(p, SCX_OPSS_DISPATCHING);
+
if (dst_dsq->id == SCX_DSQ_LOCAL) {
dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
if (src_rq != dst_rq &&
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b6cace1d0e32..9e1a28a158da 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6250,12 +6250,20 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
load = cfs_rq->sum_weight;
if (curr && curr->on_rq)
load += avg_vruntime_weight(cfs_rq, curr->load.weight);
-
- weight = avg_vruntime_weight(cfs_rq, se->load.weight);
- lag *= load + weight;
if (WARN_ON_ONCE(!load))
load = 1;
- lag = div64_long(lag, load);
+
+ weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+
+ /*
+ * Rewrite, because the product (W + w_i)*vl'_i might
+ * overflow s64:
+ *
+ * vl_i = (W + w_i)*vl'_i / W
+ * = (W*vl'_i + w_i*vl'_i) / W
+ * = vl'_i + w_i*vl'_i/W
+ */
+ lag += div64_long(lag * weight, load);
/*
* A heavy entity (relative to the tree) will pull the
diff --git a/kernel/signal.c b/kernel/signal.c
index 808980d08371..67d395986739 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3181,6 +3181,10 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
sigset_t retarget;
struct task_struct *t;
+ /* Nobody dequeues them in a dying group, see get_signal(). */
+ if (tsk->signal->flags & SIGNAL_GROUP_EXIT)
+ return;
+
sigandsets(&retarget, &tsk->signal->shared_pending.signal, which);
if (sigisemptyset(&retarget))
return;
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index db8eeaefeb00..b2f74dfdb936 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -4426,9 +4426,9 @@ subsys_initcall(ftrace_check_for_weak_functions);
static int print_rec(struct seq_file *m, unsigned long ip)
{
- unsigned long offset;
+ unsigned long offset = 0;
char str[KSYM_SYMBOL_LEN];
- char *modname;
+ char *modname = NULL;
const char *ret;
ret = kallsyms_lookup(ip, NULL, &offset, &modname, str);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 9dcbe243dbe8..479b20809fa9 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -3614,6 +3614,7 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file)
return single_release(inode, file);
}
+static bool __update_last_data_if_empty(struct trace_array *tr);
static bool update_last_data_if_empty(struct trace_array *tr);
static int tracing_open(struct inode *inode, struct file *file)
@@ -4706,7 +4707,7 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf,
int tracer_init(struct tracer *t, struct trace_array *tr)
{
tracing_reset_online_cpus(&tr->array_buffer);
- update_last_data_if_empty(tr);
+ __update_last_data_if_empty(tr);
return t->init(tr);
}
@@ -5385,8 +5386,10 @@ static int tracing_wait_pipe(struct file *filp)
return 1;
}
-static bool update_last_data_if_empty(struct trace_array *tr)
+static bool __update_last_data_if_empty(struct trace_array *tr)
{
+ lockdep_assert_held(&trace_types_lock);
+
if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
return false;
@@ -5401,6 +5404,16 @@ static bool update_last_data_if_empty(struct trace_array *tr)
return true;
}
+static bool update_last_data_if_empty(struct trace_array *tr)
+{
+ /* Avoid the lock on the read path unless this is a last boot buffer */
+ if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT))
+ return false;
+
+ guard(mutex)(&trace_types_lock);
+ return __update_last_data_if_empty(tr);
+}
+
/*
* Consumer reader.
*/
@@ -6493,7 +6506,7 @@ int tracing_set_clock(struct trace_array *tr, const char *clockstr)
ring_buffer_set_clock(tr->snapshot_buffer.buffer, trace_clocks[i].func);
tracing_reset_online_cpus(&tr->snapshot_buffer);
#endif
- update_last_data_if_empty(tr);
+ __update_last_data_if_empty(tr);
if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) {
struct trace_scratch *tscratch = tr->scratch;
@@ -9261,6 +9274,8 @@ static void trace_module_record(struct module *mod, bool add)
struct trace_array *tr;
unsigned long flags;
+ guard(mutex)(&trace_types_lock);
+
list_for_each_entry(tr, &ftrace_trace_arrays, list) {
flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT);
/* Update any persistent trace array that has already been started */
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 127880ebe83f..b09acea4602b 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -595,6 +595,12 @@ predicate_parse(const char *str, int nr_parens, int nr_preds,
top--; /* #12 */
}
}
+ /* Only the inner loop can complete a non-empty expression. */
+ if (N) {
+ ret = -EINVAL;
+ parse_error(pe, FILT_ERR_INVALID_FILTER, ptr - str);
+ goto out_free;
+ }
out:
if (top != op_stack) {
/* Too many '(' */
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index 160445760545..45a3eb6a65cd 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -467,7 +467,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field,
{ \
type *addr = (type *)(event + hist_field->field->offset); \
\
- return (u64)(unsigned long)*addr; \
+ return (u64)*addr; \
}
DEFINE_HIST_FIELD_FN(s64);
@@ -550,7 +550,7 @@ struct field_var {
};
struct field_var_hist {
- struct hist_trigger_data *hist_data;
+ struct trace_event_file *file;
char *cmd;
};
@@ -3099,8 +3099,8 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data,
return ERR_PTR(-ENOMEM);
}
- /* Save the compatible histogram information */
- var_hist->hist_data = hist_data;
+ /* Needed to remove the histogram when the target goes away */
+ var_hist->file = file;
/* Create the new histogram with our variable */
ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
@@ -3115,20 +3115,18 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data,
kfree(cmd);
+ n = target_hist_data->n_field_var_hists;
+ target_hist_data->field_var_hists[n] = var_hist;
+ target_hist_data->n_field_var_hists++;
+
/* If we can't find the variable, something went wrong */
event_var = find_synthetic_field_var(target_hist_data, subsys_name,
event_name, field_name);
if (IS_ERR_OR_NULL(event_var)) {
- kfree(var_hist->cmd);
- kfree(var_hist);
hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
return ERR_PTR(-EINVAL);
}
- n = target_hist_data->n_field_var_hists;
- target_hist_data->field_var_hists[n] = var_hist;
- target_hist_data->n_field_var_hists++;
-
return event_var;
}
@@ -6331,7 +6329,7 @@ static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
int ret;
for (i = 0; i < hist_data->n_field_var_hists; i++) {
- file = hist_data->field_var_hists[i]->hist_data->event_file;
+ file = hist_data->field_var_hists[i]->file;
cmd = hist_data->field_var_hists[i]->cmd;
ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
"!hist", "hist", cmd);
@@ -6781,7 +6779,6 @@ static void hist_unreg_all(struct trace_event_file *file)
struct event_trigger_data *test, *n;
struct hist_trigger_data *hist_data;
struct synth_event *se;
- const char *se_name;
lockdep_assert_held(&event_mutex);
@@ -6794,8 +6791,7 @@ static void hist_unreg_all(struct trace_event_file *file)
list_del_rcu(&test->list);
trace_event_trigger_enable_disable(file, 0);
- se_name = trace_event_name(file->event_call);
- se = find_synth_event(se_name);
+ se = find_synth_event_call(file->event_call);
if (se)
se->ref--;
@@ -6819,7 +6815,6 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops,
struct hist_trigger_data *hist_data;
char *param, *filter, *p, *start;
struct synth_event *se;
- const char *se_name;
bool remove;
int ret = 0;
@@ -6913,8 +6908,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops,
}
event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
- se_name = trace_event_name(file->event_call);
- se = find_synth_event(se_name);
+ se = find_synth_event_call(file->event_call);
if (se)
se->ref--;
ret = 0;
@@ -6949,8 +6943,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops,
if (ret)
goto out_unreg;
- se_name = trace_event_name(file->event_call);
- se = find_synth_event(se_name);
+ se = find_synth_event_call(file->event_call);
if (se)
se->ref++;
out:
@@ -6967,6 +6960,8 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops,
trigger_data_free(trigger_data);
+ unregister_field_var_hists(hist_data);
+
destroy_hist_data(hist_data);
goto out;
}
diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c
index dc15658a887c..e2870212e58d 100644
--- a/kernel/trace/trace_events_synth.c
+++ b/kernel/trace/trace_events_synth.c
@@ -914,6 +914,16 @@ struct synth_event *find_synth_event(const char *name)
return NULL;
}
+struct synth_event *find_synth_event_call(struct trace_event_call *call)
+{
+ struct synth_event *se;
+ const char *name;
+
+ name = trace_event_name(call);
+ se = find_synth_event(name);
+ return se && &se->call == call ? se : NULL;
+}
+
static struct trace_event_fields synth_event_fields_array[] = {
{ .type = TRACE_FUNCTION_TYPE,
.define_fields = synth_event_define_fields },
diff --git a/kernel/trace/trace_synth.h b/kernel/trace/trace_synth.h
index 43f6fb6078db..d4ebf01b7049 100644
--- a/kernel/trace/trace_synth.h
+++ b/kernel/trace/trace_synth.h
@@ -37,5 +37,6 @@ struct synth_event {
};
extern struct synth_event *find_synth_event(const char *name);
+extern struct synth_event *find_synth_event_call(struct trace_event_call *call);
#endif /* __TRACE_SYNTH_H */
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 8a4f3c75e39f..6406123e702c 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -682,7 +682,7 @@ static int syscall_copy_user_array(char *buf, const char __user *ptr,
for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) {
ptr = (char __user *)args->ptr_array[i];
- ret = __copy_from_user(buf, ptr, size);
+ ret = copy_from_user(buf, ptr, size);
args->read[i] = ret ? -1 : size;
}
return 0;
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 40714c69d1b1..0aefaef46d65 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -999,15 +999,18 @@ static void damos_commit_quota_goal_union(
}
}
-static void damos_commit_quota_goal(
+static int damos_commit_quota_goal(
struct damos_quota_goal *dst, struct damos_quota_goal *src)
{
+ if (!src->target_value)
+ return -EINVAL;
dst->metric = src->metric;
dst->target_value = src->target_value;
if (dst->metric == DAMOS_QUOTA_USER_INPUT)
dst->current_value = src->current_value;
/* keep last_psi_total as is, since it will be updated in next cycle */
damos_commit_quota_goal_union(dst, src);
+ return 0;
}
/**
@@ -1025,14 +1028,17 @@ static void damos_commit_quota_goal(
int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
{
struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal;
- int i = 0, j = 0;
+ int i = 0, j = 0, err;
damos_for_each_quota_goal_safe(dst_goal, next, dst) {
src_goal = damos_nth_quota_goal(i++, src);
- if (src_goal)
- damos_commit_quota_goal(dst_goal, src_goal);
- else
+ if (src_goal) {
+ err = damos_commit_quota_goal(dst_goal, src_goal);
+ if (err)
+ return err;
+ } else {
damos_destroy_quota_goal(dst_goal);
+ }
}
damos_for_each_quota_goal_safe(src_goal, next, src) {
if (j++ < i)
@@ -1041,7 +1047,11 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
src_goal->metric, src_goal->target_value);
if (!new_goal)
return -ENOMEM;
- damos_commit_quota_goal(new_goal, src_goal);
+ err = damos_commit_quota_goal(new_goal, src_goal);
+ if (err) {
+ damos_free_quota_goal(new_goal);
+ return err;
+ }
damos_add_quota_goal(dst, new_goal);
}
return 0;
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index 26cdad603ad0..9c7e34f18b28 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -2226,6 +2226,20 @@ static int damon_sysfs_repeat_call_fn(void *data)
return 0;
}
+static bool damon_sysfs_valid_start_ctx(struct damon_ctx *ctx)
+{
+ struct damos *s;
+ struct damos_quota_goal *g;
+
+ damon_for_each_scheme(s, ctx) {
+ damos_for_each_quota_goal(g, &s->quota) {
+ if (!g->target_value)
+ return false;
+ }
+ }
+ return true;
+}
+
static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
{
struct damon_ctx *ctx;
@@ -2251,6 +2265,11 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
kfree(repeat_call_control);
return PTR_ERR(ctx);
}
+ if (!damon_sysfs_valid_start_ctx(ctx)) {
+ kfree(repeat_call_control);
+ damon_destroy_ctx(ctx);
+ return -EINVAL;
+ }
err = damon_start(&ctx, 1, false);
if (err) {
kfree(repeat_call_control);
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 9a1b0ae07679..fa17f235d5e4 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -622,6 +622,10 @@ void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan)
BT_DBG("conn %p, psm 0x%2.2x, dcid 0x%4.4x", conn,
__le16_to_cpu(chan->psm), chan->dcid);
+ /* Caller must ensure l2cap_chan is linked to l2cap_conn only once */
+ if (WARN_ON_ONCE(chan->conn || test_bit(FLAG_DEL, &chan->flags)))
+ return;
+
conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM;
chan->conn = l2cap_conn_get(conn);
@@ -7412,7 +7416,8 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid,
}
}
- if (cid && __l2cap_get_chan_by_dcid(conn, cid)) {
+ if ((cid && __l2cap_get_chan_by_dcid(conn, cid)) || chan->conn ||
+ test_bit(FLAG_DEL, &chan->flags)) {
hci_conn_drop(hcon);
err = -EBUSY;
goto chan_unlock;
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 143654a47989..7eefb46f0692 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -109,6 +109,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i
return -EINVAL;
}
+ l2cap_chan_lock(chan);
lock_sock(sk);
if (sk->sk_state != BT_OPEN) {
@@ -174,6 +175,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i
done:
release_sock(sk);
+ l2cap_chan_unlock(chan);
return err;
}
diff --git a/security/keys/key.c b/security/keys/key.c
index a438c4508595..6ebb8691551f 100644
--- a/security/keys/key.c
+++ b/security/keys/key.c
@@ -588,11 +588,18 @@ int key_reject_and_link(struct key *key,
ret = -EBUSY;
if (keyring) {
- if (keyring->restrict_link)
- return -EPERM;
-
link_ret = __key_link_lock(keyring, &key->index_key);
if (link_ret == 0) {
+ /*
+ * Check the restriction under the keyring semaphore.
+ * keyring_restrict() installs it while holding the
+ * semaphore, so testing it beforehand races with a
+ * concurrent restriction install.
+ */
+ if (keyring->restrict_link) {
+ __key_link_end(keyring, &key->index_key, edit);
+ return -EPERM;
+ }
link_ret = __key_link_begin(keyring, &key->index_key, &edit);
if (link_ret < 0)
__key_link_end(keyring, &key->index_key, edit);
diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c
index d14ace88e529..36ffba0ebba6 100644
--- a/security/keys/keyctl.c
+++ b/security/keys/keyctl.c
@@ -677,12 +677,14 @@ long keyctl_describe_key(key_serial_t keyid,
/* calculate how much information we're going to return */
ret = -ENOMEM;
+ down_read(&key->sem);
infobuf = kasprintf(GFP_KERNEL,
"%s;%d;%d;%08x;",
key->type->name,
from_kuid_munged(current_user_ns(), key->uid),
from_kgid_munged(current_user_ns(), key->gid),
key->perm);
+ up_read(&key->sem);
if (!infobuf)
goto error2;
infolen = strlen(infobuf);
@@ -806,8 +808,12 @@ static long __keyctl_read_key(struct key *key, char *buffer, size_t buflen)
down_read(&key->sem);
ret = key_validate(key);
- if (ret == 0)
- ret = key->type->read(key, buffer, buflen);
+ if (ret == 0) {
+ if (key->type->read)
+ ret = key->type->read(key, buffer, buflen);
+ else
+ ret = -EOPNOTSUPP;
+ }
up_read(&key->sem);
return ret;
}
@@ -861,11 +867,6 @@ long keyctl_read_key(key_serial_t keyid, char __user *buffer, size_t buflen)
/* the key is probably readable - now try to read it */
can_read_key:
- if (!key->type->read) {
- ret = -EOPNOTSUPP;
- goto key_put_out;
- }
-
if (!buffer || !buflen) {
/* Get the key length from the read method */
ret = __keyctl_read_key(key, NULL, 0);
diff --git a/security/keys/keyring.c b/security/keys/keyring.c
index 15bf4af8f282..7143f3079d20 100644
--- a/security/keys/keyring.c
+++ b/security/keys/keyring.c
@@ -14,6 +14,9 @@
#include <linux/err.h>
#include <linux/user_namespace.h>
#include <linux/nsproxy.h>
+#include <linux/once.h>
+#include <linux/random.h>
+#include <linux/siphash.h>
#include <keys/keyring-type.h>
#include <keys/user-type.h>
#include <linux/assoc_array_priv.h>
@@ -147,49 +150,46 @@ static int keyring_instantiate(struct key *keyring,
}
/*
- * Multiply 64-bits by 32-bits to 96-bits and fold back to 64-bit. Ideally we'd
- * fold the carry back too, but that requires inline asm.
+ * Give key types and domains address-independent identities. Zero means that
+ * an ID has not yet been allocated. As with inode sequence numbers used by
+ * futexes, this relies on the 64-bit counter not wrapping in a machine's
+ * lifetime.
*/
-static u64 mult_64x32_and_fold(u64 x, u32 y)
+static u64 key_index_id(atomic64_t *id)
{
- u64 hi = (u64)(u32)(x >> 32) * y;
- u64 lo = (u64)(u32)(x) * y;
- return lo + ((u64)(u32)hi << 32) + (u32)(hi >> 32);
+ static atomic64_t next_id;
+ u64 old, new;
+
+ old = atomic64_read(id);
+ if (likely(old))
+ return old;
+
+ do {
+ new = atomic64_inc_return(&next_id);
+ } while (WARN_ON_ONCE(!new));
+
+ old = 0;
+ if (!atomic64_try_cmpxchg_relaxed(id, &old, new))
+ return old;
+ return new;
}
/*
- * Hash a key type and description.
+ * Hash a key type, domain and description. KEYCTL_READ exposes the tree's
+ * traversal order, so no part of the index may contain kernel addresses.
*/
static void hash_key_type_and_desc(struct keyring_index_key *index_key)
{
const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP;
const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK;
- const char *description = index_key->description;
- unsigned long hash, type;
- u32 piece;
+ static siphash_key_t hash_key __read_mostly;
+ unsigned long hash;
u64 acc;
- int n, desc_len = index_key->desc_len;
-
- type = (unsigned long)index_key->type;
- acc = mult_64x32_and_fold(type, desc_len + 13);
- acc = mult_64x32_and_fold(acc, 9207);
- piece = (unsigned long)index_key->domain_tag;
- acc = mult_64x32_and_fold(acc, piece);
- acc = mult_64x32_and_fold(acc, 9207);
-
- for (;;) {
- n = desc_len;
- if (n <= 0)
- break;
- if (n > 4)
- n = 4;
- piece = 0;
- memcpy(&piece, description, n);
- description += n;
- desc_len -= n;
- acc = mult_64x32_and_fold(acc, piece);
- acc = mult_64x32_and_fold(acc, 9207);
- }
+
+ get_random_once(&hash_key, sizeof(hash_key));
+ acc = siphash(index_key->description, index_key->desc_len, &hash_key);
+ acc = siphash_3u64(index_key->type_id, index_key->domain_id, acc,
+ &hash_key);
/* Fold the hash down to 32 bits if need be. */
hash = acc;
@@ -209,7 +209,7 @@ static void hash_key_type_and_desc(struct keyring_index_key *index_key)
/*
* Finalise an index key to include a part of the description actually in the
- * index key, to set the domain tag and to calculate the hash.
+ * index key, to set the domain tag and index IDs, and to calculate the hash.
*/
void key_set_index_key(struct keyring_index_key *index_key)
{
@@ -225,6 +225,14 @@ void key_set_index_key(struct keyring_index_key *index_key)
index_key->domain_tag = &default_domain_tag;
}
+ /* Preserve the IDs when copying an existing key's index. In
+ * particular, its type may already have been unregistered.
+ */
+ if (!index_key->type_id)
+ index_key->type_id = key_index_id(&index_key->type->index_id);
+ if (!index_key->domain_id)
+ index_key->domain_id = key_index_id(&index_key->domain_tag->index_id);
+
hash_key_type_and_desc(index_key);
}
@@ -263,15 +271,19 @@ void key_remove_domain(struct key_tag *domain_tag)
/*
* Build the next index key chunk.
*
+ * The hash and inline description are followed by the 64-bit type and domain
+ * IDs, low word first, and then by the rest of the description.
* We return it one word-sized chunk at a time.
*/
static unsigned long keyring_get_key_chunk(const void *data, int level)
{
+ const unsigned int id_chunks = sizeof(u64) / sizeof(unsigned long);
const struct keyring_index_key *index_key = data;
+ int desc_len = index_key->desc_len;
unsigned long chunk = 0;
- const u8 *d;
- int desc_len = index_key->desc_len, n = sizeof(chunk);
+ int n = sizeof(chunk);
unsigned int offset;
+ const u8 *d;
level /= ASSOC_ARRAY_KEY_CHUNK_SIZE;
switch (level) {
@@ -279,27 +291,30 @@ static unsigned long keyring_get_key_chunk(const void *data, int level)
return index_key->hash;
case 1:
return index_key->x;
- case 2:
- return (unsigned long)index_key->type;
- case 3:
- return (unsigned long)index_key->domain_tag;
- default:
- level -= 4;
- offset = sizeof(index_key->desc) + level * sizeof(long);
- if (desc_len <= offset)
- return 0;
-
- d = index_key->description + offset;
- desc_len -= offset;
- if (desc_len > n)
- desc_len = n;
- d += desc_len;
- do {
- chunk <<= 8;
- chunk |= *--d;
- } while (--desc_len > 0);
- return chunk;
}
+
+ level -= 2;
+ if (level < id_chunks)
+ return index_key->type_id >> (level * ASSOC_ARRAY_KEY_CHUNK_SIZE);
+ level -= id_chunks;
+ if (level < id_chunks)
+ return index_key->domain_id >> (level * ASSOC_ARRAY_KEY_CHUNK_SIZE);
+ level -= id_chunks;
+
+ offset = sizeof(index_key->desc) + level * sizeof(long);
+ if (desc_len <= offset)
+ return 0;
+
+ d = index_key->description + offset;
+ desc_len -= offset;
+ if (desc_len > n)
+ desc_len = n;
+ d += desc_len;
+ do {
+ chunk <<= 8;
+ chunk |= *--d;
+ } while (--desc_len > 0);
+ return chunk;
}
static unsigned long keyring_get_object_key_chunk(const void *object, int level)
@@ -331,6 +346,7 @@ static int keyring_diff_objects(const void *object, const void *data)
const struct keyring_index_key *b = data;
unsigned long seg_a, seg_b;
int level, i;
+ u64 diff;
level = 0;
seg_a = a->hash;
@@ -339,28 +355,22 @@ static int keyring_diff_objects(const void *object, const void *data)
goto differ;
level += ASSOC_ARRAY_KEY_CHUNK_SIZE / 8;
- /* The number of bits contributed by the hash is controlled by a
- * constant in the assoc_array headers. Everything else thereafter we
- * can deal with as being machine word-size dependent.
- */
+ /* The inline description contributes one machine-word-sized chunk. */
seg_a = a->x;
seg_b = b->x;
if ((seg_a ^ seg_b) != 0)
goto differ;
level += sizeof(unsigned long);
- /* The next bit may not work on big endian */
- seg_a = (unsigned long)a->type;
- seg_b = (unsigned long)b->type;
- if ((seg_a ^ seg_b) != 0)
- goto differ;
- level += sizeof(unsigned long);
+ diff = a->type_id ^ b->type_id;
+ if (diff)
+ return level * 8 + __ffs64(diff);
+ level += sizeof(u64);
- seg_a = (unsigned long)a->domain_tag;
- seg_b = (unsigned long)b->domain_tag;
- if ((seg_a ^ seg_b) != 0)
- goto differ;
- level += sizeof(unsigned long);
+ diff = a->domain_id ^ b->domain_id;
+ if (diff)
+ return level * 8 + __ffs64(diff);
+ level += sizeof(u64);
i = sizeof(a->desc);
if (a->desc_len <= i)
@@ -1158,7 +1168,7 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring)
* grants Search permission and that hasn't been revoked
*/
list_for_each_entry(keyring, &ns->keyring_name_list, name_link) {
- if (!kuid_has_mapping(ns, keyring->user->uid))
+ if (!kuid_has_mapping(ns, keyring->uid))
continue;
if (test_bit(KEY_FLAG_REVOKED, &keyring->flags))
diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
index a63c46bb2d14..76da13fa07fa 100644
--- a/security/keys/process_keys.c
+++ b/security/keys/process_keys.c
@@ -556,9 +556,8 @@ key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx)
) {
const struct cred *cred = ctx->cred;
- if (key_validate(cred->request_key_auth) == 0) {
- rka = ctx->cred->request_key_auth->payload.data[0];
-
+ rka = dereference_key_rcu(cred->request_key_auth);
+ if (rka && key_validate(cred->request_key_auth) == 0) {
//// was search_process_keyrings() [ie. recursive]
ctx->cred = rka->cred;
key_ref = search_cred_keyrings_rcu(ctx);
diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c
index 8f57c6111e7e..38ffaccf850a 100644
--- a/security/keys/trusted-keys/trusted_tpm1.c
+++ b/security/keys/trusted-keys/trusted_tpm1.c
@@ -928,8 +928,10 @@ static int trusted_tpm_unseal(struct trusted_key_payload *p, char *datablob)
ret = tpm2_unseal_trusted(chip, p, options);
else
ret = key_unseal(p, options);
- if (ret < 0)
+ if (ret < 0) {
pr_info("key_unseal failed (%d)\n", ret);
+ goto out;
+ }
if (options->pcrlock) {
ret = pcrlock(options->pcrlock);
diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c
index e8c129396b8b..fab793912fd1 100644
--- a/security/keys/trusted-keys/trusted_tpm2.c
+++ b/security/keys/trusted-keys/trusted_tpm2.c
@@ -116,7 +116,7 @@ static int tpm2_key_decode(struct trusted_key_payload *payload,
if (ctx.priv_len + ctx.pub_len > MAX_BLOB_SIZE)
return -EINVAL;
- blob = kmalloc(ctx.priv_len + ctx.pub_len + 4, GFP_KERNEL);
+ blob = kmalloc(ctx.priv_len + ctx.pub_len, GFP_KERNEL);
if (!blob)
return -ENOMEM;
diff --git a/tools/dma/Makefile b/tools/dma/Makefile
index e4abf37bf020..0c239a6b9eae 100644
--- a/tools/dma/Makefile
+++ b/tools/dma/Makefile
@@ -38,8 +38,8 @@ prepare: $(OUTPUT)include/linux/map_benchmark.h
FORCE:
DMA_MAP_BENCHMARK = dma_map_benchmark
-$(DMA_MAP_BENCHMARK): prepare FORCE
- $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $(DMA_MAP_BENCHMARK)
+$(OUTPUT)$(DMA_MAP_BENCHMARK): prepare FORCE
+ $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $@
clean:
rm -f $(ALL_PROGRAMS)
diff --git a/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc b/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc
new file mode 100644
index 000000000000..cc6df762845f
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc
@@ -0,0 +1,24 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+# description: event filter - reject incomplete logical expressions
+# requires: events/sched/sched_switch/filter
+# flags: instance
+
+filter=events/sched/sched_switch/filter
+
+for base in \
+ 'common_pid == 1' \
+ 'common_pid == 1 || common_pid == 2' \
+ 'common_pid == 1 && common_pid == 2' \
+ '!(common_pid == 1 || common_pid == 2)'
+do
+ printf '%s' "$base" > "$filter"
+ for tail in '&&' '||' '&& !' '|| !' '&& !! ' '|| !! '; do
+ if printf '%s' "$base $tail" > "$filter" 2>/dev/null; then
+ echo "Accepted incomplete filter: $base $tail"
+ exit_fail
+ fi
+ done
+done
+
+echo 0 > "$filter"
^ permalink raw reply [flat|nested] 2+ messages in thread