From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.remote-ip=100.103.45.18 ARC-Seal: i=1; d=subspace.kernel.org; s=arc20261015; a=rsa-sha256; cv=none; t=1793362366; b=i/JjH/xvv/G3O1Gpfr7JJfk7HCPE8U+qMtb39cdD5Gdt9v5PzitNHt4Xmmdkjz2vdaom g4zuopzP0apnRKWOeQLGkvAgTj6s+iEg0ESkRJKQ8riRGr2rDSfXjDEGX4+1t5S76EWrp eB5SDDtLZ+M4Ewly6L2+71BSBgTDvfdTY/3z05n7txGHjFYtfg0mocPZVrwYKKsgRz0R0 2PaOEmckRSEhqPoy00b5WOhwIlV+xtaisaAkX6fomwFswNJ4xqz2LUQxCvbZb3T2vks4R fY16z8pfE5QFRmWOH5LYKfmtb8DBnrLlwBzx0D8oHNKOMroFui8dGCxMy5p2ZpBwjWQ== ARC-Message-Signature: i=1; d=subspace.kernel.org; s=arc20261015; a=rsa-sha256; c=relaxed/relaxed; t=1793362366; h=Received:DKIM-Signature:From:To:Cc:Subject:Date:Message-ID:X-Mailer: In-Reply-To:References:MIME-Version:Content-Transfer-Encoding; bh=ci0QkwBiOudZUlUkeYQaqYBZ+4VnhyCQoTpwhsuLJlI=; b=XuPwXKyt4Aq3awbTfwaaU5Z5gK7zXTlImi7S0wIyqhtqI1f5RXnajrylvmrLvNQ8HNB1 yxMt9R0Dn87x+XK8DbENnT/OwbnPmuvd7GIt63itn58hXj1EEvPaGdx9xIhsawaE+4dJK MgVhhelH/Bxahmn8YzwfFSDba9aeZRIJkGUIrZp+nL4ARjl5sBY/bm34WRriL1FeWUrmz EkRAgmaL7/clSojUZUUb5ZwiciANwEf3tGGHImsCuEZykA8Q8fLE+0txUSYlcy0LZM1M3 x4oFidXMP1kk3cvHcsRMHTZB/ATfxMHWVlVSU6nbTKQSlMHJ0u1oNwGEr7RuH+wmEXw== ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass header.d=linuxfoundation.org header.i=@linuxfoundation.org header.a=rsa-sha256 header.s=korg header.b=m4nxy68g; arc=none smtp.remote-ip=100.103.45.18 Received: from smtp.kernel.org (quasi.space.kernel.org [100.103.45.18]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (prime256v1)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B081E606A32E; Fri, 30 Oct 2026 12:12:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key, unprotected) header.d=linuxfoundation.org header.i=@linuxfoundation.org header.a=rsa-sha256 header.s=korg header.b=m4nxy68g Received: by smtp.kernel.org (Postfix) with ESMTPSA id 61B301F00A18; Fri, 30 Oct 2026 12:12:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=korg; t=1793362365; bh=ci0QkwBiOudZUlUkeYQaqYBZ+4VnhyCQoTpwhsuLJlI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=m4nxy68gkf25ASguS+OAHRU1qFGAZBWhqytQEAxXJMlgdKec9wal9pzkXGvKbVLsp B/yuzA2D2+vn2anQ39vegnK0typ7tBppioYosfEoxwQPWHeWqcWIrK7hYjxNbqLKAK O+DMxLmFuYpRVVXkBJA9UOE/phi+XvYz3eZVuYps= From: Greg Kroah-Hartman To: linux-kernel@vger.kernel.org, akpm@linux-foundation.org, torvalds@linux-foundation.org, stable@vger.kernel.org Cc: lwn@lwn.net, jslaby@suse.cz, Greg Kroah-Hartman Subject: Re: Linux 7.2.14 Date: Fri, 30 Oct 2026 13:12:33 +0100 Message-ID: <2026103032-eligibly-exile-8edf@gregkh> X-Mailer: git-send-email 2.56.0 In-Reply-To: <2026103032-conclude-illusion-3814@gregkh> References: <2026103032-conclude-illusion-3814@gregkh> X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Post: List-Help: List-Subscribe: List-Unsubscribe: List-Owner: Precedence: list MIME-Version: 1.0 Content-Transfer-Encoding: 8bit diff --git a/Makefile b/Makefile index dfd40298a0cf..ab29f857e9ed 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 2 -SUBLEVEL = 13 +SUBLEVEL = 14 EXTRAVERSION = NAME = Baby Opossum Posse diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c index 86292965f493..0e4783ff7d22 100644 --- a/crypto/asymmetric_keys/restrict.c +++ b/crypto/asymmetric_keys/restrict.c @@ -243,10 +243,14 @@ static int key_or_keyring_common(struct key *dest_keyring, if (IS_ERR(key)) key = NULL; } else if (trusted->type == &key_type_asymmetric) { + const struct asymmetric_key_ids *kids; const struct asymmetric_key_id **signer_ids; - signer_ids = (const struct asymmetric_key_id **) - asymmetric_key_ids(trusted)->id; + kids = asymmetric_key_ids(trusted); + if (!kids) + return -ENOKEY; + + signer_ids = (const struct asymmetric_key_id **)kids->id; /* * The auth_ids come from the candidate key (the diff --git a/fs/coredump.c b/fs/coredump.c index e68a76ff92a3..19c1cfe86472 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -508,7 +508,9 @@ static int zap_threads(struct task_struct *tsk, int nr = -EAGAIN; spin_lock_irq(&tsk->sighand->siglock); - if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task) { + /* A freeze requested before the dump would be lost with TIF_SIGPENDING. */ + if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task && + !freezing(tsk) && !(tsk->jobctl & JOBCTL_TRAP_FREEZE)) { /* Allow SIGKILL, see prepare_signal() */ signal->core_state = core_state; nr = zap_process(signal, exit_code); @@ -562,6 +564,8 @@ static void coredump_finish(bool core_dumped) current->signal->core_state = NULL; spin_unlock_irq(¤t->sighand->siglock); + /* A released thread may exit and be freed before it is woken. */ + guard(rcu)(); while ((curr = next) != NULL) { next = curr->next; task = curr->task; @@ -570,6 +574,7 @@ static void coredump_finish(bool core_dumped) * ->task == NULL before we read ->next. */ smp_mb(); + /* Any wakeup now lets the thread exit, rcu keeps it alive. */ curr->task = NULL; wake_up_process(task); } @@ -577,13 +582,8 @@ static void coredump_finish(bool core_dumped) static bool dump_interrupted(void) { - /* - * SIGKILL or freezing() interrupt the coredumping. Perhaps we - * can do try_to_freeze() and check __fatal_signal_pending(), - * but then we need to teach dump_write() to restart and clear - * TIF_SIGPENDING. - */ - return fatal_signal_pending(current) || freezing(current); + /* Only SIGKILL and the freezers set it after zap_threads(). */ + return task_sigpending(current); } static void wait_for_dump_helpers(struct file *file) @@ -1196,6 +1196,8 @@ void vfs_coredump(const kernel_siginfo_t *siginfo) if (coredump_wait(siginfo->si_signo, &core_state) < 0) return; + /* Task work must not cut the dump short, see signal_pending(). */ + guard(no_notify_signal)(); scoped_with_creds(cred) do_coredump(&cn, &cprm, &argv, &argc, binfmt); coredump_cleanup(&cn, &cprm); diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index fb8e52574397..8515560e2a4b 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -620,12 +620,20 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) static vm_fault_t ntfs_filemap_page_mkwrite(struct vm_fault *vmf) { struct inode *inode = file_inode(vmf->vma->vm_file); + struct address_space *mapping = inode->i_mapping; vm_fault_t ret; sb_start_pagefault(inode->i_sb); file_update_time(vmf->vma->vm_file); + /* + * Serialize against truncate/fallocate which hold the lock + * exclusively while invalidating pagecache and changing extents. + */ + filemap_invalidate_lock_shared(mapping); ret = iomap_page_mkwrite(vmf, &ntfs_page_mkwrite_iomap_ops, NULL); + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(inode->i_sb); return ret; } @@ -1061,7 +1069,6 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le struct ntfs_volume *vol = ni->vol; int err = 0; loff_t old_size; - bool map_locked = false; if (mode & ~(NTFS_FALLOC_FL_SUPPORTED)) return -EOPNOTSUPP; @@ -1090,16 +1097,12 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le inode_lock(vi); if (NInoCompressed(ni) || NInoEncrypted(ni)) { - err = -EOPNOTSUPP; - goto out; + inode_unlock(vi); + return -EOPNOTSUPP; } inode_dio_wait(vi); - if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | - FALLOC_FL_INSERT_RANGE)) { - filemap_invalidate_lock(vi->i_mapping); - map_locked = true; - } + filemap_invalidate_lock(vi->i_mapping); switch (mode & FALLOC_FL_MODE_MASK) { case FALLOC_FL_ALLOCATE_RANGE: @@ -1124,14 +1127,13 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le err = file_modified(file); out: - if (map_locked) - filemap_invalidate_unlock(vi->i_mapping); + if (!err && mode == 0 && NInoNonResident(ni) && + offset > old_size) { + truncate_pagecache(vi, old_size); + pagecache_isize_extended(vi, old_size, offset); + } + filemap_invalidate_unlock(vi->i_mapping); if (!err) { - if (mode == 0 && NInoNonResident(ni) && - offset > old_size) { - truncate_pagecache(vi, old_size); - pagecache_isize_extended(vi, old_size, offset); - } NInoSetFileNameDirty(ni); inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi)); mark_inode_dirty(vi); diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 20116462cf3d..896ec1616f58 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -513,8 +513,13 @@ struct io_ring_ctx { u32 iowq_limits[2]; struct callback_head poll_wq_task_work; + /* deferred ->poll_wq wake up for local task_work */ + struct rcu_head poll_wq_rcu; + atomic_t poll_wq_rcu_pending; struct list_head defer_list; unsigned nr_drained; + /* drain request dispatched from ->defer_list, until it completes */ + struct io_kiocb *drain_req; /* protected by ->completion_lock */ unsigned nr_req_allocated; diff --git a/include/linux/key-type.h b/include/linux/key-type.h index bb97bd3e5af4..c4f329b6ae3f 100644 --- a/include/linux/key-type.h +++ b/include/linux/key-type.h @@ -161,6 +161,7 @@ struct key_type { const void *in, const void *in2); /* internal fields */ + atomic64_t index_id; /* Opaque keyring index ID */ struct list_head link; /* link in types list */ struct lock_class_key lock_class; /* key->sem lock class */ } __randomize_layout; diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..e40b5267658f 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -107,6 +107,7 @@ struct keyring_name; struct key_tag { struct rcu_head rcu; + atomic64_t index_id; /* Opaque keyring index ID */ refcount_t usage; bool removed; /* T when subject removed */ }; @@ -129,6 +130,8 @@ struct keyring_index_key { struct key_type *type; struct key_tag *domain_tag; /* Domain of operation */ const char *description; + u64 type_id; /* Cached index IDs */ + u64 domain_id; }; union key_payload { @@ -251,6 +254,8 @@ struct key { struct key_type *type; /* type of key */ struct key_tag *domain_tag; /* Domain of operation */ char *description; + u64 type_id; + u64 domain_id; }; }; diff --git a/include/linux/sched.h b/include/linux/sched.h index 5b36a3364ca0..e2f472e6ca94 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1805,7 +1805,7 @@ extern struct pid __rcu *cad_pid; * I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ -#define PF__HOLE__00800000 0x00800000 +#define PF_NO_NOTIFY_SIGNAL 0x00800000 /* see no_notify_signal_save() */ #define PF__HOLE__01000000 0x01000000 #define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index efa1e2d6c7a6..bfb84f025b67 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -2,6 +2,7 @@ #ifndef _LINUX_SCHED_SIGNAL_H #define _LINUX_SCHED_SIGNAL_H +#include #include #include #include @@ -384,14 +385,36 @@ static inline int task_sigpending(struct task_struct *p) return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); } +/* Prevent TIF_NOTIFY_SIGNAL from interrupting this task. */ +static inline unsigned int no_notify_signal_save(void) +{ + unsigned int flags = current->flags; + + current->flags |= PF_NO_NOTIFY_SIGNAL; + return flags; +} + +/* Restore the previous PF_NO_NOTIFY_SIGNAL state. */ +static inline void no_notify_signal_restore(unsigned int flags) +{ + current_restore_flags(flags, PF_NO_NOTIFY_SIGNAL); +} + +DEFINE_LOCK_GUARD_0(no_notify_signal, + _T->flags = no_notify_signal_save(), + no_notify_signal_restore(_T->flags), + unsigned int flags) + static inline int signal_pending(struct task_struct *p) { /* * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same * behavior in terms of ensuring that we break out of wait loops - * so that notify signal callbacks can be processed. + * so that notify signal callbacks can be processed. Not for a task + * that asked not to be interrupted by it, see no_notify_signal_save(). */ - if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) + if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)) && + likely(!(READ_ONCE(p->flags) & PF_NO_NOTIFY_SIGNAL))) return 1; return task_sigpending(p); } diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 4f706aff14fc..75a21ecfe69c 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -72,7 +72,16 @@ static bool io_cancel_cb(struct io_wq_work *work, void *data) struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_cancel_data *cd = data; - return io_cancel_req_match(req, cd); + if (!io_cancel_req_match(req, cd)) + return false; + /* + * Pending work is gone once it has matched. Work that is running may + * park the request where the same cancellation looks next. Don't make + * it skip the request as one that it has dealt with. + */ + if (cd->flags & (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_ANY)) + req->cancel_seq_set = false; + return true; } static int io_async_cancel_one(struct io_uring_task *tctx, @@ -117,6 +126,8 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, */ if (!ret) return 0; + if (ret == -EALREADY) + cd->running = true; ret = io_poll_cancel(ctx, cd, issue_flags); if (ret != -ENOENT) @@ -180,6 +191,7 @@ static int __io_async_cancel(struct io_cancel_data *cd, struct io_tctx_node *node; int ret, nr = 0; + cd->running = false; do { ret = io_try_cancel(tctx, cd, issue_flags); if (ret == -ENOENT) @@ -195,6 +207,9 @@ static int __io_async_cancel(struct io_cancel_data *cd, mutex_lock(&ctx->tctx_lock); ret = -ENOENT; list_for_each_entry(node, &ctx->tctx_list, ctx_node) { + /* has been done above, don't count what is running there */ + if (all && node->task->io_uring == tctx) + continue; ret = io_async_cancel_one(node->task->io_uring, cd); if (ret != -ENOENT) { if (!all) @@ -204,7 +219,15 @@ static int __io_async_cancel(struct io_cancel_data *cd, } mutex_unlock(&ctx->tctx_lock); io_ring_submit_unlock(ctx, issue_flags); - return all ? nr : ret; + if (all) + return nr; + /* + * The worker may be done with the work that was found running, and + * flagged, above. That isn't the same as not having found it. + */ + if (ret == -ENOENT && cd->running) + ret = -EALREADY; + return ret; } int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) @@ -308,7 +331,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) if (ret != -EALREADY) goto out; - if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) { + if (sc.timeout.tv_sec != -1 || sc.timeout.tv_nsec != -1) { struct timespec64 ts = { .tv_sec = sc.timeout.tv_sec, .tv_nsec = sc.timeout.tv_nsec @@ -636,8 +659,18 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) io_uring_drop_tctx_refs(current); /* only skip the sleep for local work this task may run */ xa_for_each(&tctx->xa, index, node) { - if (io_local_work_pending(node->ctx) && - io_allowed_defer_tw_run(node->ctx)) + if (!(node->ctx->flags & IORING_SETUP_DEFER_TASKRUN) || + !io_allowed_defer_tw_run(node->ctx)) + continue; + /* + * The wake up io_uring_try_cancel_requests() asked for + * may have been issued while we were still running. Ask + * again now that it can't be missed. + */ + atomic_set(&node->ctx->cq_wait_nr, 1); + /* pairs with the xchg() in mpscq_push() */ + smp_mb(); + if (io_local_work_pending(node->ctx)) goto end_wait; } /* diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 1b201a094303..e03a6fb0b7f9 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -11,6 +11,8 @@ struct io_cancel_data { u8 opcode; u32 flags; int seq; + /* io_try_cancel() has found it being run by io-wq, and flagged it */ + bool running; }; int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 3ae4804765b9..30027fd46045 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -154,11 +154,17 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) cq_entries = min(cq_tail - cq_head, ctx->cq_entries); for (i = 0; i < cq_entries; i++) { struct io_uring_cqe *cqe; - bool cqe32 = false; + bool cqe32 = false, two_entries = false; - cqe = &r->cqes[(cq_head & cq_mask)]; - if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) + if (ctx->flags & IORING_SETUP_CQE32) { + /* all entries have twice the size */ + cqe = &r->cqes[(cq_head & cq_mask) << 1]; cqe32 = true; + } else { + cqe = &r->cqes[(cq_head & cq_mask)]; + two_entries = cqe->flags & IORING_CQE_F_32; + cqe32 = two_entries; + } seq_printf(m, "%5u: user_data:%llu, res:%d, flags:%x", cq_head & cq_mask, cqe->user_data, cqe->res, cqe->flags); @@ -167,7 +173,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) cqe->big_cqe[0], cqe->big_cqe[1]); seq_printf(m, "\n"); cq_head++; - if (cqe32) { + if (two_entries) { cq_head++; i++; } diff --git a/io_uring/futex.c b/io_uring/futex.c index eaee14242a3a..9203998654ff 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -152,6 +152,24 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } +/* + * A wait on a private futex does not block the submitter. Even a single + * threaded task can go on and have the mm private hash installed by clone() + * or prctl(), which does not pick up waiters from the global hash. Make + * sure the private hash exists before the first waiter is queued. + */ +static int io_futex_prep_private(struct io_kiocb *req) +{ + int ret = futex_hash_allocate_default(); + + if (unlikely(ret)) + return ret; + + /* inflight tracking only needed for mm private hash */ + io_req_track_inflight(req); + return 0; +} + int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); @@ -161,9 +179,8 @@ int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(ret)) return ret; - /* inflight tracking only needed for mm private hash */ if (!(iof->futex_flags & FLAGS_SHARED)) - io_req_track_inflight(req); + return io_futex_prep_private(req); return 0; } @@ -213,10 +230,13 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } - /* inflight tracking only needed for mm private hash */ for (i = 0; i < iof->futex_nr; i++) { if (!(ifd->futexv[i].w.flags & FLAGS_SHARED)) { - io_req_track_inflight(req); + ret = io_futex_prep_private(req); + if (ret) { + kfree(ifd); + return ret; + } break; } } @@ -283,6 +303,8 @@ int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) */ __set_current_state(TASK_RUNNING); hlist_add_head(&req->hash_node, &ctx->futex_list); + if (io_wq_cancel_seen(req, issue_flags)) + __io_futex_cancel(req); } else { iof->futexv_unqueued = 1; if (woken != -1) @@ -323,6 +345,8 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) &ifd->q, NULL, NULL); if (!ret) { hlist_add_head(&req->hash_node, &ctx->futex_list); + if (io_wq_cancel_seen(req, issue_flags)) + __io_futex_cancel(req); io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index fa403ed24596..59aa1ab58492 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -55,6 +55,8 @@ struct io_worker { struct io_wq_acct *acct; struct io_wq_work *cur_work; + /* rest of the hashed chain ->cur_work was taken off the list with */ + struct io_wq_work *next_work; raw_spinlock_t lock; struct completion ref_done; @@ -466,11 +468,9 @@ static void io_wq_dec_running(struct io_worker *worker) if (!atomic_dec_and_test(&acct->nr_running)) return; - if (!worker->cur_work) - return; if (!io_acct_run_queue(acct)) return; - if (io_wq_hash_defer(worker->cur_work, acct)) { + if (worker->cur_work && io_wq_hash_defer(worker->cur_work, acct)) { raw_spin_unlock(&acct->lock); return; } @@ -582,6 +582,17 @@ static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct, return NULL; } +static void __io_assign_current_work(struct io_worker *worker, + struct io_wq_work *work) +{ + raw_spin_lock(&worker->lock); + worker->cur_work = work; + /* stays valid while in between two items of a chain */ + if (work) + worker->next_work = wq_next_work(work); + raw_spin_unlock(&worker->lock); +} + static void io_assign_current_work(struct io_worker *worker, struct io_wq_work *work) { @@ -590,9 +601,7 @@ static void io_assign_current_work(struct io_worker *worker, cond_resched(); } - raw_spin_lock(&worker->lock); - worker->cur_work = work; - raw_spin_unlock(&worker->lock); + __io_assign_current_work(worker, work); } /* @@ -623,9 +632,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct, * work list, but isn't yet discoverable as the * current work item for this worker. */ - raw_spin_lock(&worker->lock); - worker->cur_work = work; - raw_spin_unlock(&worker->lock); + __io_assign_current_work(worker, work); } raw_spin_unlock(&acct->lock); @@ -641,21 +648,28 @@ static void io_worker_handle_work(struct io_wq_acct *acct, /* handle a whole dependent link */ do { bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); - struct io_wq_work *next_hashed, *linked; + struct io_wq_work *next_hashed, *linked, *done = work; unsigned int work_flags = atomic_read(&work->flags); unsigned int hash = __io_wq_is_hashed(work_flags) ? __io_get_work_hash(work_flags) : -1U; + bool last; next_hashed = wq_next_work(work); - if (do_kill && - (work_flags & IO_WQ_WORK_UNBOUND)) + /* + * io_wq_exit_workers() signals us once, so that we + * don't sleep for ever. If that was before this work + * became the current one, the signal is gone. + */ + if (do_kill && (work_flags & IO_WQ_WORK_UNBOUND)) atomic_or(IO_WQ_WORK_CANCEL, &work->flags); + else if (do_kill) + set_notify_signal(current); io_wq_submit_work(work); io_assign_current_work(worker, NULL); - linked = io_wq_free_work(work); + last = io_wq_put_work(work, &linked); work = next_hashed; if (!work && linked && !io_wq_is_hashed(linked)) { work = linked; @@ -664,6 +678,13 @@ static void io_worker_handle_work(struct io_wq_acct *acct, io_assign_current_work(worker, work); if (linked) io_wq_enqueue(wq, linked); + /* + * Not before the link is current or pending work again. + * This may be the last time that a task is woken up + * which exits and looks for requests to cancel. + */ + if (last) + io_wq_free_work(done); if (hash != -1U && !next_hashed) { /* serialize hash clear with wake_up() */ @@ -682,6 +703,35 @@ static void io_worker_handle_work(struct io_wq_acct *acct, } while (1); } +/* + * The worker has decided to exit, and has given up its ->nr_workers slot and + * its place on the free list. io_wq_enqueue() may have picked it from that + * list, or seen it running, after it last looked at the work list. Leave + * ->nr_running as well and check again. Returns false if the worker has to + * stay. + */ +static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker) +{ + /* turns io_wq_dec_running() and the scheduler hooks into no-ops */ + clear_bit(IO_WORKER_F_UP, &worker->flags); + atomic_dec(&acct->nr_running); + if (!io_acct_run_queue(acct)) + return true; + raw_spin_unlock(&acct->lock); + + raw_spin_lock(&acct->workers_lock); + if (acct->nr_workers >= acct->max_workers) { + /* a worker was created in our place, the work is its business */ + raw_spin_unlock(&acct->workers_lock); + return true; + } + acct->nr_workers++; + raw_spin_unlock(&acct->workers_lock); + atomic_inc(&acct->nr_running); + set_bit(IO_WORKER_F_UP, &worker->flags); + return false; +} + static int io_wq_worker(void *data) { struct io_worker *worker = data; @@ -715,19 +765,41 @@ static int io_wq_worker(void *data) * idle-exit, drop the worker as well. This is used to avoid * keeping io-wq workers around for tasks that no longer have * any active io_uring instances. + * Not as long as a request of ours for another worker is + * pending, though. It is accounted as a running worker, which + * others rely on, and io_worker_exit() would cancel it. */ - if ((last_timeout && (exit_mask || acct->nr_workers > 1)) || - test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) { + if (((last_timeout && (exit_mask || acct->nr_workers > 1)) || + test_bit(IO_WQ_BIT_EXIT_ON_IDLE, &wq->state)) && + !test_bit(0, &worker->create_state)) { acct->nr_workers--; + if (test_and_clear_bit(IO_WORKER_F_FREE, &worker->flags)) + hlist_nulls_del_init_rcu(&worker->nulls_node); raw_spin_unlock(&acct->workers_lock); __set_current_state(TASK_RUNNING); - break; + if (io_worker_retire(acct, worker)) + break; + last_timeout = false; + continue; } last_timeout = false; __io_worker_idle(acct, worker); raw_spin_unlock(&acct->workers_lock); if (io_run_task_work()) continue; + /* + * io_wq_enqueue() leaves new work to us for as long as we are + * accounted as running. Get out of ->nr_running before taking + * the last look at the work list, rather than when scheduling + * out. No need to ask for another worker then, we can do it. + */ + clear_bit(IO_WORKER_F_RUNNING, &worker->flags); + atomic_dec(&acct->nr_running); + if (io_acct_run_queue(acct)) { + io_wq_worker_running(current); + io_worker_handle_work(acct, worker); + continue; + } ret = schedule_timeout(WORKER_IDLE_TIMEOUT); if (signal_pending(current)) { struct ksignal ksig; @@ -993,9 +1065,13 @@ void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable) static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) { do { + struct io_wq_work *linked; + atomic_or(IO_WQ_WORK_CANCEL, &work->flags); io_wq_submit_work(work); - work = io_wq_free_work(work); + if (io_wq_put_work(work, &linked)) + io_wq_free_work(work); + work = linked; } while (work); } @@ -1103,6 +1179,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker, static bool io_wq_worker_cancel(struct io_worker *worker, void *data) { struct io_cb_cancel_data *match = data; + struct io_wq_work *work; /* * Hold the lock to avoid ->cur_work going out of scope, caller @@ -1111,6 +1188,19 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) raw_spin_lock(&worker->lock); if (__io_wq_worker_cancel(worker, match, worker->cur_work)) match->nr_running++; + /* + * Hashed work is taken off the work list a chain at a time, what the + * worker hasn't started yet can only be found here. It doesn't move on + * without the lock. Just mark the work, it fails once it is its turn. + */ + for (work = worker->next_work; work; work = wq_next_work(work)) { + if (match->nr_running && !match->cancel_all) + break; + if (match->fn(work, match->data)) { + atomic_or(IO_WQ_WORK_CANCEL, &work->flags); + match->nr_running++; + } + } raw_spin_unlock(&worker->lock); return match->nr_running && !match->cancel_all; @@ -1319,6 +1409,8 @@ static bool io_task_work_match(struct callback_head *cb, void *data) void io_wq_exit_start(struct io_wq *wq) { set_bit(IO_WQ_BIT_EXIT, &wq->state); + /* Pairs with task_work_add() in io_queue_worker_create(). */ + smp_mb__after_atomic(); } static void io_wq_cancel_tw_create(struct io_wq *wq) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 3d8c8a50fd31..33f0c6ab7dba 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -405,17 +405,46 @@ static void io_prep_async_link(struct io_kiocb *req) } } +/* Called for every request that is issued: keep it inline */ +static __always_inline bool io_assign_file(struct io_kiocb *req, + const struct io_issue_def *def, + unsigned int issue_flags) +{ + if (req->file || !def->needs_file) + return true; + + if (req->flags & REQ_F_FIXED_FILE) + req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags); + else + req->file = io_file_get_normal(req, req->cqe.fd); + + return !!req->file; +} + static void io_queue_iowq(struct io_kiocb *req) { struct io_uring_task *tctx = req->tctx; BUG_ON(!tctx); + lockdep_assert_held(&req->ctx->uring_lock); if ((current->flags & PF_KTHREAD) || !tctx->io_wq) { io_req_task_queue_fail(req, -ECANCELED); return; } + /* + * The file is looked up as late as possible, as it may be the result + * of an earlier request of the link. This one is up now, and what kind + * of work it is depends on the file. If that fails, leave the error + * handling to the worker. + * + * Not for a splice: whatever it writes to, it may wait for its input + * for ever. It has to remain unbound work that isn't hashed. + */ + if (req->opcode != IORING_OP_SPLICE) + io_assign_file(req, &io_issue_defs[req->opcode], 0); + /* init ->work of the whole link before punting */ io_prep_async_link(req); @@ -454,12 +483,15 @@ unsigned io_linked_nr(struct io_kiocb *req) return nr; } -static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) +static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx) { - bool drain_seen = false, first = true; + bool drain_seen = false, first = true, drain_queued = false; lockdep_assert_held(&ctx->uring_lock); __io_req_caches_free(ctx); + /* nothing outside of ->defer_list is alive */ + if (ctx->nr_req_allocated == ctx->nr_drained) + ctx->drain_req = NULL; while (!list_empty(&ctx->defer_list)) { struct io_defer_entry *de = list_first_entry(&ctx->defer_list, @@ -467,14 +499,21 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) drain_seen |= de->req->flags & REQ_F_IO_DRAIN; if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained) - return; + return drain_queued; list_del_init(&de->list); ctx->nr_drained -= io_linked_nr(de->req); + if (de->req->flags & REQ_F_IO_DRAIN) { + drain_queued = true; + /* it runs alone, track its completion unless it's a link */ + if (!de->req->link) + ctx->drain_req = de->req; + } io_req_task_queue(de->req); kfree(de); first = false; } + return drain_queued; } void __io_commit_cqring_flush(struct io_ring_ctx *ctx) @@ -738,6 +777,13 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32) if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))) return false; + /* + * A single entry may be left if a 32b CQE is asked for. It must not be + * handed out later on if this fails, as the CQE is going to overflow. + */ + ctx->cqe_cached = NULL; + ctx->cqe_sentinel = NULL; + /* * Post dummy CQE if a 32b CQE is needed and there's only room for a * 16b CQE before the ring wraps. @@ -930,9 +976,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) /* * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires - * the submitter task context, IOPOLL protects with uring_lock. + * the submitter task context, IOPOLL protects with uring_lock. A drain + * request ends the drain under uring_lock as its CQE is posted. */ - if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) { + if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || + (req->flags & (REQ_F_REISSUE | REQ_F_IO_DRAIN))) { defer_complete: req->io_task_work.func = io_req_task_complete; io_req_task_work_add(req); @@ -1133,11 +1181,33 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, } while (node); } +/* + * The dispatched drain request ran alone, and anything submitted after it was + * deferred. Once it completed, new requests may run if nothing is waiting. + */ +static __cold bool io_drain_req_completed(struct io_ring_ctx *ctx) +{ + struct io_wq_work_node *node; + + __wq_list_for_each(node, &ctx->submit_state.compl_reqs) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); + + /* a requeued request hasn't completed */ + if (req == ctx->drain_req && !(req->flags & REQ_F_REISSUE)) { + ctx->drain_req = NULL; + return true; + } + } + return false; +} + void __io_submit_flush_completions(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_submit_state *state = &ctx->submit_state; struct io_wq_work_node *node; + bool drain_done = false; __io_cq_lock(ctx); __wq_list_for_each(node, &state->compl_reqs) { @@ -1159,13 +1229,21 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx) } __io_cq_unlock_post(ctx); + if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) && ctx->drain_req) + drain_done = io_drain_req_completed(ctx); + if (!wq_list_empty(&state->compl_reqs)) { io_free_batch_list(ctx, state->compl_reqs.first); INIT_WQ_LIST(&state->compl_reqs); } - if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) - io_queue_deferred(ctx); + if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) { + /* a drain request started from the list keeps the drain going */ + if (io_queue_deferred(ctx)) + drain_done = false; + if (drain_done && list_empty(&ctx->defer_list)) + ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE; + } ctx->submit_state.cq_flush = false; } @@ -1358,20 +1436,6 @@ static __cold void io_drain_req(struct io_kiocb *req) ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE; } -static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def, - unsigned int issue_flags) -{ - if (req->file || !def->needs_file) - return true; - - if (req->flags & REQ_F_FIXED_FILE) - req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags); - else - req->file = io_file_get_normal(req, req->cqe.fd); - - return !!req->file; -} - #define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT) static inline int __io_issue_sqe(struct io_kiocb *req, @@ -1418,6 +1482,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) ret = __io_issue_sqe(req, issue_flags, def); if (ret == IOU_COMPLETE) { + /* not all handlers take care of the internal restart codes */ + if (unlikely(req->cqe.res <= -ERESTARTSYS)) + req->cqe.res = io_fixup_restart_res(req->cqe.res); if (issue_flags & IO_URING_F_COMPLETE_DEFER) io_req_complete_defer(req); else @@ -1432,6 +1499,8 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (req->flags & REQ_F_IOPOLL) io_iopoll_req_issued(req, issue_flags); } + if (unlikely(ret <= -ERESTARTSYS)) + ret = io_fixup_restart_res(ret); return ret; } @@ -1454,22 +1523,35 @@ int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw) return ret; } -struct io_wq_work *io_wq_free_work(struct io_wq_work *work) +/* + * io-wq is done with @work. Returns true if it held the last reference, and + * hands out the next request of the link then. The caller has to make sure + * that cancellations can find the latter before it calls io_wq_free_work(). + */ +bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_kiocb *nxt = NULL; - if (req_ref_put_and_test_atomic(req)) { - if (req->flags & IO_REQ_LINK_FLAGS) { - struct io_ring_ctx *ctx = req->ctx; + *linked = NULL; + if (!req_ref_put_and_test_atomic(req)) + return false; - mutex_lock(&ctx->uring_lock); - nxt = io_req_find_next(req); - mutex_unlock(&ctx->uring_lock); - } - io_free_req(req); + if (req->flags & IO_REQ_LINK_FLAGS) { + struct io_ring_ctx *ctx = req->ctx; + + mutex_lock(&ctx->uring_lock); + nxt = io_req_find_next(req); + mutex_unlock(&ctx->uring_lock); } - return nxt ? &nxt->work : NULL; + if (nxt) + *linked = &nxt->work; + return true; +} + +void io_wq_free_work(struct io_wq_work *work) +{ + io_free_req(container_of(work, struct io_kiocb, work)); } void io_wq_submit_work(struct io_wq_work *work) @@ -1480,7 +1562,7 @@ void io_wq_submit_work(struct io_wq_work *work) bool needs_poll = false; int ret = 0, err = -ECANCELED; - /* one will be dropped by io_wq_free_work() after returning to io-wq */ + /* one will be dropped by io_wq_put_work() after returning to io-wq */ if (!(req->flags & REQ_F_REFCOUNT)) __io_req_set_refcount(req, 2); else @@ -1758,17 +1840,27 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, * A 128b op on a non-128b SQ requires mixed SQE support as * well as 2 contiguous entries. */ - if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || *left < 2 || + if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || (unsigned)(sqe - ctx->sq_sqes) >= ctx->sq_entries - 1) return io_init_fail_req(req, -EINVAL); + if (*left >= 2) { + /* + * A 128b operation on a mixed SQ uses two entries, so + * we have to increment the head and cached refs, and + * decrement what's left. + */ + current->io_uring->cached_refs++; + (*left)--; + } else if (!(ctx->flags & IORING_SETUP_SQPOLL) || + !__io_sqring_entries(ctx)) { + return io_init_fail_req(req, -EINVAL); + } /* - * A 128b operation on a mixed SQ uses two entries, so we have - * to increment the head and cached refs, and decrement what's - * left. + * else: the SQPOLL thread caps the number of entries it submits + * in one go, which is none of the application's business. Take + * the 2nd entry on top. */ - current->io_uring->cached_refs++; ctx->cached_sq_head++; - (*left)--; } if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) { @@ -2297,6 +2389,7 @@ struct io_tctx_exit { struct callback_head task_work; struct completion completion; struct io_ring_ctx *ctx; + bool in_cancel; }; static __cold void io_tctx_exit_cb(struct callback_head *cb) @@ -2311,7 +2404,8 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb) * tctx can be NULL if the queueing of this task_work raced with * work cancelation off the exec path. */ - if (tctx && !atomic_read(&tctx->in_cancel)) + work->in_cancel = tctx && atomic_read(&tctx->in_cancel); + if (tctx && !work->in_cancel) io_uring_del_tctx_node((unsigned long)work->ctx); complete(&work->completion); } @@ -2405,6 +2499,14 @@ static __cold void io_ring_exit_work(struct work_struct *work) * wait is marked as interruptible. */ wait_for_completion_interruptible(&exit.completion); + /* + * The task removes the node when it is done with its + * cancellation. Give it the time for that: it doesn't get back + * to it as long as it finds new task_work each time it has run + * the old one, and we may be one of many who send it. + */ + if (exit.in_cancel) + schedule_timeout_idle(HZ / 20); mutex_lock(&ctx->uring_lock); mutex_lock(&ctx->tctx_lock); } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index cb736b815422..684553242c41 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -203,7 +203,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx); void __io_submit_flush_completions(struct io_ring_ctx *ctx); -struct io_wq_work *io_wq_free_work(struct io_wq_work *work); +bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked); +void io_wq_free_work(struct io_wq_work *work); void io_wq_submit_work(struct io_wq_work *work); void io_free_req(struct io_kiocb *req); @@ -348,6 +349,24 @@ static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, req->big_cqe.extra2 = extra2; } +static inline ssize_t io_fixup_restart_res(ssize_t ret) +{ + switch (ret) { + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + /* + * We can't just restart the syscall, since previously + * submitted sqes may already be in progress. Just fail + * this IO with EINTR. + */ + return -EINTR; + default: + return ret; + } +} + static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, struct io_kiocb *req) { @@ -410,6 +429,29 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, lockdep_assert_held(&ctx->uring_lock); } +/* + * For a handler that is run by io-wq and parks the request on a list where + * cancellations find it, rather than completing it. Whoever tried to cancel + * the request before that found it running. It has flagged the work item, + * has searched the list in vain, and won't come back. To be called with + * the lock the search takes held: if the search came first, the flag is + * visible. For most lists that is ->uring_lock, timeouts use ->timeout_lock. + */ +static inline bool __io_wq_cancel_seen(struct io_kiocb *req, + unsigned int issue_flags) +{ + return (issue_flags & IO_URING_F_IOWQ) && + (atomic_read(&req->work.flags) & IO_WQ_WORK_CANCEL); +} + +static inline bool io_wq_cancel_seen(struct io_kiocb *req, + unsigned int issue_flags) +{ + lockdep_assert_held(&req->ctx->uring_lock); + + return __io_wq_cancel_seen(req, issue_flags); +} + static inline void io_commit_cqring(struct io_ring_ctx *ctx) { /* order cqe stores with ring update */ diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 5065a3eff809..fd0c9c49c13e 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -203,7 +203,7 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, if (unlikely(tail == head)) return sel; - if (head + 1 == tail) + if ((__u16)(head + 1) == tail) req->flags |= REQ_F_BL_EMPTY; buf = io_ring_head_to_buf(br, head, bl->mask); diff --git a/io_uring/loop.c b/io_uring/loop.c index bbbb6ef14e6a..ca185d9c0624 100644 --- a/io_uring/loop.c +++ b/io_uring/loop.c @@ -9,12 +9,6 @@ static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail); } -static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) -{ - atomic_set(&ctx->cq_wait_nr, nr_wait); - set_current_state(TASK_INTERRUPTIBLE); -} - static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) { __set_current_state(TASK_RUNNING); @@ -24,7 +18,7 @@ static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, unsigned nr_wait) { - io_loop_wait_start(ctx, nr_wait); + io_cq_wait_arm(ctx, nr_wait); if (unlikely(io_local_work_pending(ctx) || io_loop_nr_cqes(ctx, lp) <= 0) || diff --git a/io_uring/mpscq.h b/io_uring/mpscq.h index f910526766fd..da9474497cc8 100644 --- a/io_uring/mpscq.h +++ b/io_uring/mpscq.h @@ -25,9 +25,10 @@ * its predecessor. In between, the tail end of the queue is not yet reachable * from the head. mpscq_pop() detects this and returns NULL, while mpscq_empty() * reports false. The consumer must not treat such a NULL as "queue empty" - it - * should retry later. The window is two instructions wide, but a producer can - * be preempted inside it, so the consumer must not spin on it while holding - * resources the producer might need to make progress. + * should retry later. The window is two instructions wide and the producer + * can't be preempted inside it, but it can be interrupted, so the consumer must + * not spin on it while holding resources the producer might need to make + * progress. * * The consumer side only supports a single consumer at a time, callers must * provide their own serialization for it. The stub node is what allows the @@ -69,6 +70,12 @@ static inline bool mpscq_push(struct mpscq *q, struct llist_node *node) struct llist_node *prev; node->next = NULL; + /* + * All the consumer can do about a node that isn't linked yet is to try + * again. If it is what preempted us, and we don't get to run as long + * as it does, that never ends. + */ + preempt_disable(); /* * xchg() implies a full barrier, so the initialization of the * entry (including ->next above) is visible before the node can @@ -77,6 +84,7 @@ static inline bool mpscq_push(struct mpscq *q, struct llist_node *node) */ prev = xchg(&q->tail, node); WRITE_ONCE(prev->next, node); + preempt_enable(); return prev == &q->stub; } diff --git a/io_uring/napi.c b/io_uring/napi.c index bfc771445912..ca0fe6f1871b 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -6,7 +6,7 @@ #ifdef CONFIG_NET_RX_BUSY_POLL /* Timeout for cleanout of stale entries. */ -#define NAPI_TIMEOUT (60 * SEC_CONVERSION) +#define NAPI_TIMEOUT (60 * HZ) struct io_napi_entry { unsigned int napi_id; @@ -378,6 +378,8 @@ void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) if (iowq->timeout != KTIME_MAX) { ktime_t dt = ktime_sub(iowq->timeout, io_get_time(ctx)); + /* an absolute timeout may have passed already */ + dt = max_t(ktime_t, dt, 0); iowq->napi_busy_poll_dt = min_t(u64, iowq->napi_busy_poll_dt, dt); } diff --git a/io_uring/net.c b/io_uring/net.c index 37fc7baa55b2..e982ce1fa4de 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -511,6 +511,24 @@ static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) return nbufs; } +/* + * A receive that is retried after a partial transfer keeps writing to what it + * has mapped. io_net_kbuf_recyle() only commits what was filled so far, which + * takes a buffer out of circulation if it's a single one that is consumed as a + * whole. The tail of an incrementally consumed buffer, or the yet unused + * buffers of a bundle, would be up for grabs. They can't be committed in + * advance either, the CQE has no way to tell if they end up not being used. + */ +static bool io_net_kbuf_can_retry(struct io_kiocb *req, + struct io_buffer_list *bl, + struct io_async_msghdr *kmsg) +{ + /* not ring provided, or committed at selection time */ + if (!(req->flags & REQ_F_BUFFERS_COMMIT)) + return true; + return !(bl->flags & IOBL_INC) && iter_is_ubuf(&kmsg->msg.msg_iter); +} + static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, struct io_async_msghdr *kmsg, int len) { @@ -1122,7 +1140,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); return IOU_RETRY; } - if (ret > 0 && io_net_retry(sock, flags)) { + if (ret > 0 && io_net_retry(sock, flags) && + io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) { sr->done_io += ret; return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); } @@ -1209,6 +1228,8 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); if (!sel->addr) return -ENOBUFS; + /* ring provided buffers can be up to 4G, keep len an int */ + len = min_t(size_t, len, MAX_RW_COUNT); sr->buf = sel->addr; sr->len = len; map_ubuf: @@ -1283,7 +1304,8 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); return IOU_RETRY; } - if (ret > 0 && io_net_retry(sock, flags)) { + if (ret > 0 && io_net_retry(sock, flags) && + io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) { sr->len -= ret; sr->buf += ret; sr->done_io += ret; diff --git a/io_uring/openclose.c b/io_uring/openclose.c index c71242915dad..24cf3ce2e2bc 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -375,13 +375,16 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, fds[1] = alloc_slot ? ret : slot - 1; files[1] = NULL; - io_ring_submit_unlock(ctx, issue_flags); - - if (!copy_to_user(p->fds, fds, sizeof(fds))) + /* + * Hang on to the lock. If this fails, the slots have to be cleared, and + * they must still hold what was put there. + */ + if (!copy_to_user(p->fds, fds, sizeof(fds))) { + io_ring_submit_unlock(ctx, issue_flags); return 0; + } ret = -EFAULT; - io_ring_submit_lock(ctx, issue_flags); err: if (fds[0] != -1) io_fixed_fd_remove(ctx, fds[0]); diff --git a/io_uring/poll.c b/io_uring/poll.c index 5447a7c24dce..4f04bda42f7f 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -241,12 +241,14 @@ static inline void io_poll_execute(struct io_kiocb *req, int res, */ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) { + bool again; int v; if (unlikely(tw.cancel)) return -ECANCELED; do { + again = false; v = atomic_read(&req->poll_refs); if (unlikely(v != 1)) { @@ -310,9 +312,15 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) } else { int ret; - /* multiple refs and HUP, ensure we loop once more */ - if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) - v--; + /* + * Multiple refs and HUP, ensure we loop once more. Drop + * down to one ref now, so the next pass ends unless a + * wake up came in. + */ + if ((req->cqe.res & (POLLHUP | POLLRDHUP)) && v != 1) { + atomic_sub(v - 1, &req->poll_refs); + again = true; + } ret = io_poll_issue(req, tw); if (ret == IOU_COMPLETE) @@ -330,7 +338,8 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) * Release all references, retry if someone tried to restart * task_work while we were executing it. */ - } while (atomic_sub_return(v, &req->poll_refs) & IO_POLL_REF_MASK); + v &= IO_POLL_REF_MASK; + } while (again || atomic_cmpxchg(&req->poll_refs, v, 0) != v); io_napi_add(req); return IOU_POLL_NO_ACTION; @@ -617,6 +626,18 @@ static int __io_arm_poll_handler(struct io_kiocb *req, io_poll_add_hash(req, issue_flags); + /* + * Whoever tried to cancel the request while io-wq was running it has + * flagged the work item. If it searched the hash table before the + * request got there it found nothing, and it won't come back. Both + * the search and the insertion are done with ->uring_lock held, and + * the flag is set before the search. If the search came first, + * the flag is visible by now. + */ + if (ipt->owning && + (atomic_read(&req->work.flags) & IO_WQ_WORK_CANCEL)) + io_poll_mark_cancelled(req); + if (mask && (poll->events & EPOLLET) && io_poll_can_finish_inline(req, ipt)) { __io_poll_execute(req, mask, 0); @@ -906,6 +927,15 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) ipt.pt._qproc = io_poll_queue_proc; + /* + * A poll update or a spurious wake up arms the request again. Forget + * the entries of the previous round like io_arm_apoll() does, a stale + * second entry would be taken for a third wait queue. + */ + if (req_has_async_data(req)) + io_req_async_data_free(req); + req->flags &= ~(REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL); + ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); if (ret > 0) { io_req_set_res(req, ipt.result_mask, 0); diff --git a/io_uring/register.c b/io_uring/register.c index dab3eb4258df..2ce36623e25a 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -496,6 +496,17 @@ static void io_register_free_rings(struct io_ring_ctx *ctx, IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED) +static bool io_sqe_is_128(struct io_ring_ctx *ctx, + const struct io_uring_sqe *sqe) +{ + u8 opcode = READ_ONCE(sqe->opcode); + + if (!(ctx->flags & IORING_SETUP_SQE_MIXED) || opcode >= IORING_OP_LAST) + return false; + opcode = array_index_nospec(opcode, IORING_OP_LAST); + return io_issue_defs[opcode].is_128; +} + static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) { struct io_ctx_config config; @@ -505,6 +516,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) struct io_uring_params *p = &config.p; struct io_rings_layout *rl = &config.layout; u32 *o_sq_array, *n_sq_array = NULL; + bool cqe32_tail = false; + unsigned int sqe128_src = 0; int ret; memset(&config, 0, sizeof(config)); @@ -603,13 +616,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) old_head = READ_ONCE(o.rings->sq.head); if (tail - old_head > p->sq_entries) goto overflow; - for (i = old_head; i < tail; i++) { + for (i = old_head; i != tail; i++) { + bool sqe128_tail = sqe128_src; unsigned int dst, src; size_t sq_size; dst = i & (p->sq_entries - 1); src = i & (ctx->sq_entries - 1); - if (n_sq_array) { + if (sqe128_tail) { + /* it follows the first half, whatever the array says */ + src = sqe128_src; + sqe128_src = 0; + if (n_sq_array) + WRITE_ONCE(n_sq_array[dst], dst); + } else if (n_sq_array) { src = READ_ONCE(o_sq_array[src]); if (unlikely(src >= ctx->sq_entries)) { WRITE_ONCE(n_sq_array[dst], UINT_MAX); @@ -618,6 +638,19 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) WRITE_ONCE(n_sq_array[dst], dst); } + /* + * A 128b SQE on a SQE_MIXED ring takes two contiguous entries + * and must not wrap. A smaller ring may place it on the last + * entry, fail the resize if that happens. + */ + if (!sqe128_tail && i + 1 != tail && + src < ctx->sq_entries - 1 && + io_sqe_is_128(ctx, &o.sq_sqes[src])) { + if (dst == p->sq_entries - 1) + goto overflow; + sqe128_src = src + 1; + } + sq_size = sizeof(struct io_uring_sqe); if (ctx->flags & IORING_SETUP_SQE128) { dst <<= 1; @@ -640,7 +673,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ret = -EOVERFLOW; goto out; } - for (i = old_head; i < tail; i++) { + for (i = old_head; i != tail; i++) { unsigned index, dst_mask, src_mask; size_t cq_size; @@ -654,6 +687,21 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) src_mask = (ctx->cq_entries << 1) - 1; dst_mask = (p->cq_entries << 1) - 1; } + /* + * A 32b CQE on a CQE_MIXED ring takes two contiguous entries + * and must not wrap. A smaller ring may place it on the last + * entry, fail the resize if that happens. + */ + if (ctx->flags & IORING_SETUP_CQE_MIXED) { + if (cqe32_tail) { + cqe32_tail = false; + } else if (READ_ONCE(o.rings->cqes[index & src_mask].flags) & + IORING_CQE_F_32) { + if ((index & dst_mask) == dst_mask) + goto overflow; + cqe32_tail = true; + } + } memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); } WRITE_ONCE(n.rings->cq.head, old_head); diff --git a/io_uring/rw.c b/io_uring/rw.c index 45a96e6546fc..a00b89868cb3 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -624,24 +624,6 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) smp_store_release(&req->iopoll_completed, 1); } -static inline ssize_t io_fixup_restart_res(ssize_t ret) -{ - switch (ret) { - case -ERESTARTSYS: - case -ERESTARTNOINTR: - case -ERESTARTNOHAND: - case -ERESTART_RESTARTBLOCK: - /* - * We can't just restart the syscall, since previously - * submitted sqes may already be in progress. Just fail - * this IO with EINTR. - */ - return -EINTR; - default: - return ret; - } -} - static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 2460bd605266..2991eaec256d 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -380,6 +380,13 @@ static int io_sq_thread(void *data) */ smp_mb__after_atomic(); + /* + * Nothing gets submitted before the ring is + * enabled, which comes with a wake up. + */ + if (READ_ONCE(ctx->flags) & IORING_SETUP_R_DISABLED) + continue; + if (io_sqring_entries(ctx)) { needs_sched = false; break; diff --git a/io_uring/timeout.c b/io_uring/timeout.c index c4dd26cf342d..ca03cb95ad8d 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -171,6 +171,14 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) raw_spin_lock_irq(&ctx->timeout_lock); seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); + /* + * A fired timeout bumps ->cq_timeouts before its CQE is posted, which + * leaves the sequence behind the last flush until it is. No event has + * happened then, see io_timeout(). + */ + if ((s32)(seq - ctx->cq_last_tm_flush) < 0) + seq = ctx->cq_last_tm_flush; + list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { struct io_kiocb *req = cmd_to_io_kiocb(timeout); u32 events_needed, events_got; @@ -321,22 +329,29 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) return HRTIMER_NORESTART; } -static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, - struct io_cancel_data *cd) +static struct io_kiocb *io_timeout_find(struct io_ring_ctx *ctx, + struct io_cancel_data *cd) __must_hold(&ctx->timeout_lock) { struct io_timeout *timeout; - struct io_timeout_data *io; - struct io_kiocb *req = NULL; list_for_each_entry(timeout, &ctx->timeout_list, list) { - struct io_kiocb *tmp = cmd_to_io_kiocb(timeout); + struct io_kiocb *req = cmd_to_io_kiocb(timeout); - if (io_cancel_req_match(tmp, cd)) { - req = tmp; - break; - } + if (io_cancel_req_match(req, cd)) + return req; } + return NULL; +} + +static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, + struct io_cancel_data *cd) + __must_hold(&ctx->timeout_lock) +{ + struct io_kiocb *req = io_timeout_find(ctx, cd); + struct io_timeout *timeout; + struct io_timeout_data *io; + if (!req) return ERR_PTR(-ENOENT); @@ -386,6 +401,9 @@ static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw) }; ret = io_try_cancel(req->tctx, &cd, 0); + /* a number of requests for futex and waitid */ + if (ret > 0) + ret = 0; } else { ret = -ECANCELED; } @@ -458,6 +476,8 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, io = req->async_data; if (hrtimer_try_to_cancel(&io->timer) == -1) return -EALREADY; + if (mode == HRTIMER_MODE_ABS) + ts = timens_ktime_to_host(io_timeout_get_clock(io), ts); hrtimer_setup(&io->timer, io_link_timeout_fn, io_timeout_get_clock(io), mode); hrtimer_start(&io->timer, ts, mode); return 0; @@ -468,15 +488,26 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, __must_hold(&ctx->timeout_lock) { struct io_cancel_data cd = { .ctx = ctx, .data = user_data, }; - struct io_kiocb *req = io_timeout_extract(ctx, &cd); + struct io_kiocb *req = io_timeout_find(ctx, &cd); struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data; - if (IS_ERR(req)) - return PTR_ERR(req); + if (!req) + return -ENOENT; - timeout->off = 0; /* noseq */ data = req->async_data; + /* multishot requests only make sense with rel values */ + if ((data->flags & IORING_TIMEOUT_MULTISHOT) && mode == HRTIMER_MODE_ABS) + return -EINVAL; + + req = io_timeout_extract(ctx, &cd); + if (IS_ERR(req)) + return PTR_ERR(req); + /* for a multishot request this isn't a sequence, keep the repeats */ + if (!(data->flags & IORING_TIMEOUT_MULTISHOT)) + timeout->off = 0; /* noseq */ + if (mode == HRTIMER_MODE_ABS) + time = timens_ktime_to_host(io_timeout_get_clock(data), time); data->time = time; list_add_tail(&timeout->list, &ctx->timeout_list); @@ -509,7 +540,12 @@ int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) IORING_TIMEOUT_ABS | IORING_TIMEOUT_IMMEDIATE_ARG)) return -EINVAL; - ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2), tr->flags); + /* + * The clock is that of the timeout to update, an absolute time + * is converted to the host's view once that has been found. + */ + ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2), + tr->flags & ~IORING_TIMEOUT_ABS); if (ret) return ret; } else if (tr->flags) { @@ -645,11 +681,19 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags) struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; struct io_timeout_data *data = req->async_data; + struct io_timeout *first; struct list_head *entry; - u32 tail, off = timeout->off; + u32 tail, base, off = timeout->off; raw_spin_lock_irq(&ctx->timeout_lock); + /* the list is searched under ->timeout_lock, see io_wq_cancel_seen() */ + if (__io_wq_cancel_seen(req, issue_flags)) { + raw_spin_unlock_irq(&ctx->timeout_lock); + io_req_task_queue_fail(req, -ECANCELED); + return IOU_ISSUE_SKIP_COMPLETE; + } + /* * sqe->off holds how many events that need to occur for this * timeout event to be satisfied. If it isn't set, then this is @@ -661,13 +705,27 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags) } tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); - timeout->target_seq = tail + off; - /* Update the last seq here in case io_flush_timeouts() hasn't. - * This is safe because ->completion_lock is held, and submissions - * and completions are never mixed in the same ->completion_lock section. + /* + * Update the last seq here in case io_flush_timeouts() hasn't, it only + * gets called once sequences are in use. Leave it alone if there are + * timeouts waiting for one. CQEs may have been posted since the last + * flush, and a target that has been reached must not fall behind it. */ - ctx->cq_last_tm_flush = tail; + first = list_first_entry_or_null(&ctx->timeout_list, struct io_timeout, + list); + if (!first || io_is_timeout_noseq(cmd_to_io_kiocb(first))) + ctx->cq_last_tm_flush = tail; + base = ctx->cq_last_tm_flush; + + /* + * A fired timeout bumps ->cq_timeouts before its CQE is posted, which + * leaves the sequence behind the last flush until it is. Nothing else + * separates the two then, so count from the flush. + */ + if ((s32)(tail - base) < 0) + tail = base; + timeout->target_seq = tail + off; /* * Insertion sort, ensuring the first entry in the list is always @@ -679,8 +737,8 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags) if (io_is_timeout_noseq(nxt)) continue; - /* nxt.seq is behind @tail, otherwise would've been completed */ - if (off >= nextt->target_seq - tail) + /* no target is behind the last flush, see io_flush_timeouts() */ + if (timeout->target_seq - base >= nextt->target_seq - base) break; } add: diff --git a/io_uring/tw.c b/io_uring/tw.c index f9affd68dd26..0b5066f97224 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -148,6 +148,37 @@ static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) } } +static void io_poll_wq_wake_rcu(struct rcu_head *rcu) +{ + struct io_ring_ctx *ctx = container_of(rcu, struct io_ring_ctx, + poll_wq_rcu); + + atomic_set(&ctx->poll_wq_rcu_pending, 0); + io_poll_wq_wake(ctx); + percpu_ref_put(&ctx->refs); +} + +/* + * io_uring_poll() reports pending local work as EPOLLIN, as it takes a trip to + * the kernel to turn it into CQEs. Let those know who are asleep already. + */ +static void io_local_work_wake_poll(struct io_ring_ctx *ctx, unsigned int flags) +{ + if (!(flags & IOU_F_TWQ_IN_WAKE)) { + io_poll_wq_wake(ctx); + return; + } + /* + * Called from a wake up handler with some wait queue lock held, this + * one could lead back to it. Do it from a clean context. + */ + if (!wq_has_sleeper(&ctx->poll_wq) || + atomic_xchg(&ctx->poll_wq_rcu_pending, 1)) + return; + percpu_ref_get(&ctx->refs); + call_rcu_hurry(&ctx->poll_wq_rcu, io_poll_wq_wake_rcu); +} + void io_req_local_work_add(struct io_kiocb *req, unsigned flags) { struct io_ring_ctx *ctx = req->ctx; @@ -165,7 +196,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) /* * The xchg() in mpscq_push() implies a full barrier, which pairs with - * the barrier in set_current_state() on the io_cqring_wait() side. This + * the barrier in io_cq_wait_arm() on the io_cqring_wait() side. This * ensures that either we see the updated ->cq_wait_nr, or waiters going * to sleep will observe the work added to the list, which is similar to * the wait/wake task state sync. @@ -174,6 +205,9 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) io_ctx_mark_taskrun(ctx); if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); + /* whoever sets the flag wakes ->poll_wq afterwards */ + if (data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED) + io_local_work_wake_poll(ctx, flags); } /* diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 7c1337f29a5e..f2301c455de2 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -87,7 +87,7 @@ static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd, cmd->flags &= ~IORING_URING_CMD_CANCELABLE; io_ring_submit_lock(ctx, issue_flags); - hlist_del(&req->hash_node); + hlist_del_init(&req->hash_node); io_ring_submit_unlock(ctx, issue_flags); } diff --git a/io_uring/wait.c b/io_uring/wait.c index a48523c2f356..fb1c082993a7 100644 --- a/io_uring/wait.c +++ b/io_uring/wait.c @@ -3,6 +3,7 @@ * Waiting for completion events */ #include +#include #include #include #include @@ -26,7 +27,8 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, */ if (io_should_wake(iowq) || io_has_work(iowq->ctx)) return autoremove_wake_function(curr, mode, wake_flags, key); - return -1; + /* keep going, a waiter further down may be content with less */ + return 0; } int io_run_task_work_sig(struct io_ring_ctx *ctx) @@ -76,9 +78,16 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) if (iowq->timeout == KTIME_MAX || ktime_compare(iowq->min_timeout, iowq->timeout) >= 0) goto out_wake; - /* work we may need to run, wake function will see if we need to wake */ - if (io_has_work(ctx)) - goto out_wake; + /* + * Work we need to run. It isn't known if that is going to post a CQE, + * so this isn't a timeout. Carry on with the normal one if it doesn't. + */ + if (io_has_work(ctx)) { + iowq->min_timeout = 0; + iowq->cq_tail = iowq->cq_min_tail + 1; + wake_up_process(iowq->wq.private); + return HRTIMER_NORESTART; + } /* got events since we started waiting, min timeout is done */ scoped_guard(rcu) { struct io_rings *rings = io_get_rings(ctx); @@ -112,23 +121,32 @@ static enum hrtimer_restart io_cqring_min_timer_wakeup(struct hrtimer *timer) return io_cqring_timer_wakeup(timer); } +/* Same policy as for hrtimer sleepers, see __hrtimer_setup_sleeper(). */ +static enum hrtimer_mode io_cqring_timer_mode(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && rt_or_dl_task_policy(current)) + return HRTIMER_MODE_ABS_HARD; + return HRTIMER_MODE_ABS; +} + static int io_cqring_schedule_timeout(struct io_wait_queue *iowq, clockid_t clock_id, ktime_t start_time) { + enum hrtimer_mode mode = io_cqring_timer_mode(); ktime_t timeout; if (iowq->min_timeout) { timeout = ktime_add_ns(iowq->min_timeout, start_time); hrtimer_setup_on_stack(&iowq->t, io_cqring_min_timer_wakeup, clock_id, - HRTIMER_MODE_ABS); + mode); } else { timeout = iowq->timeout; hrtimer_setup_on_stack(&iowq->t, io_cqring_timer_wakeup, clock_id, - HRTIMER_MODE_ABS); + mode); } hrtimer_set_expires_range_ns(&iowq->t, timeout, 0); - hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS); + hrtimer_start_expires(&iowq->t, mode); if (!READ_ONCE(iowq->hit_timeout)) schedule(); @@ -257,8 +275,7 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, unsigned long check_cq; if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { - atomic_set(&ctx->cq_wait_nr, nr_wait); - set_current_state(TASK_INTERRUPTIBLE); + io_cq_wait_arm(ctx, nr_wait); } else { prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); @@ -291,9 +308,18 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, check_cq = READ_ONCE(ctx->check_cq); if (unlikely(check_cq)) { - /* let the caller flush overflows, retry */ - if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) + if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) { io_cqring_do_overflow_flush(ctx); + /* + * No room for what has overflown. It takes the + * application to change that, so return to it. + */ + if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, + &ctx->check_cq)) { + ret = 0; + break; + } + } if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) { ret = -EBADR; break; diff --git a/io_uring/wait.h b/io_uring/wait.h index b7b9c46b1b01..eb5ae8a6fc94 100644 --- a/io_uring/wait.h +++ b/io_uring/wait.h @@ -14,6 +14,21 @@ */ #define IO_CQ_WAKE_INIT (-1) +/* + * Start a wait cycle of the current task. The add that uses up ->cq_wait_nr + * may be one that queued its entry long ago and only now gets to look at the + * count. As nobody else wakes us after that, it has to find us sleeping, hence + * the task state must be visible before the count is. The barrier pairs with + * the xchg() in mpscq_push(), see io_req_local_work_add(). + */ +static inline void io_cq_wait_arm(struct io_ring_ctx *ctx, int nr_wait) +{ + __set_current_state(TASK_INTERRUPTIBLE); + atomic_set_release(&ctx->cq_wait_nr, nr_wait); + /* order the above against looking at the work list */ + smp_mb(); +} + struct ext_arg { size_t argsz; struct timespec64 ts; diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 2b756017423f..172cf770b29b 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -154,9 +154,12 @@ static bool __io_waitid_cancel(struct io_kiocb *req, bool copy_si) /* * Mark us canceled regardless of ownership. This will prevent a - * potential retry from a spurious wakeup. + * potential retry from a spurious wakeup. Only the first cancel + * attempt should take a reference for the ownership handoff. */ - atomic_or(IO_WAITID_CANCEL_FLAG, &iw->refs); + if (atomic_fetch_or(IO_WAITID_CANCEL_FLAG, &iw->refs) & + IO_WAITID_CANCEL_FLAG) + return false; /* claim ownership */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) @@ -265,12 +268,18 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, return 0; list_del_init(&wait->entry); - smp_store_release(&iw->head, NULL); - /* cancel is in progress */ + /* + * Cancel or issue is in progress. Leave ->head alone, so that whoever + * owns the request has to take the wait queue lock, which we hold, + * before it can complete and free it. + */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return 1; + /* pairs with the smp_load_acquire() in io_waitid_remove_wq() */ + smp_store_release(&iw->head, NULL); + req->io_task_work.func = io_waitid_cb; __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); return 1; @@ -343,6 +352,8 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags) * a waitqueue callback, or if someone cancels it. */ if (!io_waitid_drop_issue_ref(req)) { + if (io_wq_cancel_seen(req, issue_flags)) + __io_waitid_cancel(req, true); io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; } diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 0a9e8a58b99a..6324d9196df6 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1474,7 +1474,7 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov, cqe->flags |= IORING_CQE_F_32; area = io_zcrx_iov_to_area(niov); - offset = off + (net_iov_idx(niov) << ifq->niov_shift); + offset = off + ((u64)net_iov_idx(niov) << ifq->niov_shift); rcqe = (struct io_uring_zcrx_cqe *)(cqe + 1); rcqe->off = offset + ((u64)area->area_id << IORING_ZCRX_AREA_SHIFT); rcqe->__pad = 0; diff --git a/kernel/exit.c b/kernel/exit.c index f7c48f1c404f..256f9ab9cbfe 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -979,10 +979,11 @@ void __noreturn do_exit(long code) exit_sem(tsk); exit_shm(tsk); - exit_files(tsk); - exit_fs(tsk); + /* Hang the tty up before the last close of it can clear the session. */ if (group_dead) disassociate_ctty(1); + exit_files(tsk); + exit_fs(tsk); exit_nsproxy_namespaces(tsk); exit_task_work(tsk); exit_thread(tsk); diff --git a/kernel/fork.c b/kernel/fork.c index 854448152696..26485ba0dac7 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1658,6 +1658,7 @@ static int copy_files(u64 clone_flags, struct task_struct *tsk, if (clone_flags & CLONE_FILES) { atomic_inc(&oldf->count); + tsk->files = oldf; return 0; } @@ -2181,6 +2182,8 @@ __latent_entropy struct task_struct *copy_process( INIT_LIST_HEAD(&p->sibling); rcu_copy_process(p); p->vfork_done = NULL; + /* Set by copy_files(), exit_files() on the error path skips NULL. */ + p->files = NULL; spin_lock_init(&p->alloc_lock); init_sigpending(&p->pending); @@ -2282,7 +2285,7 @@ __latent_entropy struct task_struct *copy_process( goto bad_fork_cleanup_semundo; retval = copy_fs(clone_flags, p); if (retval) - goto bad_fork_cleanup_files; + goto bad_fork_cleanup_semundo; retval = copy_sighand(clone_flags, p); if (retval) goto bad_fork_cleanup_fs; @@ -2596,8 +2599,6 @@ __latent_entropy struct task_struct *copy_process( __cleanup_sighand(p->sighand); bad_fork_cleanup_fs: exit_fs(p); /* blocking */ -bad_fork_cleanup_files: - exit_files(p); /* blocking */ bad_fork_cleanup_semundo: exit_sem(p); bad_fork_cleanup_security: @@ -2608,6 +2609,8 @@ __latent_entropy struct task_struct *copy_process( perf_event_free_task(p); bad_fork_sched_cancel_fork: sched_cancel_fork(p); + /* ->release() of a file may need scx_fork_rwsem for write. */ + exit_files(p); /* blocking */ bad_fork_cleanup_policy: lockdep_free_task(p); #ifdef CONFIG_NUMA diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 4ade0f37ed54..127a5fd23e70 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -238,9 +238,10 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns) * kernel_wait4() will also block until our children traced from the * parent namespace are detached and become EXIT_DEAD. */ + /* Task work must not busy-loop the reaper, see signal_pending(). */ + guard(no_notify_signal)(); do { clear_thread_flag(TIF_SIGPENDING); - clear_thread_flag(TIF_NOTIFY_SIGNAL); rc = kernel_wait4(-1, NULL, __WALL, NULL); } while (rc != -ECHILD); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index d041645d9d17..4e9822a87aab 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -1227,6 +1227,12 @@ int ptrace_request(struct task_struct *child, long request, case PTRACE_SETSIGMASK: { sigset_t new_set; + /* A user worker only ever takes SIGKILL and SIGSTOP. */ + if (child->flags & PF_USER_WORKER) { + ret = -EPERM; + break; + } + if (addr != sizeof(sigset_t)) { ret = -EINVAL; break; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3ee16f7f22cd..c117cf172dfc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2292,6 +2292,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) list_add(&p->scx.dsq_list.node, &dst_dsq->list); @@ -2462,8 +2464,13 @@ static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, raw_spin_rq_lock(src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); + if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); + return true; + } + + return false; } static bool consume_remote_task(struct rq *this_rq, @@ -2511,6 +2518,9 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(src_rq); + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); + if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); if (src_rq != dst_rq && diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b6cace1d0e32..9e1a28a158da 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6250,12 +6250,20 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) load = cfs_rq->sum_weight; if (curr && curr->on_rq) load += avg_vruntime_weight(cfs_rq, curr->load.weight); - - weight = avg_vruntime_weight(cfs_rq, se->load.weight); - lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; - lag = div64_long(lag, load); + + weight = avg_vruntime_weight(cfs_rq, se->load.weight); + + /* + * Rewrite, because the product (W + w_i)*vl'_i might + * overflow s64: + * + * vl_i = (W + w_i)*vl'_i / W + * = (W*vl'_i + w_i*vl'_i) / W + * = vl'_i + w_i*vl'_i/W + */ + lag += div64_long(lag * weight, load); /* * A heavy entity (relative to the tree) will pull the diff --git a/kernel/signal.c b/kernel/signal.c index 808980d08371..67d395986739 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -3181,6 +3181,10 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which) sigset_t retarget; struct task_struct *t; + /* Nobody dequeues them in a dying group, see get_signal(). */ + if (tsk->signal->flags & SIGNAL_GROUP_EXIT) + return; + sigandsets(&retarget, &tsk->signal->shared_pending.signal, which); if (sigisemptyset(&retarget)) return; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index db8eeaefeb00..b2f74dfdb936 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -4426,9 +4426,9 @@ subsys_initcall(ftrace_check_for_weak_functions); static int print_rec(struct seq_file *m, unsigned long ip) { - unsigned long offset; + unsigned long offset = 0; char str[KSYM_SYMBOL_LEN]; - char *modname; + char *modname = NULL; const char *ret; ret = kallsyms_lookup(ip, NULL, &offset, &modname, str); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 9dcbe243dbe8..479b20809fa9 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3614,6 +3614,7 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file) return single_release(inode, file); } +static bool __update_last_data_if_empty(struct trace_array *tr); static bool update_last_data_if_empty(struct trace_array *tr); static int tracing_open(struct inode *inode, struct file *file) @@ -4706,7 +4707,7 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf, int tracer_init(struct tracer *t, struct trace_array *tr) { tracing_reset_online_cpus(&tr->array_buffer); - update_last_data_if_empty(tr); + __update_last_data_if_empty(tr); return t->init(tr); } @@ -5385,8 +5386,10 @@ static int tracing_wait_pipe(struct file *filp) return 1; } -static bool update_last_data_if_empty(struct trace_array *tr) +static bool __update_last_data_if_empty(struct trace_array *tr) { + lockdep_assert_held(&trace_types_lock); + if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) return false; @@ -5401,6 +5404,16 @@ static bool update_last_data_if_empty(struct trace_array *tr) return true; } +static bool update_last_data_if_empty(struct trace_array *tr) +{ + /* Avoid the lock on the read path unless this is a last boot buffer */ + if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) + return false; + + guard(mutex)(&trace_types_lock); + return __update_last_data_if_empty(tr); +} + /* * Consumer reader. */ @@ -6493,7 +6506,7 @@ int tracing_set_clock(struct trace_array *tr, const char *clockstr) ring_buffer_set_clock(tr->snapshot_buffer.buffer, trace_clocks[i].func); tracing_reset_online_cpus(&tr->snapshot_buffer); #endif - update_last_data_if_empty(tr); + __update_last_data_if_empty(tr); if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { struct trace_scratch *tscratch = tr->scratch; @@ -9261,6 +9274,8 @@ static void trace_module_record(struct module *mod, bool add) struct trace_array *tr; unsigned long flags; + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT); /* Update any persistent trace array that has already been started */ diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 127880ebe83f..b09acea4602b 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -595,6 +595,12 @@ predicate_parse(const char *str, int nr_parens, int nr_preds, top--; /* #12 */ } } + /* Only the inner loop can complete a non-empty expression. */ + if (N) { + ret = -EINVAL; + parse_error(pe, FILT_ERR_INVALID_FILTER, ptr - str); + goto out_free; + } out: if (top != op_stack) { /* Too many '(' */ diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 160445760545..45a3eb6a65cd 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -467,7 +467,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field, { \ type *addr = (type *)(event + hist_field->field->offset); \ \ - return (u64)(unsigned long)*addr; \ + return (u64)*addr; \ } DEFINE_HIST_FIELD_FN(s64); @@ -550,7 +550,7 @@ struct field_var { }; struct field_var_hist { - struct hist_trigger_data *hist_data; + struct trace_event_file *file; char *cmd; }; @@ -3099,8 +3099,8 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, return ERR_PTR(-ENOMEM); } - /* Save the compatible histogram information */ - var_hist->hist_data = hist_data; + /* Needed to remove the histogram when the target goes away */ + var_hist->file = file; /* Create the new histogram with our variable */ ret = event_hist_trigger_parse(&trigger_hist_cmd, file, @@ -3115,20 +3115,18 @@ create_field_var_hist(struct hist_trigger_data *target_hist_data, kfree(cmd); + n = target_hist_data->n_field_var_hists; + target_hist_data->field_var_hists[n] = var_hist; + target_hist_data->n_field_var_hists++; + /* If we can't find the variable, something went wrong */ event_var = find_synthetic_field_var(target_hist_data, subsys_name, event_name, field_name); if (IS_ERR_OR_NULL(event_var)) { - kfree(var_hist->cmd); - kfree(var_hist); hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); return ERR_PTR(-EINVAL); } - n = target_hist_data->n_field_var_hists; - target_hist_data->field_var_hists[n] = var_hist; - target_hist_data->n_field_var_hists++; - return event_var; } @@ -6331,7 +6329,7 @@ static void unregister_field_var_hists(struct hist_trigger_data *hist_data) int ret; for (i = 0; i < hist_data->n_field_var_hists; i++) { - file = hist_data->field_var_hists[i]->hist_data->event_file; + file = hist_data->field_var_hists[i]->file; cmd = hist_data->field_var_hists[i]->cmd; ret = event_hist_trigger_parse(&trigger_hist_cmd, file, "!hist", "hist", cmd); @@ -6781,7 +6779,6 @@ static void hist_unreg_all(struct trace_event_file *file) struct event_trigger_data *test, *n; struct hist_trigger_data *hist_data; struct synth_event *se; - const char *se_name; lockdep_assert_held(&event_mutex); @@ -6794,8 +6791,7 @@ static void hist_unreg_all(struct trace_event_file *file) list_del_rcu(&test->list); trace_event_trigger_enable_disable(file, 0); - se_name = trace_event_name(file->event_call); - se = find_synth_event(se_name); + se = find_synth_event_call(file->event_call); if (se) se->ref--; @@ -6819,7 +6815,6 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, struct hist_trigger_data *hist_data; char *param, *filter, *p, *start; struct synth_event *se; - const char *se_name; bool remove; int ret = 0; @@ -6913,8 +6908,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, } event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); - se_name = trace_event_name(file->event_call); - se = find_synth_event(se_name); + se = find_synth_event_call(file->event_call); if (se) se->ref--; ret = 0; @@ -6949,8 +6943,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, if (ret) goto out_unreg; - se_name = trace_event_name(file->event_call); - se = find_synth_event(se_name); + se = find_synth_event_call(file->event_call); if (se) se->ref++; out: @@ -6967,6 +6960,8 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, trigger_data_free(trigger_data); + unregister_field_var_hists(hist_data); + destroy_hist_data(hist_data); goto out; } diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index dc15658a887c..e2870212e58d 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -914,6 +914,16 @@ struct synth_event *find_synth_event(const char *name) return NULL; } +struct synth_event *find_synth_event_call(struct trace_event_call *call) +{ + struct synth_event *se; + const char *name; + + name = trace_event_name(call); + se = find_synth_event(name); + return se && &se->call == call ? se : NULL; +} + static struct trace_event_fields synth_event_fields_array[] = { { .type = TRACE_FUNCTION_TYPE, .define_fields = synth_event_define_fields }, diff --git a/kernel/trace/trace_synth.h b/kernel/trace/trace_synth.h index 43f6fb6078db..d4ebf01b7049 100644 --- a/kernel/trace/trace_synth.h +++ b/kernel/trace/trace_synth.h @@ -37,5 +37,6 @@ struct synth_event { }; extern struct synth_event *find_synth_event(const char *name); +extern struct synth_event *find_synth_event_call(struct trace_event_call *call); #endif /* __TRACE_SYNTH_H */ diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 8a4f3c75e39f..6406123e702c 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -682,7 +682,7 @@ static int syscall_copy_user_array(char *buf, const char __user *ptr, for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) { ptr = (char __user *)args->ptr_array[i]; - ret = __copy_from_user(buf, ptr, size); + ret = copy_from_user(buf, ptr, size); args->read[i] = ret ? -1 : size; } return 0; diff --git a/mm/damon/core.c b/mm/damon/core.c index 40714c69d1b1..0aefaef46d65 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -999,15 +999,18 @@ static void damos_commit_quota_goal_union( } } -static void damos_commit_quota_goal( +static int damos_commit_quota_goal( struct damos_quota_goal *dst, struct damos_quota_goal *src) { + if (!src->target_value) + return -EINVAL; dst->metric = src->metric; dst->target_value = src->target_value; if (dst->metric == DAMOS_QUOTA_USER_INPUT) dst->current_value = src->current_value; /* keep last_psi_total as is, since it will be updated in next cycle */ damos_commit_quota_goal_union(dst, src); + return 0; } /** @@ -1025,14 +1028,17 @@ static void damos_commit_quota_goal( int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) { struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal; - int i = 0, j = 0; + int i = 0, j = 0, err; damos_for_each_quota_goal_safe(dst_goal, next, dst) { src_goal = damos_nth_quota_goal(i++, src); - if (src_goal) - damos_commit_quota_goal(dst_goal, src_goal); - else + if (src_goal) { + err = damos_commit_quota_goal(dst_goal, src_goal); + if (err) + return err; + } else { damos_destroy_quota_goal(dst_goal); + } } damos_for_each_quota_goal_safe(src_goal, next, src) { if (j++ < i) @@ -1041,7 +1047,11 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src) src_goal->metric, src_goal->target_value); if (!new_goal) return -ENOMEM; - damos_commit_quota_goal(new_goal, src_goal); + err = damos_commit_quota_goal(new_goal, src_goal); + if (err) { + damos_free_quota_goal(new_goal); + return err; + } damos_add_quota_goal(dst, new_goal); } return 0; diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 26cdad603ad0..9c7e34f18b28 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -2226,6 +2226,20 @@ static int damon_sysfs_repeat_call_fn(void *data) return 0; } +static bool damon_sysfs_valid_start_ctx(struct damon_ctx *ctx) +{ + struct damos *s; + struct damos_quota_goal *g; + + damon_for_each_scheme(s, ctx) { + damos_for_each_quota_goal(g, &s->quota) { + if (!g->target_value) + return false; + } + } + return true; +} + static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) { struct damon_ctx *ctx; @@ -2251,6 +2265,11 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) kfree(repeat_call_control); return PTR_ERR(ctx); } + if (!damon_sysfs_valid_start_ctx(ctx)) { + kfree(repeat_call_control); + damon_destroy_ctx(ctx); + return -EINVAL; + } err = damon_start(&ctx, 1, false); if (err) { kfree(repeat_call_control); diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 9a1b0ae07679..fa17f235d5e4 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -622,6 +622,10 @@ void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan) BT_DBG("conn %p, psm 0x%2.2x, dcid 0x%4.4x", conn, __le16_to_cpu(chan->psm), chan->dcid); + /* Caller must ensure l2cap_chan is linked to l2cap_conn only once */ + if (WARN_ON_ONCE(chan->conn || test_bit(FLAG_DEL, &chan->flags))) + return; + conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM; chan->conn = l2cap_conn_get(conn); @@ -7412,7 +7416,8 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid, } } - if (cid && __l2cap_get_chan_by_dcid(conn, cid)) { + if ((cid && __l2cap_get_chan_by_dcid(conn, cid)) || chan->conn || + test_bit(FLAG_DEL, &chan->flags)) { hci_conn_drop(hcon); err = -EBUSY; goto chan_unlock; diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 143654a47989..7eefb46f0692 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -109,6 +109,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i return -EINVAL; } + l2cap_chan_lock(chan); lock_sock(sk); if (sk->sk_state != BT_OPEN) { @@ -174,6 +175,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i done: release_sock(sk); + l2cap_chan_unlock(chan); return err; } diff --git a/security/keys/key.c b/security/keys/key.c index a438c4508595..6ebb8691551f 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -588,11 +588,18 @@ int key_reject_and_link(struct key *key, ret = -EBUSY; if (keyring) { - if (keyring->restrict_link) - return -EPERM; - link_ret = __key_link_lock(keyring, &key->index_key); if (link_ret == 0) { + /* + * Check the restriction under the keyring semaphore. + * keyring_restrict() installs it while holding the + * semaphore, so testing it beforehand races with a + * concurrent restriction install. + */ + if (keyring->restrict_link) { + __key_link_end(keyring, &key->index_key, edit); + return -EPERM; + } link_ret = __key_link_begin(keyring, &key->index_key, &edit); if (link_ret < 0) __key_link_end(keyring, &key->index_key, edit); diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index d14ace88e529..36ffba0ebba6 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -677,12 +677,14 @@ long keyctl_describe_key(key_serial_t keyid, /* calculate how much information we're going to return */ ret = -ENOMEM; + down_read(&key->sem); infobuf = kasprintf(GFP_KERNEL, "%s;%d;%d;%08x;", key->type->name, from_kuid_munged(current_user_ns(), key->uid), from_kgid_munged(current_user_ns(), key->gid), key->perm); + up_read(&key->sem); if (!infobuf) goto error2; infolen = strlen(infobuf); @@ -806,8 +808,12 @@ static long __keyctl_read_key(struct key *key, char *buffer, size_t buflen) down_read(&key->sem); ret = key_validate(key); - if (ret == 0) - ret = key->type->read(key, buffer, buflen); + if (ret == 0) { + if (key->type->read) + ret = key->type->read(key, buffer, buflen); + else + ret = -EOPNOTSUPP; + } up_read(&key->sem); return ret; } @@ -861,11 +867,6 @@ long keyctl_read_key(key_serial_t keyid, char __user *buffer, size_t buflen) /* the key is probably readable - now try to read it */ can_read_key: - if (!key->type->read) { - ret = -EOPNOTSUPP; - goto key_put_out; - } - if (!buffer || !buflen) { /* Get the key length from the read method */ ret = __keyctl_read_key(key, NULL, 0); diff --git a/security/keys/keyring.c b/security/keys/keyring.c index 15bf4af8f282..7143f3079d20 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -147,49 +150,46 @@ static int keyring_instantiate(struct key *keyring, } /* - * Multiply 64-bits by 32-bits to 96-bits and fold back to 64-bit. Ideally we'd - * fold the carry back too, but that requires inline asm. + * Give key types and domains address-independent identities. Zero means that + * an ID has not yet been allocated. As with inode sequence numbers used by + * futexes, this relies on the 64-bit counter not wrapping in a machine's + * lifetime. */ -static u64 mult_64x32_and_fold(u64 x, u32 y) +static u64 key_index_id(atomic64_t *id) { - u64 hi = (u64)(u32)(x >> 32) * y; - u64 lo = (u64)(u32)(x) * y; - return lo + ((u64)(u32)hi << 32) + (u32)(hi >> 32); + static atomic64_t next_id; + u64 old, new; + + old = atomic64_read(id); + if (likely(old)) + return old; + + do { + new = atomic64_inc_return(&next_id); + } while (WARN_ON_ONCE(!new)); + + old = 0; + if (!atomic64_try_cmpxchg_relaxed(id, &old, new)) + return old; + return new; } /* - * Hash a key type and description. + * Hash a key type, domain and description. KEYCTL_READ exposes the tree's + * traversal order, so no part of the index may contain kernel addresses. */ static void hash_key_type_and_desc(struct keyring_index_key *index_key) { const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP; const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK; - const char *description = index_key->description; - unsigned long hash, type; - u32 piece; + static siphash_key_t hash_key __read_mostly; + unsigned long hash; u64 acc; - int n, desc_len = index_key->desc_len; - - type = (unsigned long)index_key->type; - acc = mult_64x32_and_fold(type, desc_len + 13); - acc = mult_64x32_and_fold(acc, 9207); - piece = (unsigned long)index_key->domain_tag; - acc = mult_64x32_and_fold(acc, piece); - acc = mult_64x32_and_fold(acc, 9207); - - for (;;) { - n = desc_len; - if (n <= 0) - break; - if (n > 4) - n = 4; - piece = 0; - memcpy(&piece, description, n); - description += n; - desc_len -= n; - acc = mult_64x32_and_fold(acc, piece); - acc = mult_64x32_and_fold(acc, 9207); - } + + get_random_once(&hash_key, sizeof(hash_key)); + acc = siphash(index_key->description, index_key->desc_len, &hash_key); + acc = siphash_3u64(index_key->type_id, index_key->domain_id, acc, + &hash_key); /* Fold the hash down to 32 bits if need be. */ hash = acc; @@ -209,7 +209,7 @@ static void hash_key_type_and_desc(struct keyring_index_key *index_key) /* * Finalise an index key to include a part of the description actually in the - * index key, to set the domain tag and to calculate the hash. + * index key, to set the domain tag and index IDs, and to calculate the hash. */ void key_set_index_key(struct keyring_index_key *index_key) { @@ -225,6 +225,14 @@ void key_set_index_key(struct keyring_index_key *index_key) index_key->domain_tag = &default_domain_tag; } + /* Preserve the IDs when copying an existing key's index. In + * particular, its type may already have been unregistered. + */ + if (!index_key->type_id) + index_key->type_id = key_index_id(&index_key->type->index_id); + if (!index_key->domain_id) + index_key->domain_id = key_index_id(&index_key->domain_tag->index_id); + hash_key_type_and_desc(index_key); } @@ -263,15 +271,19 @@ void key_remove_domain(struct key_tag *domain_tag) /* * Build the next index key chunk. * + * The hash and inline description are followed by the 64-bit type and domain + * IDs, low word first, and then by the rest of the description. * We return it one word-sized chunk at a time. */ static unsigned long keyring_get_key_chunk(const void *data, int level) { + const unsigned int id_chunks = sizeof(u64) / sizeof(unsigned long); const struct keyring_index_key *index_key = data; + int desc_len = index_key->desc_len; unsigned long chunk = 0; - const u8 *d; - int desc_len = index_key->desc_len, n = sizeof(chunk); + int n = sizeof(chunk); unsigned int offset; + const u8 *d; level /= ASSOC_ARRAY_KEY_CHUNK_SIZE; switch (level) { @@ -279,27 +291,30 @@ static unsigned long keyring_get_key_chunk(const void *data, int level) return index_key->hash; case 1: return index_key->x; - case 2: - return (unsigned long)index_key->type; - case 3: - return (unsigned long)index_key->domain_tag; - default: - level -= 4; - offset = sizeof(index_key->desc) + level * sizeof(long); - if (desc_len <= offset) - return 0; - - d = index_key->description + offset; - desc_len -= offset; - if (desc_len > n) - desc_len = n; - d += desc_len; - do { - chunk <<= 8; - chunk |= *--d; - } while (--desc_len > 0); - return chunk; } + + level -= 2; + if (level < id_chunks) + return index_key->type_id >> (level * ASSOC_ARRAY_KEY_CHUNK_SIZE); + level -= id_chunks; + if (level < id_chunks) + return index_key->domain_id >> (level * ASSOC_ARRAY_KEY_CHUNK_SIZE); + level -= id_chunks; + + offset = sizeof(index_key->desc) + level * sizeof(long); + if (desc_len <= offset) + return 0; + + d = index_key->description + offset; + desc_len -= offset; + if (desc_len > n) + desc_len = n; + d += desc_len; + do { + chunk <<= 8; + chunk |= *--d; + } while (--desc_len > 0); + return chunk; } static unsigned long keyring_get_object_key_chunk(const void *object, int level) @@ -331,6 +346,7 @@ static int keyring_diff_objects(const void *object, const void *data) const struct keyring_index_key *b = data; unsigned long seg_a, seg_b; int level, i; + u64 diff; level = 0; seg_a = a->hash; @@ -339,28 +355,22 @@ static int keyring_diff_objects(const void *object, const void *data) goto differ; level += ASSOC_ARRAY_KEY_CHUNK_SIZE / 8; - /* The number of bits contributed by the hash is controlled by a - * constant in the assoc_array headers. Everything else thereafter we - * can deal with as being machine word-size dependent. - */ + /* The inline description contributes one machine-word-sized chunk. */ seg_a = a->x; seg_b = b->x; if ((seg_a ^ seg_b) != 0) goto differ; level += sizeof(unsigned long); - /* The next bit may not work on big endian */ - seg_a = (unsigned long)a->type; - seg_b = (unsigned long)b->type; - if ((seg_a ^ seg_b) != 0) - goto differ; - level += sizeof(unsigned long); + diff = a->type_id ^ b->type_id; + if (diff) + return level * 8 + __ffs64(diff); + level += sizeof(u64); - seg_a = (unsigned long)a->domain_tag; - seg_b = (unsigned long)b->domain_tag; - if ((seg_a ^ seg_b) != 0) - goto differ; - level += sizeof(unsigned long); + diff = a->domain_id ^ b->domain_id; + if (diff) + return level * 8 + __ffs64(diff); + level += sizeof(u64); i = sizeof(a->desc); if (a->desc_len <= i) @@ -1158,7 +1168,7 @@ struct key *find_keyring_by_name(const char *name, bool uid_keyring) * grants Search permission and that hasn't been revoked */ list_for_each_entry(keyring, &ns->keyring_name_list, name_link) { - if (!kuid_has_mapping(ns, keyring->user->uid)) + if (!kuid_has_mapping(ns, keyring->uid)) continue; if (test_bit(KEY_FLAG_REVOKED, &keyring->flags)) diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index a63c46bb2d14..76da13fa07fa 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -556,9 +556,8 @@ key_ref_t search_process_keyrings_rcu(struct keyring_search_context *ctx) ) { const struct cred *cred = ctx->cred; - if (key_validate(cred->request_key_auth) == 0) { - rka = ctx->cred->request_key_auth->payload.data[0]; - + rka = dereference_key_rcu(cred->request_key_auth); + if (rka && key_validate(cred->request_key_auth) == 0) { //// was search_process_keyrings() [ie. recursive] ctx->cred = rka->cred; key_ref = search_cred_keyrings_rcu(ctx); diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index 8f57c6111e7e..38ffaccf850a 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -928,8 +928,10 @@ static int trusted_tpm_unseal(struct trusted_key_payload *p, char *datablob) ret = tpm2_unseal_trusted(chip, p, options); else ret = key_unseal(p, options); - if (ret < 0) + if (ret < 0) { pr_info("key_unseal failed (%d)\n", ret); + goto out; + } if (options->pcrlock) { ret = pcrlock(options->pcrlock); diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c index e8c129396b8b..fab793912fd1 100644 --- a/security/keys/trusted-keys/trusted_tpm2.c +++ b/security/keys/trusted-keys/trusted_tpm2.c @@ -116,7 +116,7 @@ static int tpm2_key_decode(struct trusted_key_payload *payload, if (ctx.priv_len + ctx.pub_len > MAX_BLOB_SIZE) return -EINVAL; - blob = kmalloc(ctx.priv_len + ctx.pub_len + 4, GFP_KERNEL); + blob = kmalloc(ctx.priv_len + ctx.pub_len, GFP_KERNEL); if (!blob) return -ENOMEM; diff --git a/tools/dma/Makefile b/tools/dma/Makefile index e4abf37bf020..0c239a6b9eae 100644 --- a/tools/dma/Makefile +++ b/tools/dma/Makefile @@ -38,8 +38,8 @@ prepare: $(OUTPUT)include/linux/map_benchmark.h FORCE: DMA_MAP_BENCHMARK = dma_map_benchmark -$(DMA_MAP_BENCHMARK): prepare FORCE - $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $(DMA_MAP_BENCHMARK) +$(OUTPUT)$(DMA_MAP_BENCHMARK): prepare FORCE + $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $@ clean: rm -f $(ALL_PROGRAMS) diff --git a/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc b/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc new file mode 100644 index 000000000000..cc6df762845f --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/filter/event-filter-incomplete.tc @@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: event filter - reject incomplete logical expressions +# requires: events/sched/sched_switch/filter +# flags: instance + +filter=events/sched/sched_switch/filter + +for base in \ + 'common_pid == 1' \ + 'common_pid == 1 || common_pid == 2' \ + 'common_pid == 1 && common_pid == 2' \ + '!(common_pid == 1 || common_pid == 2)' +do + printf '%s' "$base" > "$filter" + for tail in '&&' '||' '&& !' '|| !' '&& !! ' '|| !! '; do + if printf '%s' "$base $tail" > "$filter" 2>/dev/null; then + echo "Accepted incomplete filter: $base $tail" + exit_fail + fi + done +done + +echo 0 > "$filter"