* [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS
@ 2026-11-14 15:05 Aibing Chen
2026-11-15 0:18 ` Jason Gunthorpe
2026-11-16 7:54 ` [PATCH net v2] net/rds: defer CM id destruction to a workqueue Aibing Chen
0 siblings, 2 replies; 4+ messages in thread
From: Aibing Chen @ 2026-11-14 15:05 UTC (permalink / raw)
To: jgg, leon, achender
Cc: davem, edumazet, kuba, pabeni, horms, linux-rdma, netdev,
rds-devel, linux-kernel, hdanton, hackerowskiy,
syzbot+ccdb5058c3df2d94811c, Aibing Chen
syzbot reported a possible circular locking dependency between
rds_trans_sem and the RDMA CM handler mutex:
CPU0 (bind path) CPU1 (CM callback path)
---- ----
down_read(&rds_trans_sem) mutex_lock(&handler_mutex)
rds_ib_laddr_check() cma_ib_req_handler()
rdma_destroy_id() rds_ib_cm_handle_connect()
mutex_lock(&handler_mutex) rds_conn_create()
rds_trans_get_preferred()
down_read(&rds_trans_sem)
rds_ib_laddr_check_cm() destroys the CM id from inside
rds_trans_get_preferred()'s read-side critical section, while CM
callbacks create RDS connections (taking the same read lock) from
under handler_mutex.
Break the cycle by destroying the CM id asynchronously: add
rdma_destroy_id_wq(), which queues the destruction on cma_wq, and
switch rds_ib_laddr_check_cm() to it. The destroy work is embedded
in struct rdma_id_private, which keeps the id alive until the work
runs; rds does not touch cm_id after the destroy call.
This formalizes the approach proposed and syzbot-tested by Hillf
Danton; Eduard Zateev additionally verified the fix on a real QEMU
system (KASAN + lockdep + RDMA_RXE, active and passive RDS
connections established without any lockdep output).
Reported-by: syzbot+ccdb5058c3df2d94811c@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=ccdb5058c3df2d94811c
Suggested-by: Hillf Danton <hdanton@sina.com>
Tested-by: syzbot+ccdb5058c3df2d94811c@syzkaller.appspotmail.com
Tested-by: Eduard Zateev <hackerowskiy@gmail.com>
Signed-off-by: Aibing Chen <chenaibing@cmss.chinamobile.com>
---
drivers/infiniband/core/cma.c | 19 +++++++++++++++++++
drivers/infiniband/core/cma_priv.h | 1 +
include/rdma/rdma_cm.h | 1 +
net/rds/ib.c | 2 +-
4 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 337a49d1a..d10aaf51e 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -2126,6 +2126,25 @@ void rdma_destroy_id(struct rdma_cm_id *id)
}
EXPORT_SYMBOL(rdma_destroy_id);
+static void destroy_id_work(struct work_struct *work)
+{
+ struct rdma_id_private *id_priv =
+ container_of(work, struct rdma_id_private, destroy_work);
+
+ mutex_lock(&id_priv->handler_mutex);
+ destroy_id_handler_unlock(id_priv);
+}
+
+void rdma_destroy_id_wq(struct rdma_cm_id *id)
+{
+ struct rdma_id_private *id_priv =
+ container_of(id, struct rdma_id_private, id);
+
+ INIT_WORK(&id_priv->destroy_work, destroy_id_work);
+ queue_work(cma_wq, &id_priv->destroy_work);
+}
+EXPORT_SYMBOL(rdma_destroy_id_wq);
+
static int cma_rep_recv(struct rdma_id_private *id_priv)
{
int ret;
diff --git a/drivers/infiniband/core/cma_priv.h b/drivers/infiniband/core/cma_priv.h
index 04332eb82..95ff8bbd4 100644
--- a/drivers/infiniband/core/cma_priv.h
+++ b/drivers/infiniband/core/cma_priv.h
@@ -110,6 +110,7 @@ struct rdma_id_private {
*/
struct rdma_restrack_entry res;
struct rdma_ucm_ece ece;
+ struct work_struct destroy_work;
};
#if IS_ENABLED(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS)
diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h
index d639ff889..ac1c1d917 100644
--- a/include/rdma/rdma_cm.h
+++ b/include/rdma/rdma_cm.h
@@ -167,6 +167,7 @@ struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
* asynchronous operations associated with the id.
*/
void rdma_destroy_id(struct rdma_cm_id *id);
+void rdma_destroy_id_wq(struct rdma_cm_id *id);
/**
* rdma_restrict_node_type - Restrict an RDMA identifier to specific
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 4ea9838d0..1290747cd 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -484,7 +484,7 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr,
cm_id->device ? cm_id->device->node_type : -1);
out:
- rdma_destroy_id(cm_id);
+ rdma_destroy_id_wq(cm_id);
return ret;
}
--
2.50.1.windows.1
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS
2026-11-14 15:05 [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS Aibing Chen
@ 2026-11-15 0:18 ` Jason Gunthorpe
2026-11-16 6:47 ` Allison Henderson
2026-11-16 7:54 ` [PATCH net v2] net/rds: defer CM id destruction to a workqueue Aibing Chen
1 sibling, 1 reply; 4+ messages in thread
From: Jason Gunthorpe @ 2026-11-15 0:18 UTC (permalink / raw)
To: Aibing Chen
Cc: leon, achender, davem, edumazet, kuba, pabeni, horms, linux-rdma,
netdev, rds-devel, linux-kernel, hdanton, hackerowskiy,
syzbot+ccdb5058c3df2d94811c
On Sat, Nov 14, 2026 at 11:05:41PM +0800, Aibing Chen wrote:
> syzbot reported a possible circular locking dependency between
> rds_trans_sem and the RDMA CM handler mutex:
>
> CPU0 (bind path) CPU1 (CM callback path)
> ---- ----
> down_read(&rds_trans_sem) mutex_lock(&handler_mutex)
> rds_ib_laddr_check() cma_ib_req_handler()
> rdma_destroy_id() rds_ib_cm_handle_connect()
> mutex_lock(&handler_mutex) rds_conn_create()
> rds_trans_get_preferred()
> down_read(&rds_trans_sem)
>
> rds_ib_laddr_check_cm() destroys the CM id from inside
> rds_trans_get_preferred()'s read-side critical section, while CM
> callbacks create RDS connections (taking the same read lock) from
> under handler_mutex.
>
> Break the cycle by destroying the CM id asynchronously: add
> rdma_destroy_id_wq(), which queues the destruction on cma_wq, and
> switch rds_ib_laddr_check_cm() to it. The destroy work is embedded
> in struct rdma_id_private, which keeps the id alive until the work
> runs; rds does not touch cm_id after the destroy call.
This doesn't seem like it should be the core codes problem? Why can't
rds do the workqueue?
Jason
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS
2026-11-15 0:18 ` Jason Gunthorpe
@ 2026-11-16 6:47 ` Allison Henderson
0 siblings, 0 replies; 4+ messages in thread
From: Allison Henderson @ 2026-11-16 6:47 UTC (permalink / raw)
To: Jason Gunthorpe, Aibing Chen
Cc: leon, davem, edumazet, kuba, pabeni, horms, linux-rdma, netdev,
rds-devel, linux-kernel, hdanton, hackerowskiy,
syzbot+ccdb5058c3df2d94811c
On Sat, 2026-11-14 at 20:18 -0400, Jason Gunthorpe wrote:
> On Sat, Nov 14, 2026 at 11:05:41PM +0800, Aibing Chen wrote:
> > syzbot reported a possible circular locking dependency between
> > rds_trans_sem and the RDMA CM handler mutex:
> >
> > CPU0 (bind path) CPU1 (CM callback path)
> > ---- ----
> > down_read(&rds_trans_sem) mutex_lock(&handler_mutex)
> > rds_ib_laddr_check() cma_ib_req_handler()
> > rdma_destroy_id() rds_ib_cm_handle_connect()
> > mutex_lock(&handler_mutex) rds_conn_create()
> > rds_trans_get_preferred()
> > down_read(&rds_trans_sem)
> >
> > rds_ib_laddr_check_cm() destroys the CM id from inside
> > rds_trans_get_preferred()'s read-side critical section, while CM
> > callbacks create RDS connections (taking the same read lock) from
> > under handler_mutex.
> >
> > Break the cycle by destroying the CM id asynchronously: add
> > rdma_destroy_id_wq(), which queues the destruction on cma_wq, and
> > switch rds_ib_laddr_check_cm() to it. The destroy work is embedded
> > in struct rdma_id_private, which keeps the id alive until the work
> > runs; rds does not touch cm_id after the destroy call.
>
> This doesn't seem like it should be the core codes problem? Why can't
> rds do the workqueue?
>
> Jason
Hi Aibing,
Aibing, thanks for chasing this down. I think Jason is right, this
should probably be in the rds side. This solution doesnt quite cover
all the locking issues though. rds_conn_create calls
rds_trans_get_preferred() which falls through to rds_ib_laddr_check_cm()
when not local. And then we get a rdma_destroy_id while the listen id's
handler_mutex is still held.
I think a better way to do this might be to create a new work item and
then deffer the rdma_destroy_id to be completed later by a worker. The
work item would just hold the cm_id queued on rds_wq, and then the handler
just calls rdma_destroy_id() and frees itself. alloc the new work item at
the top of rds_ib_laddr_check_cm and then queue the new work item in place of
the rdma_destroy_id. You'll also need to add another flush_workqueue(rds_wq)
after rds_trans_unregister() in rds_ib_exit().
Other nits: this patch should have a fixes tag, and target the net tree
like this: "[PATCH net v2] net/rds ...". And I think the fixes tag is:
ec16227e1414 ("RDS/IB: Infiniband transport"). Drop the tested-by until it
is tested by syzbot and/or Eduard, and be sure to update your change log
below the dashed line.
Thanks for working on this!
Allion
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH net v2] net/rds: defer CM id destruction to a workqueue
2026-11-14 15:05 [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS Aibing Chen
2026-11-15 0:18 ` Jason Gunthorpe
@ 2026-11-16 7:54 ` Aibing Chen
1 sibling, 0 replies; 4+ messages in thread
From: Aibing Chen @ 2026-11-16 7:54 UTC (permalink / raw)
To: Allison Henderson
Cc: David S . Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Simon Horman, netdev, linux-rdma, rds-devel, linux-kernel
rds_ib_laddr_check_cm() destroys the RDMA CM id it created while
rds_trans_get_preferred() holds rds_trans_sem for reading. The bind
path then takes rds_trans_sem and afterwards handler_mutex in
rdma_destroy_id(), while the CM callback path takes handler_mutex in
cma_ib_req_handler() and afterwards rds_trans_sem through
rds_conn_create(). lockdep reports the resulting cycle.
Queue the destruction on rds_wq instead, so that rds_trans_sem is
released before handler_mutex is taken. The work item allocated here
owns the cm_id until rdma_destroy_id() has run; if the allocation fails,
the id is destroyed synchronously as before.
rds_ib_exit() flushes rds_wq after rds_trans_unregister() so that no
queued destroy runs after the module text is gone.
Reported-by: syzbot+ccdb5058c3df2d94811c@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=ccdb5058c3df2d94811c
Suggested-by: Hillf Danton <hdanton@sina.com>
Suggested-by: Jason Gunthorpe <jgg@ziepe.ca>
Suggested-by: Allison Henderson <achender@kernel.org>
Fixes: ec16227e1414 ("RDS/IB: Infiniband transport")
Cc: stable@vger.kernel.org
Signed-off-by: Aibing Chen <chenaibing@cmss.chinamobile.com>
---
v2:
- move the workqueue from RDMA core into RDS, as suggested in review.
- flush rds_wq in rds_ib_exit() so a queued destroy cannot run after
the module text is freed.
- add the Fixes tag, drop the Tested-by tags.
net/rds/ib.c | 37 ++++++++++++++++++++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 4ea9838d0..66af4a72a 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -391,6 +391,20 @@ static void rds6_ib_ic_info(struct socket *sock, unsigned int len,
}
#endif
+struct rds_ib_destroy_work {
+ struct work_struct work;
+ struct rdma_cm_id *cm_id;
+};
+
+static void rds_ib_destroy_id_work(struct work_struct *work)
+{
+ struct rds_ib_destroy_work *w =
+ container_of(work, struct rds_ib_destroy_work, work);
+
+ rdma_destroy_id(w->cm_id);
+ kfree(w);
+}
+
/*
* Early RDS/IB was built to only bind to an address if there is an IPoIB
* device with that address set.
@@ -411,6 +425,7 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr,
#endif
struct sockaddr_in sin;
struct sockaddr *sa;
+ struct rds_ib_destroy_work *w;
bool isv4;
isv4 = ipv6_addr_v4mapped(addr);
@@ -419,6 +434,15 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr,
NULL, RDMA_PS_TCP, IB_QPT_RC);
if (IS_ERR(cm_id))
return PTR_ERR(cm_id);
+
+ /*
+ * The id is destroyed from rds_wq: rds_trans_get_preferred() calls this
+ * function with rds_trans_sem held for reading, and rdma_destroy_id()
+ * takes handler_mutex, which the CM callback path acquires before
+ * taking rds_trans_sem. If the allocation fails, fall back to the
+ * synchronous destroy.
+ */
+ w = kzalloc_obj(*w, GFP_KERNEL);
ret = rdma_restrict_node_type(cm_id, RDMA_NODE_IB_CA);
if (ret)
goto out;
@@ -484,7 +508,13 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr,
cm_id->device ? cm_id->device->node_type : -1);
out:
- rdma_destroy_id(cm_id);
+ if (w) {
+ INIT_WORK(&w->work, rds_ib_destroy_id_work);
+ w->cm_id = cm_id;
+ queue_work(rds_wq, &w->work);
+ } else {
+ rdma_destroy_id(cm_id);
+ }
return ret;
}
@@ -542,6 +572,11 @@ void rds_ib_exit(void)
rds_ib_sysctl_exit();
rds_ib_recv_exit();
rds_trans_unregister(&rds_ib_transport);
+ /*
+ * rds_ib_laddr_check_cm() may have queued rdma_destroy_id() on rds_wq;
+ * wait for it before the module text goes away.
+ */
+ flush_workqueue(rds_wq);
rds_ib_mr_exit();
}
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2026-11-16 7:54 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-14 15:05 [PATCH] RDMA/cma: add rdma_destroy_id_wq() and use it from RDS Aibing Chen
2026-11-15 0:18 ` Jason Gunthorpe
2026-11-16 6:47 ` Allison Henderson
2026-11-16 7:54 ` [PATCH net v2] net/rds: defer CM id destruction to a workqueue Aibing Chen
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®