From: Ionut Nechita <ionut.nechita@windriver.com>
To: ceph-devel@vger.kernel.org
Cc: Ilya Dryomov <idryomov@gmail.com>,
Alex Markuze <amarkuze@redhat.com>,
Viacheslav Dubeyko <slava@dubeyko.com>,
Xiubo Li <xiubo.li@clyso.com>,
linux-kernel@vger.kernel.org
Subject: [PATCH v2 6/7] ceph: request a full mdsmap when an MDS stays unreachable
Date: Fri, 20 Nov 2026 09:34:22 +0200 [thread overview]
Message-ID: <20261120073431.53636-7-ionut.nechita@windriver.com> (raw)
In-Reply-To: <20261120073431.53636-1-ionut.nechita@windriver.com>
The client subscribes to incremental mdsmap updates starting at
current_epoch + 1. If an MDS restarts at a different address while
the monitor session is also down (e.g. during a rolling upgrade in a
containerized deployment), the client can miss that update and keep
reconnecting to the stale address indefinitely. This was observed
with the mdsmap stuck at epoch 53 while the cluster was at epoch 68,
the session HUNG and monc reporting "have mdsmap 53 want 54+".
Nothing currently reacts to this: mds_con_ops has no .fault callback,
and a session going HUNG doesn't touch the subscription either.
Add a .fault callback that asks for a full mdsmap (start epoch 0) on
every MDS_CON_FAIL_REFRESH_MDSMAP consecutive faults, resetting the
count on any session message, and do the same when a session is
marked HUNG, which covers a connection that went to STANDBY and thus
never faults.
Signed-off-by: Ionut Nechita <ionut.nechita@windriver.com>
---
fs/ceph/mds_client.c | 48 ++++++++++++++++++++++++++++++++++++++++++++
fs/ceph/mds_client.h | 2 ++
2 files changed, 50 insertions(+)
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 4460f614050e4..3c7ecf3147e6f 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -72,6 +72,13 @@ static void ceph_mdsc_reset_workfn(struct work_struct *work);
static const struct ceph_connection_operations mds_con_ops;
+/*
+ * Consecutive MDS connection faults after which we ask the monitor for
+ * a full mdsmap. With the messenger backoff (250ms doubling up to 15s)
+ * ten faults take about a minute, comparable to mount_timeout.
+ */
+#define MDS_CON_FAIL_REFRESH_MDSMAP 10
+
static void ceph_metric_bind_session(struct ceph_mds_client *mdsc,
struct ceph_mds_session *session)
{
@@ -4651,6 +4658,9 @@ static void handle_session(struct ceph_mds_session *session,
ceph_session_op_name(op), session,
ceph_session_state_name(session->s_state), seq);
+ /* the connection works, see mds_fault() */
+ session->s_con_failures = 0;
+
if (session->s_state == CEPH_MDS_SESSION_HUNG) {
session->s_state = CEPH_MDS_SESSION_OPEN;
pr_info_client(cl, "mds%d came back\n", session->s_mds);
@@ -6285,6 +6295,18 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
ceph_force_reconnect(fsc->sb);
}
+/*
+ * Subscribe to the mdsmap from epoch 0 so that the monitor sends the
+ * current full map rather than waiting for an incremental update.
+ */
+static void request_full_mdsmap(struct ceph_mds_client *mdsc)
+{
+ struct ceph_mon_client *monc = &mdsc->fsc->client->monc;
+
+ if (ceph_monc_want_map(monc, CEPH_SUB_MDSMAP, 0, true))
+ ceph_monc_renew_subs(monc);
+}
+
bool check_session_state(struct ceph_mds_session *s)
{
struct ceph_client *cl = s->s_mdsc->fsc->client;
@@ -6294,6 +6316,12 @@ bool check_session_state(struct ceph_mds_session *s)
if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
s->s_state = CEPH_MDS_SESSION_HUNG;
pr_info_client(cl, "mds%d hung\n", s->s_mds);
+ /*
+ * The MDS may have come back at a different
+ * address while the connection sat in STANDBY,
+ * which doesn't fault.
+ */
+ request_full_mdsmap(s->s_mdsc);
}
break;
case CEPH_MDS_SESSION_CLOSING:
@@ -7525,12 +7553,32 @@ static int mds_check_message_signature(struct ceph_msg *msg)
return ceph_auth_check_message_signature(auth, msg);
}
+/*
+ * If the MDS keeps failing to connect, our mdsmap may be stale (e.g. the
+ * MDS restarted at a different address and the incremental update was
+ * missed while the monitor session was down). Periodically ask for a
+ * full map.
+ */
+static void mds_fault(struct ceph_connection *con)
+{
+ struct ceph_mds_session *s = con->private;
+
+ if (++s->s_con_failures < MDS_CON_FAIL_REFRESH_MDSMAP)
+ return;
+
+ s->s_con_failures = 0;
+ doutc(s->s_mdsc->fsc->client,
+ "mds%d keeps faulting, requesting full mdsmap\n", s->s_mds);
+ request_full_mdsmap(s->s_mdsc);
+}
+
static const struct ceph_connection_operations mds_con_ops = {
.get = mds_get_con,
.put = mds_put_con,
.alloc_msg = mds_alloc_msg,
.dispatch = mds_dispatch,
.peer_reset = mds_peer_reset,
+ .fault = mds_fault,
.get_authorizer = mds_get_authorizer,
.add_authorizer_challenge = mds_add_authorizer_challenge,
.verify_authorizer_reply = mds_verify_authorizer_reply,
diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
index 31abbb3b3bdb3..46e13446afcc7 100644
--- a/fs/ceph/mds_client.h
+++ b/fs/ceph/mds_client.h
@@ -301,6 +301,8 @@ struct ceph_mds_session {
struct list_head s_unsafe; /* unsafe requests */
struct xarray s_delegated_inos;
atomic_t s_num_deleg_inos;
+
+ u8 s_con_failures; /* consecutive connection faults */
};
/*
--
2.56.0
next prev parent reply other threads:[~2026-11-20 7:35 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 7:34 [PATCH v2 0/7] ceph/libceph: bound sync waits and speed up recovery after network disruption Ionut Nechita
2026-11-20 7:34 ` [PATCH v2 1/7] libceph: don't back off exponentially on EADDRNOTAVAIL Ionut Nechita
2026-11-20 7:34 ` [PATCH v2 2/7] ceph: bound the ceph_mdsc_sync() wait by mount_timeout Ionut Nechita
2026-11-20 7:34 ` [PATCH v2 3/7] ceph: bound the ceph_osdc_sync() " Ionut Nechita
2026-11-20 7:34 ` [PATCH v2 4/7] ceph: bound the safe-completion wait in ceph_lock_wait_for_completion() Ionut Nechita
2026-11-20 7:34 ` [PATCH v2 5/7] ceph: set default timeout for MDS requests Ionut Nechita
2026-11-20 7:34 ` Ionut Nechita [this message]
2026-11-20 7:34 ` [PATCH v2 7/7] libceph: don't escalate monitor hunt backoff on EADDRNOTAVAIL Ionut Nechita
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120073431.53636-7-ionut.nechita@windriver.com \
--to=ionut.nechita@windriver.com \
--cc=amarkuze@redhat.com \
--cc=ceph-devel@vger.kernel.org \
--cc=idryomov@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=slava@dubeyko.com \
--cc=xiubo.li@clyso.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®