mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Ionut Nechita <ionut.nechita@windriver.com>
To: ceph-devel@vger.kernel.org
Cc: Ilya Dryomov <idryomov@gmail.com>,
	Alex Markuze <amarkuze@redhat.com>,
	Viacheslav Dubeyko <slava@dubeyko.com>,
	Xiubo Li <xiubo.li@clyso.com>,
	linux-kernel@vger.kernel.org
Subject: [PATCH v2 2/7] ceph: bound the ceph_mdsc_sync() wait by mount_timeout
Date: Fri, 20 Nov 2026 09:34:18 +0200	[thread overview]
Message-ID: <20261120073431.53636-3-ionut.nechita@windriver.com> (raw)
In-Reply-To: <20261120073431.53636-1-ionut.nechita@windriver.com>

When the MDS is unreachable, sync(2) and syncfs(2) block forever in
ceph_mdsc_sync(): flush_mdlog_and_wait_mdsc_unsafe_requests() waits
for r_safe_completion and wait_caps_flush() waits for the cap flushes
without any bound.  The diagnostic loop added to wait_caps_flush()
makes the stall visible, but the task still can't make progress and
keeps tripping the hung task detector:

  INFO: task sync:514917 blocked for more than 983 seconds.
  Call Trace:
   ceph_mdsc_sync+0x4d6/0x5a0 [ceph]
   ceph_sync_fs+0x31/0x130 [ceph]
   iterate_supers+0x97/0x100
   ksys_sync+0x32/0xb0

Bound both waits by a single mount_timeout deadline, like
wait_requests() and ceph_mdsc_close_sessions() already do, and
return -ETIMEDOUT from ceph_sync_fs().  Nothing is discarded: unsafe
requests and dirty caps stay queued and are flushed once the MDS is
reachable again.  mount_timeout=0 keeps the current wait-forever
behaviour.

Signed-off-by: Ionut Nechita <ionut.nechita@windriver.com>
---
 fs/ceph/mds_client.c | 84 +++++++++++++++++++++++++++++++++-----------
 fs/ceph/mds_client.h |  2 +-
 fs/ceph/super.c      |  5 +--
 3 files changed, 67 insertions(+), 24 deletions(-)

diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index 3791fc627714c..d02fe1e34527b 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -2520,11 +2520,24 @@ static void dump_cap_flushes(struct ceph_mds_client *mdsc, u64 want_tid)
 }
 
 /*
- * Wait for all cap flushes through @want_flush_tid to complete.
- * Periodically dumps pending cap flush state for diagnostics.
+ * Time left until the sync deadline, or MAX_SCHEDULE_TIMEOUT if
+ * mount_timeout=0 (wait forever).
  */
-static void wait_caps_flush(struct ceph_mds_client *mdsc,
-			    u64 want_flush_tid)
+static unsigned long sync_time_left(struct ceph_mds_client *mdsc,
+				    unsigned long deadline)
+{
+	if (!mdsc->fsc->client->options->mount_timeout)
+		return MAX_SCHEDULE_TIMEOUT;
+	return time_before(jiffies, deadline) ? deadline - jiffies : 0;
+}
+
+/*
+ * Wait for all cap flushes through @want_flush_tid to complete or for
+ * @deadline to pass.  Periodically dumps pending cap flush state for
+ * diagnostics.
+ */
+static int wait_caps_flush(struct ceph_mds_client *mdsc,
+			   u64 want_flush_tid, unsigned long deadline)
 {
 	struct ceph_client *cl = mdsc->fsc->client;
 	int i = 0;
@@ -2532,22 +2545,31 @@ static void wait_caps_flush(struct ceph_mds_client *mdsc,
 
 	doutc(cl, "want %llu\n", want_flush_tid);
 
-	do {
+	while (true) {
 		/* 60 * HZ fits in a long on all supported architectures. */
 		ret = wait_event_timeout(mdsc->cap_flushing_wq,
 			   check_caps_flush(mdsc, want_flush_tid),
-			   CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC * HZ);
-		if (ret == 0) {
-			if (i < CEPH_CAP_FLUSH_MAX_DUMP_ITERS)
-				dump_cap_flushes(mdsc, want_flush_tid);
-			else if (i == CEPH_CAP_FLUSH_MAX_DUMP_ITERS)
-				pr_info_client(cl,
-					       "still waiting for cap flushes; suppressing further dumps\n");
-			i++;
+			   min_t(unsigned long, sync_time_left(mdsc, deadline),
+				 CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC * HZ));
+		if (ret)
+			break;
+
+		if (i < CEPH_CAP_FLUSH_MAX_DUMP_ITERS)
+			dump_cap_flushes(mdsc, want_flush_tid);
+		else if (i == CEPH_CAP_FLUSH_MAX_DUMP_ITERS)
+			pr_info_client(cl,
+				       "still waiting for cap flushes; suppressing further dumps\n");
+		i++;
+
+		if (!sync_time_left(mdsc, deadline)) {
+			doutc(cl, "timed out waiting for flush tid %llu\n",
+			      want_flush_tid);
+			return -ETIMEDOUT;
 		}
-	} while (ret == 0);
+	}
 
 	doutc(cl, "ok, flushed thru %llu\n", want_flush_tid);
+	return 0;
 }
 
 /*
@@ -6737,13 +6759,15 @@ void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc)
 /*
  * flush the mdlog and wait for all write mds requests to flush.
  */
-static void flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *mdsc,
-						 u64 want_tid)
+static int flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *mdsc,
+						      u64 want_tid,
+						      unsigned long deadline)
 {
 	struct ceph_client *cl = mdsc->fsc->client;
 	struct ceph_mds_request *req = NULL, *nextreq;
 	struct ceph_mds_session *last_session = NULL;
 	struct rb_node *n;
+	unsigned long left;
 
 	mutex_lock(&mdsc->mutex);
 	doutc(cl, "want %lld\n", want_tid);
@@ -6782,7 +6806,18 @@ static void flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *md
 			}
 			doutc(cl, "wait on %llu (want %llu)\n",
 			      req->r_tid, want_tid);
-			wait_for_completion(&req->r_safe_completion);
+			left = sync_time_left(mdsc, deadline);
+			left = wait_for_completion_timeout(&req->r_safe_completion,
+							   left);
+			if (!left) {
+				doutc(cl, "timed out waiting for tid %llu\n",
+				      req->r_tid);
+				ceph_mdsc_put_request(req);
+				if (nextreq)
+					ceph_mdsc_put_request(nextreq);
+				ceph_put_mds_session(last_session);
+				return -ETIMEDOUT;
+			}
 
 			mutex_lock(&mdsc->mutex);
 			ceph_mdsc_put_request(req);
@@ -6800,15 +6835,18 @@ static void flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *md
 	mutex_unlock(&mdsc->mutex);
 	ceph_put_mds_session(last_session);
 	doutc(cl, "done\n");
+	return 0;
 }
 
-void ceph_mdsc_sync(struct ceph_mds_client *mdsc)
+int ceph_mdsc_sync(struct ceph_mds_client *mdsc)
 {
 	struct ceph_client *cl = mdsc->fsc->client;
+	unsigned long deadline = jiffies + cl->options->mount_timeout;
 	u64 want_tid, want_flush;
+	int ret;
 
 	if (READ_ONCE(mdsc->fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN)
-		return;
+		return 0;
 
 	doutc(cl, "sync\n");
 	mutex_lock(&mdsc->mutex);
@@ -6829,8 +6867,12 @@ void ceph_mdsc_sync(struct ceph_mds_client *mdsc)
 
 	doutc(cl, "sync want tid %lld flush_seq %lld\n", want_tid, want_flush);
 
-	flush_mdlog_and_wait_mdsc_unsafe_requests(mdsc, want_tid);
-	wait_caps_flush(mdsc, want_flush);
+	ret = flush_mdlog_and_wait_mdsc_unsafe_requests(mdsc, want_tid,
+							deadline);
+	if (ret)
+		return ret;
+
+	return wait_caps_flush(mdsc, want_flush, deadline);
 }
 
 /*
diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
index 4749438d94e1f..31abbb3b3bdb3 100644
--- a/fs/ceph/mds_client.h
+++ b/fs/ceph/mds_client.h
@@ -635,7 +635,7 @@ extern void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc);
 extern void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc);
 extern void ceph_mdsc_destroy(struct ceph_fs_client *fsc);
 
-extern void ceph_mdsc_sync(struct ceph_mds_client *mdsc);
+extern int ceph_mdsc_sync(struct ceph_mds_client *mdsc);
 
 extern void ceph_invalidate_dir_request(struct ceph_mds_request *req);
 extern int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req,
diff --git a/fs/ceph/super.c b/fs/ceph/super.c
index 7b5ef76e430d7..7936b76aa3cc0 100644
--- a/fs/ceph/super.c
+++ b/fs/ceph/super.c
@@ -126,6 +126,7 @@ static int ceph_sync_fs(struct super_block *sb, int wait)
 {
 	struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb);
 	struct ceph_client *cl = fsc->client;
+	int ret;
 
 	if (!wait) {
 		doutc(cl, "(non-blocking)\n");
@@ -137,9 +138,9 @@ static int ceph_sync_fs(struct super_block *sb, int wait)
 
 	doutc(cl, "(blocking)\n");
 	ceph_osdc_sync(&fsc->client->osdc);
-	ceph_mdsc_sync(fsc->mdsc);
+	ret = ceph_mdsc_sync(fsc->mdsc);
 	doutc(cl, "(blocking) done\n");
-	return 0;
+	return ret;
 }
 
 /*
-- 
2.56.0


  parent reply	other threads:[~2026-11-20  7:35 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-20  7:34 [PATCH v2 0/7] ceph/libceph: bound sync waits and speed up recovery after network disruption Ionut Nechita
2026-11-20  7:34 ` [PATCH v2 1/7] libceph: don't back off exponentially on EADDRNOTAVAIL Ionut Nechita
2026-11-20  7:34 ` Ionut Nechita [this message]
2026-11-20  7:34 ` [PATCH v2 3/7] ceph: bound the ceph_osdc_sync() wait by mount_timeout Ionut Nechita
2026-11-20  7:34 ` [PATCH v2 4/7] ceph: bound the safe-completion wait in ceph_lock_wait_for_completion() Ionut Nechita
2026-11-20  7:34 ` [PATCH v2 5/7] ceph: set default timeout for MDS requests Ionut Nechita
2026-11-20  7:34 ` [PATCH v2 6/7] ceph: request a full mdsmap when an MDS stays unreachable Ionut Nechita
2026-11-20  7:34 ` [PATCH v2 7/7] libceph: don't escalate monitor hunt backoff on EADDRNOTAVAIL Ionut Nechita

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261120073431.53636-3-ionut.nechita@windriver.com \
    --to=ionut.nechita@windriver.com \
    --cc=amarkuze@redhat.com \
    --cc=ceph-devel@vger.kernel.org \
    --cc=idryomov@gmail.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=slava@dubeyko.com \
    --cc=xiubo.li@clyso.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®