[PATCH 3/3] SUNRPC: reconnect client transports after resuming from hibernation

[Andrew Pam <andrew@sericyb.com.au>]

With rpciod and xprtiod frozen across hibernation, no RPC is sent after
the snapshot. TCP can still acknowledge data the server sends then,
though: typically a reply that was in flight when the snapshot was
taken. The restored image has not seen that data, and the server, which
saw it acknowledged, will not send it again. The client then waits
indefinitely for replies that never arrive.

Captured in a QEMU test (below): on the one cycle of six that hung, the
client's TCP acknowledgement went backwards by 200 bytes after the
restore, and the requests outstanding (WRITE, COMMIT, GETATTR) stayed
queued on xprt_pending with no further progress.

After resuming, disconnect every client transport. Requests in flight
are retransmitted on the new connection, and the server answers any it
has already executed from its reply cache. Because nothing was sent
after the snapshot, NFSv4.1 session slot sequence ids still agree on
both sides.

The disconnect is done from a PM notifier at PM_POST_HIBERNATION, which
runs in the restored kernel once everything has thawed. It is also sent
when hibernation fails, where the reconnect is unnecessary but harmless.
xprt_put() may sleep, so references to the transports are collected
under the client list lock and RCU, and released afterwards.

Tested in the QEMU setup described in the first patch of this series,
with 20 hibernations in test_resume mode back to back while NFS writes
(8 MB, fsync) and reads ran continuously. Without the series the client
hung on the first cycle; with only the first two patches, on the sixth.
With this patch all 20 cycles completed and I/O progressed on every
one. The capture shows the client's acknowledgement going backwards on
two of the old connections, and the client reconnecting and carrying on
both times. The single-hibernation test is unchanged apart from one
extra connection per resume: nothing reached the server during the
image write, and the client resumed without errors.

Drafted with an AI coding assistant (Claude), which also ran the tests.

Signed-off-by: Andrew Pam <andrew@sericyb.com.au>
---
 net/sunrpc/clnt.c        | 85 ++++++++++++++++++++++++++++++++++++++++
 net/sunrpc/sunrpc.h      |  2 +
 net/sunrpc/sunrpc_syms.c |  7 ++++
 3 files changed, 94 insertions(+)

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 6cedc824cf82..a488be2891be 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -32,6 +32,8 @@
 #include <linux/in.h>
 #include <linux/in6.h>
 #include <linux/un.h>
+#include <linux/suspend.h>
+#include <net/net_namespace.h>
 
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/addr.h>
@@ -878,6 +880,89 @@ int rpc_clnt_iterate_for_each_xprt(struct rpc_clnt *clnt,
 }
 EXPORT_SYMBOL_GPL(rpc_clnt_iterate_for_each_xprt);
 
+struct rpc_pm_xprt {
+	struct list_head	list;
+	struct rpc_xprt		*xprt;
+};
+
+/*
+ * After hibernation takes its snapshot, the kernel keeps running with the
+ * network up while it writes the image. rpciod and xprtiod are frozen by
+ * then, so no RPC is sent, but TCP still acknowledges anything the server
+ * sends, such as a reply that was in flight when the snapshot was taken.
+ * The restored image has not seen that data, and the server will not send
+ * it again, so the connection cannot make progress.
+ *
+ * Once the system has resumed, disconnect every client transport. Requests
+ * in flight are retransmitted on a new connection, and the server answers
+ * any it has already executed from its reply cache. Because nothing was
+ * sent after the snapshot, NFSv4.1 session slot sequence ids still agree.
+ */
+static int rpc_pm_notify(struct notifier_block *nb, unsigned long action,
+			 void *data)
+{
+	struct rpc_pm_xprt *entry, *tmp;
+	LIST_HEAD(xprts);
+	struct net *net;
+
+	if (action != PM_POST_HIBERNATION)
+		return NOTIFY_DONE;
+
+	/* xprt_put() may sleep, so collect references first */
+	down_read(&net_rwsem);
+	for_each_net(net) {
+		struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
+		struct rpc_clnt *clnt;
+
+		spin_lock(&sn->rpc_client_lock);
+		rcu_read_lock();
+		list_for_each_entry(clnt, &sn->all_clients, cl_clients) {
+			struct rpc_xprt_switch *xps;
+			struct rpc_xprt *xprt;
+
+			xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch);
+			if (!xps)
+				continue;
+			list_for_each_entry_rcu(xprt, &xps->xps_xprt_list,
+						xprt_switch) {
+				entry = kmalloc_obj(*entry, GFP_ATOMIC);
+				if (!entry)
+					break;
+				entry->xprt = xprt_get(xprt);
+				if (!entry->xprt) {
+					kfree(entry);
+					continue;
+				}
+				list_add(&entry->list, &xprts);
+			}
+		}
+		rcu_read_unlock();
+		spin_unlock(&sn->rpc_client_lock);
+	}
+	up_read(&net_rwsem);
+
+	list_for_each_entry_safe(entry, tmp, &xprts, list) {
+		xprt_force_disconnect(entry->xprt);
+		xprt_put(entry->xprt);
+		kfree(entry);
+	}
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block rpc_pm_nb = {
+	.notifier_call = rpc_pm_notify,
+};
+
+int rpc_pm_notifier_register(void)
+{
+	return register_pm_notifier(&rpc_pm_nb);
+}
+
+void rpc_pm_notifier_unregister(void)
+{
+	unregister_pm_notifier(&rpc_pm_nb);
+}
+
 /*
  * Kill all tasks for the given client.
  * XXX: kill their descendants as well?
diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h
index e3c6e3b63f0b..6d7581a115ce 100644
--- a/net/sunrpc/sunrpc.h
+++ b/net/sunrpc/sunrpc.h
@@ -40,6 +40,8 @@ struct svc_serv;
 struct svc_rqst;
 int rpc_clients_notifier_register(void);
 void rpc_clients_notifier_unregister(void);
+int rpc_pm_notifier_register(void);
+void rpc_pm_notifier_unregister(void);
 void auth_domain_cleanup(void);
 void svc_sock_update_bufs(struct svc_serv *serv);
 enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp);
diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c
index 1a3884a0376a..d32786f51ae5 100644
--- a/net/sunrpc/sunrpc_syms.c
+++ b/net/sunrpc/sunrpc_syms.c
@@ -116,6 +116,10 @@ init_sunrpc(void)
 	if (err)
 		goto out6;
 
+	err = rpc_pm_notifier_register();
+	if (err)
+		goto out7;
+
 	sunrpc_debugfs_init();
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 	rpc_register_sysctl();
@@ -124,6 +128,8 @@ init_sunrpc(void)
 	init_socket_xprt();	/* clnt sock transport */
 	return 0;
 
+out7:
+	genl_unregister_family(&sunrpc_nl_family);
 out6:
 	rpc_sysfs_exit();
 out5:
@@ -141,6 +147,7 @@ init_sunrpc(void)
 static void __exit
 cleanup_sunrpc(void)
 {
+	rpc_pm_notifier_unregister();
 	genl_unregister_family(&sunrpc_nl_family);
 	rpc_sysfs_exit();
 	rpc_cleanup_clids();
-- 
2.53.0