[PATCH net 0/2] netlink: fix a dump livelock with a notification queued

[PATCH net 0/2] netlink: fix a dump livelock with a notification queued

[Jhonata Poma-Hansen via B4 Relay]

A netlink dump that starts while data is still charged to the reader's
socket fails with ENOBUFS when the reader's receive buffer is small,
even though a minimum size skb would fit. The dump stays marked as
running, every recvmsg() retries it, and a reader that peeks before
reading, as libnl does, spins forever.

I hit this with "iw dev wlan0 scan" on an arm64 board with a 6.6 based
kernel: iw hung at 100% CPU in 5 to 10 scans out of 15. A kprobe trace
showed the scan dump starting with the NEW_SCAN_RESULTS notification
still charged to iw's 16KiB socket, because iw had preempted the sender
before netlink_broadcast() returned.

Patch 1 retries the dump skb with the minimum size before failing.
Patch 2 adds a selftest which starts a link dump with a link
notification queued and reads it like libnl does.

Tested on net 734bef548a96 in a KVM guest: without patch 1 the new test
fails and a standalone reproducer livelocks in 3 runs out of 3; with
it, netlink-dumps passes 5 of 5 and the reproducer completes. The
existing netlink-dumps tests pass with and without the fix.

---
Jhonata Poma-Hansen (2):
      netlink: retry a dump skb that does not fit with the minimum size
      selftests: net: netlink-dumps: start a dump with a notification queued

 net/netlink/af_netlink.c                    | 11 ++++
 tools/testing/selftests/net/netlink-dumps.c | 93 +++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
---
base-commit: 734bef548a96e8774d6eef5dfa2026c6edc4e80b
change-id: 20261023-netlink-dump-livelock-e83cf13f977a

Best regards,
--  
Jhonata Poma-Hansen <jhonata.poma@gmail.com>

[PATCH net 1/2] netlink: retry a dump skb that does not fit with the minimum size

[Jhonata Poma-Hansen via B4 Relay]

From: Jhonata Poma-Hansen <jhonata.poma@gmail.com>

netlink_dump() sizes the skb after the largest buffer the reader passed
to recvmsg(), up to 32KiB. Since commit ae8f160e7eb2 ("netlink: Fix
wraparounds of sk->sk_rmem_alloc.") the skb is only queued if it fits
in sk_rcvbuf, or if nothing else is charged to the socket.

A reader with a small receive buffer that still has data charged to
its socket when a dump starts gets ENOBUFS, even when a minimum size
skb would fit. The dump stays marked as running and every recvmsg()
retries it, so a reader that peeks before reading, as libnl does,
never consumes the data at the head of the queue and spins forever.

"iw dev wlan0 scan" hits this: its socket has an 8KiB buffer, it joins
the nl80211 multicast groups, and it dumps the scan results as soon as
it has read the NEW_SCAN_RESULTS notification. When it preempts the
sender, the notification is still charged to its socket until
netlink_broadcast() returns, the dump fails to start and iw spins at
100% CPU.

Retry with alloc_min_size before failing the dump.

Fixes: ae8f160e7eb2 ("netlink: Fix wraparounds of sk->sk_rmem_alloc.")
Signed-off-by: Jhonata Poma-Hansen <jhonata.poma@gmail.com>
---
 net/netlink/af_netlink.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index a904dc3bc..b750e79fe 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -2318,10 +2318,21 @@ static int netlink_dump(struct sock *sk, bool lock_taken)
 	if (!skb)
 		goto errout_skb;
 
+retry:
 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
 	rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);
 	if (rmem != skb->truesize && rmem >= rcvbuf) {
 		atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
+		/* A skb sized after max_recvmsg_len may not fit next to data
+		 * still charged to the socket, while a minimum size one does.
+		 */
+		if (alloc_size > alloc_min_size) {
+			kfree_skb(skb);
+			alloc_size = alloc_min_size;
+			skb = alloc_skb(alloc_size, GFP_KERNEL);
+			if (skb)
+				goto retry;
+		}
 		goto errout_skb;
 	}
 

-- 
2.54.0

[PATCH net 2/2] selftests: net: netlink-dumps: start a dump with a notification queued

[Jhonata Poma-Hansen via B4 Relay]

From: Jhonata Poma-Hansen <jhonata.poma@gmail.com>

Add a test which starts a link dump on a socket with a small receive
buffer while a link notification is still queued, after a large read
made the kernel attempt large dump skbs. The test reads like libnl
does, peeking before each read, and expects the dump to complete.

Signed-off-by: Jhonata Poma-Hansen <jhonata.poma@gmail.com>
---
 tools/testing/selftests/net/netlink-dumps.c | 93 +++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/tools/testing/selftests/net/netlink-dumps.c b/tools/testing/selftests/net/netlink-dumps.c
index 51129c564..6f66a478d 100644
--- a/tools/testing/selftests/net/netlink-dumps.c
+++ b/tools/testing/selftests/net/netlink-dumps.c
@@ -3,8 +3,10 @@
 #define _GNU_SOURCE
 
 #include <fcntl.h>
+#include <sched.h>
 #include <stdio.h>
 #include <string.h>
+#include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
@@ -260,4 +262,95 @@ TEST(close_with_ref)
 	usleep(100 * 1000);
 }
 
+static int lo_set_up(int sock, bool up)
+{
+	struct ifreq ifr = {};
+
+	strcpy(ifr.ifr_name, "lo");
+	if (ioctl(sock, SIOCGIFFLAGS, &ifr))
+		return -1;
+	if (up)
+		ifr.ifr_flags |= IFF_UP;
+	else
+		ifr.ifr_flags &= ~IFF_UP;
+	return ioctl(sock, SIOCSIFFLAGS, &ifr);
+}
+
+static const struct {
+	struct nlmsghdr nlhdr;
+	struct ifinfomsg ifm;
+} dump_links = {
+	.nlhdr = {
+		.nlmsg_len	= sizeof(dump_links),
+		.nlmsg_type	= RTM_GETLINK,
+		.nlmsg_flags	= NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP,
+		.nlmsg_seq	= 1,
+	},
+	.ifm = {
+		.ifi_family	= AF_UNSPEC,
+	},
+};
+
+// A dump started while a notification is still queued must make progress
+// for a reader which peeks before reading, like libnl does
+TEST(dump_with_queued_ntf)
+{
+	struct sockaddr_nl addr = {
+		.nl_family	= AF_NETLINK,
+		.nl_groups	= RTMGRP_LINK,
+	};
+	int netlink_sock, inet_sock, i;
+	int rcvbuf = 8192;
+	bool done = false;
+	char buf[32768];
+	ssize_t n;
+
+	ASSERT_EQ(unshare(CLONE_NEWNET), 0);
+
+	inet_sock = socket(AF_INET, SOCK_DGRAM, 0);
+	ASSERT_GE(inet_sock, 0);
+	netlink_sock = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	ASSERT_GE(netlink_sock, 0);
+	ASSERT_EQ(setsockopt(netlink_sock, SOL_SOCKET, SO_RCVBUF,
+			     &rcvbuf, sizeof(rcvbuf)), 0);
+	ASSERT_EQ(bind(netlink_sock, (struct sockaddr *)&addr,
+		       sizeof(addr)), 0);
+
+	// A large read makes the kernel attempt large dump skbs
+	ASSERT_EQ(lo_set_up(inet_sock, true), 0);
+	n = recv(netlink_sock, buf, sizeof(buf), 0);
+	ASSERT_GE(n, (ssize_t)sizeof(struct nlmsghdr));
+
+	// Leave one notification queued, then start the dump
+	ASSERT_EQ(lo_set_up(inet_sock, false), 0);
+	n = send(netlink_sock, &dump_links, sizeof(dump_links), 0);
+	ASSERT_EQ(n, sizeof(dump_links));
+
+	for (i = 0; i < 100 && !done; i++) {
+		struct nlmsghdr *nlh;
+
+		n = recv(netlink_sock, buf, sizeof(buf),
+			 MSG_PEEK | MSG_TRUNC | MSG_DONTWAIT);
+		if (n < 0)
+			continue;
+		n = recv(netlink_sock, buf, sizeof(buf), MSG_DONTWAIT);
+		if (n < 0)
+			continue;
+
+		for (nlh = (struct nlmsghdr *)buf; NLMSG_OK(nlh, n);
+		     nlh = NLMSG_NEXT(nlh, n)) {
+			if (nlh->nlmsg_seq != dump_links.nlhdr.nlmsg_seq)
+				continue;
+			if (nlh->nlmsg_type == NLMSG_ERROR)
+				EXPECT_EQ(*(int *)NLMSG_DATA(nlh), 0);
+			if (nlh->nlmsg_type == NLMSG_DONE)
+				done = true;
+		}
+	}
+	EXPECT_TRUE(done);
+
+	close(netlink_sock);
+	close(inet_sock);
+}
+
 TEST_HARNESS_MAIN

-- 
2.54.0