mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
* [BUG] bpf, sockmap: data left in ingress_msg is dropped without moving copied_seq, and TLS RX then warns
@ 2026-11-19  3:04 Kyuwon Cho
  0 siblings, 0 replies; only message in thread
From: Kyuwon Cho @ 2026-11-19  3:04 UTC (permalink / raw)
  To: John Fastabend, Jakub Sitnicki, Jakub Kicinski, Sabrina Dubroca
  Cc: David S. Miller, Eric Dumazet, Paolo Abeni, Simon Horman,
	Jiayuan Chen, Alexei Starovoitov, netdev, bpf, linux-kernel

[-- Attachment #1: Type: text/plain, Size: 3080 bytes --]

Hi John, Jakub, Jakub, Sabrina,

While testing v7.4-rc3 with an LLM-guided test generator that we are
developing as a research prototype, we found that setting up TLS RX on
a TCP socket right after it was removed from a sockmap can find the
socket's receive queue and copied_seq out of step.
tls_strp_read_sock() sees tcp_inq() > 0, but tcp_recv_skb() finds no
skb at copied_seq:

  WARNING: net/tls/tls_strp.c:468 at tls_strp_load_anchor_with_queue+0x2ba/0x380, CPU#0: test/114
  CPU: 0 UID: 0 PID: 114 Comm: test Not tainted 7.4.0-rc3 #1 PREEMPT(lazy)
  RIP: 0010:tls_strp_load_anchor_with_queue (net/tls/tls_strp.c:468)
  Call Trace:
   <TASK>
   tls_strp_check_rcv (net/tls/tls_strp.c:527 net/tls/tls_strp.c:562 net/tls/tls_strp.c:557)
   tls_setsockopt (net/tls/tls_main.c:769 net/tls/tls_main.c:881 net/tls/tls_main.c:920)
   do_sock_setsockopt (net/socket.c:2399)
   __sys_setsockopt (net/socket.c:2424)
   __x64_sys_setsockopt (net/socket.c:2430 net/socket.c:2427)
   do_syscall_64 (arch/x86/entry/syscall_64.c:61 arch/x86/entry/syscall_64.c:89)
   entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:121)
   </TASK>

	first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
	if (WARN_ON_ONCE(!first))
		return;

tls_strp_load_anchor_with_queue() then returns without loading the
anchor, and tls_strp_read_sock() goes on to tls_rx_msg_size() with it;
in some runs that is a KASAN null-ptr-deref in tls_strp_check_rcv()
from the same setsockopt(). In others, a later recv() on the socket
hits the "TCP recvmsg seq # bug" WARN in tcp_recvmsg_locked().

The test puts the accepted end of a loopback TCP connection into a
sockmap with a stream verdict program that returns SK_PASS, while a
thread writes into the other end. It then takes the socket out of the
sockmap (element delete, element overwrite, or closing the map) and
at once sets TCP_ULP "tls" and TLS_RX on it. 460e6486617c ("tls: reject
the combination of TLS and sockmap") refuses the TLS keys only while
sk_psock(sk) is set, so this is allowed once the psock is gone.

With SK_PASS, the verdict path moves the data off sk_receive_queue into
psock->ingress_msg, and copied_seq only advances when it is read
through tcp_bpf_recvmsg(). When the socket leaves the sockmap with data
still in ingress_msg, sk_psock_destroy() frees it in
__sk_psock_purge_ingress_msg() and nothing moves copied_seq, so
tcp_inq() keeps counting bytes that are no longer on the socket. On a
kernel that logs the purge, every warning came with 512 or 576 unread
bytes dropped from ingress_msg; the data is lost as well.

The reproducer runs as root (the sockmap and the SK_SKB program need
CAP_NET_ADMIN and CAP_BPF) on an x86_64 QEMU guest with 2 vCPUs and
KASAN (gcc 13.3); it reports one of the warnings above in every run.

Checked against v7.4-rc3 (aebb72744764), and against the bpf tree
(4b4eb7f44ee0) with the sockmap fixes queued there ("bpf, sockmap:
Fixes for verdicts, redirects, corks and recvmsg()"), where it
reproduces the same way.

The reproducer (repro.c) and the kernel config are attached.

Thanks,
Kyuwon Cho

[-- Attachment #2: repro.c --]
[-- Type: text/plain, Size: 3836 bytes --]

// A TCP socket in a sockmap with a stream verdict program (SK_PASS) receives data from a writer thread. The test
// removes the socket from the sockmap (element delete, element overwrite, or closing the map) and immediately
// installs the TLS ULP and TLS RX keys on it. 460e6486617c only refuses TLS while sk_psock(sk) is set, so this
// succeeds once the psock is gone; a verdict sk_data_ready already in flight can then dequeue skbs from the
// socket's receive queue that TLS now owns.
// Runs as root (sockmap needs CAP_NET_ADMIN/CAP_BPF). Expect a NULL dereference in tls_strp_check_rcv(), or a
// WARNING in tls_strp_load_anchor_with_queue() or tcp_recvmsg_locked(), depending on timing.
#define _GNU_SOURCE
#include <linux/bpf.h>
#include <linux/tls.h>
#include <net/if.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>

#ifndef SOL_TLS
#define SOL_TLS 282
#endif

static int bpf(int cmd, union bpf_attr *a)
{
	return syscall(__NR_bpf, cmd, a, sizeof(*a));
}

static int sockmap_with_verdict(int prog)
{
	union bpf_attr a = { .map_type = BPF_MAP_TYPE_SOCKMAP, .key_size = 4, .value_size = 4, .max_entries = 4 };
	int map = bpf(BPF_MAP_CREATE, &a);

	memset(&a, 0, sizeof(a));
	a.target_fd = map;
	a.attach_bpf_fd = prog;
	a.attach_type = BPF_SK_SKB_STREAM_VERDICT;
	bpf(BPF_PROG_ATTACH, &a);
	return map;
}

static int map_set(int map, uint32_t key, int sock, int cmd)
{
	uint32_t val = sock;
	union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key, .value = (uintptr_t)&val };

	return bpf(cmd, &a);
}

static void tcp_pair(int *c, int *s)
{
	struct sockaddr_in a = { .sin_family = AF_INET, .sin_addr.s_addr = htonl(INADDR_LOOPBACK) };
	socklen_t al = sizeof(a);
	int l = socket(AF_INET, SOCK_STREAM, 0);

	bind(l, (void *)&a, sizeof(a));
	getsockname(l, (void *)&a, &al);
	listen(l, 1);
	*c = socket(AF_INET, SOCK_STREAM, 0);
	connect(*c, (void *)&a, sizeof(a));
	*s = accept(l, NULL, NULL);
	close(l);
}

static void *writer(void *p)
{
	/* A TLS 1.2 application-data header with a long length keeps a partial record queued. */
	char buf[64] = { 0x17, 3, 3, 0, 0x40 };

	for (int i = 0; i < 200; i++)
		if (send(*(int *)p, buf, sizeof(buf), MSG_NOSIGNAL) < 0)
			break;
	return NULL;
}

int main(void)
{
	struct bpf_insn insns[] = { { .code = 0xb7, .imm = 1 }, { .code = 0x95 } };	/* r0 = SK_PASS; exit */
	union bpf_attr a = { .prog_type = BPF_PROG_TYPE_SK_SKB, .insn_cnt = 2, .insns = (uintptr_t)insns,
			     .license = (uintptr_t)"GPL" };
	struct ifreq ifr = { .ifr_name = "lo", .ifr_flags = IFF_UP };
	struct tls12_crypto_info_aes_gcm_128 ci = { .info = { TLS_1_2_VERSION, TLS_CIPHER_AES_GCM_128 } };
	int s0 = socket(AF_INET, SOCK_DGRAM, 0), prog = bpf(BPF_PROG_LOAD, &a);
	unsigned seed = 1;
	time_t t0 = time(NULL);

	ioctl(s0, SIOCSIFFLAGS, &ifr);
	for (int i = 0; time(NULL) - t0 < 100; i++) {
		int map = sockmap_with_verdict(prog), c, s, c2 = -1, s2 = -1;
		pthread_t t;
		char drain[4096];

		tcp_pair(&c, &s);
		map_set(map, 0, s, BPF_MAP_UPDATE_ELEM);
		pthread_create(&t, NULL, writer, &c);
		for (volatile unsigned k = rand_r(&seed) % 20000; k; k--)
			;
		if (i % 3 == 0)
			map_set(map, 0, s, BPF_MAP_DELETE_ELEM);
		if (i % 3 == 1) {
			tcp_pair(&c2, &s2);
			map_set(map, 0, s2, BPF_MAP_UPDATE_ELEM);
		}
		if (i % 3 == 2) {
			close(map);
			map = -1;
		}
		if (!setsockopt(s, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls")))
			setsockopt(s, SOL_TLS, TLS_RX, &ci, sizeof(ci));
		pthread_join(t, NULL);
		for (int k = 0; k < 128 && recv(s, drain, sizeof(drain), MSG_DONTWAIT) > 0; k++)
			;
		close(s2);
		close(c2);
		close(map);
		close(s);
		close(c);
	}
	printf("done\n");
	return 0;
}

[-- Attachment #3: config.gz --]
[-- Type: application/gzip, Size: 38966 bytes --]

^ permalink raw reply	[flat|nested] only message in thread

only message in thread, other threads:[~2026-11-19  3:04 UTC | newest]

Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-19  3:04 [BUG] bpf, sockmap: data left in ingress_msg is dropped without moving copied_seq, and TLS RX then warns Kyuwon Cho

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®