mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
* [BUG] md: md_error() dereferences a NULL mddev->pers when SET_DISK_FAULTY races with STOP_ARRAY
@ 2026-11-16 19:42 Kyuwon Cho
  0 siblings, 0 replies; only message in thread
From: Kyuwon Cho @ 2026-11-16 19:42 UTC (permalink / raw)
  To: Song Liu, Yu Kuai; +Cc: Li Nan, Xiao Ni, NeilBrown, linux-raid, linux-kernel

[-- Attachment #1: Type: text/plain, Size: 7817 bytes --]

Hi Song, Kuai,

While testing v7.4-rc3 with an LLM-guided test generator that we are
developing as a research prototype, we hit a NULL pointer dereference in
md_error() when the SET_DISK_FAULTY ioctl races with STOP_ARRAY on a
RAID1 array:

  md: md320: resync interrupted.
  md/raid1:md320: Disk failure on loop0, disabling device.
  md/raid1:md320: Operation continuing on 1 devices.
  Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000: 0000 [#1] SMP KASAN NOPTI
  KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007]
  CPU: 0 UID: 0 PID: 1712 Comm: test Not tainted 7.4.0-rc3 #1 PREEMPT(lazy) 
  RIP: 0010:md_error+0xf1/0x480
  Call Trace:
   <TASK>
   md_ioctl+0x278/0x3940
   blkdev_ioctl+0x3dc/0x570
   __x64_sys_ioctl+0x134/0x1c0
   do_syscall_64+0x116/0x630
   entry_SYSCALL_64_after_hwframe+0x77/0x7f
   </TASK>

md_ioctl() handles SET_DISK_FAULTY before it takes reconfig_mutex, since
1ca69c4bc4b1 ("md: avoid taking the mutex on some ioctls."):

	case SET_DISK_FAULTY:
		return set_disk_faulty(mddev, new_decode_dev(arg));

set_disk_faulty() checks mddev->pers, and md_error() reads mddev->pers
again for every use:

	if (!mddev->pers || !mddev->pers->error_handler)
		return;
	mddev->pers->error_handler(mddev, rdev);

	if (mddev->pers->head.id == ID_RAID0 ||
	    mddev->pers->head.id == ID_LINEAR)
		return;

A concurrent STOP_ARRAY runs __md_stop() under reconfig_mutex, which sets
mddev->pers to NULL and then frees the personality's private data. In
the oops above raid1_error() has already run (the "Disk failure" lines),
and md_error+0xf1 is the KASAN check of the mddev->pers->head.id load
after ->error_handler() returns, with pers now NULL. The same window
would also let ->error_handler() run while __md_stop() frees
mddev->private; we have not seen that happen.

Handling SET_DISK_FAULTY after mddev_lock() in md_ioctl(), as STOP_ARRAY
is, makes the problem go away here: the reproducer below oopsed in
3 of 6 runs before that change and in 0 of 6 runs after it. We have
not checked whether the reasons for the mutex-free path in 1ca69c4bc4b1
still apply; reading pers once and keeping it valid against __md_stop()
would be another way.

The reproducer needs root (CAP_SYS_ADMIN for the md ioctls). It builds a
RAID1 from two loop devices and, on each array, issues SET_DISK_FAULTY on
a member from one thread and STOP_ARRAY from another, with small random
delays, for up to 6000 arrays or 110 seconds. On an x86_64 QEMU guest
with 2 vCPUs and KASAN (gcc 13.3, config attached), it oopses in about
half of the runs.

Checked against v7.4-rc3 (aebb72744764); set_disk_faulty() and
md_error() are the same in the md tree's md-7.4 branch. We found no
earlier report on syzbot or on linux-raid. The closest is the
md_start_sync() NULL pers fix posted on 2026-11-06, which is a different
path.

// Run as root. Builds a RAID1 on loop0/loop1 again and again and, on each array,
// races SET_DISK_FAULTY on loop0 (thread A) against STOP_ARRAY (thread B).
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <time.h>
#include <unistd.h>

#define MD_MAJOR 9
#define LOOP_SET_FD 0x4C00
#define MD_DISK_ACTIVE 1
#define MD_DISK_SYNC 2

typedef struct {
	int major_version, minor_version, patch_version;
	unsigned int ctime;
	int level, size, nr_disks, raid_disks, md_minor, not_persistent;
	unsigned int utime;
	int state, active_disks, working_disks, failed_disks, spare_disks;
	int layout, chunk_size;
} mdu_array_info_t;

typedef struct { int number, major, minor, raid_disk, state; } mdu_disk_info_t;
typedef struct { int personality, chunk_size, max_fault; } mdu_param_t;

#define ADD_NEW_DISK	_IOW(MD_MAJOR, 0x21, mdu_disk_info_t)
#define SET_ARRAY_INFO	_IOW(MD_MAJOR, 0x23, mdu_array_info_t)
#define SET_DISK_FAULTY	_IO(MD_MAJOR, 0x29)
#define RUN_ARRAY	_IOW(MD_MAJOR, 0x30, mdu_param_t)
#define STOP_ARRAY	_IO(MD_MAJOR, 0x32)

#define MAX_ITERS	6000
#define TIME_BUDGET	110

#define DELAY_A_NS	20000000LL	/* faulter starts 0..20 ms after the go flag */
#define DELAY_B_NS	2000000LL	/* stopper starts 0..2 ms after the go flag */

static int md_fd;
static atomic_int go;
static long long go_ns, delayA_ns, delayB_ns;

static long long now_ns(void)
{
	struct timespec ts;

	clock_gettime(CLOCK_MONOTONIC, &ts);
	return ts.tv_sec * 1000000000LL + ts.tv_nsec;
}

static void wait_until(long long t)
{
	while (now_ns() < t)
		;
}

static int open_node(const char *name, int maj, int min, int flags)
{
	char path[64];
	int fd;

	snprintf(path, sizeof(path), "/dev/%s", name);
	fd = open(path, flags);
	if (fd >= 0)
		return fd;
	snprintf(path, sizeof(path), "/tmp/%s", name);
	mknod(path, S_IFBLK | 0600, makedev(maj, min));
	return open(path, flags);
}

static void setup_loops(void)
{
	for (int i = 0; i < 2; i++) {
		char name[16], img[32];
		int ffd, lfd;

		snprintf(name, sizeof(name), "loop%d", i);
		snprintf(img, sizeof(img), "/tmp/disk%d.img", i);
		ffd = open(img, O_RDWR | O_CREAT, 0600);
		if (ffd < 0 || ftruncate(ffd, 16 << 20) < 0) {
			perror("backing file");
			exit(1);
		}
		lfd = open_node(name, 7, i, O_RDWR);
		if (lfd < 0 || ioctl(lfd, LOOP_SET_FD, ffd) < 0) {
			perror("LOOP_SET_FD");
			exit(1);
		}
		close(ffd);
		close(lfd);
	}
}

static int setup_array(int md, int minor)
{
	mdu_array_info_t ai;
	mdu_param_t p;

	memset(&ai, 0, sizeof(ai));
	ai.major_version = 0;
	ai.minor_version = 90;
	ai.level = 1;
	ai.size = 8192;
	ai.nr_disks = 2;
	ai.raid_disks = 2;
	ai.md_minor = minor;
	ai.not_persistent = 1;
	if (ioctl(md, SET_ARRAY_INFO, &ai) < 0)
		return -errno;
	for (int i = 0; i < 2; i++) {
		mdu_disk_info_t di;

		memset(&di, 0, sizeof(di));
		di.number = i;
		di.major = 7;
		di.minor = i;
		di.raid_disk = i;
		di.state = (1 << MD_DISK_ACTIVE) | (1 << MD_DISK_SYNC);
		if (ioctl(md, ADD_NEW_DISK, &di) < 0)
			return -errno;
	}
	memset(&p, 0, sizeof(p));
	if (ioctl(md, RUN_ARRAY, &p) < 0)
		return -errno;
	return 0;
}

static void *faulter(void *arg)
{
	(void)arg;
	while (!atomic_load(&go))
		;
	wait_until(go_ns + delayA_ns);
	ioctl(md_fd, SET_DISK_FAULTY, (unsigned long)((7 << 8) | 0));	/* loop0 */
	return NULL;
}

static void *stopper(void *arg)
{
	(void)arg;
	while (!atomic_load(&go))
		;
	wait_until(go_ns + delayB_ns);
	for (int k = 0; k < 3; k++)
		if (ioctl(md_fd, STOP_ARRAY, 0) == 0)
			break;
	return NULL;
}

int main(void)
{
	unsigned int seed = 1234;
	time_t t0 = time(NULL);
	int iters, ok = 0;

	setup_loops();
	for (iters = 0; iters < MAX_ITERS && time(NULL) - t0 < TIME_BUDGET; iters++) {
		pthread_t a, b;
		char name[16];
		int r, minor = 1 + iters;

		/* a fresh minor per run: a closed md node is not reopened (ENXIO) */
		snprintf(name, sizeof(name), "md%d", minor);
		md_fd = open_node(name, MD_MAJOR, minor, O_RDWR);
		if (md_fd < 0) {
			perror("open md");
			break;
		}
		r = setup_array(md_fd, minor);
		if (r) {
			if (iters < 3)
				fprintf(stderr, "setup failed: %s\n", strerror(-r));
			ioctl(md_fd, STOP_ARRAY, 0);
			close(md_fd);
			continue;
		}
		ok++;
		seed = seed * 1103515245u + 12345u;
		delayA_ns = (long long)((seed >> 4) % 1000000u) * DELAY_A_NS / 1000000;
		seed = seed * 1103515245u + 12345u;
		delayB_ns = (long long)((seed >> 4) % 1000000u) * DELAY_B_NS / 1000000;
		atomic_store(&go, 0);
		pthread_create(&a, NULL, faulter, NULL);
		pthread_create(&b, NULL, stopper, NULL);
		go_ns = now_ns();
		atomic_store(&go, 1);
		pthread_join(a, NULL);
		pthread_join(b, NULL);
		ioctl(md_fd, STOP_ARRAY, 0);	/* no-op if the racer already stopped it */
		close(md_fd);
	}
	printf("done: iters %d setup_ok %d\n", iters, ok);
	fflush(stdout);
	return 0;
}

Thanks,
Kyuwon Cho

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 37687 bytes --]

^ permalink raw reply	[flat|nested] only message in thread

only message in thread, other threads:[~2026-11-16 19:42 UTC | newest]

Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-16 19:42 [BUG] md: md_error() dereferences a NULL mddev->pers when SET_DISK_FAULTY races with STOP_ARRAY Kyuwon Cho

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®