* [BUG] md: md_error() dereferences a NULL mddev->pers when SET_DISK_FAULTY races with STOP_ARRAY
@ 2026-11-16 19:42 Kyuwon Cho
0 siblings, 0 replies; only message in thread
From: Kyuwon Cho @ 2026-11-16 19:42 UTC (permalink / raw)
To: Song Liu, Yu Kuai; +Cc: Li Nan, Xiao Ni, NeilBrown, linux-raid, linux-kernel
[-- Attachment #1: Type: text/plain, Size: 7817 bytes --]
Hi Song, Kuai,
While testing v7.4-rc3 with an LLM-guided test generator that we are
developing as a research prototype, we hit a NULL pointer dereference in
md_error() when the SET_DISK_FAULTY ioctl races with STOP_ARRAY on a
RAID1 array:
md: md320: resync interrupted.
md/raid1:md320: Disk failure on loop0, disabling device.
md/raid1:md320: Operation continuing on 1 devices.
Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000: 0000 [#1] SMP KASAN NOPTI
KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007]
CPU: 0 UID: 0 PID: 1712 Comm: test Not tainted 7.4.0-rc3 #1 PREEMPT(lazy)
RIP: 0010:md_error+0xf1/0x480
Call Trace:
<TASK>
md_ioctl+0x278/0x3940
blkdev_ioctl+0x3dc/0x570
__x64_sys_ioctl+0x134/0x1c0
do_syscall_64+0x116/0x630
entry_SYSCALL_64_after_hwframe+0x77/0x7f
</TASK>
md_ioctl() handles SET_DISK_FAULTY before it takes reconfig_mutex, since
1ca69c4bc4b1 ("md: avoid taking the mutex on some ioctls."):
case SET_DISK_FAULTY:
return set_disk_faulty(mddev, new_decode_dev(arg));
set_disk_faulty() checks mddev->pers, and md_error() reads mddev->pers
again for every use:
if (!mddev->pers || !mddev->pers->error_handler)
return;
mddev->pers->error_handler(mddev, rdev);
if (mddev->pers->head.id == ID_RAID0 ||
mddev->pers->head.id == ID_LINEAR)
return;
A concurrent STOP_ARRAY runs __md_stop() under reconfig_mutex, which sets
mddev->pers to NULL and then frees the personality's private data. In
the oops above raid1_error() has already run (the "Disk failure" lines),
and md_error+0xf1 is the KASAN check of the mddev->pers->head.id load
after ->error_handler() returns, with pers now NULL. The same window
would also let ->error_handler() run while __md_stop() frees
mddev->private; we have not seen that happen.
Handling SET_DISK_FAULTY after mddev_lock() in md_ioctl(), as STOP_ARRAY
is, makes the problem go away here: the reproducer below oopsed in
3 of 6 runs before that change and in 0 of 6 runs after it. We have
not checked whether the reasons for the mutex-free path in 1ca69c4bc4b1
still apply; reading pers once and keeping it valid against __md_stop()
would be another way.
The reproducer needs root (CAP_SYS_ADMIN for the md ioctls). It builds a
RAID1 from two loop devices and, on each array, issues SET_DISK_FAULTY on
a member from one thread and STOP_ARRAY from another, with small random
delays, for up to 6000 arrays or 110 seconds. On an x86_64 QEMU guest
with 2 vCPUs and KASAN (gcc 13.3, config attached), it oopses in about
half of the runs.
Checked against v7.4-rc3 (aebb72744764); set_disk_faulty() and
md_error() are the same in the md tree's md-7.4 branch. We found no
earlier report on syzbot or on linux-raid. The closest is the
md_start_sync() NULL pers fix posted on 2026-11-06, which is a different
path.
// Run as root. Builds a RAID1 on loop0/loop1 again and again and, on each array,
// races SET_DISK_FAULTY on loop0 (thread A) against STOP_ARRAY (thread B).
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/sysmacros.h>
#include <time.h>
#include <unistd.h>
#define MD_MAJOR 9
#define LOOP_SET_FD 0x4C00
#define MD_DISK_ACTIVE 1
#define MD_DISK_SYNC 2
typedef struct {
int major_version, minor_version, patch_version;
unsigned int ctime;
int level, size, nr_disks, raid_disks, md_minor, not_persistent;
unsigned int utime;
int state, active_disks, working_disks, failed_disks, spare_disks;
int layout, chunk_size;
} mdu_array_info_t;
typedef struct { int number, major, minor, raid_disk, state; } mdu_disk_info_t;
typedef struct { int personality, chunk_size, max_fault; } mdu_param_t;
#define ADD_NEW_DISK _IOW(MD_MAJOR, 0x21, mdu_disk_info_t)
#define SET_ARRAY_INFO _IOW(MD_MAJOR, 0x23, mdu_array_info_t)
#define SET_DISK_FAULTY _IO(MD_MAJOR, 0x29)
#define RUN_ARRAY _IOW(MD_MAJOR, 0x30, mdu_param_t)
#define STOP_ARRAY _IO(MD_MAJOR, 0x32)
#define MAX_ITERS 6000
#define TIME_BUDGET 110
#define DELAY_A_NS 20000000LL /* faulter starts 0..20 ms after the go flag */
#define DELAY_B_NS 2000000LL /* stopper starts 0..2 ms after the go flag */
static int md_fd;
static atomic_int go;
static long long go_ns, delayA_ns, delayB_ns;
static long long now_ns(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec * 1000000000LL + ts.tv_nsec;
}
static void wait_until(long long t)
{
while (now_ns() < t)
;
}
static int open_node(const char *name, int maj, int min, int flags)
{
char path[64];
int fd;
snprintf(path, sizeof(path), "/dev/%s", name);
fd = open(path, flags);
if (fd >= 0)
return fd;
snprintf(path, sizeof(path), "/tmp/%s", name);
mknod(path, S_IFBLK | 0600, makedev(maj, min));
return open(path, flags);
}
static void setup_loops(void)
{
for (int i = 0; i < 2; i++) {
char name[16], img[32];
int ffd, lfd;
snprintf(name, sizeof(name), "loop%d", i);
snprintf(img, sizeof(img), "/tmp/disk%d.img", i);
ffd = open(img, O_RDWR | O_CREAT, 0600);
if (ffd < 0 || ftruncate(ffd, 16 << 20) < 0) {
perror("backing file");
exit(1);
}
lfd = open_node(name, 7, i, O_RDWR);
if (lfd < 0 || ioctl(lfd, LOOP_SET_FD, ffd) < 0) {
perror("LOOP_SET_FD");
exit(1);
}
close(ffd);
close(lfd);
}
}
static int setup_array(int md, int minor)
{
mdu_array_info_t ai;
mdu_param_t p;
memset(&ai, 0, sizeof(ai));
ai.major_version = 0;
ai.minor_version = 90;
ai.level = 1;
ai.size = 8192;
ai.nr_disks = 2;
ai.raid_disks = 2;
ai.md_minor = minor;
ai.not_persistent = 1;
if (ioctl(md, SET_ARRAY_INFO, &ai) < 0)
return -errno;
for (int i = 0; i < 2; i++) {
mdu_disk_info_t di;
memset(&di, 0, sizeof(di));
di.number = i;
di.major = 7;
di.minor = i;
di.raid_disk = i;
di.state = (1 << MD_DISK_ACTIVE) | (1 << MD_DISK_SYNC);
if (ioctl(md, ADD_NEW_DISK, &di) < 0)
return -errno;
}
memset(&p, 0, sizeof(p));
if (ioctl(md, RUN_ARRAY, &p) < 0)
return -errno;
return 0;
}
static void *faulter(void *arg)
{
(void)arg;
while (!atomic_load(&go))
;
wait_until(go_ns + delayA_ns);
ioctl(md_fd, SET_DISK_FAULTY, (unsigned long)((7 << 8) | 0)); /* loop0 */
return NULL;
}
static void *stopper(void *arg)
{
(void)arg;
while (!atomic_load(&go))
;
wait_until(go_ns + delayB_ns);
for (int k = 0; k < 3; k++)
if (ioctl(md_fd, STOP_ARRAY, 0) == 0)
break;
return NULL;
}
int main(void)
{
unsigned int seed = 1234;
time_t t0 = time(NULL);
int iters, ok = 0;
setup_loops();
for (iters = 0; iters < MAX_ITERS && time(NULL) - t0 < TIME_BUDGET; iters++) {
pthread_t a, b;
char name[16];
int r, minor = 1 + iters;
/* a fresh minor per run: a closed md node is not reopened (ENXIO) */
snprintf(name, sizeof(name), "md%d", minor);
md_fd = open_node(name, MD_MAJOR, minor, O_RDWR);
if (md_fd < 0) {
perror("open md");
break;
}
r = setup_array(md_fd, minor);
if (r) {
if (iters < 3)
fprintf(stderr, "setup failed: %s\n", strerror(-r));
ioctl(md_fd, STOP_ARRAY, 0);
close(md_fd);
continue;
}
ok++;
seed = seed * 1103515245u + 12345u;
delayA_ns = (long long)((seed >> 4) % 1000000u) * DELAY_A_NS / 1000000;
seed = seed * 1103515245u + 12345u;
delayB_ns = (long long)((seed >> 4) % 1000000u) * DELAY_B_NS / 1000000;
atomic_store(&go, 0);
pthread_create(&a, NULL, faulter, NULL);
pthread_create(&b, NULL, stopper, NULL);
go_ns = now_ns();
atomic_store(&go, 1);
pthread_join(a, NULL);
pthread_join(b, NULL);
ioctl(md_fd, STOP_ARRAY, 0); /* no-op if the racer already stopped it */
close(md_fd);
}
printf("done: iters %d setup_ok %d\n", iters, ok);
fflush(stdout);
return 0;
}
Thanks,
Kyuwon Cho
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 37687 bytes --]
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2026-11-16 19:42 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2026-11-16 19:42 [BUG] md: md_error() dereferences a NULL mddev->pers when SET_DISK_FAULTY races with STOP_ARRAY Kyuwon Cho
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®