Re: [PATCH] mm: nommu: set MMF_OOM_SKIP in exit_mmap()

[Alexis Bouzigues <BouziguesAlexis@JohnDeere.com>]

On Wed, Oct 14, 2026 at 09:50:30AM +0200, Michal Hocko wrote:
> > Set MMF_OOM_SKIP in the NOMMU exit_mmap() once the VMAs are freed, as
> > the MMU version does.
> Is this a real problem that you can actually hit?

Yes, without this change the oom killer can fail to make progress to
resolve a global oom. This gets seeded by a parent PID making an
allocation that triggers oom killer before reaping an exited child oom
victim. The parent pid expected to reap the victim indefinitely retries
in the page allocator because the oom killer does not mark a new victim
until the old victim is reaped.

A /init reproducer is provided below. I tested this patch on both the
reproducer and original benchmark where I encountered the livelocks.

---8<---
#include <poll.h>
#include <stdio.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <signal.h>
#include <unistd.h>

#define TEST_TIME_MS 10000

static char *self;

static void hog(void)
{
 for (;;) {
  char *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE,
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (p != MAP_FAILED) memset(p, 0xa5, 4096);
 }
}

static pid_t spawn(char *role, int *rfd)
{
 char *argv[] = { self, role, NULL };
 char *envp[] = { NULL };
 int p[2];
 pid_t pid;

 if (pipe(p)) return -1;

 pid = vfork();
 if (pid == 0) {
  close(p[0]);
  execve(self, argv, envp);
  _exit(127);
 }

 close(p[1]);
 *rfd = p[0];

 return pid;
}

static long wait_eof(int fd, long timeout_ms)
{
 struct pollfd pfd = { .fd = fd, .events = POLLIN };
 long ms;

 for (ms = 0; timeout_ms < 0 || ms < timeout_ms; ms += 100)
  if (poll(&pfd, 1, 100) > 0)
   return ms;

 return -1;
}

static void parent(void)
{
 int fd;
 pid_t child = spawn("child", &fd);

 if (child < 0 || wait_eof(fd, -1) < 0)
  _exit(1);

 hog();
}

int main(int argc, char **argv)
{
 int fd;
 long ms;
 pid_t p;

 self = argv[0];

 if (argc > 1 && !strcmp(argv[1], "child")) hog();
 if (argc > 1 && !strcmp(argv[1], "parent")) parent();

 p = spawn("parent", &fd);
 if (p < 0)
  return 1;

 ms = wait_eof(fd, TEST_TIME_MS);
 if (ms < 0) kill(p, SIGKILL);

 int status = 0;

 if (!(waitpid(p, &status, 0) == p && WIFSIGNALED(status) &&
       WTERMSIG(status) == SIGKILL))
  printf("Parent pid exited unexpectedly\n");
 else if (ms >= 0)
  printf("Parent pid was killed\n");
 else
  printf("Parent pid was still alive!\n");

 if (getpid() == 1)
  for (;;)
   poll(NULL, 0, -1);

 return ms < 0;
}
--->8---