Re: [PATCH 0/8][for -mm] mem_notify v6

[KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>]

[-- Attachment #1: Type: text/plain, Size: 1854 bytes --]

> madvise can be replaced with munmap and the same behavior occurs.
> 
> --- test.c.orig 2008-04-17 11:41:47.000000000 -0700
> +++ test.c 2008-04-17 11:44:04.000000000 -0700
> @@ -127,7 +127,7 @@
>     /* Release FREE_CHUNK pages. */
> 
>     for (i = 0; i < FREE_CHUNK; i++) {
> -       int r = madvise(p + page*PAGESIZE, PAGESIZE, MADV_DONTNEED);
> +       int r = munmap(p + page*PAGESIZE, PAGESIZE);
>         if (r == -1) {
>             perror("madvise");
>             exit(1);
> 
> Here's what I'm seeing on my system.  This is with munmap, but I see
> the same thing with madvise.  First, /proc/meminfo on my system before
> running the test:

Oh sorry my bad!
I investigated again and found 2 problem in your test program.

1. text segment isn't locked.

   if strong memory pressure happned, kernel may drop program text region.
   then your test program suddenly slow down.

   please use mlockall(MCL_CURRENT) before large buffer allocation.

2. repeat open/close to /proc/meminfo.
   
   in the fact, open(2) system call use a bit memory.
   if call open(2) in strong memory pressure, doesn't return until
   memory freed enough.
   thus, it cause slow down your program sometimes.

attached changed test program :)
it works well on my test environment.


> If it's possible to get a notification when MemFree + Cached + Mapped
> (I'm not sure whether this is the right formula) falls below some
> threshold, so that the program has time to find memory to discard
> before the system runs out, that would prevent the oom -- as long as
> the application(s) can ensure that there is not too much memory
> allocated while it is looking for memory to free.   But at least the
> threshold would give it a reasonable amount of time to handle the
> notification.

your proposal is interesting.
but I hope try to my attached test program at first.

[-- Attachment #2: tom.c --]
[-- Type: application/octet-stream, Size: 3604 bytes --]

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <poll.h>
#include <sched.h>
#include <time.h>
#include <unistd.h>
#include <pthread.h>

#define PAGESIZE (64*1024)

/* How many pages we've mmap'd. */
static long pages;

/* Pointer to mmap'd memory used as a circular buffer.  One thread
   touches pages, another thread releases them on notification. */
static char *p;

/* How many pages to touch each 5ms.  This makes at most 2000
   pages/sec. */
#define TOUCH_CHUNK 10

/* How many pages to free when we're notified.  With a 100ms FREE_DELAY,
   we can free ~9110 pages/sec, or perhaps only 5*911 = 4555 pages/sec if we're
   notified only 5 times/sec. */
#define FREE_CHUNK 911

/* Delay in milliseconds before freeing pages, to simulate latency while finding
   pages to free. */
#define FREE_DELAY 100

static void touch(void);
static int release(void *arg);
static void release_pages(void);
static void show_meminfo(void);
static void* _release (void *arg);

int
main (int argc, char **argv)
{
	pthread_t thr;

	mlockall(MCL_CURRENT); /* lock text*/
	setvbuf(stdout, (char *)NULL, _IOLBF, 0);

	pages = atol(argv[1]) * 1024 * 1024 / PAGESIZE;

	p = mmap(NULL, pages * PAGESIZE, PROT_READ | PROT_WRITE,
		 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, 0, 0);
	if (p == MAP_FAILED) {
		perror("mmap");
		exit(1);
	}

	if(pthread_create(&thr, NULL, _release, NULL)) {
		perror("pthread_create");
		exit(1);
	}


	touch();
	return 0;
}

static void
touch (void)
{
	long page = 0;

	while (1) {
		int i;
		struct timespec t;
		for (i = 0; i < TOUCH_CHUNK; i++) {
			p[page * PAGESIZE] = 1;
			if (++page >= pages) {
				page = 0;
			}
		}

#if 1
		t.tv_sec = 0;
		t.tv_nsec = 5 * 1000 * 1000; /* 5ms */
		if (nanosleep(&t, NULL) == -1) {
			perror("nanosleep");
		}
#endif
	}
}

static int
release (void *arg)
{
	int fd = open("/dev/mem_notify", O_RDONLY);
	if (fd == -1) {
		perror("open(/dev/mem_notify)");
		exit(1);
	}

	while (1) {
		struct pollfd pfd;
		int nfds;

		pfd.fd = fd;
		pfd.events = POLLIN;

		printf("poll\n");
		nfds = poll(&pfd, 1, -1);
		if (nfds == -1) {
			perror("poll");
			exit(1);
		}
		printf("notify\n");
		if (nfds == 1) {
			struct timespec t;
			t.tv_sec = 0;
			t.tv_nsec = FREE_DELAY * 1000 * 1000;
			if (nanosleep(&t, NULL) == -1) {
				perror("nanosleep");
			}
			printf("wakeup\n");
			release_pages();
			printf("time: %ld\n", time(NULL));
//			show_meminfo();
		}
	}
}

static void*
_release (void *arg)
{
	release(arg);
	return NULL;
}

static void
release_pages (void)
{
	/* Index of the next page to free. */
	static long page = 0;
	int i;

	/* Release FREE_CHUNK pages. */

	for (i = 0; i < FREE_CHUNK; i++) {
		int r = madvise(p + page*PAGESIZE, PAGESIZE, MADV_DONTNEED);
		if (r == -1) {
			perror("madvise");
			exit(1);
		}
//		printf("free %p\n", p + page*PAGESIZE);
		if (++page >= pages) {
			page = 0;
		}
	}
}

static void
show_meminfo (void)
{
	char buffer[2000];
	int fd;
	ssize_t n;

	fd = open("/proc/meminfo", O_RDONLY);
	if (fd == -1) {
		perror("open(/proc/meminfo)");
		exit(1);
	}

	n = read(fd, buffer, sizeof(buffer));
	if (n == -1) {
		perror("read(/proc/meminfo)");
		exit(1);
	}

	n = write(1, buffer, n);
	if (n == -1) {
		perror("write(stdout)");
		exit(1);
	}

	if (close(fd) == -1) {
		perror("close(/proc/meminfo)");
		exit(1);
	}
}