Re: Linux 2.6.31 - very swap-happy with plenty of free RAM

[KOSAKI Motohiro <kosaki.motohiro@gmail.com>]

[-- Attachment #1: Type: text/plain, Size: 3366 bytes --]

Hi Dan,

2009/11/18 Dan Merillat <dan.merillat@gmail.com>:
> procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu----
>  r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa
>  1  1      0 774404  49644 1713932    0    0     8    44 2680 5105 23 42  0 35
>  2  1      0 774900  49448 1713668    0    0     8    28 2870 5101 22 41  0 36
>
> swapon /dev/vg1/swap
>
>  2  2  34460 770808  47824 1709852    0 34460   952 34472 3174 5601 21 48  2 29
>  1  1 129716 759904  47460 1699364    0 95316   272 95532 3093 5641 16 49  7 29
>  1  4 196728 756680  47436 1693616   64 67080    92 67104 2980 5569 18 49 24  8
>  2  1 246212 754076  47396 1689240   32 49652   220 49760 3241 5405 19 51  1 29
>  2  3 282404 791648  47272 1686252   16 36284   240 36392 3088 6281 18 52  9 22
>  1  4 299464 847200  47260 1685208  324 17296   324 17320 3190 6199 23 46  0 30
>  2  5 302316 854384  47256 1685884  944 3804  1948  3812 2723 5297 20 45  5 31
> 11  3 303436 861700  47384 1686400 1188 1900  2084  1912 2615 4593 21 51  9 20
> swapoff -a
>  2  6 301740 863436  47384 1687048 1860    4  2368   128 2541 4591 21 63  0 15
>  3  4 300076 865916  47384 1687604 1672    0  2120   156 2673 5208 19 63  0 18
>  2  5 297676 866288  47380 1687988 2396    0  2668   188 2632 5259 17 70  2 12
>  2  2 295556 866784  47380 1687956 2352    0  2352     0 2621 5316 16 72  9  4
>  5  5 293168 867156  47384 1688120 2344    0  2344    20 2651 5469 18 79  0  4
>  2  4 291108 870504  47396 1688260 2036    0  2172   160 2381 4817 15 67  0 18
>  3  3 289416 870612  47400 1688476 2028    0  2260   148 2602 5214 18 69  0 13
>  3  3 287528 871928  47400 1688556 1928    0  2136     0 2470 5096 19 68  3 10
>  2  4 285740 873896  47408 1688892 2352    0  2764   172 2490 5194 18 66  0 17
>
> 1.7gb of disk cache, 750 meg of free RAM, and it swaps out 300meg in a
> matter of seconds when given access to a swapfile, then thrashes the
> disk like crazy because that RAM was actively in use. From an
> interactivity standpoint the machine is unusable - 1-2 second pauses
> on mouse movement, 10-15 seconds for the window manager to change
> window focus, keypresses take 1-2 seconds to show up, etc.  When I
> issue a swapoff it takes 10-15  minutes for it to slowly pull it all
> back in - once it's done I run fine.
>
> The machine has 4gb of RAM (64bit), and runs 3 VMware VMs with a total
> of 1024mb allocated to them.   VMware uses file-backed RAM (for some
> ungodly stupid reason) which is on a separate spindle (/dev/sdb) from
> swap (/dev/sda).  I've got one VM pointed to a file-backed RAM on a
> tmpfs (512mb) that's probably got quite a bit of memory activity.
>
> The other main RAM hogs are the usual suspects - firefox, Xorg,
> thunderbird, openoffice, pidgin, etc, the usual junk on a busy Linux
> desktop.
>
> All the sysctl tuneables should be defaults, there's no distro or
> local changes to any vm settings (mmap_min_addr=0 for Wine,
> shmmax=256mb, some ipv6 tweaks)
>
> Ideas on where to start looking?  If this is a vmware issue I'll take
> it to them.

Umm, very strange.
I made two debug patch. can you please apply it and post following
command output?

% cat /proc/meminfo
% cat /proc/vmstat
% cat /proc/zoneinfo
# cat /proc/filecache | sort -nr -k3 |head -30

[-- Attachment #2: proc-filecache-v2.patch --]
[-- Type: application/octet-stream, Size: 10880 bytes --]

diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt
index 94b9f20..bad2506 100644
--- a/Documentation/filesystems/proc.txt
+++ b/Documentation/filesystems/proc.txt
@@ -414,6 +414,7 @@ Table 1-5: Kernel info in /proc
  driver	     Various drivers grouped here, currently rtc (2.4)
  execdomains Execdomains, related to security			(2.4)
  fb	     Frame Buffer devices				(2.4)
+ filecache   Query/drop in-memory file cache
  fs	     File system parameters, currently nfs/exports	(2.4)
  ide         Directory containing info about the IDE subsystem 
  interrupts  Interrupt usage                                   
@@ -770,6 +771,39 @@ Provides counts of softirq handlers serviced since boot time, for each cpu.
  HRTIMER:          0          0          0          0
      RCU:       1678       1769       2178       2250
 
+..............................................................................
+
+filecache:
+
+Provides access to the in-memory file cache.
+
+To list an index of all cached files:
+
+    cat /proc/filecache
+
+The output looks like:
+
+    # filecache 1.0
+    #      ino       size   cached cached%  state   refcnt  dev             file
+       1026334         91       92    100   --      66      03:02(hda2)     /lib/ld-2.3.6.so
+        233608       1242      972     78   --      66      03:02(hda2)     /lib/tls/libc-2.3.6.so
+         65203        651      476     73   --      1       03:02(hda2)     /bin/bash
+       1026445        261      160     61   --      10      03:02(hda2)     /lib/libncurses.so.5.5
+        235427         10       12    100   --      44      03:02(hda2)     /lib/tls/libdl-2.3.6.so
+
+
+	ino:		inode number
+	size:		inode size in KB
+	cached:		cached size in KB
+	cached%:	percent of file data cached
+	state1:		'-' clean; 'd' metadata dirty; 'D' data dirty
+	state2:		'-' unlocked; 'L' locked, normally indicates file being written out
+	refcnt:		file reference count, it's an in-kernel one, not exactly open count
+	dev:		major:minor numbers in hex, followed by a descriptive device name
+	file:		file path _inside_ the filesystem. There are several special names:
+			'(noname)':	the file name is not available
+			'(03:02)':	the file is a block device file of major:minor
+			'...(deleted)': the named file has been deleted from the disk
 
 1.3 IDE devices in /proc/ide
 ----------------------------
diff --git a/fs/dcache.c b/fs/dcache.c
index ddd1986..fa1c3ce 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -1927,7 +1927,11 @@ char *__d_path(const struct path *path, struct path *root,
 
 		if (dentry == root->dentry && vfsmnt == root->mnt)
 			break;
-		if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
+
+		if (!vfsmnt) {
+			if (IS_ROOT(dentry))
+				break; 
+		} else if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
 			/* Global root? */
 			if (vfsmnt->mnt_parent == vfsmnt) {
 				goto global_root;
diff --git a/fs/proc/Makefile b/fs/proc/Makefile
index 11a7b5c..c532d59 100644
--- a/fs/proc/Makefile
+++ b/fs/proc/Makefile
@@ -19,6 +19,7 @@ proc-y	+= stat.o
 proc-y	+= uptime.o
 proc-y	+= version.o
 proc-y	+= softirqs.o
+proc-y	+= filecache.o
 proc-$(CONFIG_PROC_SYSCTL)	+= proc_sysctl.o
 proc-$(CONFIG_NET)		+= proc_net.o
 proc-$(CONFIG_PROC_KCORE)	+= kcore.o
diff --git a/fs/proc/filecache.c b/fs/proc/filecache.c
new file mode 100644
index 0000000..0ca121c
--- /dev/null
+++ b/fs/proc/filecache.c
@@ -0,0 +1,334 @@
+/*
+ * fs/proc/filecache.c
+ *
+ * Copyright (C) 2006, 2007 Fengguang Wu <wfg@mail.ustc.edu.cn>
+ * Copyright (C) 2009 KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/radix-tree.h>
+#include <linux/page-flags.h>
+#include <linux/pagevec.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
+#include <linux/writeback.h>
+#include <linux/buffer_head.h>
+#include <linux/parser.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/file.h>
+#include <linux/namei.h>
+#include <linux/module.h>
+#include <linux/backing-dev.h>
+
+#include <asm/uaccess.h>
+
+/* Internal buffer sizes. The larger the more effcient. */
+#define SBUF_SIZE	(128<<10)
+#define IWIN_PAGE_ORDER	3
+#define IWIN_SIZE	((PAGE_SIZE<<IWIN_PAGE_ORDER) / sizeof(struct inode *))
+
+struct fcache_read_state {
+	/* inode at last pos */
+	struct {
+		unsigned long pos;
+		struct inode *inode;
+	} ipos;
+
+	/* inode window */
+	struct {
+		unsigned long cursor;
+		unsigned long origin;
+		unsigned long size;
+		struct inode **inodes;
+	} iwin;
+};
+
+static int may_show_inode(struct fcache_read_state *s, struct inode *inode)
+{
+	if (!atomic_read(&inode->i_count))
+		return 0;
+	if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE))
+		return 0;
+	if (!inode->i_mapping)
+		return 0;
+
+	if (!inode->i_mapping->nrpages)
+		return 0;
+
+	if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
+	      S_ISLNK(inode->i_mode) || S_ISBLK(inode->i_mode)))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * Full: there are more data following.
+ */
+static int iwin_full(struct fcache_read_state *s)
+{
+	return !s->iwin.cursor ||
+		s->iwin.cursor > s->iwin.origin + s->iwin.size;
+}
+
+static int iwin_push(struct fcache_read_state *s, struct inode *inode)
+{
+	if (!may_show_inode(s, inode))
+		return 0;
+
+	s->iwin.cursor++;
+
+	if (s->iwin.size >= IWIN_SIZE)
+		return 1;
+
+	if (s->iwin.cursor > s->iwin.origin)
+		s->iwin.inodes[s->iwin.size++] = inode;
+	return 0;
+}
+
+/*
+ * Travease the inode lists in order - newest first.
+ * And fill @s->iwin.inodes with inodes positioned in [@pos, @pos+IWIN_SIZE).
+ */
+static int iwin_fill(struct fcache_read_state *s, unsigned long pos)
+{
+	struct inode *inode;
+	struct backing_dev_info *bdi;
+
+	s->iwin.origin = pos;
+	s->iwin.cursor = 0;
+	s->iwin.size = 0;
+
+	spin_lock_bh(&bdi_lock);
+	list_for_each_entry(bdi, &bdi_list, bdi_list) {
+		list_for_each_entry(inode, &bdi->wb.b_dirty, i_list) {
+			if (iwin_push(s, inode))
+				goto out_full_unlock;
+		}
+		list_for_each_entry(inode, &bdi->wb.b_io, i_list) {
+			if (iwin_push(s, inode))
+				goto out_full_unlock;
+		}
+	}
+	spin_unlock_bh(&bdi_lock);
+
+	list_for_each_entry(inode, &inode_in_use, i_list) {
+		if (iwin_push(s, inode))
+			goto out_full;
+		;
+	}
+
+	list_for_each_entry(inode, &inode_unused, i_list) {
+		if (iwin_push(s, inode))
+			goto out_full;
+	}
+
+	return 0;
+
+out_full_unlock:
+	spin_unlock_bh(&bdi_lock);
+out_full:
+	return 1;
+}
+
+static struct inode* iwin_inode(struct fcache_read_state *s, unsigned long pos)
+{
+	if ((iwin_full(s) && pos >= s->iwin.origin + s->iwin.size)
+			  || pos < s->iwin.origin)
+		iwin_fill(s, pos);
+
+	if (pos >= s->iwin.cursor)
+		return NULL;
+
+	s->ipos.pos = pos;
+	s->ipos.inode = s->iwin.inodes[pos - s->iwin.origin];
+	BUG_ON(!s->ipos.inode);
+
+	return s->ipos.inode;
+}
+
+static void show_inode(struct seq_file *m, struct inode *inode)
+{
+	char state[] = "--"; /* dirty, locked */
+	struct dentry *dentry;
+	unsigned long size = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
+	unsigned long nrpages;
+	int percent = 0;
+	int refcnt;
+	int len;
+
+	if (inode->i_mapping)
+		nrpages = inode->i_mapping->nrpages;
+	else {
+		nrpages = 0;
+		WARN_ON(1);
+	}
+
+	if (size)
+		percent = 100 * nrpages / size;
+
+	if (inode->i_state & (I_DIRTY_DATASYNC|I_DIRTY_PAGES))
+		state[0] = 'D';
+	else if (inode->i_state & I_DIRTY_SYNC)
+		state[0] = 'd';
+
+	if (inode->i_state & I_LOCK)
+		state[1] = 'L';
+
+	refcnt = 0;
+	list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
+		refcnt += atomic_read(&dentry->d_count);
+	}
+
+	seq_printf(m, "%10lu %10lu %8lu %7d %6d %5s %02x:%02x(%.5s)%n",
+		   inode->i_ino,
+		   size << (PAGE_CACHE_SHIFT - 10),
+		   nrpages << (PAGE_CACHE_SHIFT - 10),
+		   percent,
+		   refcnt,
+		   state,
+		   MAJOR(inode->i_sb->s_dev),
+		   MINOR(inode->i_sb->s_dev),
+		   inode->i_sb->s_id,
+		   &len);
+
+	seq_printf(m, "%*c", 65 - len, ' ');
+
+	seq_printf(m, "%.7o ",
+		   inode->i_mode);
+
+	if (list_empty(&inode->i_dentry)) {
+		if (!atomic_read(&inode->i_count))
+			seq_puts(m, "(noname)\n");
+		else
+			seq_printf(m, "(%02x:%02x)\n",
+					imajor(inode), iminor(inode));
+	} else {
+		struct path path = {
+			.mnt = NULL,
+			.dentry = list_entry(inode->i_dentry.next,
+					     struct dentry, d_alias)
+		};
+
+		seq_path(m, &path, "\t\n\\");
+		seq_putc(m, '\n');
+	}
+}
+
+static int ii_show(struct seq_file *m, void *v)
+{
+	unsigned long index = *(loff_t *) v;
+	struct fcache_read_state *s = m->private;
+        struct inode *inode;
+
+	if (index == 0) {
+		seq_puts(m, "#      ino       size   cached cached% "
+			 "refcnt state dev              mode file\n");
+	}
+
+        inode = iwin_inode(s, index);
+	show_inode(m, inode);
+
+	return 0;
+}
+
+static void *ii_start(struct seq_file *m, loff_t *pos)
+{
+	struct fcache_read_state *s = m->private;
+
+	s->iwin.size = 0;
+	s->iwin.inodes = (struct inode **)
+				__get_free_pages(GFP_KERNEL, IWIN_PAGE_ORDER);
+	if (!s->iwin.inodes)
+		return NULL;
+
+	spin_lock(&inode_lock);
+
+	return iwin_inode(s, *pos) ? pos : NULL;
+}
+
+static void *ii_next(struct seq_file *m, void *v, loff_t *pos)
+{
+	struct fcache_read_state *s = m->private;
+
+	(*pos)++;
+	return iwin_inode(s, *pos) ? pos : NULL;
+}
+
+static void ii_stop(struct seq_file *m, void *v)
+{
+	struct fcache_read_state *s = m->private;
+
+	if (!s->iwin.inodes)
+		return;		/* we don't have inode_lock */
+
+	spin_unlock(&inode_lock);
+	free_pages((unsigned long) s->iwin.inodes, IWIN_PAGE_ORDER);
+}
+
+static struct seq_operations ii_op = {
+	.start  = ii_start,
+	.next   = ii_next,
+	.stop   = ii_stop,
+	.show   = ii_show,
+};
+
+/*
+ * Proc file operations.
+ */
+static int filecache_open(struct inode *inode, struct file *proc_file)
+{
+	struct seq_file *m;
+	struct fcache_read_state *s;
+	int ret;
+
+	s = kmalloc(sizeof(*s), GFP_KERNEL);
+	if (!s) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = seq_open(proc_file, &ii_op);
+	if (ret) {
+		kfree(s);
+		goto out;
+	}
+
+	m = proc_file->private_data;
+	m->private = s;
+
+out:
+	return ret;
+}
+
+static int filecache_release(struct inode *inode, struct file *proc_file)
+{
+	struct seq_file *seq = proc_file->private_data;
+	struct fcache_read_state *s = seq ? seq->private : NULL;
+	int ret;
+
+	kfree(s);
+	ret = seq_release(inode, proc_file);
+	return ret;
+}
+
+static struct file_operations proc_filecache_fops = {
+	.open		= filecache_open,
+	.release	= filecache_release,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+};
+
+
+static __init int filecache_init(void)
+{
+	proc_create("filecache", 0600, NULL, &proc_filecache_fops);
+	return 0;
+}
+module_init(filecache_init);

[-- Attachment #3: 0001-Add-recent-rotated-scanned-info-to-proc-zoneinfo.patch --]
[-- Type: application/octet-stream, Size: 1049 bytes --]

From 53d1d55e95591891102cd7f1c3404f447e83dfd0 Mon Sep 17 00:00:00 2001
From: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Date: Thu, 19 Nov 2009 23:19:39 +0900
Subject: [PATCH] Add recent-rotated/scanned info to /proc/zoneinfo

Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
---
 mm/vmstat.c |   11 +++++++++++
 1 files changed, 11 insertions(+), 0 deletions(-)

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 889254f..9bf9451 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -765,6 +765,17 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
 		   zone->prev_priority,
 		   zone->zone_start_pfn,
 		   zone->inactive_ratio);
+
+	seq_printf(m,
+		   "\n  recent_rotated_anon: %lu"
+		   "\n  recent_scanned_anon: %lu"
+		   "\n  recent_rotated_file: %lu"
+		   "\n  recent_scanned_file: %lu",
+		   zone->reclaim_stat.recent_rotated[0],
+		   zone->reclaim_stat.recent_scanned[0],
+		   zone->reclaim_stat.recent_rotated[1],
+		   zone->reclaim_stat.recent_scanned[1]);
+
 	seq_putc(m, '\n');
 }
 
-- 
1.6.2.5