Zone reclaim V4 [1/3]: resurrect may_swap

Zone reclaim V4 [1/3]: resurrect may_swap

[Christoph Lameter]

Resurrect may_swap in struct scan_control

Patch against 2.6.15-rc5-mm3 to undo the patch to remove may_writepage.

Not needed for 2.6.14 / 2.6.15-rc6.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.15-rc5-mm3/mm/vmscan.c
===================================================================
--- linux-2.6.15-rc5-mm3.orig/mm/vmscan.c	2005-12-20 15:46:51.000000000 -0800
+++ linux-2.6.15-rc5-mm3/mm/vmscan.c	2005-12-21 12:32:43.000000000 -0800
@@ -71,6 +71,9 @@ struct scan_control {
 
 	int may_writepage;
 
+	/* Can pages be swapped as part of reclaim? */
+	int may_swap;
+
 	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
 	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
 	 * In this context, it doesn't matter that we scan the
@@ -457,6 +460,8 @@ static int shrink_list(struct list_head 
 		 * Try to allocate it some swap space here.
 		 */
 		if (PageAnon(page) && !PageSwapCache(page)) {
+			if (!sc->may_swap)
+				goto keep_locked;
 			if (!add_to_swap(page, GFP_ATOMIC))
 				goto activate_locked;
 		}
@@ -1180,6 +1185,7 @@ int try_to_free_pages(struct zone **zone
 
 	sc.gfp_mask = gfp_mask;
 	sc.may_writepage = 0;
+	sc.may_swap = 1;
 
 	count_event(ALLOCSTALL);
 
@@ -1282,6 +1288,7 @@ loop_again:
 	total_reclaimed = 0;
 	sc.gfp_mask = GFP_KERNEL;
 	sc.may_writepage = 0;
+	sc.may_swap = 1;
 	sc.nr_mapped = global_page_state(NR_MAPPED);
 
 	count_event(PAGEOUTRUN);

Zone reclaim V4 [2/3]: Reclaim logic based on zoned counters

[Christoph Lameter]

Zone reclaim allows the reclaiming of pages from a zone if the number of free
pages falls below the watermark even if other zones still have enough pages
available. Zone reclaim is of particular importance for NUMA machines. It can
be more beneficial to reclaim a page than taking the performance penalties
that come with allocating a page on a remote zone.

Zone reclaim is enabled if the maximum distance to another node is higher
than RECLAIM_DISTANCE, which may be defined by an arch. By default
RECLAIM_DISTANCE is 20 meaning the distance to another node in the
same component (enclosure or motherboard).

The zone_reclaim function already exists in 2.6.14/2.6.15-rc6. It was
removed in mm because zone reclaim was not usable. This patch makes it
work.

Patch depends on the first 4 patches of the zoned counter patchset.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.15-rc5-mm3/mm/page_alloc.c
===================================================================
--- linux-2.6.15-rc5-mm3.orig/mm/page_alloc.c	2005-12-20 15:46:53.000000000 -0800
+++ linux-2.6.15-rc5-mm3/mm/page_alloc.c	2005-12-21 12:33:49.000000000 -0800
@@ -1181,7 +1181,9 @@ get_page_from_freelist(gfp_t gfp_mask, u
 				mark = (*z)->pages_high;
 			if (!zone_watermark_ok(*z, order, mark,
 				    classzone_idx, alloc_flags))
-				continue;
+				if (!zone_reclaim_mode ||
+			        	!zone_reclaim(*z, gfp_mask, order))
+						continue;
 		}
 
 		page = buffered_rmqueue(zonelist, *z, order, gfp_mask);
@@ -1908,13 +1910,22 @@ static void __init build_zonelists(pg_da
 	prev_node = local_node;
 	nodes_clear(used_mask);
 	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
+		int distance = node_distance(local_node, node);
+
+		/*
+		 * If another node is sufficiently far away then it is better
+		 * to reclaim pages in a zone before going off node.
+		 */
+		if (distance > RECLAIM_DISTANCE)
+			zone_reclaim_mode = 1;
+
 		/*
 		 * We don't want to pressure a particular node.
 		 * So adding penalty to the first node in same
 		 * distance group to make it round-robin.
 		 */
-		if (node_distance(local_node, node) !=
-				node_distance(local_node, prev_node))
+
+		if (distance != node_distance(local_node, prev_node))
 			node_load[node] += load;
 		prev_node = node;
 		load--;
Index: linux-2.6.15-rc5-mm3/include/linux/swap.h
===================================================================
--- linux-2.6.15-rc5-mm3.orig/include/linux/swap.h	2005-12-16 11:44:09.000000000 -0800
+++ linux-2.6.15-rc5-mm3/include/linux/swap.h	2005-12-21 12:33:50.000000000 -0800
@@ -172,6 +172,17 @@ extern void swap_setup(void);
 
 /* linux/mm/vmscan.c */
 extern int try_to_free_pages(struct zone **, gfp_t);
+#ifdef CONFIG_NUMA
+extern int zone_reclaim_mode;
+extern int zone_reclaim(struct zone *, gfp_t, unsigned int);
+#else
+#define zone_reclaim_mode 0
+static inline int zone_reclaim(struct zone *z, gfp_t mask,
+				unsigned int order)
+{
+	return 0;
+}
+#endif
 extern int shrink_all_memory(int);
 extern int vm_swappiness;
 
Index: linux-2.6.15-rc5-mm3/include/linux/topology.h
===================================================================
--- linux-2.6.15-rc5-mm3.orig/include/linux/topology.h	2005-12-16 11:44:09.000000000 -0800
+++ linux-2.6.15-rc5-mm3/include/linux/topology.h	2005-12-21 12:33:50.000000000 -0800
@@ -56,6 +56,9 @@
 #define REMOTE_DISTANCE		20
 #define node_distance(from,to)	((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
 #endif
+#ifndef RECLAIM_DISTANCE
+#define RECLAIM_DISTANCE 20
+#endif
 #ifndef PENALTY_FOR_NODE_WITH_CPUS
 #define PENALTY_FOR_NODE_WITH_CPUS	(1)
 #endif
Index: linux-2.6.15-rc5-mm3/mm/vmscan.c
===================================================================
--- linux-2.6.15-rc5-mm3.orig/mm/vmscan.c	2005-12-21 12:32:43.000000000 -0800
+++ linux-2.6.15-rc5-mm3/mm/vmscan.c	2005-12-21 12:33:50.000000000 -0800
@@ -1586,3 +1586,60 @@ static int __init kswapd_init(void)
 }
 
 module_init(kswapd_init)
+
+#ifdef CONFIG_NUMA
+/*
+ * Zone reclaim mode
+ *
+ * If non-zero call zone_reclaim when the number of free pages falls below
+ * the watermarks.
+ */
+int zone_reclaim_mode __read_mostly;
+
+/*
+ * Try to free up some pages from this zone through reclaim.
+ */
+int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
+{
+	struct scan_control sc;
+	int nr_pages = 1 << order;
+	struct task_struct *p = current;
+	struct reclaim_state reclaim_state;
+
+	if (!(gfp_mask & __GFP_WAIT) ||
+	    zone->zone_pgdat->node_id != numa_node_id() ||
+	    zone->all_unreclaimable ||
+	    atomic_read(&zone->reclaim_in_progress) > 0)
+		return 0;
+
+	/*
+	 * Check if there is a reasonable amount of recoverable memory before
+	 * doing the scan.
+	 */
+	if (zone_page_state(zone, NR_PAGECACHE) <=
+			zone_page_state(zone, NR_MAPPED) + nr_pages)
+		return 0;
+
+	sc.gfp_mask = gfp_mask;
+	sc.may_writepage = 0;
+	sc.may_swap = 0;
+	sc.nr_mapped = global_page_state(NR_MAPPED);
+	sc.nr_scanned = 0;
+	sc.nr_reclaimed = 0;
+	sc.priority = 0;
+	disable_swap_token();
+
+	sc.swap_cluster_max = max(nr_pages, SWAP_CLUSTER_MAX);
+
+	cond_resched();
+	p->flags |= PF_MEMALLOC;
+	reclaim_state.reclaimed_slab = 0;
+	p->reclaim_state = &reclaim_state;
+	shrink_zone(zone, &sc);
+	p->reclaim_state = NULL;
+	current->flags &= ~PF_MEMALLOC;
+	cond_resched();
+	return sc.nr_reclaimed >= (1 << order);
+}
+#endif
+

Zone reclaim V4 [3/3]: Alternate logic without zoned counters

[Christoph Lameter]

Reduce frequency of unsuccessful zone reclaim attempts

This is a fallback patch if zoned vm counters are not available. In that
case no check for reclaimable pages can be made before starting the scan.

The scan may have to occur for every off node allocation once the node
is full. In order to deal with that situation we note the last time a
scan has failed and only retry once per tick.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.15-rc5-mm2/mm/vmscan.c
===================================================================
--- linux-2.6.15-rc5-mm2.orig/mm/vmscan.c	2005-12-12 16:50:20.000000000 -0800
+++ linux-2.6.15-rc5-mm2/mm/vmscan.c	2005-12-12 16:50:21.000000000 -0800
@@ -1842,6 +1842,16 @@ int zone_reclaim(struct zone *zone, gfp_
 	    atomic_read(&zone->reclaim_in_progress) > 0)
 		return 0;
 
+	/*
+	 * If an unsuccessful zone reclaim occurred in this tick then we
+	 * already needed to go off before. Our local purity is already
+	 * tainted and its likely that the scan for easily reclaimable pages
+	 * will be a waste of time. Continue off node allocations for the
+	 * duration of this tick.
+	 */
+	if (zone->last_unsuccessful_zone_reclaim == jiffies)
+		return 0;
+
 	sc.gfp_mask = gfp_mask;
 	sc.may_writepage = 0;
 	sc.nr_mapped = read_page_state(nr_mapped);
@@ -1859,6 +1869,8 @@ int zone_reclaim(struct zone *zone, gfp_
 	shrink_zone(zone, &sc);
 	p->reclaim_state = NULL;
 	current->flags &= ~PF_MEMALLOC;
+	if (sc.nr_reclaimed == 0)
+		zone->last_unsuccessful_zone_reclaim = jiffies;
 	cond_resched();
 	return sc.nr_reclaimed >= (1 << order);
 }
Index: linux-2.6.15-rc5-mm2/include/linux/mmzone.h
===================================================================
--- linux-2.6.15-rc5-mm2.orig/include/linux/mmzone.h	2005-12-12 09:10:34.000000000 -0800
+++ linux-2.6.15-rc5-mm2/include/linux/mmzone.h	2005-12-12 16:50:21.000000000 -0800
@@ -157,6 +157,8 @@ struct zone {
 	/* A count of how many reclaimers are scanning this zone */
 	atomic_t		reclaim_in_progress;
 
+	unsigned long		last_unsuccessful_zone_reclaim;
+
 	/*
 	 * prev_priority holds the scanning priority for this zone.  It is
 	 * defined as the scanning priority at which we achieved our reclaim